1256 lines
49 KiB
JavaScript
1256 lines
49 KiB
JavaScript
let processingData = [];
|
|
let currentSessionId = null; // Store session ID for report download
|
|
let resultsMap = null;
|
|
let markers = [];
|
|
let eventListenersInitialized = false;
|
|
|
|
// Check and display geocoding provider status
|
|
async function checkGeocodingProviders() {
|
|
const statusElement = document.getElementById('provider-status');
|
|
if (!statusElement) return;
|
|
|
|
try {
|
|
const response = await fetch('/api/geocode/provider-status', {
|
|
method: 'GET',
|
|
credentials: 'same-origin',
|
|
headers: {
|
|
'Accept': 'application/json',
|
|
'Content-Type': 'application/json'
|
|
}
|
|
});
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`HTTP error! status: ${response.status}`);
|
|
}
|
|
|
|
const data = await response.json();
|
|
|
|
if (data.success) {
|
|
const providers = data.providers;
|
|
let statusHTML = '<div class="provider-list">';
|
|
|
|
providers.forEach(provider => {
|
|
const icon = provider.available ? '✅' : '❌';
|
|
const status = provider.available ? 'Available' : 'Not configured';
|
|
const className = provider.available ? 'provider-available' : 'provider-unavailable';
|
|
|
|
statusHTML += `
|
|
<div class="provider-item ${className}">
|
|
<span class="provider-icon">${icon}</span>
|
|
<strong>${provider.name}</strong>: ${status}
|
|
${provider.name === 'Mapbox' && provider.available ?
|
|
'<span class="provider-premium">🌟 Premium</span>' : ''}
|
|
</div>
|
|
`;
|
|
});
|
|
|
|
statusHTML += '</div>';
|
|
|
|
// Add configuration note if no premium providers
|
|
const hasPremium = providers.some(p => p.available && ['Mapbox', 'LocationIQ'].includes(p.name));
|
|
if (!hasPremium) {
|
|
statusHTML += `
|
|
<div class="provider-note">
|
|
<p><strong>💡 Tip:</strong> For better geocoding accuracy, configure a premium provider:</p>
|
|
<ul>
|
|
<li><strong>Mapbox</strong>: Add <code>MAPBOX_ACCESS_TOKEN</code> to your .env file</li>
|
|
<li><strong>LocationIQ</strong>: Add <code>LOCATIONIQ_API_KEY</code> to your .env file</li>
|
|
</ul>
|
|
</div>
|
|
`;
|
|
} else if (providers.find(p => p.name === 'Mapbox' && p.available)) {
|
|
statusHTML += `
|
|
<div class="provider-note" style="background: #d1ecf1; border-color: #bee5eb;">
|
|
<p><strong>🌟 Mapbox Configured!</strong> Using premium geocoding for better accuracy.</p>
|
|
</div>
|
|
`;
|
|
}
|
|
|
|
statusElement.innerHTML = statusHTML;
|
|
} else {
|
|
statusElement.innerHTML = '<span class="error">❌ Failed to check provider status</span>';
|
|
}
|
|
} catch (error) {
|
|
console.error('Failed to check geocoding providers:', error);
|
|
statusElement.innerHTML = `
|
|
<div class="provider-error">
|
|
<span class="error">⚠️ Unable to check provider status</span>
|
|
<small>Using fallback providers: Nominatim, Photon, ArcGIS</small>
|
|
</div>
|
|
`;
|
|
}
|
|
}
|
|
|
|
// Utility function to show status messages
|
|
function showDataConvertStatus(message, type = 'info') {
|
|
// Try to use the global showStatus from admin.js if available
|
|
if (typeof window.showStatus === 'function') {
|
|
return window.showStatus(message, type);
|
|
}
|
|
|
|
// Fallback to console
|
|
console.log(`[${type.toUpperCase()}] ${message}`);
|
|
|
|
// Try to display in status container if it exists
|
|
const statusContainer = document.getElementById('status-container');
|
|
if (statusContainer) {
|
|
const statusEl = document.createElement('div');
|
|
statusEl.className = `status-message status-${type}`;
|
|
statusEl.textContent = message;
|
|
statusContainer.appendChild(statusEl);
|
|
|
|
// Auto-remove after 5 seconds
|
|
setTimeout(() => {
|
|
if (statusEl.parentNode) {
|
|
statusEl.parentNode.removeChild(statusEl);
|
|
}
|
|
}, 5000);
|
|
}
|
|
}
|
|
|
|
// Initialize when DOM is loaded
|
|
document.addEventListener('DOMContentLoaded', () => {
|
|
console.log('Data convert JS loaded');
|
|
// Don't auto-initialize, wait for section to be activated
|
|
|
|
// Make setupDataConvertEventListeners available globally for admin.js
|
|
window.setupDataConvertEventListeners = setupDataConvertEventListeners;
|
|
});
|
|
|
|
function setupDataConvertEventListeners() {
|
|
console.log('Setting up data convert event listeners...');
|
|
|
|
// Prevent duplicate event listeners
|
|
if (eventListenersInitialized) {
|
|
console.log('Event listeners already initialized, skipping...');
|
|
return;
|
|
}
|
|
|
|
// Check geocoding provider status when section loads
|
|
checkGeocodingProviders();
|
|
|
|
const fileInput = document.getElementById('csv-file-input');
|
|
const browseBtn = document.getElementById('browse-btn');
|
|
const uploadArea = document.getElementById('upload-area');
|
|
const uploadForm = document.getElementById('csv-upload-form');
|
|
const clearBtn = document.getElementById('clear-upload-btn');
|
|
const saveResultsBtn = document.getElementById('save-results-btn');
|
|
const newUploadBtn = document.getElementById('new-upload-btn');
|
|
|
|
console.log('Elements found:', {
|
|
fileInput: !!fileInput,
|
|
browseBtn: !!browseBtn,
|
|
uploadArea: !!uploadArea,
|
|
uploadForm: !!uploadForm,
|
|
clearBtn: !!clearBtn,
|
|
saveResultsBtn: !!saveResultsBtn,
|
|
newUploadBtn: !!newUploadBtn
|
|
});
|
|
|
|
// File input change
|
|
if (fileInput) {
|
|
fileInput.addEventListener('change', handleFileSelect);
|
|
}
|
|
|
|
// Browse button
|
|
if (browseBtn) {
|
|
browseBtn.addEventListener('click', () => {
|
|
console.log('Browse button clicked');
|
|
fileInput?.click();
|
|
});
|
|
}
|
|
|
|
// Drag and drop
|
|
if (uploadArea) {
|
|
uploadArea.addEventListener('dragover', handleDragOver);
|
|
uploadArea.addEventListener('dragleave', handleDragLeave);
|
|
uploadArea.addEventListener('drop', handleDrop);
|
|
uploadArea.addEventListener('click', () => fileInput?.click());
|
|
}
|
|
|
|
// Form submission
|
|
if (uploadForm) {
|
|
uploadForm.addEventListener('submit', handleCSVUpload);
|
|
}
|
|
|
|
// Clear button
|
|
if (clearBtn) {
|
|
clearBtn.addEventListener('click', clearUpload);
|
|
}
|
|
|
|
// Save results button - ADD DATA TO MAP
|
|
if (saveResultsBtn) {
|
|
saveResultsBtn.addEventListener('click', saveGeocodedResults);
|
|
console.log('Save results button event listener attached');
|
|
}
|
|
|
|
// New upload button
|
|
if (newUploadBtn) {
|
|
newUploadBtn.addEventListener('click', resetToUpload);
|
|
}
|
|
|
|
// Scan & Geocode Database button
|
|
const scanGeocodeBtn = document.getElementById('scan-geocode-btn');
|
|
const cancelScanBtn = document.getElementById('cancel-scan-btn');
|
|
const downloadScanReportBtn = document.getElementById('download-scan-report-btn');
|
|
const newScanBtn = document.getElementById('new-scan-btn');
|
|
|
|
if (scanGeocodeBtn) {
|
|
scanGeocodeBtn.addEventListener('click', startDatabaseScan);
|
|
console.log('Scan geocode button event listener attached');
|
|
}
|
|
|
|
if (cancelScanBtn) {
|
|
cancelScanBtn.addEventListener('click', cancelDatabaseScan);
|
|
}
|
|
|
|
if (downloadScanReportBtn) {
|
|
downloadScanReportBtn.addEventListener('click', downloadScanReport);
|
|
}
|
|
|
|
if (newScanBtn) {
|
|
newScanBtn.addEventListener('click', resetScanInterface);
|
|
}
|
|
|
|
// Mark as initialized
|
|
eventListenersInitialized = true;
|
|
console.log('Data convert event listeners initialized successfully');
|
|
}
|
|
|
|
function handleFileSelect(e) {
|
|
const file = e.target.files[0];
|
|
if (file) {
|
|
displayFileInfo(file);
|
|
}
|
|
}
|
|
|
|
function handleDragOver(e) {
|
|
e.preventDefault();
|
|
e.currentTarget.classList.add('drag-over');
|
|
}
|
|
|
|
function handleDragLeave(e) {
|
|
e.preventDefault();
|
|
e.currentTarget.classList.remove('drag-over');
|
|
}
|
|
|
|
function handleDrop(e) {
|
|
e.preventDefault();
|
|
e.currentTarget.classList.remove('drag-over');
|
|
|
|
const files = e.dataTransfer.files;
|
|
if (files.length > 0) {
|
|
const file = files[0];
|
|
if (file.type === 'text/csv' || file.name.endsWith('.csv')) {
|
|
document.getElementById('csv-file-input').files = files;
|
|
displayFileInfo(file);
|
|
} else {
|
|
showDataConvertStatus('Please upload a CSV file', 'error');
|
|
}
|
|
}
|
|
}
|
|
|
|
function displayFileInfo(file) {
|
|
document.getElementById('file-info').style.display = 'block';
|
|
document.getElementById('file-name').textContent = file.name;
|
|
document.getElementById('file-size').textContent = formatFileSize(file.size);
|
|
document.getElementById('process-csv-btn').disabled = false;
|
|
}
|
|
|
|
function formatFileSize(bytes) {
|
|
if (bytes === 0) return '0 Bytes';
|
|
const k = 1024;
|
|
const sizes = ['Bytes', 'KB', 'MB'];
|
|
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
|
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
|
|
}
|
|
|
|
async function handleCSVUpload(e) {
|
|
e.preventDefault();
|
|
|
|
const fileInput = document.getElementById('csv-file-input');
|
|
const file = fileInput?.files[0];
|
|
|
|
if (!file) {
|
|
showDataConvertStatus('Please select a CSV file', 'error');
|
|
return;
|
|
}
|
|
|
|
// Reset processing data
|
|
processingData = [];
|
|
|
|
// Show processing section
|
|
const uploadSection = document.getElementById('upload-section');
|
|
const processingSection = document.getElementById('processing-section');
|
|
|
|
if (!uploadSection || !processingSection) {
|
|
console.error('Required DOM elements not found');
|
|
showDataConvertStatus('Interface error: required elements not found', 'error');
|
|
return;
|
|
}
|
|
|
|
uploadSection.style.display = 'none';
|
|
processingSection.style.display = 'block';
|
|
|
|
// Initialize results map
|
|
initializeResultsMap();
|
|
|
|
// Create form data
|
|
const formData = new FormData();
|
|
formData.append('csvFile', file);
|
|
|
|
try {
|
|
// Send the file and handle SSE response
|
|
const response = await fetch('/api/admin/data-convert/process-csv', {
|
|
method: 'POST',
|
|
body: formData
|
|
});
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`HTTP error! status: ${response.status}`);
|
|
}
|
|
|
|
// Read the response as a stream for SSE
|
|
const reader = response.body.getReader();
|
|
const decoder = new TextDecoder();
|
|
let buffer = ''; // Buffer to accumulate partial data
|
|
|
|
while (true) {
|
|
const { done, value } = await reader.read();
|
|
if (done) break;
|
|
|
|
// Decode the chunk and add to buffer
|
|
buffer += decoder.decode(value, { stream: true });
|
|
|
|
// Split buffer by lines and process complete lines
|
|
const lines = buffer.split('\n');
|
|
|
|
// Keep the last potentially incomplete line in the buffer
|
|
buffer = lines.pop() || '';
|
|
|
|
for (const line of lines) {
|
|
if (line.startsWith('data: ')) {
|
|
try {
|
|
const jsonData = line.substring(6).trim(); // Remove 'data: ' prefix and trim
|
|
if (jsonData && jsonData !== '') {
|
|
const data = JSON.parse(jsonData);
|
|
handleProcessingUpdate(data);
|
|
}
|
|
} catch (parseError) {
|
|
console.warn('Failed to parse SSE data:', parseError);
|
|
console.warn('Problematic line:', line);
|
|
console.warn('JSON data:', line.substring(6));
|
|
}
|
|
} else if (line.trim() === '') {
|
|
// Empty line, ignore
|
|
continue;
|
|
} else if (line.trim() !== '' && !line.startsWith('event:') && !line.startsWith('id:')) {
|
|
// Unexpected line format
|
|
console.warn('Unexpected SSE line format:', line);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Process any remaining data in buffer
|
|
if (buffer.trim()) {
|
|
const line = buffer;
|
|
if (line.startsWith('data: ')) {
|
|
try {
|
|
const jsonData = line.substring(6).trim();
|
|
if (jsonData && jsonData !== '') {
|
|
const data = JSON.parse(jsonData);
|
|
handleProcessingUpdate(data);
|
|
}
|
|
} catch (parseError) {
|
|
console.warn('Failed to parse final SSE data:', parseError);
|
|
console.warn('Final buffer content:', buffer);
|
|
}
|
|
}
|
|
}
|
|
|
|
} catch (error) {
|
|
console.error('CSV processing error:', error);
|
|
showDataConvertStatus(error.message || 'Failed to process CSV', 'error');
|
|
resetToUpload();
|
|
}
|
|
}
|
|
|
|
// Enhanced processing update handler
|
|
function handleProcessingUpdate(data) {
|
|
console.log('Processing update:', data);
|
|
|
|
// Validate data structure
|
|
if (!data || typeof data !== 'object') {
|
|
console.warn('Invalid data received:', data);
|
|
return;
|
|
}
|
|
|
|
switch (data.type) {
|
|
case 'start':
|
|
updateProgress(0, data.total);
|
|
document.getElementById('current-address').textContent = 'Starting geocoding process...';
|
|
break;
|
|
|
|
case 'progress':
|
|
updateProgress(data.current, data.total);
|
|
|
|
// Show current address with status
|
|
if (data.status === 'failed') {
|
|
document.getElementById('current-address').innerHTML = `<span style="color: red;">✗ ${data.currentAddress || data.address}</span>`;
|
|
} else if (data.status === 'processing') {
|
|
document.getElementById('current-address').innerHTML = `<span style="color: blue;">⟳ Processing: ${data.currentAddress || data.address}</span>`;
|
|
} else {
|
|
document.getElementById('current-address').innerHTML = `<span style="color: green;">✓ ${data.currentAddress || data.address}</span>`;
|
|
}
|
|
break;
|
|
|
|
case 'geocoded':
|
|
// Check if result has warnings
|
|
const isWarning = data.status === 'warning' || (data.data && data.data.is_malformed);
|
|
const confidence = data.confidence || (data.data && data.data.confidence_score) || 100;
|
|
const warnings = data.warnings || (data.data && data.data.warnings) || [];
|
|
|
|
// Mark data with appropriate status and add to processing data
|
|
const successData = {
|
|
...data.data,
|
|
geocode_success: true,
|
|
confidence_score: confidence,
|
|
warnings: Array.isArray(warnings) ? warnings.join('; ') : warnings
|
|
};
|
|
processingData.push(successData);
|
|
|
|
// Add to table with appropriate status
|
|
const resultStatus = isWarning ? 'warning' : 'success';
|
|
addResultToTable(successData, resultStatus, confidence, warnings);
|
|
addMarkerToMap(successData, isWarning);
|
|
updateProgress(data.index + 1, data.total);
|
|
break;
|
|
|
|
case 'error':
|
|
// Mark as failed and add to processing data
|
|
const errorData = { ...data.data, geocode_success: false };
|
|
processingData.push(errorData);
|
|
addResultToTable(errorData, 'error');
|
|
break;
|
|
|
|
case 'complete':
|
|
console.log('Received complete event:', data);
|
|
currentSessionId = data.sessionId; // Store session ID for report download
|
|
|
|
// Show comprehensive completion message
|
|
let completionMessage = `Complete! Processed ${data.total} addresses:\n`;
|
|
completionMessage += `✓ ${data.successful || 0} successful\n`;
|
|
if (data.warnings > 0) {
|
|
completionMessage += `⚠ ${data.warnings} with warnings (low confidence)\n`;
|
|
}
|
|
if (data.malformed > 0) {
|
|
completionMessage += `🔍 ${data.malformed} potentially malformed (need review)\n`;
|
|
}
|
|
completionMessage += `✗ ${data.errors || data.failed || 0} failed`;
|
|
|
|
document.getElementById('current-address').innerHTML = completionMessage.replace(/\n/g, '<br>');
|
|
|
|
onProcessingComplete(data);
|
|
break;
|
|
|
|
case 'fatal_error':
|
|
showDataConvertStatus(data.message, 'error');
|
|
resetToUpload();
|
|
break;
|
|
|
|
default:
|
|
console.warn('Unknown data type received:', data.type);
|
|
}
|
|
}
|
|
|
|
function updateProgress(current, total) {
|
|
const percentage = (current / total) * 100;
|
|
document.getElementById('progress-bar-fill').style.width = percentage + '%';
|
|
document.getElementById('progress-current').textContent = current;
|
|
document.getElementById('progress-total').textContent = total;
|
|
}
|
|
|
|
function initializeResultsMap() {
|
|
const mapContainer = document.getElementById('results-map');
|
|
if (!mapContainer) return;
|
|
|
|
// Initialize map
|
|
resultsMap = L.map('results-map').setView([53.5461, -113.4938], 11);
|
|
|
|
L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', {
|
|
attribution: '© OpenStreetMap contributors'
|
|
}).addTo(resultsMap);
|
|
|
|
document.getElementById('results-preview').style.display = 'block';
|
|
|
|
// Fix map sizing
|
|
setTimeout(() => {
|
|
resultsMap.invalidateSize();
|
|
}, 100);
|
|
}
|
|
|
|
function addResultToTable(data, status, confidence = null, warnings = []) {
|
|
const tbody = document.getElementById('results-tbody');
|
|
const row = document.createElement('tr');
|
|
|
|
// Set row class based on status
|
|
if (status === 'success') {
|
|
row.className = 'result-success';
|
|
} else if (status === 'warning') {
|
|
row.className = 'result-warning';
|
|
} else {
|
|
row.className = 'result-error';
|
|
}
|
|
|
|
if (status === 'success' || status === 'warning') {
|
|
// Add confidence indicator for successful geocoding
|
|
let statusIcon = status === 'warning' ?
|
|
`<span class="status-icon warning" title="Low confidence result">⚠</span>` :
|
|
`<span class="status-icon success">✓</span>`;
|
|
|
|
if (confidence !== null && confidence < 100) {
|
|
statusIcon += ` <small>(${Math.round(confidence)}%)</small>`;
|
|
}
|
|
|
|
let addressCell = escapeHtml(data.geocoded_address || '');
|
|
if (warnings && warnings.length > 0) {
|
|
const warningText = Array.isArray(warnings) ? warnings.join(', ') : warnings;
|
|
addressCell += `<br><small style="color: orange;" title="${escapeHtml(warningText)}">⚠ ${escapeHtml(warningText)}</small>`;
|
|
}
|
|
|
|
row.innerHTML = `
|
|
<td>${statusIcon}</td>
|
|
<td>${escapeHtml(data.address || data.Address || '')}</td>
|
|
<td>${addressCell}</td>
|
|
<td>${data.latitude.toFixed(6)}, ${data.longitude.toFixed(6)}</td>
|
|
<td>${data.provider ? escapeHtml(data.provider) : 'N/A'}</td>
|
|
`;
|
|
} else {
|
|
row.innerHTML = `
|
|
<td><span class="status-icon error">✗</span></td>
|
|
<td>${escapeHtml(data.address || data.Address || '')}</td>
|
|
<td colspan="3" class="error-message">${escapeHtml(data.geocode_error || data.error || 'Geocoding failed')}</td>
|
|
`;
|
|
}
|
|
|
|
tbody.appendChild(row);
|
|
}
|
|
|
|
function addMarkerToMap(data, isWarning = false) {
|
|
if (!resultsMap || !data.latitude || !data.longitude) return;
|
|
|
|
// Choose marker color based on status
|
|
const markerColor = isWarning ? 'orange' : 'green';
|
|
|
|
const marker = L.marker([data.latitude, data.longitude], {
|
|
icon: L.divIcon({
|
|
className: `custom-marker ${isWarning ? 'warning-marker' : 'success-marker'}`,
|
|
html: `<div style="background-color: ${markerColor}; width: 12px; height: 12px; border-radius: 50%; border: 2px solid white; box-shadow: 0 1px 3px rgba(0,0,0,0.3);"></div>`,
|
|
iconSize: [16, 16],
|
|
iconAnchor: [8, 8]
|
|
})
|
|
});
|
|
|
|
// Create popup content with warning information
|
|
let popupContent = `<strong>${escapeHtml(data.geocoded_address || data.address)}</strong><br>`;
|
|
popupContent += `${data.latitude.toFixed(6)}, ${data.longitude.toFixed(6)}`;
|
|
|
|
if (data.provider) {
|
|
popupContent += `<br><small>Provider: ${data.provider}</small>`;
|
|
}
|
|
|
|
if (isWarning && data.confidence_score) {
|
|
popupContent += `<br><span style="color: orange;">⚠ Confidence: ${Math.round(data.confidence_score)}%</span>`;
|
|
}
|
|
|
|
if (isWarning && data.warnings) {
|
|
popupContent += `<br><small style="color: orange;">${escapeHtml(data.warnings)}</small>`;
|
|
}
|
|
|
|
marker.bindPopup(popupContent);
|
|
marker.addTo(resultsMap);
|
|
markers.push(marker);
|
|
|
|
// Adjust map bounds to show all markers
|
|
if (markers.length > 0) {
|
|
const group = new L.featureGroup(markers);
|
|
resultsMap.fitBounds(group.getBounds().pad(0.1));
|
|
}
|
|
}
|
|
|
|
function onProcessingComplete(data) {
|
|
console.log('Processing complete called with data:', data);
|
|
|
|
const processedCount = data.processed || data.successful || processingData.filter(item => item.geocode_success !== false).length;
|
|
|
|
// Show comprehensive processing actions with download option
|
|
if (processedCount > 0) {
|
|
console.log('Showing processing actions for', processedCount, 'successful items');
|
|
const actionsDiv = document.getElementById('processing-actions');
|
|
if (actionsDiv) {
|
|
actionsDiv.style.display = 'block';
|
|
console.log('Processing actions div is now visible');
|
|
} else {
|
|
console.error('processing-actions div not found!');
|
|
}
|
|
}
|
|
|
|
// Add download report button if session ID is available
|
|
if (currentSessionId) {
|
|
showDownloadReportButton(data);
|
|
}
|
|
|
|
// Update final message with detailed statistics
|
|
let finalMessage = `Geocoding Complete!\n\n`;
|
|
finalMessage += `📊 Summary:\n`;
|
|
finalMessage += `• Total Processed: ${data.total || 0}\n`;
|
|
finalMessage += `• ✅ Successful: ${data.successful || 0}\n`;
|
|
|
|
if (data.warnings > 0) {
|
|
finalMessage += `• ⚠️ Warnings: ${data.warnings} (low confidence, review recommended)\n`;
|
|
}
|
|
|
|
if (data.malformed > 0) {
|
|
finalMessage += `• 🔍 Potentially Malformed: ${data.malformed} (need manual review)\n`;
|
|
}
|
|
|
|
finalMessage += `• ❌ Failed: ${data.errors || data.failed || 0}\n\n`;
|
|
|
|
if (data.warnings > 0 || data.malformed > 0) {
|
|
finalMessage += `⚠️ Note: ${(data.warnings || 0) + (data.malformed || 0)} addresses may need manual verification.\n`;
|
|
finalMessage += `Please download the detailed report for review.`;
|
|
} else if ((data.successful || 0) > 0) {
|
|
finalMessage += `✅ All geocoded addresses appear to have high confidence results!`;
|
|
}
|
|
|
|
showDataConvertStatus(finalMessage, (data.errors || data.failed || 0) > 0 ? 'warning' : 'success');
|
|
}
|
|
|
|
// Add download report button function
|
|
function showDownloadReportButton(data) {
|
|
const processingActions = document.getElementById('processing-actions');
|
|
if (!processingActions) return;
|
|
|
|
// Check if download button already exists
|
|
let downloadBtn = document.getElementById('download-report-btn');
|
|
if (!downloadBtn) {
|
|
downloadBtn = document.createElement('button');
|
|
downloadBtn.id = 'download-report-btn';
|
|
downloadBtn.className = 'btn btn-info';
|
|
downloadBtn.innerHTML = '📄 Download Detailed Report';
|
|
downloadBtn.addEventListener('click', downloadProcessingReport);
|
|
|
|
// Insert download button before save results button
|
|
const saveBtn = document.getElementById('save-results-btn');
|
|
if (saveBtn && saveBtn.parentNode === processingActions) {
|
|
processingActions.insertBefore(downloadBtn, saveBtn);
|
|
} else {
|
|
processingActions.appendChild(downloadBtn);
|
|
}
|
|
}
|
|
|
|
// Update button text with statistics
|
|
const warningsCount = (data.warnings || 0) + (data.malformed || 0);
|
|
if (warningsCount > 0) {
|
|
downloadBtn.innerHTML = `📄 Download Report (${warningsCount} need review)`;
|
|
downloadBtn.className = 'btn btn-warning';
|
|
}
|
|
}
|
|
|
|
// Download processing report function
|
|
async function downloadProcessingReport() {
|
|
if (!currentSessionId) {
|
|
showDataConvertStatus('No report available to download', 'error');
|
|
return;
|
|
}
|
|
|
|
try {
|
|
const response = await fetch(`/api/admin/data-convert/download-report/${currentSessionId}`);
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`HTTP error! status: ${response.status}`);
|
|
}
|
|
|
|
// Get the filename from the response headers
|
|
const filename = response.headers.get('content-disposition')
|
|
?.split('filename=')[1]
|
|
?.replace(/"/g, '') || `geocoding-report-${currentSessionId}.txt`;
|
|
|
|
// Create blob and download
|
|
const blob = await response.blob();
|
|
const url = window.URL.createObjectURL(blob);
|
|
const a = document.createElement('a');
|
|
a.href = url;
|
|
a.download = filename;
|
|
document.body.appendChild(a);
|
|
a.click();
|
|
window.URL.revokeObjectURL(url);
|
|
document.body.removeChild(a);
|
|
|
|
showDataConvertStatus('Report downloaded successfully', 'success');
|
|
|
|
} catch (error) {
|
|
console.error('Download report error:', error);
|
|
showDataConvertStatus('Failed to download report: ' + error.message, 'error');
|
|
}
|
|
}
|
|
async function saveGeocodedResults() {
|
|
const successfulData = processingData.filter(item => item.geocode_success !== false && item.latitude && item.longitude);
|
|
|
|
if (successfulData.length === 0) {
|
|
showDataConvertStatus('No successfully geocoded data to save', 'error');
|
|
return;
|
|
}
|
|
|
|
// Disable save button and show progress
|
|
const saveBtn = document.getElementById('save-results-btn');
|
|
const originalText = saveBtn.textContent;
|
|
saveBtn.disabled = true;
|
|
saveBtn.textContent = 'Adding to map...';
|
|
|
|
// Show saving status
|
|
showDataConvertStatus(`Adding ${successfulData.length} locations to map...`, 'info');
|
|
|
|
let successCount = 0;
|
|
let failedCount = 0;
|
|
const errors = [];
|
|
|
|
try {
|
|
// Use the bulk save endpoint instead of individual location creation
|
|
// This is more efficient and avoids rate limiting issues
|
|
const response = await fetch('/api/admin/data-convert/save-geocoded', {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json'
|
|
},
|
|
body: JSON.stringify({ data: successfulData })
|
|
});
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`HTTP error! status: ${response.status}`);
|
|
}
|
|
|
|
const result = await response.json();
|
|
|
|
if (result.success) {
|
|
successCount = result.results.success;
|
|
failedCount = result.results.failed;
|
|
|
|
// Show detailed errors if any
|
|
if (result.results.errors && result.results.errors.length > 0) {
|
|
result.results.errors.forEach((error, index) => {
|
|
errors.push(`Location ${index + 1}: ${error.error}`);
|
|
});
|
|
}
|
|
|
|
console.log(`Bulk save completed: ${successCount} successful, ${failedCount} failed`);
|
|
} else {
|
|
throw new Error(result.error || 'Bulk save failed');
|
|
}
|
|
|
|
// Show final results
|
|
if (successCount > 0) {
|
|
const message = `Successfully added ${successCount} locations to map.` +
|
|
(failedCount > 0 ? ` ${failedCount} failed.` : '');
|
|
showDataConvertStatus(message, failedCount > 0 ? 'warning' : 'success');
|
|
|
|
// Update UI to show completion
|
|
saveBtn.textContent = `Added ${successCount} locations!`;
|
|
setTimeout(() => {
|
|
saveBtn.style.display = 'none';
|
|
document.getElementById('new-upload-btn').style.display = 'inline-block';
|
|
}, 3000);
|
|
|
|
} else {
|
|
throw new Error('Failed to add any locations to the map');
|
|
}
|
|
|
|
// Log errors if any
|
|
if (errors.length > 0) {
|
|
console.error('Import errors:', errors);
|
|
}
|
|
|
|
} catch (error) {
|
|
console.error('Save error:', error);
|
|
showDataConvertStatus('Failed to add data to map: ' + error.message, 'error');
|
|
saveBtn.disabled = false;
|
|
saveBtn.textContent = originalText;
|
|
}
|
|
}
|
|
|
|
// Utility function to escape HTML
|
|
function escapeHtml(text) {
|
|
const map = {
|
|
'&': '&',
|
|
'<': '<',
|
|
'>': '>',
|
|
'"': '"',
|
|
"'": '''
|
|
};
|
|
return text ? text.replace(/[&<>"']/g, function(m) { return map[m]; }) : '';
|
|
}
|
|
|
|
// Clear upload and reset form
|
|
function clearUpload() {
|
|
const fileInput = document.getElementById('csv-file-input');
|
|
const fileInfo = document.getElementById('file-info');
|
|
const processBtn = document.getElementById('process-csv-btn');
|
|
|
|
if (fileInput) fileInput.value = '';
|
|
if (fileInfo) fileInfo.style.display = 'none';
|
|
if (processBtn) processBtn.disabled = true;
|
|
}
|
|
|
|
// Reset to upload state
|
|
function resetToUpload() {
|
|
console.log('Resetting to upload state');
|
|
|
|
const uploadSection = document.getElementById('upload-section');
|
|
const processingSection = document.getElementById('processing-section');
|
|
const actionsDiv = document.getElementById('processing-actions');
|
|
const resultsPreview = document.getElementById('results-preview');
|
|
const saveBtn = document.getElementById('save-results-btn');
|
|
const newUploadBtn = document.getElementById('new-upload-btn');
|
|
|
|
if (uploadSection) uploadSection.style.display = 'block';
|
|
if (processingSection) processingSection.style.display = 'none';
|
|
if (actionsDiv) actionsDiv.style.display = 'none';
|
|
if (resultsPreview) resultsPreview.style.display = 'none';
|
|
|
|
// Reset buttons
|
|
if (saveBtn) {
|
|
saveBtn.style.display = 'inline-block';
|
|
saveBtn.disabled = false;
|
|
saveBtn.textContent = 'Add Data to Map';
|
|
}
|
|
if (newUploadBtn) {
|
|
newUploadBtn.style.display = 'none';
|
|
}
|
|
|
|
// Clear any existing data
|
|
processingData = [];
|
|
currentSessionId = null; // Reset session ID
|
|
|
|
// Remove download report button if it exists
|
|
const downloadBtn = document.getElementById('download-report-btn');
|
|
if (downloadBtn) {
|
|
downloadBtn.remove();
|
|
}
|
|
if (markers && markers.length > 0) {
|
|
markers.forEach(marker => {
|
|
if (resultsMap && marker) {
|
|
try {
|
|
resultsMap.removeLayer(marker);
|
|
} catch (e) {
|
|
console.warn('Error removing marker:', e);
|
|
}
|
|
}
|
|
});
|
|
}
|
|
markers = [];
|
|
|
|
// Reset results table
|
|
const tbody = document.getElementById('results-tbody');
|
|
if (tbody) {
|
|
tbody.innerHTML = '';
|
|
}
|
|
|
|
// Reset progress
|
|
const progressFill = document.getElementById('progress-bar-fill');
|
|
const progressCurrent = document.getElementById('progress-current');
|
|
const progressTotal = document.getElementById('progress-total');
|
|
const currentAddress = document.getElementById('current-address');
|
|
|
|
if (progressFill) progressFill.style.width = '0%';
|
|
if (progressCurrent) progressCurrent.textContent = '0';
|
|
if (progressTotal) progressTotal.textContent = '0';
|
|
if (currentAddress) currentAddress.textContent = '';
|
|
|
|
// Destroy existing map if it exists
|
|
if (resultsMap) {
|
|
try {
|
|
resultsMap.remove();
|
|
resultsMap = null;
|
|
} catch (e) {
|
|
console.warn('Error removing map:', e);
|
|
}
|
|
}
|
|
|
|
// Reset form
|
|
clearUpload();
|
|
}
|
|
|
|
// Show download report button
|
|
function showDownloadReportButton(totalRecords, errorCount) {
|
|
const actionsDiv = document.getElementById('processing-actions');
|
|
if (!actionsDiv) return;
|
|
|
|
// Check if button already exists
|
|
let downloadBtn = document.getElementById('download-report-btn');
|
|
if (!downloadBtn) {
|
|
downloadBtn = document.createElement('button');
|
|
downloadBtn.id = 'download-report-btn';
|
|
downloadBtn.type = 'button';
|
|
downloadBtn.className = 'btn btn-info';
|
|
downloadBtn.addEventListener('click', downloadProcessingReport);
|
|
|
|
// Insert before the save results button
|
|
const saveBtn = document.getElementById('save-results-btn');
|
|
if (saveBtn) {
|
|
actionsDiv.insertBefore(downloadBtn, saveBtn);
|
|
} else {
|
|
actionsDiv.appendChild(downloadBtn);
|
|
}
|
|
}
|
|
|
|
// Update button text with error info
|
|
if (errorCount > 0) {
|
|
downloadBtn.textContent = `📄 Download Full Report (${totalRecords} records, ${errorCount} errors)`;
|
|
downloadBtn.title = `Download complete processing report including ${errorCount} error records for review`;
|
|
} else {
|
|
downloadBtn.textContent = `📄 Download Processing Report (${totalRecords} records)`;
|
|
downloadBtn.title = 'Download complete processing report';
|
|
}
|
|
|
|
downloadBtn.style.marginRight = '10px'; // Add some spacing
|
|
}
|
|
|
|
// Download processing report
|
|
async function downloadProcessingReport() {
|
|
if (!currentSessionId) {
|
|
showDataConvertStatus('No processing session available for report generation', 'error');
|
|
return;
|
|
}
|
|
|
|
const downloadBtn = document.getElementById('download-report-btn');
|
|
const originalText = downloadBtn.textContent;
|
|
|
|
try {
|
|
downloadBtn.disabled = true;
|
|
downloadBtn.textContent = '📄 Generating Report...';
|
|
|
|
showDataConvertStatus('Generating processing report...', 'info');
|
|
|
|
// Create download link
|
|
const downloadUrl = `/api/admin/data-convert/download-report/${currentSessionId}`;
|
|
|
|
// Create a temporary anchor element and trigger download
|
|
const link = document.createElement('a');
|
|
link.href = downloadUrl;
|
|
link.download = `geocoding-report-${currentSessionId}.csv`;
|
|
link.style.display = 'none';
|
|
|
|
document.body.appendChild(link);
|
|
link.click();
|
|
document.body.removeChild(link);
|
|
|
|
showDataConvertStatus('Processing report downloaded successfully', 'success');
|
|
|
|
// Update button text
|
|
downloadBtn.textContent = '📄 Report Downloaded';
|
|
setTimeout(() => {
|
|
downloadBtn.textContent = originalText;
|
|
downloadBtn.disabled = false;
|
|
}, 3000);
|
|
|
|
} catch (error) {
|
|
console.error('Download report error:', error);
|
|
showDataConvertStatus('Failed to download processing report: ' + error.message, 'error');
|
|
downloadBtn.textContent = originalText;
|
|
downloadBtn.disabled = false;
|
|
}
|
|
}
|
|
|
|
// === DATABASE SCAN & GEOCODE FUNCTIONALITY ===
|
|
|
|
async function startDatabaseScan() {
|
|
try {
|
|
const scanBtn = document.getElementById('scan-geocode-btn');
|
|
const cancelBtn = document.getElementById('cancel-scan-btn');
|
|
const processingSection = document.getElementById('scan-processing-section');
|
|
const scanPhase = document.getElementById('scan-phase');
|
|
const scanStatus = document.getElementById('scan-status');
|
|
const progressContainer = document.getElementById('scan-progress-container');
|
|
const summary = document.getElementById('scan-summary');
|
|
const resultsPreview = document.getElementById('scan-results-preview');
|
|
const actionsCompleted = document.getElementById('scan-actions-completed');
|
|
|
|
// Show processing UI and hide scan button
|
|
if (scanBtn) scanBtn.style.display = 'none';
|
|
if (cancelBtn) cancelBtn.style.display = 'inline-block';
|
|
if (processingSection) processingSection.style.display = 'block';
|
|
|
|
// Reset UI state
|
|
if (progressContainer) progressContainer.style.display = 'none';
|
|
if (summary) summary.style.display = 'none';
|
|
if (resultsPreview) resultsPreview.style.display = 'none';
|
|
if (actionsCompleted) actionsCompleted.style.display = 'none';
|
|
|
|
// Clear previous results
|
|
const scanResultsTable = document.getElementById('scan-results-tbody');
|
|
if (scanResultsTable) scanResultsTable.innerHTML = '';
|
|
|
|
console.log('Starting database scan and geocode...');
|
|
|
|
// Make POST request for Server-Sent Events
|
|
const response = await fetch('/api/admin/data-convert/scan-and-geocode', {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json',
|
|
'Accept': 'text/event-stream'
|
|
}
|
|
});
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
}
|
|
|
|
// Process the stream manually
|
|
const reader = response.body.getReader();
|
|
const decoder = new TextDecoder();
|
|
|
|
let scanSessionId = null;
|
|
let scanResults = [];
|
|
|
|
try {
|
|
while (true) {
|
|
const { done, value } = await reader.read();
|
|
if (done) break;
|
|
|
|
const chunk = decoder.decode(value);
|
|
const lines = chunk.split('\n');
|
|
|
|
for (const line of lines) {
|
|
if (line.startsWith('data: ') && line.trim() !== 'data: ') {
|
|
try {
|
|
const data = JSON.parse(line.substring(6));
|
|
console.log('Scan event received:', data);
|
|
|
|
switch (data.type) {
|
|
case 'status':
|
|
if (scanStatus) scanStatus.textContent = data.message;
|
|
if (data.sessionId) {
|
|
scanSessionId = data.sessionId;
|
|
console.log('Initial scan session ID set:', scanSessionId);
|
|
}
|
|
break;
|
|
|
|
case 'scanning':
|
|
if (scanStatus) scanStatus.textContent = data.message;
|
|
break;
|
|
|
|
case 'scan_complete':
|
|
if (scanStatus) scanStatus.textContent = data.message;
|
|
|
|
// Update scan summary
|
|
document.getElementById('total-records').textContent = data.total || 0;
|
|
document.getElementById('need-geocoding').textContent = data.needingGeocode || 0;
|
|
document.getElementById('successfully-geocoded').textContent = '0';
|
|
document.getElementById('failed-geocoded').textContent = '0';
|
|
|
|
if (summary) summary.style.display = 'block';
|
|
|
|
if (data.needingGeocode > 0) {
|
|
// Show progress bar and update phase
|
|
if (progressContainer) progressContainer.style.display = 'block';
|
|
if (scanPhase) {
|
|
scanPhase.innerHTML = '<h4>Phase 2: Geocoding Addresses</h4>';
|
|
}
|
|
|
|
// Update progress totals
|
|
document.getElementById('scan-progress-total').textContent = data.needingGeocode;
|
|
}
|
|
break;
|
|
|
|
case 'progress':
|
|
// Update progress bar
|
|
const progressPercent = (data.current / data.total) * 100;
|
|
const progressBar = document.getElementById('scan-progress-bar-fill');
|
|
if (progressBar) progressBar.style.width = `${progressPercent}%`;
|
|
|
|
document.getElementById('scan-progress-current').textContent = data.current;
|
|
document.getElementById('scan-current-address').textContent = `Processing: ${data.currentAddress}`;
|
|
break;
|
|
|
|
case 'geocoded':
|
|
// Update success counter
|
|
const successCount = parseInt(document.getElementById('successfully-geocoded').textContent) + 1;
|
|
document.getElementById('successfully-geocoded').textContent = successCount;
|
|
|
|
// Add to results preview
|
|
addScanResultToTable(data.data, data.status);
|
|
scanResults.push(data.data);
|
|
|
|
// Show results preview if not already visible
|
|
if (resultsPreview) resultsPreview.style.display = 'block';
|
|
break;
|
|
|
|
case 'error':
|
|
if (data.data) {
|
|
// Update failed counter
|
|
const failedCount = parseInt(document.getElementById('failed-geocoded').textContent) + 1;
|
|
document.getElementById('failed-geocoded').textContent = failedCount;
|
|
|
|
// Add to results preview
|
|
addScanResultToTable(data.data, 'error');
|
|
scanResults.push(data.data);
|
|
} else {
|
|
// General error
|
|
if (scanStatus) scanStatus.textContent = `Error: ${data.message}`;
|
|
}
|
|
break;
|
|
|
|
case 'complete':
|
|
if (scanStatus) scanStatus.textContent = data.message;
|
|
|
|
// Show completed actions
|
|
if (actionsCompleted) actionsCompleted.style.display = 'block';
|
|
|
|
// Store session ID for report download from the completion message
|
|
const finalSessionId = data.sessionId || data.results?.sessionId || scanSessionId;
|
|
if (finalSessionId) {
|
|
const downloadBtn = document.getElementById('download-scan-report-btn');
|
|
if (downloadBtn) {
|
|
downloadBtn.dataset.sessionId = finalSessionId;
|
|
console.log('Stored scan session ID for report download:', finalSessionId);
|
|
}
|
|
} else {
|
|
console.warn('No session ID available for scan report download');
|
|
}
|
|
|
|
// Hide cancel button
|
|
if (cancelBtn) cancelBtn.style.display = 'none';
|
|
|
|
// Exit the loop when complete
|
|
return;
|
|
}
|
|
|
|
} catch (error) {
|
|
console.error('Error parsing scan event:', error);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} catch (error) {
|
|
console.error('Stream processing error:', error);
|
|
if (scanStatus) scanStatus.textContent = 'Error processing stream';
|
|
} finally {
|
|
// Clean up reader
|
|
reader.releaseLock();
|
|
}
|
|
|
|
} catch (error) {
|
|
console.error('Error starting database scan:', error);
|
|
alert('Failed to start database scan: ' + error.message);
|
|
}
|
|
}
|
|
|
|
function cancelDatabaseScan() {
|
|
console.log('Cancelling database scan...');
|
|
|
|
// Note: With fetch streams, we can't easily cancel mid-stream
|
|
// but we can reset the UI to let user know cancellation was requested
|
|
|
|
// Reset UI
|
|
const scanBtn = document.getElementById('scan-geocode-btn');
|
|
const cancelBtn = document.getElementById('cancel-scan-btn');
|
|
const scanStatus = document.getElementById('scan-status');
|
|
|
|
if (scanBtn) scanBtn.style.display = 'inline-block';
|
|
if (cancelBtn) cancelBtn.style.display = 'none';
|
|
if (scanStatus) scanStatus.textContent = 'Scan cancelled by user.';
|
|
}
|
|
|
|
function addScanResultToTable(result, status) {
|
|
const tbody = document.getElementById('scan-results-tbody');
|
|
if (!tbody) return;
|
|
|
|
// Limit table to last 10 results
|
|
while (tbody.children.length >= 10) {
|
|
tbody.removeChild(tbody.firstChild);
|
|
}
|
|
|
|
const row = document.createElement('tr');
|
|
row.className = `status-${status}`;
|
|
|
|
const statusIcon = status === 'success' ? '✅' :
|
|
status === 'warning' ? '⚠️' : '❌';
|
|
|
|
const coordinates = result.latitude && result.longitude ?
|
|
`${result.latitude}, ${result.longitude}` : 'Failed';
|
|
|
|
const confidence = result.confidence_score !== undefined ?
|
|
`${result.confidence_score}%` : 'N/A';
|
|
|
|
row.innerHTML = `
|
|
<td>${statusIcon} ${status.toUpperCase()}</td>
|
|
<td>${result.address || 'Unknown'}</td>
|
|
<td>${coordinates}</td>
|
|
<td>${confidence}</td>
|
|
<td>${result.provider || 'N/A'}</td>
|
|
`;
|
|
|
|
tbody.appendChild(row);
|
|
}
|
|
|
|
async function downloadScanReport() {
|
|
const downloadBtn = document.getElementById('download-scan-report-btn');
|
|
const sessionId = downloadBtn?.dataset.sessionId;
|
|
|
|
if (!sessionId) {
|
|
alert('No processing session available for report generation');
|
|
return;
|
|
}
|
|
|
|
try {
|
|
const response = await fetch(`/api/admin/data-convert/download-report/${sessionId}`);
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`HTTP error! status: ${response.status}`);
|
|
}
|
|
|
|
// Get the filename from the response headers
|
|
const filename = response.headers.get('content-disposition')
|
|
?.split('filename=')[1]
|
|
?.replace(/"/g, '') || `scan-geocoding-report-${sessionId}.txt`;
|
|
|
|
// Create blob and download
|
|
const blob = await response.blob();
|
|
const url = window.URL.createObjectURL(blob);
|
|
const a = document.createElement('a');
|
|
a.href = url;
|
|
a.download = filename;
|
|
document.body.appendChild(a);
|
|
a.click();
|
|
window.URL.revokeObjectURL(url);
|
|
document.body.removeChild(a);
|
|
|
|
// Show success message in scan status area
|
|
const scanStatus = document.getElementById('scan-status');
|
|
if (scanStatus) {
|
|
scanStatus.innerHTML = '✅ Report downloaded successfully';
|
|
}
|
|
|
|
} catch (error) {
|
|
console.error('Download scan report error:', error);
|
|
// Show error message in scan status area
|
|
const scanStatus = document.getElementById('scan-status');
|
|
if (scanStatus) {
|
|
scanStatus.innerHTML = `❌ Failed to download report: ${error.message}`;
|
|
} else {
|
|
alert(`Failed to download report: ${error.message}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
function resetScanInterface() {
|
|
// Reset to initial state
|
|
const scanBtn = document.getElementById('scan-geocode-btn');
|
|
const cancelBtn = document.getElementById('cancel-scan-btn');
|
|
const processingSection = document.getElementById('scan-processing-section');
|
|
|
|
if (scanBtn) scanBtn.style.display = 'inline-block';
|
|
if (cancelBtn) cancelBtn.style.display = 'none';
|
|
if (processingSection) processingSection.style.display = 'none';
|
|
}
|