let processingData = []; let currentSessionId = null; // Store session ID for report download let resultsMap = null; let markers = []; let eventListenersInitialized = false; // Check and display geocoding provider status async function checkGeocodingProviders() { const statusElement = document.getElementById('provider-status'); if (!statusElement) return; try { const response = await fetch('/api/geocode/provider-status', { method: 'GET', credentials: 'same-origin', headers: { 'Accept': 'application/json', 'Content-Type': 'application/json' } }); if (!response.ok) { throw new Error(`HTTP error! status: ${response.status}`); } const data = await response.json(); if (data.success) { const providers = data.providers; let statusHTML = '
'; providers.forEach(provider => { const icon = provider.available ? '✅' : '❌'; const status = provider.available ? 'Available' : 'Not configured'; const className = provider.available ? 'provider-available' : 'provider-unavailable'; statusHTML += `
${icon} ${provider.name}: ${status} ${provider.name === 'Mapbox' && provider.available ? '🌟 Premium' : ''}
`; }); statusHTML += '
'; // Add configuration note if no premium providers const hasPremium = providers.some(p => p.available && ['Mapbox', 'LocationIQ'].includes(p.name)); if (!hasPremium) { statusHTML += `

💡 Tip: For better geocoding accuracy, configure a premium provider:

`; } else if (providers.find(p => p.name === 'Mapbox' && p.available)) { statusHTML += `

🌟 Mapbox Configured! Using premium geocoding for better accuracy.

`; } statusElement.innerHTML = statusHTML; } else { statusElement.innerHTML = '❌ Failed to check provider status'; } } catch (error) { console.error('Failed to check geocoding providers:', error); statusElement.innerHTML = `
⚠️ Unable to check provider status Using fallback providers: Nominatim, Photon, ArcGIS
`; } } // Utility function to show status messages function showDataConvertStatus(message, type = 'info') { // Try to use the global showStatus from admin.js if available if (typeof window.showStatus === 'function') { return window.showStatus(message, type); } // Fallback to console console.log(`[${type.toUpperCase()}] ${message}`); // Try to display in status container if it exists const statusContainer = document.getElementById('status-container'); if (statusContainer) { const statusEl = document.createElement('div'); statusEl.className = `status-message status-${type}`; statusEl.textContent = message; statusContainer.appendChild(statusEl); // Auto-remove after 5 seconds setTimeout(() => { if (statusEl.parentNode) { statusEl.parentNode.removeChild(statusEl); } }, 5000); } } // Initialize when DOM is loaded document.addEventListener('DOMContentLoaded', () => { console.log('Data convert JS loaded'); // Don't auto-initialize, wait for section to be activated // Make setupDataConvertEventListeners available globally for admin.js window.setupDataConvertEventListeners = setupDataConvertEventListeners; }); function setupDataConvertEventListeners() { console.log('Setting up data convert event listeners...'); // Prevent duplicate event listeners if (eventListenersInitialized) { console.log('Event listeners already initialized, skipping...'); return; } // Check geocoding provider status when section loads checkGeocodingProviders(); const fileInput = document.getElementById('csv-file-input'); const browseBtn = document.getElementById('browse-btn'); const uploadArea = document.getElementById('upload-area'); const uploadForm = document.getElementById('csv-upload-form'); const clearBtn = document.getElementById('clear-upload-btn'); const saveResultsBtn = document.getElementById('save-results-btn'); const newUploadBtn = document.getElementById('new-upload-btn'); console.log('Elements found:', { fileInput: !!fileInput, browseBtn: !!browseBtn, uploadArea: !!uploadArea, uploadForm: !!uploadForm, clearBtn: !!clearBtn, saveResultsBtn: !!saveResultsBtn, newUploadBtn: !!newUploadBtn }); // File input change if (fileInput) { fileInput.addEventListener('change', handleFileSelect); } // Browse button if (browseBtn) { browseBtn.addEventListener('click', () => { console.log('Browse button clicked'); fileInput?.click(); }); } // Drag and drop if (uploadArea) { uploadArea.addEventListener('dragover', handleDragOver); uploadArea.addEventListener('dragleave', handleDragLeave); uploadArea.addEventListener('drop', handleDrop); uploadArea.addEventListener('click', () => fileInput?.click()); } // Form submission if (uploadForm) { uploadForm.addEventListener('submit', handleCSVUpload); } // Clear button if (clearBtn) { clearBtn.addEventListener('click', clearUpload); } // Save results button - ADD DATA TO MAP if (saveResultsBtn) { saveResultsBtn.addEventListener('click', saveGeocodedResults); console.log('Save results button event listener attached'); } // New upload button if (newUploadBtn) { newUploadBtn.addEventListener('click', resetToUpload); } // Mark as initialized eventListenersInitialized = true; console.log('Data convert event listeners initialized successfully'); } function handleFileSelect(e) { const file = e.target.files[0]; if (file) { displayFileInfo(file); } } function handleDragOver(e) { e.preventDefault(); e.currentTarget.classList.add('drag-over'); } function handleDragLeave(e) { e.preventDefault(); e.currentTarget.classList.remove('drag-over'); } function handleDrop(e) { e.preventDefault(); e.currentTarget.classList.remove('drag-over'); const files = e.dataTransfer.files; if (files.length > 0) { const file = files[0]; if (file.type === 'text/csv' || file.name.endsWith('.csv')) { document.getElementById('csv-file-input').files = files; displayFileInfo(file); } else { showDataConvertStatus('Please upload a CSV file', 'error'); } } } function displayFileInfo(file) { document.getElementById('file-info').style.display = 'block'; document.getElementById('file-name').textContent = file.name; document.getElementById('file-size').textContent = formatFileSize(file.size); document.getElementById('process-csv-btn').disabled = false; } function formatFileSize(bytes) { if (bytes === 0) return '0 Bytes'; const k = 1024; const sizes = ['Bytes', 'KB', 'MB']; const i = Math.floor(Math.log(bytes) / Math.log(k)); return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]; } async function handleCSVUpload(e) { e.preventDefault(); const fileInput = document.getElementById('csv-file-input'); const file = fileInput?.files[0]; if (!file) { showDataConvertStatus('Please select a CSV file', 'error'); return; } // Reset processing data processingData = []; // Show processing section const uploadSection = document.getElementById('upload-section'); const processingSection = document.getElementById('processing-section'); if (!uploadSection || !processingSection) { console.error('Required DOM elements not found'); showDataConvertStatus('Interface error: required elements not found', 'error'); return; } uploadSection.style.display = 'none'; processingSection.style.display = 'block'; // Initialize results map initializeResultsMap(); // Create form data const formData = new FormData(); formData.append('csvFile', file); try { // Send the file and handle SSE response const response = await fetch('/api/admin/data-convert/process-csv', { method: 'POST', body: formData }); if (!response.ok) { throw new Error(`HTTP error! status: ${response.status}`); } // Read the response as a stream for SSE const reader = response.body.getReader(); const decoder = new TextDecoder(); let buffer = ''; // Buffer to accumulate partial data while (true) { const { done, value } = await reader.read(); if (done) break; // Decode the chunk and add to buffer buffer += decoder.decode(value, { stream: true }); // Split buffer by lines and process complete lines const lines = buffer.split('\n'); // Keep the last potentially incomplete line in the buffer buffer = lines.pop() || ''; for (const line of lines) { if (line.startsWith('data: ')) { try { const jsonData = line.substring(6).trim(); // Remove 'data: ' prefix and trim if (jsonData && jsonData !== '') { const data = JSON.parse(jsonData); handleProcessingUpdate(data); } } catch (parseError) { console.warn('Failed to parse SSE data:', parseError); console.warn('Problematic line:', line); console.warn('JSON data:', line.substring(6)); } } else if (line.trim() === '') { // Empty line, ignore continue; } else if (line.trim() !== '' && !line.startsWith('event:') && !line.startsWith('id:')) { // Unexpected line format console.warn('Unexpected SSE line format:', line); } } } // Process any remaining data in buffer if (buffer.trim()) { const line = buffer; if (line.startsWith('data: ')) { try { const jsonData = line.substring(6).trim(); if (jsonData && jsonData !== '') { const data = JSON.parse(jsonData); handleProcessingUpdate(data); } } catch (parseError) { console.warn('Failed to parse final SSE data:', parseError); console.warn('Final buffer content:', buffer); } } } } catch (error) { console.error('CSV processing error:', error); showDataConvertStatus(error.message || 'Failed to process CSV', 'error'); resetToUpload(); } } // Enhanced processing update handler function handleProcessingUpdate(data) { console.log('Processing update:', data); // Validate data structure if (!data || typeof data !== 'object') { console.warn('Invalid data received:', data); return; } switch (data.type) { case 'start': updateProgress(0, data.total); document.getElementById('current-address').textContent = 'Starting geocoding process...'; break; case 'progress': updateProgress(data.current, data.total); // Show current address with status if (data.status === 'failed') { document.getElementById('current-address').innerHTML = `✗ ${data.currentAddress || data.address}`; } else if (data.status === 'processing') { document.getElementById('current-address').innerHTML = `⟳ Processing: ${data.currentAddress || data.address}`; } else { document.getElementById('current-address').innerHTML = `✓ ${data.currentAddress || data.address}`; } break; case 'geocoded': // Check if result has warnings const isWarning = data.status === 'warning' || (data.data && data.data.is_malformed); const confidence = data.confidence || (data.data && data.data.confidence_score) || 100; const warnings = data.warnings || (data.data && data.data.warnings) || []; // Mark data with appropriate status and add to processing data const successData = { ...data.data, geocode_success: true, confidence_score: confidence, warnings: Array.isArray(warnings) ? warnings.join('; ') : warnings }; processingData.push(successData); // Add to table with appropriate status const resultStatus = isWarning ? 'warning' : 'success'; addResultToTable(successData, resultStatus, confidence, warnings); addMarkerToMap(successData, isWarning); updateProgress(data.index + 1, data.total); break; case 'error': // Mark as failed and add to processing data const errorData = { ...data.data, geocode_success: false }; processingData.push(errorData); addResultToTable(errorData, 'error'); break; case 'complete': console.log('Received complete event:', data); currentSessionId = data.sessionId; // Store session ID for report download // Show comprehensive completion message let completionMessage = `Complete! Processed ${data.total} addresses:\n`; completionMessage += `✓ ${data.successful || 0} successful\n`; if (data.warnings > 0) { completionMessage += `⚠ ${data.warnings} with warnings (low confidence)\n`; } if (data.malformed > 0) { completionMessage += `🔍 ${data.malformed} potentially malformed (need review)\n`; } completionMessage += `✗ ${data.errors || data.failed || 0} failed`; document.getElementById('current-address').innerHTML = completionMessage.replace(/\n/g, '
'); onProcessingComplete(data); break; case 'fatal_error': showDataConvertStatus(data.message, 'error'); resetToUpload(); break; default: console.warn('Unknown data type received:', data.type); } } function updateProgress(current, total) { const percentage = (current / total) * 100; document.getElementById('progress-bar-fill').style.width = percentage + '%'; document.getElementById('progress-current').textContent = current; document.getElementById('progress-total').textContent = total; } function initializeResultsMap() { const mapContainer = document.getElementById('results-map'); if (!mapContainer) return; // Initialize map resultsMap = L.map('results-map').setView([53.5461, -113.4938], 11); L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', { attribution: '© OpenStreetMap contributors' }).addTo(resultsMap); document.getElementById('results-preview').style.display = 'block'; // Fix map sizing setTimeout(() => { resultsMap.invalidateSize(); }, 100); } function addResultToTable(data, status, confidence = null, warnings = []) { const tbody = document.getElementById('results-tbody'); const row = document.createElement('tr'); // Set row class based on status if (status === 'success') { row.className = 'result-success'; } else if (status === 'warning') { row.className = 'result-warning'; } else { row.className = 'result-error'; } if (status === 'success' || status === 'warning') { // Add confidence indicator for successful geocoding let statusIcon = status === 'warning' ? `` : ``; if (confidence !== null && confidence < 100) { statusIcon += ` (${Math.round(confidence)}%)`; } let addressCell = escapeHtml(data.geocoded_address || ''); if (warnings && warnings.length > 0) { const warningText = Array.isArray(warnings) ? warnings.join(', ') : warnings; addressCell += `
⚠ ${escapeHtml(warningText)}`; } row.innerHTML = ` ${statusIcon} ${escapeHtml(data.address || data.Address || '')} ${addressCell} ${data.latitude.toFixed(6)}, ${data.longitude.toFixed(6)} ${data.provider ? escapeHtml(data.provider) : 'N/A'} `; } else { row.innerHTML = ` ${escapeHtml(data.address || data.Address || '')} ${escapeHtml(data.geocode_error || data.error || 'Geocoding failed')} `; } tbody.appendChild(row); } function addMarkerToMap(data, isWarning = false) { if (!resultsMap || !data.latitude || !data.longitude) return; // Choose marker color based on status const markerColor = isWarning ? 'orange' : 'green'; const marker = L.marker([data.latitude, data.longitude], { icon: L.divIcon({ className: `custom-marker ${isWarning ? 'warning-marker' : 'success-marker'}`, html: `
`, iconSize: [16, 16], iconAnchor: [8, 8] }) }); // Create popup content with warning information let popupContent = `${escapeHtml(data.geocoded_address || data.address)}
`; popupContent += `${data.latitude.toFixed(6)}, ${data.longitude.toFixed(6)}`; if (data.provider) { popupContent += `
Provider: ${data.provider}`; } if (isWarning && data.confidence_score) { popupContent += `
⚠ Confidence: ${Math.round(data.confidence_score)}%`; } if (isWarning && data.warnings) { popupContent += `
${escapeHtml(data.warnings)}`; } marker.bindPopup(popupContent); marker.addTo(resultsMap); markers.push(marker); // Adjust map bounds to show all markers if (markers.length > 0) { const group = new L.featureGroup(markers); resultsMap.fitBounds(group.getBounds().pad(0.1)); } } function onProcessingComplete(data) { console.log('Processing complete called with data:', data); const processedCount = data.processed || data.successful || processingData.filter(item => item.geocode_success !== false).length; // Show comprehensive processing actions with download option if (processedCount > 0) { console.log('Showing processing actions for', processedCount, 'successful items'); const actionsDiv = document.getElementById('processing-actions'); if (actionsDiv) { actionsDiv.style.display = 'block'; console.log('Processing actions div is now visible'); } else { console.error('processing-actions div not found!'); } } // Add download report button if session ID is available if (currentSessionId) { showDownloadReportButton(data); } // Update final message with detailed statistics let finalMessage = `Geocoding Complete!\n\n`; finalMessage += `📊 Summary:\n`; finalMessage += `• Total Processed: ${data.total || 0}\n`; finalMessage += `• ✅ Successful: ${data.successful || 0}\n`; if (data.warnings > 0) { finalMessage += `• ⚠️ Warnings: ${data.warnings} (low confidence, review recommended)\n`; } if (data.malformed > 0) { finalMessage += `• 🔍 Potentially Malformed: ${data.malformed} (need manual review)\n`; } finalMessage += `• ❌ Failed: ${data.errors || data.failed || 0}\n\n`; if (data.warnings > 0 || data.malformed > 0) { finalMessage += `⚠️ Note: ${(data.warnings || 0) + (data.malformed || 0)} addresses may need manual verification.\n`; finalMessage += `Please download the detailed report for review.`; } else if ((data.successful || 0) > 0) { finalMessage += `✅ All geocoded addresses appear to have high confidence results!`; } showDataConvertStatus(finalMessage, (data.errors || data.failed || 0) > 0 ? 'warning' : 'success'); } // Add download report button function function showDownloadReportButton(data) { const processingActions = document.getElementById('processing-actions'); if (!processingActions) return; // Check if download button already exists let downloadBtn = document.getElementById('download-report-btn'); if (!downloadBtn) { downloadBtn = document.createElement('button'); downloadBtn.id = 'download-report-btn'; downloadBtn.className = 'btn btn-info'; downloadBtn.innerHTML = '📄 Download Detailed Report'; downloadBtn.addEventListener('click', downloadProcessingReport); // Insert download button before save results button const saveBtn = document.getElementById('save-results-btn'); if (saveBtn && saveBtn.parentNode === processingActions) { processingActions.insertBefore(downloadBtn, saveBtn); } else { processingActions.appendChild(downloadBtn); } } // Update button text with statistics const warningsCount = (data.warnings || 0) + (data.malformed || 0); if (warningsCount > 0) { downloadBtn.innerHTML = `📄 Download Report (${warningsCount} need review)`; downloadBtn.className = 'btn btn-warning'; } } // Download processing report function async function downloadProcessingReport() { if (!currentSessionId) { showDataConvertStatus('No report available to download', 'error'); return; } try { const response = await fetch(`/api/admin/data-convert/download-report/${currentSessionId}`); if (!response.ok) { throw new Error(`HTTP error! status: ${response.status}`); } // Get the filename from the response headers const filename = response.headers.get('content-disposition') ?.split('filename=')[1] ?.replace(/"/g, '') || `geocoding-report-${currentSessionId}.txt`; // Create blob and download const blob = await response.blob(); const url = window.URL.createObjectURL(blob); const a = document.createElement('a'); a.href = url; a.download = filename; document.body.appendChild(a); a.click(); window.URL.revokeObjectURL(url); document.body.removeChild(a); showDataConvertStatus('Report downloaded successfully', 'success'); } catch (error) { console.error('Download report error:', error); showDataConvertStatus('Failed to download report: ' + error.message, 'error'); } } async function saveGeocodedResults() { const successfulData = processingData.filter(item => item.geocode_success !== false && item.latitude && item.longitude); if (successfulData.length === 0) { showDataConvertStatus('No successfully geocoded data to save', 'error'); return; } // Disable save button and show progress const saveBtn = document.getElementById('save-results-btn'); const originalText = saveBtn.textContent; saveBtn.disabled = true; saveBtn.textContent = 'Adding to map...'; // Show saving status showDataConvertStatus(`Adding ${successfulData.length} locations to map...`, 'info'); let successCount = 0; let failedCount = 0; const errors = []; try { // Use the bulk save endpoint instead of individual location creation // This is more efficient and avoids rate limiting issues const response = await fetch('/api/admin/data-convert/save-geocoded', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ data: successfulData }) }); if (!response.ok) { throw new Error(`HTTP error! status: ${response.status}`); } const result = await response.json(); if (result.success) { successCount = result.results.success; failedCount = result.results.failed; // Show detailed errors if any if (result.results.errors && result.results.errors.length > 0) { result.results.errors.forEach((error, index) => { errors.push(`Location ${index + 1}: ${error.error}`); }); } console.log(`Bulk save completed: ${successCount} successful, ${failedCount} failed`); } else { throw new Error(result.error || 'Bulk save failed'); } // Show final results if (successCount > 0) { const message = `Successfully added ${successCount} locations to map.` + (failedCount > 0 ? ` ${failedCount} failed.` : ''); showDataConvertStatus(message, failedCount > 0 ? 'warning' : 'success'); // Update UI to show completion saveBtn.textContent = `Added ${successCount} locations!`; setTimeout(() => { saveBtn.style.display = 'none'; document.getElementById('new-upload-btn').style.display = 'inline-block'; }, 3000); } else { throw new Error('Failed to add any locations to the map'); } // Log errors if any if (errors.length > 0) { console.error('Import errors:', errors); } } catch (error) { console.error('Save error:', error); showDataConvertStatus('Failed to add data to map: ' + error.message, 'error'); saveBtn.disabled = false; saveBtn.textContent = originalText; } } // Utility function to escape HTML function escapeHtml(text) { const map = { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }; return text ? text.replace(/[&<>"']/g, function(m) { return map[m]; }) : ''; } // Clear upload and reset form function clearUpload() { const fileInput = document.getElementById('csv-file-input'); const fileInfo = document.getElementById('file-info'); const processBtn = document.getElementById('process-csv-btn'); if (fileInput) fileInput.value = ''; if (fileInfo) fileInfo.style.display = 'none'; if (processBtn) processBtn.disabled = true; } // Reset to upload state function resetToUpload() { console.log('Resetting to upload state'); const uploadSection = document.getElementById('upload-section'); const processingSection = document.getElementById('processing-section'); const actionsDiv = document.getElementById('processing-actions'); const resultsPreview = document.getElementById('results-preview'); const saveBtn = document.getElementById('save-results-btn'); const newUploadBtn = document.getElementById('new-upload-btn'); if (uploadSection) uploadSection.style.display = 'block'; if (processingSection) processingSection.style.display = 'none'; if (actionsDiv) actionsDiv.style.display = 'none'; if (resultsPreview) resultsPreview.style.display = 'none'; // Reset buttons if (saveBtn) { saveBtn.style.display = 'inline-block'; saveBtn.disabled = false; saveBtn.textContent = 'Add Data to Map'; } if (newUploadBtn) { newUploadBtn.style.display = 'none'; } // Clear any existing data processingData = []; currentSessionId = null; // Reset session ID // Remove download report button if it exists const downloadBtn = document.getElementById('download-report-btn'); if (downloadBtn) { downloadBtn.remove(); } if (markers && markers.length > 0) { markers.forEach(marker => { if (resultsMap && marker) { try { resultsMap.removeLayer(marker); } catch (e) { console.warn('Error removing marker:', e); } } }); } markers = []; // Reset results table const tbody = document.getElementById('results-tbody'); if (tbody) { tbody.innerHTML = ''; } // Reset progress const progressFill = document.getElementById('progress-bar-fill'); const progressCurrent = document.getElementById('progress-current'); const progressTotal = document.getElementById('progress-total'); const currentAddress = document.getElementById('current-address'); if (progressFill) progressFill.style.width = '0%'; if (progressCurrent) progressCurrent.textContent = '0'; if (progressTotal) progressTotal.textContent = '0'; if (currentAddress) currentAddress.textContent = ''; // Destroy existing map if it exists if (resultsMap) { try { resultsMap.remove(); resultsMap = null; } catch (e) { console.warn('Error removing map:', e); } } // Reset form clearUpload(); } // Show download report button function showDownloadReportButton(totalRecords, errorCount) { const actionsDiv = document.getElementById('processing-actions'); if (!actionsDiv) return; // Check if button already exists let downloadBtn = document.getElementById('download-report-btn'); if (!downloadBtn) { downloadBtn = document.createElement('button'); downloadBtn.id = 'download-report-btn'; downloadBtn.type = 'button'; downloadBtn.className = 'btn btn-info'; downloadBtn.addEventListener('click', downloadProcessingReport); // Insert before the save results button const saveBtn = document.getElementById('save-results-btn'); if (saveBtn) { actionsDiv.insertBefore(downloadBtn, saveBtn); } else { actionsDiv.appendChild(downloadBtn); } } // Update button text with error info if (errorCount > 0) { downloadBtn.textContent = `📄 Download Full Report (${totalRecords} records, ${errorCount} errors)`; downloadBtn.title = `Download complete processing report including ${errorCount} error records for review`; } else { downloadBtn.textContent = `📄 Download Processing Report (${totalRecords} records)`; downloadBtn.title = 'Download complete processing report'; } downloadBtn.style.marginRight = '10px'; // Add some spacing } // Download processing report async function downloadProcessingReport() { if (!currentSessionId) { showDataConvertStatus('No processing session available for report generation', 'error'); return; } const downloadBtn = document.getElementById('download-report-btn'); const originalText = downloadBtn.textContent; try { downloadBtn.disabled = true; downloadBtn.textContent = '📄 Generating Report...'; showDataConvertStatus('Generating processing report...', 'info'); // Create download link const downloadUrl = `/api/admin/data-convert/download-report/${currentSessionId}`; // Create a temporary anchor element and trigger download const link = document.createElement('a'); link.href = downloadUrl; link.download = `geocoding-report-${currentSessionId}.csv`; link.style.display = 'none'; document.body.appendChild(link); link.click(); document.body.removeChild(link); showDataConvertStatus('Processing report downloaded successfully', 'success'); // Update button text downloadBtn.textContent = '📄 Report Downloaded'; setTimeout(() => { downloadBtn.textContent = originalText; downloadBtn.disabled = false; }, 3000); } catch (error) { console.error('Download report error:', error); showDataConvertStatus('Failed to download processing report: ' + error.message, 'error'); downloadBtn.textContent = originalText; downloadBtn.disabled = false; } }