diff --git a/map/app/controllers/dataConvertController.js b/map/app/controllers/dataConvertController.js index f55a87e..40355f5 100644 --- a/map/app/controllers/dataConvertController.js +++ b/map/app/controllers/dataConvertController.js @@ -15,6 +15,7 @@ class DataConvertController { this.parseCSV = this.parseCSV.bind(this); this.saveGeocodedData = this.saveGeocodedData.bind(this); this.downloadReport = this.downloadReport.bind(this); + this.scanAndGeocode = this.scanAndGeocode.bind(this); } // Process CSV upload and geocode addresses with SSE progress updates @@ -152,7 +153,10 @@ class DataConvertController { if (geocodeResult && geocodeResult.coordinates) { // Check if result is malformed const isMalformed = geocodeResult.validation && geocodeResult.validation.isMalformed; - const confidence = geocodeResult.validation ? geocodeResult.validation.confidence : 100; + // Use combined confidence for best overall assessment + const confidence = geocodeResult.combinedConfidence !== undefined ? + geocodeResult.combinedConfidence : + (geocodeResult.validation ? geocodeResult.validation.confidence : 100); const warnings = geocodeResult.validation ? geocodeResult.validation.warnings : []; const processedRow = { @@ -165,6 +169,8 @@ class DataConvertController { geocode_status: isMalformed ? 'WARNING' : 'SUCCESS', geocode_error: '', confidence_score: confidence, + provider_confidence: geocodeResult.providerConfidence || null, + validation_confidence: geocodeResult.validation ? geocodeResult.validation.confidence : null, warnings: warnings.join('; '), is_malformed: isMalformed, provider: geocodeResult.provider || 'Unknown', @@ -333,6 +339,8 @@ class DataConvertController { latitude: parseFloat(location.latitude), longitude: parseFloat(location.longitude), Address: originalAddress, // Always use the original address from CSV + 'Geocode Confidence': location.confidence_score || null, // Add confidence score + 'Geocode Provider': location.provider || null, // Add provider name created_by_user: req.session.userEmail || 'csv_import', last_updated_by_user: req.session.userEmail || 'csv_import' }; @@ -367,7 +375,7 @@ class DataConvertController { const lowerKey = key.toLowerCase(); // Skip already processed fields - if (['latitude', 'longitude', 'geo-location', 'geocoded_address', 'geocode_success', 'address', 'csv_filename'].includes(lowerKey)) { + if (['latitude', 'longitude', 'geo-location', 'geocoded_address', 'geocode_success', 'address', 'csv_filename', 'confidence_score', 'provider_confidence', 'validation_confidence', 'warnings', 'is_malformed', 'provider', 'row_number', 'geocode_status', 'geocode_error'].includes(lowerKey)) { return; } @@ -402,6 +410,21 @@ class DataConvertController { noteParts.push(`Geocoded as: ${geocodedAddress}`); } + // Add confidence information if available + if (location.confidence_score !== undefined && location.confidence_score !== null) { + noteParts.push(`Geocode confidence: ${location.confidence_score}%`); + } + + // Add provider information if available + if (location.provider) { + noteParts.push(`Provider: ${location.provider}`); + } + + // Add warnings if present + if (location.warnings && location.warnings.trim()) { + noteParts.push(`Warnings: ${location.warnings}`); + } + locationData[targetField] = noteParts.join(' | '); } else { locationData[targetField] = location[key]; @@ -418,6 +441,21 @@ class DataConvertController { noteParts.push(`Geocoded as: ${geocodedAddress}`); } + // Add confidence information if available + if (location.confidence_score !== undefined && location.confidence_score !== null) { + noteParts.push(`Geocode confidence: ${location.confidence_score}%`); + } + + // Add provider information if available + if (location.provider) { + noteParts.push(`Provider: ${location.provider}`); + } + + // Add warnings if present + if (location.warnings && location.warnings.trim()) { + noteParts.push(`Warnings: ${location.warnings}`); + } + locationData['Notes'] = noteParts.join(' | '); } @@ -455,6 +493,7 @@ class DataConvertController { async downloadReport(req, res) { try { const { sessionId } = req.params; + const format = req.query.format || 'csv'; // Default to CSV, support 'txt' for backward compatibility if (!sessionId || !processingResults.has(sessionId)) { return res.status(404).json({ @@ -466,16 +505,27 @@ class DataConvertController { const results = processingResults.get(sessionId); const { filename, timestamp, allResults, summary } = results; - // Generate comprehensive report content - const reportContent = this.generateComprehensiveReport(allResults, filename, timestamp, summary); + let reportContent, contentType, fileExtension; - // Set headers for text download - const reportFilename = `geocoding-report-${sessionId}.txt`; - res.setHeader('Content-Type', 'text/plain'); + if (format === 'csv') { + // Generate CSV report + reportContent = this.generateReportCSV(allResults, filename, timestamp, summary); + contentType = 'text/csv'; + fileExtension = 'csv'; + } else { + // Generate text report (backward compatibility) + reportContent = this.generateComprehensiveReport(allResults, filename, timestamp, summary); + contentType = 'text/plain'; + fileExtension = 'txt'; + } + + // Set headers for download + const reportFilename = `geocoding-report-${sessionId}.${fileExtension}`; + res.setHeader('Content-Type', contentType); res.setHeader('Content-Disposition', `attachment; filename="${reportFilename}"`); res.setHeader('Cache-Control', 'no-cache'); - logger.info(`Generating comprehensive report for session ${sessionId}: ${allResults.length} records`); + logger.info(`Generating ${format.toUpperCase()} report for session ${sessionId}: ${allResults.length} records`); res.send(reportContent); @@ -651,6 +701,290 @@ class DataConvertController { return stringField; } + + // Scan NocoDB database for records missing geo-location data and geocode them + async scanAndGeocode(req, res) { + try { + const sessionId = Date.now().toString(); + + // Set up SSE headers + res.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + 'Connection': 'keep-alive', + 'X-Accel-Buffering': 'no' + }); + + logger.info(`Starting database scan for missing geo-location data (session: ${sessionId})`); + + // Send initial status + res.write(`data: ${JSON.stringify({ + type: 'status', + message: 'Scanning database for records missing geo-location data...', + sessionId: sessionId + })}\n\n`); + res.flush && res.flush(); + + // Fetch all records from NocoDB + let allRecords = []; + let offset = 0; + const limit = 100; // Process in batches + let hasMoreRecords = true; + + while (hasMoreRecords) { + try { + const response = await nocodbService.getAll(config.nocodb.tableId, { limit, offset }); + + if (response && response.list && response.list.length > 0) { + allRecords.push(...response.list); + offset += limit; + + // Send progress update + res.write(`data: ${JSON.stringify({ + type: 'scanning', + message: `Fetched ${allRecords.length} records from database...`, + count: allRecords.length + })}\n\n`); + res.flush && res.flush(); + + // Check if we've fetched all records + hasMoreRecords = response.list.length === limit; + } else { + hasMoreRecords = false; + } + } catch (error) { + logger.error('Error fetching records from NocoDB:', error); + res.write(`data: ${JSON.stringify({ + type: 'error', + message: `Error fetching records: ${error.message}` + })}\n\n`); + res.end(); + return; + } + } + + logger.info(`Database scan complete: found ${allRecords.length} total records`); + + // Filter records that need geocoding + const recordsNeedingGeocode = allRecords.filter(record => { + // Check if record is missing geo-location data + const hasGeoLocation = record['Geo-Location'] && + record['Geo-Location'].trim() !== '' && + record['Geo-Location'] !== 'null'; + const hasCoordinates = (record.latitude && record.longitude) || + (record.Latitude && record.Longitude); + const hasAddress = record.Address || record.address || record.ADDRESS; + + return !hasGeoLocation && !hasCoordinates && hasAddress; + }); + + const totalToGeocode = recordsNeedingGeocode.length; + + logger.info(`Found ${totalToGeocode} records needing geocoding`); + + // Send summary + res.write(`data: ${JSON.stringify({ + type: 'scan_complete', + message: `Scan complete: ${totalToGeocode} records need geocoding`, + total: allRecords.length, + needingGeocode: totalToGeocode + })}\n\n`); + res.flush && res.flush(); + + if (totalToGeocode === 0) { + res.write(`data: ${JSON.stringify({ + type: 'complete', + message: 'No records found that need geocoding. All records already have location data!', + results: { success: 0, failed: 0, skipped: allRecords.length } + })}\n\n`); + res.end(); + return; + } + + // Process geocoding + const results = { + success: 0, + failed: 0, + errors: [], + sessionId: sessionId + }; + + const allResults = []; + let processedCount = 0; + + for (const record of recordsNeedingGeocode) { + try { + processedCount++; + const address = record.Address || record.address || record.ADDRESS; + + // Send progress update + res.write(`data: ${JSON.stringify({ + type: 'progress', + current: processedCount, + total: totalToGeocode, + currentAddress: address, + status: 'processing' + })}\n\n`); + res.flush && res.flush(); + + logger.info(`Geocoding ${processedCount}/${totalToGeocode}: ${address} (Record ID: ${record.id || record.Id || record.ID})`); + + // Geocode the address + const geocodeResult = await forwardGeocode(address); + + if (geocodeResult && geocodeResult.coordinates) { + // Check if result is malformed + const isMalformed = geocodeResult.validation && geocodeResult.validation.isMalformed; + // Use combined confidence for best overall assessment + const confidence = geocodeResult.combinedConfidence !== undefined ? + geocodeResult.combinedConfidence : + (geocodeResult.validation ? geocodeResult.validation.confidence : 100); + const warnings = geocodeResult.validation ? geocodeResult.validation.warnings : []; + + // Update the record in NocoDB + const updateData = { + 'Geo-Location': `${geocodeResult.coordinates.lat};${geocodeResult.coordinates.lng}`, + latitude: geocodeResult.coordinates.lat, + longitude: geocodeResult.coordinates.lng, + 'Geocode Confidence': confidence, + 'Geocode Provider': geocodeResult.provider || 'Unknown', + last_updated_by_user: req.session?.userEmail || 'scan_geocode' + }; + + // Update the record in NocoDB + await nocodbService.update(config.nocodb.tableId, record.id || record.Id || record.ID, updateData); + + const processedRecord = { + id: record.id || record.Id || record.ID, + address: address, + latitude: geocodeResult.coordinates.lat, + longitude: geocodeResult.coordinates.lng, + confidence_score: confidence, + provider: geocodeResult.provider || 'Unknown', + status: isMalformed ? 'WARNING' : 'SUCCESS', + warnings: warnings.join('; ') + }; + + allResults.push(processedRecord); + results.success++; + + // Send success update + const successMessage = { + type: 'geocoded', + data: processedRecord, + index: processedCount - 1, + status: isMalformed ? 'warning' : 'success', + confidence: confidence, + warnings: warnings + }; + + logger.info(`✓ Successfully geocoded and updated: ${address} (Confidence: ${confidence}%)`); + res.write(`data: ${JSON.stringify(successMessage)}\n\n`); + res.flush && res.flush(); + + } else { + throw new Error('Geocoding failed - no coordinates returned'); + } + + } catch (error) { + logger.error(`Failed to geocode record ${processedCount}/${totalToGeocode}:`, error.message); + + const errorRecord = { + id: record.id || record.Id || record.ID, + address: record.Address || record.address || record.ADDRESS, + error: error.message, + status: 'ERROR' + }; + + allResults.push(errorRecord); + results.failed++; + results.errors.push({ + address: errorRecord.address, + error: error.message + }); + + // Send error update + res.write(`data: ${JSON.stringify({ + type: 'error', + data: errorRecord, + index: processedCount - 1, + message: `Failed to geocode: ${errorRecord.address}` + })}\n\n`); + res.flush && res.flush(); + } + + // Rate limiting to be nice to geocoding APIs + if (processedCount < totalToGeocode) { + await new Promise(resolve => setTimeout(resolve, 500)); // 0.5 second delay between requests + } + } + + // Calculate summary statistics for report + const successful = allResults.filter(r => r.status === 'SUCCESS').length; + const warnings = allResults.filter(r => r.status === 'WARNING').length; + const failed = allResults.filter(r => r.status === 'ERROR').length; + const malformed = allResults.filter(r => r.warnings && r.warnings.includes('malformed')).length; + const total = successful + warnings + failed; + + // Transform scan results to match CSV processing format for report generation + const transformedResults = allResults.map(result => ({ + // Original format fields + address: result.address, + Address: result.address, + geocoded_address: result.address, // For scan, this is the same + latitude: result.latitude, + longitude: result.longitude, + 'Geo-Location': result.latitude && result.longitude ? `${result.latitude};${result.longitude}` : '', + confidence_score: result.confidence_score, + provider: result.provider, + + // Status mapping + geocode_success: result.status !== 'ERROR', + geocode_status: result.status, + geocode_error: result.error || '', + is_malformed: result.warnings && result.warnings.includes('malformed'), + warnings: result.warnings || '', + + // Scan-specific fields + record_id: result.id, + source: 'database_scan', + row_number: result.id // Use record ID as row number for scan + })); + + // Store results for potential report download + processingResults.set(sessionId, { + filename: 'database_scan', + timestamp: new Date().toISOString(), + allResults: transformedResults, + summary: { + total: total, + successful: successful, + warnings: warnings, + failed: failed, + malformed: malformed + } + }); + + // Send completion message + logger.info(`Database scan and geocoding completed: ${results.success} successful, ${results.failed} failed`); + + res.write(`data: ${JSON.stringify({ + type: 'complete', + message: `Scan and geocode completed! Successfully updated ${results.success} records, ${results.failed} failed.`, + results: results, + sessionId: sessionId + })}\n\n`); + res.end(); + + } catch (error) { + logger.error('Database scan error:', error); + res.write(`data: ${JSON.stringify({ + type: 'error', + message: `Database scan failed: ${error.message}` + })}\n\n`); + res.end(); + } + } } module.exports = new DataConvertController(); diff --git a/map/app/public/admin.html b/map/app/public/admin.html index 623e538..3360762 100644 --- a/map/app/public/admin.html +++ b/map/app/public/admin.html @@ -1255,6 +1255,116 @@ + + +
+
+

🔍 Scan & Geocode Database

+

Scan your existing database for records that are missing location data and automatically geocode them.

+ +
+

What this does:

+
    +
  • Scans all records in your database
    • +
  • Finds records with addresses but no geo-location data
    • +
  • Automatically geocodes those addresses using the same multi-provider system
    • +
  • Updates records with coordinates, confidence scores, and provider information
    • +
  • Provides detailed progress tracking and error reporting
    • +
+ +
+

⚠ī¸ Important Notes:

+
    +
  • This will modify existing records in your database
    • +
  • Only processes records that have an address but no coordinates
    • +
  • Rate limited to be respectful to geocoding APIs (0.5 seconds between requests)
    • +
  • You can download a detailed report when completed
    • +
+
+
+ +
+ + +
+
+ + +
diff --git a/map/app/public/js/data-convert.js b/map/app/public/js/data-convert.js index 922a366..381ff50 100644 --- a/map/app/public/js/data-convert.js +++ b/map/app/public/js/data-convert.js @@ -189,6 +189,29 @@ function setupDataConvertEventListeners() { newUploadBtn.addEventListener('click', resetToUpload); } + // Scan & Geocode Database button + const scanGeocodeBtn = document.getElementById('scan-geocode-btn'); + const cancelScanBtn = document.getElementById('cancel-scan-btn'); + const downloadScanReportBtn = document.getElementById('download-scan-report-btn'); + const newScanBtn = document.getElementById('new-scan-btn'); + + if (scanGeocodeBtn) { + scanGeocodeBtn.addEventListener('click', startDatabaseScan); + console.log('Scan geocode button event listener attached'); + } + + if (cancelScanBtn) { + cancelScanBtn.addEventListener('click', cancelDatabaseScan); + } + + if (downloadScanReportBtn) { + downloadScanReportBtn.addEventListener('click', downloadScanReport); + } + + if (newScanBtn) { + newScanBtn.addEventListener('click', resetScanInterface); + } + // Mark as initialized eventListenersInitialized = true; console.log('Data convert event listeners initialized successfully'); @@ -937,3 +960,296 @@ async function downloadProcessingReport() { downloadBtn.disabled = false; } } + +// === DATABASE SCAN & GEOCODE FUNCTIONALITY === + +async function startDatabaseScan() { + try { + const scanBtn = document.getElementById('scan-geocode-btn'); + const cancelBtn = document.getElementById('cancel-scan-btn'); + const processingSection = document.getElementById('scan-processing-section'); + const scanPhase = document.getElementById('scan-phase'); + const scanStatus = document.getElementById('scan-status'); + const progressContainer = document.getElementById('scan-progress-container'); + const summary = document.getElementById('scan-summary'); + const resultsPreview = document.getElementById('scan-results-preview'); + const actionsCompleted = document.getElementById('scan-actions-completed'); + + // Show processing UI and hide scan button + if (scanBtn) scanBtn.style.display = 'none'; + if (cancelBtn) cancelBtn.style.display = 'inline-block'; + if (processingSection) processingSection.style.display = 'block'; + + // Reset UI state + if (progressContainer) progressContainer.style.display = 'none'; + if (summary) summary.style.display = 'none'; + if (resultsPreview) resultsPreview.style.display = 'none'; + if (actionsCompleted) actionsCompleted.style.display = 'none'; + + // Clear previous results + const scanResultsTable = document.getElementById('scan-results-tbody'); + if (scanResultsTable) scanResultsTable.innerHTML = ''; + + console.log('Starting database scan and geocode...'); + + // Make POST request for Server-Sent Events + const response = await fetch('/api/admin/data-convert/scan-and-geocode', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'text/event-stream' + } + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + // Process the stream manually + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + + let scanSessionId = null; + let scanResults = []; + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + const chunk = decoder.decode(value); + const lines = chunk.split('\n'); + + for (const line of lines) { + if (line.startsWith('data: ') && line.trim() !== 'data: ') { + try { + const data = JSON.parse(line.substring(6)); + console.log('Scan event received:', data); + + switch (data.type) { + case 'status': + if (scanStatus) scanStatus.textContent = data.message; + if (data.sessionId) { + scanSessionId = data.sessionId; + console.log('Initial scan session ID set:', scanSessionId); + } + break; + + case 'scanning': + if (scanStatus) scanStatus.textContent = data.message; + break; + + case 'scan_complete': + if (scanStatus) scanStatus.textContent = data.message; + + // Update scan summary + document.getElementById('total-records').textContent = data.total || 0; + document.getElementById('need-geocoding').textContent = data.needingGeocode || 0; + document.getElementById('successfully-geocoded').textContent = '0'; + document.getElementById('failed-geocoded').textContent = '0'; + + if (summary) summary.style.display = 'block'; + + if (data.needingGeocode > 0) { + // Show progress bar and update phase + if (progressContainer) progressContainer.style.display = 'block'; + if (scanPhase) { + scanPhase.innerHTML = '

Phase 2: Geocoding Addresses

'; + } + + // Update progress totals + document.getElementById('scan-progress-total').textContent = data.needingGeocode; + } + break; + + case 'progress': + // Update progress bar + const progressPercent = (data.current / data.total) * 100; + const progressBar = document.getElementById('scan-progress-bar-fill'); + if (progressBar) progressBar.style.width = `${progressPercent}%`; + + document.getElementById('scan-progress-current').textContent = data.current; + document.getElementById('scan-current-address').textContent = `Processing: ${data.currentAddress}`; + break; + + case 'geocoded': + // Update success counter + const successCount = parseInt(document.getElementById('successfully-geocoded').textContent) + 1; + document.getElementById('successfully-geocoded').textContent = successCount; + + // Add to results preview + addScanResultToTable(data.data, data.status); + scanResults.push(data.data); + + // Show results preview if not already visible + if (resultsPreview) resultsPreview.style.display = 'block'; + break; + + case 'error': + if (data.data) { + // Update failed counter + const failedCount = parseInt(document.getElementById('failed-geocoded').textContent) + 1; + document.getElementById('failed-geocoded').textContent = failedCount; + + // Add to results preview + addScanResultToTable(data.data, 'error'); + scanResults.push(data.data); + } else { + // General error + if (scanStatus) scanStatus.textContent = `Error: ${data.message}`; + } + break; + + case 'complete': + if (scanStatus) scanStatus.textContent = data.message; + + // Show completed actions + if (actionsCompleted) actionsCompleted.style.display = 'block'; + + // Store session ID for report download from the completion message + const finalSessionId = data.sessionId || data.results?.sessionId || scanSessionId; + if (finalSessionId) { + const downloadBtn = document.getElementById('download-scan-report-btn'); + if (downloadBtn) { + downloadBtn.dataset.sessionId = finalSessionId; + console.log('Stored scan session ID for report download:', finalSessionId); + } + } else { + console.warn('No session ID available for scan report download'); + } + + // Hide cancel button + if (cancelBtn) cancelBtn.style.display = 'none'; + + // Exit the loop when complete + return; + } + + } catch (error) { + console.error('Error parsing scan event:', error); + } + } + } + } + } catch (error) { + console.error('Stream processing error:', error); + if (scanStatus) scanStatus.textContent = 'Error processing stream'; + } finally { + // Clean up reader + reader.releaseLock(); + } + + } catch (error) { + console.error('Error starting database scan:', error); + alert('Failed to start database scan: ' + error.message); + } +} + +function cancelDatabaseScan() { + console.log('Cancelling database scan...'); + + // Note: With fetch streams, we can't easily cancel mid-stream + // but we can reset the UI to let user know cancellation was requested + + // Reset UI + const scanBtn = document.getElementById('scan-geocode-btn'); + const cancelBtn = document.getElementById('cancel-scan-btn'); + const scanStatus = document.getElementById('scan-status'); + + if (scanBtn) scanBtn.style.display = 'inline-block'; + if (cancelBtn) cancelBtn.style.display = 'none'; + if (scanStatus) scanStatus.textContent = 'Scan cancelled by user.'; +} + +function addScanResultToTable(result, status) { + const tbody = document.getElementById('scan-results-tbody'); + if (!tbody) return; + + // Limit table to last 10 results + while (tbody.children.length >= 10) { + tbody.removeChild(tbody.firstChild); + } + + const row = document.createElement('tr'); + row.className = `status-${status}`; + + const statusIcon = status === 'success' ? '✅' : + status === 'warning' ? '⚠ī¸' : '❌'; + + const coordinates = result.latitude && result.longitude ? + `${result.latitude}, ${result.longitude}` : 'Failed'; + + const confidence = result.confidence_score !== undefined ? + `${result.confidence_score}%` : 'N/A'; + + row.innerHTML = ` + ${statusIcon} ${status.toUpperCase()} + ${result.address || 'Unknown'} + ${coordinates} + ${confidence} + ${result.provider || 'N/A'} + `; + + tbody.appendChild(row); +} + +async function downloadScanReport() { + const downloadBtn = document.getElementById('download-scan-report-btn'); + const sessionId = downloadBtn?.dataset.sessionId; + + if (!sessionId) { + alert('No processing session available for report generation'); + return; + } + + try { + const response = await fetch(`/api/admin/data-convert/download-report/${sessionId}`); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + // Get the filename from the response headers + const filename = response.headers.get('content-disposition') + ?.split('filename=')[1] + ?.replace(/"/g, '') || `scan-geocoding-report-${sessionId}.txt`; + + // Create blob and download + const blob = await response.blob(); + const url = window.URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = filename; + document.body.appendChild(a); + a.click(); + window.URL.revokeObjectURL(url); + document.body.removeChild(a); + + // Show success message in scan status area + const scanStatus = document.getElementById('scan-status'); + if (scanStatus) { + scanStatus.innerHTML = '✅ Report downloaded successfully'; + } + + } catch (error) { + console.error('Download scan report error:', error); + // Show error message in scan status area + const scanStatus = document.getElementById('scan-status'); + if (scanStatus) { + scanStatus.innerHTML = `❌ Failed to download report: ${error.message}`; + } else { + alert(`Failed to download report: ${error.message}`); + } + } +} + +function resetScanInterface() { + // Reset to initial state + const scanBtn = document.getElementById('scan-geocode-btn'); + const cancelBtn = document.getElementById('cancel-scan-btn'); + const processingSection = document.getElementById('scan-processing-section'); + + if (scanBtn) scanBtn.style.display = 'inline-block'; + if (cancelBtn) cancelBtn.style.display = 'none'; + if (processingSection) processingSection.style.display = 'none'; +} diff --git a/map/app/routes/dataConvert.js b/map/app/routes/dataConvert.js index e963a22..464ad8e 100644 --- a/map/app/routes/dataConvert.js +++ b/map/app/routes/dataConvert.js @@ -27,4 +27,7 @@ router.post('/save-geocoded', dataConvertController.saveGeocodedData); // Download processing report router.get('/download-report/:sessionId', dataConvertController.downloadReport); +// Scan database for records missing geo-location data and geocode them +router.post('/scan-and-geocode', dataConvertController.scanAndGeocode); + module.exports = router; diff --git a/map/build-nocodb.sh b/map/build-nocodb.sh index 0740b52..4c5e484 100755 --- a/map/build-nocodb.sh +++ b/map/build-nocodb.sh @@ -917,6 +917,32 @@ create_locations_table() { "title": "last_updated_by_user", "uidt": "SingleLineText", "rqd": false + }, + { + "column_name": "geocode_confidence", + "title": "Geocode Confidence", + "uidt": "Number", + "rqd": false, + "meta": { + "min": 0, + "max": 100 + } + }, + { + "column_name": "geocode_provider", + "title": "Geocode Provider", + "uidt": "SingleSelect", + "rqd": false, + "colOptions": { + "options": [ + {"title": "Mapbox", "color": "#2563eb"}, + {"title": "Nominatim", "color": "#059669"}, + {"title": "Photon", "color": "#dc2626"}, + {"title": "LocationIQ", "color": "#7c3aed"}, + {"title": "ArcGIS", "color": "#ea580c"}, + {"title": "Unknown", "color": "#6b7280"} + ] + } } ] }'