New update for the geo-coding system to include system to automatically scan the nocodb locations to build geo-locations

map/app/controllers/dataConvertController.js

@@ -15,6 +15,7 @@ class DataConvertController {
         this.parseCSV = this.parseCSV.bind(this);
         this.saveGeocodedData = this.saveGeocodedData.bind(this);
         this.downloadReport = this.downloadReport.bind(this);
+        this.scanAndGeocode = this.scanAndGeocode.bind(this);
     }
 
     // Process CSV upload and geocode addresses with SSE progress updates
@@ -152,7 +153,10 @@ class DataConvertController {
                     if (geocodeResult && geocodeResult.coordinates) {
                         // Check if result is malformed
                         const isMalformed = geocodeResult.validation && geocodeResult.validation.isMalformed;
-                        const confidence = geocodeResult.validation ? geocodeResult.validation.confidence : 100;
+                        // Use combined confidence for best overall assessment
+                        const confidence = geocodeResult.combinedConfidence !== undefined ? 
+                                         geocodeResult.combinedConfidence : 
+                                         (geocodeResult.validation ? geocodeResult.validation.confidence : 100);
                         const warnings = geocodeResult.validation ? geocodeResult.validation.warnings : [];
                         
                         const processedRow = {
@@ -165,6 +169,8 @@ class DataConvertController {
                             geocode_status: isMalformed ? 'WARNING' : 'SUCCESS',
                             geocode_error: '',
                             confidence_score: confidence,
+                            provider_confidence: geocodeResult.providerConfidence || null,
+                            validation_confidence: geocodeResult.validation ? geocodeResult.validation.confidence : null,
                             warnings: warnings.join('; '),
                             is_malformed: isMalformed,
                             provider: geocodeResult.provider || 'Unknown',
@@ -333,6 +339,8 @@ class DataConvertController {
                         latitude: parseFloat(location.latitude),
                         longitude: parseFloat(location.longitude),
                         Address: originalAddress, // Always use the original address from CSV
+                        'Geocode Confidence': location.confidence_score || null, // Add confidence score
+                        'Geocode Provider': location.provider || null, // Add provider name
                         created_by_user: req.session.userEmail || 'csv_import',
                         last_updated_by_user: req.session.userEmail || 'csv_import'
                     };
@@ -367,7 +375,7 @@ class DataConvertController {
                         const lowerKey = key.toLowerCase();
                         
                         // Skip already processed fields
-                        if (['latitude', 'longitude', 'geo-location', 'geocoded_address', 'geocode_success', 'address', 'csv_filename'].includes(lowerKey)) {
+                        if (['latitude', 'longitude', 'geo-location', 'geocoded_address', 'geocode_success', 'address', 'csv_filename', 'confidence_score', 'provider_confidence', 'validation_confidence', 'warnings', 'is_malformed', 'provider', 'row_number', 'geocode_status', 'geocode_error'].includes(lowerKey)) {
                             return;
                         }
 
@@ -402,6 +410,21 @@ class DataConvertController {
                                     noteParts.push(`Geocoded as: ${geocodedAddress}`);
                                 }
                                 
+                                // Add confidence information if available
+                                if (location.confidence_score !== undefined && location.confidence_score !== null) {
+                                    noteParts.push(`Geocode confidence: ${location.confidence_score}%`);
+                                }
+                                
+                                // Add provider information if available
+                                if (location.provider) {
+                                    noteParts.push(`Provider: ${location.provider}`);
+                                }
+                                
+                                // Add warnings if present
+                                if (location.warnings && location.warnings.trim()) {
+                                    noteParts.push(`Warnings: ${location.warnings}`);
+                                }
+                                
                                 locationData[targetField] = noteParts.join(' | ');
                             } else {
                                 locationData[targetField] = location[key];
@@ -418,6 +441,21 @@ class DataConvertController {
                             noteParts.push(`Geocoded as: ${geocodedAddress}`);
                         }
                         
+                        // Add confidence information if available
+                        if (location.confidence_score !== undefined && location.confidence_score !== null) {
+                            noteParts.push(`Geocode confidence: ${location.confidence_score}%`);
+                        }
+                        
+                        // Add provider information if available
+                        if (location.provider) {
+                            noteParts.push(`Provider: ${location.provider}`);
+                        }
+                        
+                        // Add warnings if present
+                        if (location.warnings && location.warnings.trim()) {
+                            noteParts.push(`Warnings: ${location.warnings}`);
+                        }
+                        
                         locationData['Notes'] = noteParts.join(' | ');
                     }
 
@@ -455,6 +493,7 @@ class DataConvertController {
     async downloadReport(req, res) {
         try {
             const { sessionId } = req.params;
+            const format = req.query.format || 'csv'; // Default to CSV, support 'txt' for backward compatibility
             
             if (!sessionId || !processingResults.has(sessionId)) {
                 return res.status(404).json({
@@ -466,16 +505,27 @@ class DataConvertController {
             const results = processingResults.get(sessionId);
             const { filename, timestamp, allResults, summary } = results;
 
-            // Generate comprehensive report content
+            let reportContent, contentType, fileExtension;
-            const reportContent = this.generateComprehensiveReport(allResults, filename, timestamp, summary);
             
-            // Set headers for text download
+            if (format === 'csv') {
-            const reportFilename = `geocoding-report-${sessionId}.txt`;
+                // Generate CSV report
-            res.setHeader('Content-Type', 'text/plain');
+                reportContent = this.generateReportCSV(allResults, filename, timestamp, summary);
+                contentType = 'text/csv';
+                fileExtension = 'csv';
+            } else {
+                // Generate text report (backward compatibility)
+                reportContent = this.generateComprehensiveReport(allResults, filename, timestamp, summary);
+                contentType = 'text/plain';
+                fileExtension = 'txt';
+            }
+            
+            // Set headers for download
+            const reportFilename = `geocoding-report-${sessionId}.${fileExtension}`;
+            res.setHeader('Content-Type', contentType);
             res.setHeader('Content-Disposition', `attachment; filename="${reportFilename}"`);
             res.setHeader('Cache-Control', 'no-cache');
 
-            logger.info(`Generating comprehensive report for session ${sessionId}: ${allResults.length} records`);
+            logger.info(`Generating ${format.toUpperCase()} report for session ${sessionId}: ${allResults.length} records`);
 
             res.send(reportContent);
 
@@ -651,6 +701,290 @@ class DataConvertController {
         
         return stringField;
     }
+
+    // Scan NocoDB database for records missing geo-location data and geocode them
+    async scanAndGeocode(req, res) {
+        try {
+            const sessionId = Date.now().toString();
+            
+            // Set up SSE headers
+            res.writeHead(200, {
+                'Content-Type': 'text/event-stream',
+                'Cache-Control': 'no-cache',
+                'Connection': 'keep-alive',
+                'X-Accel-Buffering': 'no'
+            });
+
+            logger.info(`Starting database scan for missing geo-location data (session: ${sessionId})`);
+
+            // Send initial status
+            res.write(`data: ${JSON.stringify({ 
+                type: 'status', 
+                message: 'Scanning database for records missing geo-location data...',
+                sessionId: sessionId 
+            })}\n\n`);
+            res.flush && res.flush();
+
+            // Fetch all records from NocoDB
+            let allRecords = [];
+            let offset = 0;
+            const limit = 100; // Process in batches
+            let hasMoreRecords = true;
+
+            while (hasMoreRecords) {
+                try {
+                    const response = await nocodbService.getAll(config.nocodb.tableId, { limit, offset });
+                    
+                    if (response && response.list && response.list.length > 0) {
+                        allRecords.push(...response.list);
+                        offset += limit;
+                        
+                        // Send progress update
+                        res.write(`data: ${JSON.stringify({ 
+                            type: 'scanning', 
+                            message: `Fetched ${allRecords.length} records from database...`,
+                            count: allRecords.length
+                        })}\n\n`);
+                        res.flush && res.flush();
+                        
+                        // Check if we've fetched all records
+                        hasMoreRecords = response.list.length === limit;
+                    } else {
+                        hasMoreRecords = false;
+                    }
+                } catch (error) {
+                    logger.error('Error fetching records from NocoDB:', error);
+                    res.write(`data: ${JSON.stringify({ 
+                        type: 'error', 
+                        message: `Error fetching records: ${error.message}` 
+                    })}\n\n`);
+                    res.end();
+                    return;
+                }
+            }
+
+            logger.info(`Database scan complete: found ${allRecords.length} total records`);
+
+            // Filter records that need geocoding
+            const recordsNeedingGeocode = allRecords.filter(record => {
+                // Check if record is missing geo-location data
+                const hasGeoLocation = record['Geo-Location'] && 
+                                     record['Geo-Location'].trim() !== '' &&
+                                     record['Geo-Location'] !== 'null';
+                const hasCoordinates = (record.latitude && record.longitude) ||
+                                     (record.Latitude && record.Longitude);
+                const hasAddress = record.Address || record.address || record.ADDRESS;
+                
+                return !hasGeoLocation && !hasCoordinates && hasAddress;
+            });
+
+            const totalToGeocode = recordsNeedingGeocode.length;
+
+            logger.info(`Found ${totalToGeocode} records needing geocoding`);
+
+            // Send summary
+            res.write(`data: ${JSON.stringify({ 
+                type: 'scan_complete', 
+                message: `Scan complete: ${totalToGeocode} records need geocoding`,
+                total: allRecords.length,
+                needingGeocode: totalToGeocode 
+            })}\n\n`);
+            res.flush && res.flush();
+
+            if (totalToGeocode === 0) {
+                res.write(`data: ${JSON.stringify({ 
+                    type: 'complete', 
+                    message: 'No records found that need geocoding. All records already have location data!',
+                    results: { success: 0, failed: 0, skipped: allRecords.length }
+                })}\n\n`);
+                res.end();
+                return;
+            }
+
+            // Process geocoding
+            const results = {
+                success: 0,
+                failed: 0,
+                errors: [],
+                sessionId: sessionId
+            };
+
+            const allResults = [];
+            let processedCount = 0;
+
+            for (const record of recordsNeedingGeocode) {
+                try {
+                    processedCount++;
+                    const address = record.Address || record.address || record.ADDRESS;
+                    
+                    // Send progress update
+                    res.write(`data: ${JSON.stringify({ 
+                        type: 'progress', 
+                        current: processedCount,
+                        total: totalToGeocode,
+                        currentAddress: address,
+                        status: 'processing'
+                    })}\n\n`);
+                    res.flush && res.flush();
+
+                    logger.info(`Geocoding ${processedCount}/${totalToGeocode}: ${address} (Record ID: ${record.id || record.Id || record.ID})`);
+
+                    // Geocode the address
+                    const geocodeResult = await forwardGeocode(address);
+                    
+                    if (geocodeResult && geocodeResult.coordinates) {
+                        // Check if result is malformed
+                        const isMalformed = geocodeResult.validation && geocodeResult.validation.isMalformed;
+                        // Use combined confidence for best overall assessment
+                        const confidence = geocodeResult.combinedConfidence !== undefined ? 
+                                         geocodeResult.combinedConfidence : 
+                                         (geocodeResult.validation ? geocodeResult.validation.confidence : 100);
+                        const warnings = geocodeResult.validation ? geocodeResult.validation.warnings : [];
+
+                        // Update the record in NocoDB
+                        const updateData = {
+                            'Geo-Location': `${geocodeResult.coordinates.lat};${geocodeResult.coordinates.lng}`,
+                            latitude: geocodeResult.coordinates.lat,
+                            longitude: geocodeResult.coordinates.lng,
+                            'Geocode Confidence': confidence,
+                            'Geocode Provider': geocodeResult.provider || 'Unknown',
+                            last_updated_by_user: req.session?.userEmail || 'scan_geocode'
+                        };
+
+                        // Update the record in NocoDB
+                        await nocodbService.update(config.nocodb.tableId, record.id || record.Id || record.ID, updateData);
+                        
+                        const processedRecord = {
+                            id: record.id || record.Id || record.ID,
+                            address: address,
+                            latitude: geocodeResult.coordinates.lat,
+                            longitude: geocodeResult.coordinates.lng,
+                            confidence_score: confidence,
+                            provider: geocodeResult.provider || 'Unknown',
+                            status: isMalformed ? 'WARNING' : 'SUCCESS',
+                            warnings: warnings.join('; ')
+                        };
+                        
+                        allResults.push(processedRecord);
+                        results.success++;
+
+                        // Send success update
+                        const successMessage = { 
+                            type: 'geocoded', 
+                            data: processedRecord,
+                            index: processedCount - 1,
+                            status: isMalformed ? 'warning' : 'success',
+                            confidence: confidence,
+                            warnings: warnings
+                        };
+                        
+                        logger.info(`✓ Successfully geocoded and updated: ${address} (Confidence: ${confidence}%)`);
+                        res.write(`data: ${JSON.stringify(successMessage)}\n\n`);
+                        res.flush && res.flush();
+                        
+                    } else {
+                        throw new Error('Geocoding failed - no coordinates returned');
+                    }
+                    
+                } catch (error) {
+                    logger.error(`Failed to geocode record ${processedCount}/${totalToGeocode}:`, error.message);
+                    
+                    const errorRecord = {
+                        id: record.id || record.Id || record.ID,
+                        address: record.Address || record.address || record.ADDRESS,
+                        error: error.message,
+                        status: 'ERROR'
+                    };
+                    
+                    allResults.push(errorRecord);
+                    results.failed++;
+                    results.errors.push({
+                        address: errorRecord.address,
+                        error: error.message
+                    });
+
+                    // Send error update
+                    res.write(`data: ${JSON.stringify({ 
+                        type: 'error', 
+                        data: errorRecord,
+                        index: processedCount - 1,
+                        message: `Failed to geocode: ${errorRecord.address}` 
+                    })}\n\n`);
+                    res.flush && res.flush();
+                }
+
+                // Rate limiting to be nice to geocoding APIs
+                if (processedCount < totalToGeocode) {
+                    await new Promise(resolve => setTimeout(resolve, 500)); // 0.5 second delay between requests
+                }
+            }
+
+            // Calculate summary statistics for report
+            const successful = allResults.filter(r => r.status === 'SUCCESS').length;
+            const warnings = allResults.filter(r => r.status === 'WARNING').length; 
+            const failed = allResults.filter(r => r.status === 'ERROR').length;
+            const malformed = allResults.filter(r => r.warnings && r.warnings.includes('malformed')).length;
+            const total = successful + warnings + failed;
+
+            // Transform scan results to match CSV processing format for report generation
+            const transformedResults = allResults.map(result => ({
+                // Original format fields
+                address: result.address,
+                Address: result.address, 
+                geocoded_address: result.address, // For scan, this is the same
+                latitude: result.latitude,
+                longitude: result.longitude,
+                'Geo-Location': result.latitude && result.longitude ? `${result.latitude};${result.longitude}` : '',
+                confidence_score: result.confidence_score,
+                provider: result.provider,
+                
+                // Status mapping
+                geocode_success: result.status !== 'ERROR',
+                geocode_status: result.status,
+                geocode_error: result.error || '',
+                is_malformed: result.warnings && result.warnings.includes('malformed'),
+                warnings: result.warnings || '',
+                
+                // Scan-specific fields
+                record_id: result.id,
+                source: 'database_scan',
+                row_number: result.id // Use record ID as row number for scan
+            }));
+
+            // Store results for potential report download
+            processingResults.set(sessionId, {
+                filename: 'database_scan',
+                timestamp: new Date().toISOString(),
+                allResults: transformedResults,
+                summary: {
+                    total: total,
+                    successful: successful,
+                    warnings: warnings,
+                    failed: failed,
+                    malformed: malformed
+                }
+            });
+
+            // Send completion message
+            logger.info(`Database scan and geocoding completed: ${results.success} successful, ${results.failed} failed`);
+            
+            res.write(`data: ${JSON.stringify({ 
+                type: 'complete', 
+                message: `Scan and geocode completed! Successfully updated ${results.success} records, ${results.failed} failed.`,
+                results: results,
+                sessionId: sessionId
+            })}\n\n`);
+            res.end();
+
+        } catch (error) {
+            logger.error('Database scan error:', error);
+            res.write(`data: ${JSON.stringify({ 
+                type: 'error', 
+                message: `Database scan failed: ${error.message}` 
+            })}\n\n`);
+            res.end();
+        }
+    }
 }
 
 module.exports = new DataConvertController();

map/app/public/admin.html

@@ -1255,6 +1255,116 @@
                             </div>
                         </div>
                     </div>
+                    
+                    <!-- Scan & Geocode Existing Records -->
+                    <div class="scan-geocode-container" style="margin-top: 30px;">
+                        <div class="info-box">
+                            <h3>🔍 Scan & Geocode Database</h3>
+                            <p>Scan your existing database for records that are missing location data and automatically geocode them.</p>
+                            
+                            <div class="scan-info">
+                                <h4>What this does:</h4>
+                                <ul>
+                                    <li>Scans all records in your database</li>
+                                    <li>Finds records with addresses but no geo-location data</li>
+                                    <li>Automatically geocodes those addresses using the same multi-provider system</li>
+                                    <li>Updates records with coordinates, confidence scores, and provider information</li>
+                                    <li>Provides detailed progress tracking and error reporting</li>
+                                </ul>
+                                
+                                <div class="scan-warning">
+                                    <h4>⚠️ Important Notes:</h4>
+                                    <ul>
+                                        <li>This will modify existing records in your database</li>
+                                        <li>Only processes records that have an address but no coordinates</li>
+                                        <li>Rate limited to be respectful to geocoding APIs (0.5 seconds between requests)</li>
+                                        <li>You can download a detailed report when completed</li>
+                                    </ul>
+                                </div>
+                            </div>
+                            
+                            <div class="scan-actions">
+                                <button type="button" class="btn btn-primary" id="scan-geocode-btn">
+                                    🔍 Start Database Scan & Geocode
+                                </button>
+                                <button type="button" class="btn btn-secondary" id="cancel-scan-btn" style="display: none;">
+                                    ⏹️ Cancel Scan
+                                </button>
+                            </div>
+                        </div>
+                        
+                        <div class="scan-processing-section" id="scan-processing-section" style="display: none;">
+                            <h3>Database Scan Progress</h3>
+                            
+                            <div class="scan-phase" id="scan-phase">
+                                <h4>Phase 1: Database Scan</h4>
+                                <p id="scan-status">Scanning database for records...</p>
+                            </div>
+                            
+                            <div class="progress-bar-container" id="scan-progress-container" style="display: none;">
+                                <div class="progress-bar">
+                                    <div class="progress-bar-fill" id="scan-progress-bar-fill"></div>
+                                </div>
+                                <p class="progress-text">
+                                    <span id="scan-progress-current">0</span> / 
+                                    <span id="scan-progress-total">0</span> addresses geocoded
+                                </p>
+                            </div>
+                            
+                            <div class="processing-status" id="scan-processing-status">
+                                <p id="scan-current-address"></p>
+                            </div>
+                            
+                            <div class="scan-summary" id="scan-summary" style="display: none;">
+                                <h4>Scan Summary</h4>
+                                <div class="summary-stats">
+                                    <div class="stat-item">
+                                        <span class="stat-label">Total Records:</span>
+                                        <span class="stat-value" id="total-records">0</span>
+                                    </div>
+                                    <div class="stat-item">
+                                        <span class="stat-label">Need Geocoding:</span>
+                                        <span class="stat-value" id="need-geocoding">0</span>
+                                    </div>
+                                    <div class="stat-item">
+                                        <span class="stat-label">Successfully Geocoded:</span>
+                                        <span class="stat-value" id="successfully-geocoded">0</span>
+                                    </div>
+                                    <div class="stat-item">
+                                        <span class="stat-label">Failed:</span>
+                                        <span class="stat-value" id="failed-geocoded">0</span>
+                                    </div>
+                                </div>
+                            </div>
+                            
+                            <div class="scan-results-preview" id="scan-results-preview" style="display: none;">
+                                <h4>Recent Results</h4>
+                                <div class="scan-results-table-container">
+                                    <table class="results-table" id="scan-results-table">
+                                        <thead>
+                                            <tr>
+                                                <th>Status</th>
+                                                <th>Address</th>
+                                                <th>Coordinates</th>
+                                                <th>Confidence</th>
+                                                <th>Provider</th>
+                                            </tr>
+                                        </thead>
+                                        <tbody id="scan-results-tbody"></tbody>
+                                    </table>
+                                </div>
+                            </div>
+                            
+                            <div class="scan-actions-completed" id="scan-actions-completed" style="display: none;">
+                                <button type="button" class="btn btn-primary" id="download-scan-report-btn">
+                                    📄 Download Detailed Report
+                                </button>
+                                <button type="button" class="btn btn-secondary" id="new-scan-btn">
+                                    🔍 Start New Scan
+                                </button>
+                            </div>
+                        </div>
+                    </div>
                 </section>
 
                 <!-- Email Lists (Listmonk) Section -->

map/app/public/js/data-convert.js

@@ -189,6 +189,29 @@ function setupDataConvertEventListeners() {
         newUploadBtn.addEventListener('click', resetToUpload);
     }
     
+    // Scan & Geocode Database button
+    const scanGeocodeBtn = document.getElementById('scan-geocode-btn');
+    const cancelScanBtn = document.getElementById('cancel-scan-btn');
+    const downloadScanReportBtn = document.getElementById('download-scan-report-btn');
+    const newScanBtn = document.getElementById('new-scan-btn');
+    
+    if (scanGeocodeBtn) {
+        scanGeocodeBtn.addEventListener('click', startDatabaseScan);
+        console.log('Scan geocode button event listener attached');
+    }
+    
+    if (cancelScanBtn) {
+        cancelScanBtn.addEventListener('click', cancelDatabaseScan);
+    }
+    
+    if (downloadScanReportBtn) {
+        downloadScanReportBtn.addEventListener('click', downloadScanReport);
+    }
+    
+    if (newScanBtn) {
+        newScanBtn.addEventListener('click', resetScanInterface);
+    }
+    
     // Mark as initialized
     eventListenersInitialized = true;
     console.log('Data convert event listeners initialized successfully');
@@ -937,3 +960,296 @@ async function downloadProcessingReport() {
         downloadBtn.disabled = false;
     }
 }
+
+// === DATABASE SCAN & GEOCODE FUNCTIONALITY ===
+
+async function startDatabaseScan() {
+    try {
+        const scanBtn = document.getElementById('scan-geocode-btn');
+        const cancelBtn = document.getElementById('cancel-scan-btn');
+        const processingSection = document.getElementById('scan-processing-section');
+        const scanPhase = document.getElementById('scan-phase');
+        const scanStatus = document.getElementById('scan-status');
+        const progressContainer = document.getElementById('scan-progress-container');
+        const summary = document.getElementById('scan-summary');
+        const resultsPreview = document.getElementById('scan-results-preview');
+        const actionsCompleted = document.getElementById('scan-actions-completed');
+        
+        // Show processing UI and hide scan button
+        if (scanBtn) scanBtn.style.display = 'none';
+        if (cancelBtn) cancelBtn.style.display = 'inline-block';
+        if (processingSection) processingSection.style.display = 'block';
+        
+        // Reset UI state
+        if (progressContainer) progressContainer.style.display = 'none';
+        if (summary) summary.style.display = 'none';
+        if (resultsPreview) resultsPreview.style.display = 'none';
+        if (actionsCompleted) actionsCompleted.style.display = 'none';
+        
+        // Clear previous results
+        const scanResultsTable = document.getElementById('scan-results-tbody');
+        if (scanResultsTable) scanResultsTable.innerHTML = '';
+        
+        console.log('Starting database scan and geocode...');
+        
+        // Make POST request for Server-Sent Events
+        const response = await fetch('/api/admin/data-convert/scan-and-geocode', {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Accept': 'text/event-stream'
+            }
+        });
+        
+        if (!response.ok) {
+            throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+        }
+        
+        // Process the stream manually
+        const reader = response.body.getReader();
+        const decoder = new TextDecoder();
+        
+        let scanSessionId = null;
+        let scanResults = [];
+        
+        try {
+            while (true) {
+                const { done, value } = await reader.read();
+                if (done) break;
+                
+                const chunk = decoder.decode(value);
+                const lines = chunk.split('\n');
+                
+                for (const line of lines) {
+                    if (line.startsWith('data: ') && line.trim() !== 'data: ') {
+                        try {
+                            const data = JSON.parse(line.substring(6));
+                            console.log('Scan event received:', data);
+                            
+                            switch (data.type) {
+                                case 'status':
+                                    if (scanStatus) scanStatus.textContent = data.message;
+                                    if (data.sessionId) {
+                                        scanSessionId = data.sessionId;
+                                        console.log('Initial scan session ID set:', scanSessionId);
+                                    }
+                                    break;
+                                    
+                                case 'scanning':
+                                    if (scanStatus) scanStatus.textContent = data.message;
+                                    break;
+                                    
+                                case 'scan_complete':
+                                    if (scanStatus) scanStatus.textContent = data.message;
+                                    
+                                    // Update scan summary
+                                    document.getElementById('total-records').textContent = data.total || 0;
+                                    document.getElementById('need-geocoding').textContent = data.needingGeocode || 0;
+                                    document.getElementById('successfully-geocoded').textContent = '0';
+                                    document.getElementById('failed-geocoded').textContent = '0';
+                                    
+                                    if (summary) summary.style.display = 'block';
+                                    
+                                    if (data.needingGeocode > 0) {
+                                        // Show progress bar and update phase
+                                        if (progressContainer) progressContainer.style.display = 'block';
+                                        if (scanPhase) {
+                                            scanPhase.innerHTML = '<h4>Phase 2: Geocoding Addresses</h4>';
+                                        }
+                                        
+                                        // Update progress totals
+                                        document.getElementById('scan-progress-total').textContent = data.needingGeocode;
+                                    }
+                                    break;
+                                    
+                                case 'progress':
+                                    // Update progress bar
+                                    const progressPercent = (data.current / data.total) * 100;
+                                    const progressBar = document.getElementById('scan-progress-bar-fill');
+                                    if (progressBar) progressBar.style.width = `${progressPercent}%`;
+                                    
+                                    document.getElementById('scan-progress-current').textContent = data.current;
+                                    document.getElementById('scan-current-address').textContent = `Processing: ${data.currentAddress}`;
+                                    break;
+                                    
+                                case 'geocoded':
+                                    // Update success counter
+                                    const successCount = parseInt(document.getElementById('successfully-geocoded').textContent) + 1;
+                                    document.getElementById('successfully-geocoded').textContent = successCount;
+                                    
+                                    // Add to results preview
+                                    addScanResultToTable(data.data, data.status);
+                                    scanResults.push(data.data);
+                                    
+                                    // Show results preview if not already visible
+                                    if (resultsPreview) resultsPreview.style.display = 'block';
+                                    break;
+                                    
+                                case 'error':
+                                    if (data.data) {
+                                        // Update failed counter
+                                        const failedCount = parseInt(document.getElementById('failed-geocoded').textContent) + 1;
+                                        document.getElementById('failed-geocoded').textContent = failedCount;
+                                        
+                                        // Add to results preview
+                                        addScanResultToTable(data.data, 'error');
+                                        scanResults.push(data.data);
+                                    } else {
+                                        // General error
+                                        if (scanStatus) scanStatus.textContent = `Error: ${data.message}`;
+                                    }
+                                    break;
+                                    
+                                case 'complete':
+                                    if (scanStatus) scanStatus.textContent = data.message;
+                                    
+                                    // Show completed actions
+                                    if (actionsCompleted) actionsCompleted.style.display = 'block';
+                                    
+                                    // Store session ID for report download from the completion message
+                                    const finalSessionId = data.sessionId || data.results?.sessionId || scanSessionId;
+                                    if (finalSessionId) {
+                                        const downloadBtn = document.getElementById('download-scan-report-btn');
+                                        if (downloadBtn) {
+                                            downloadBtn.dataset.sessionId = finalSessionId;
+                                            console.log('Stored scan session ID for report download:', finalSessionId);
+                                        }
+                                    } else {
+                                        console.warn('No session ID available for scan report download');
+                                    }
+                                    
+                                    // Hide cancel button
+                                    if (cancelBtn) cancelBtn.style.display = 'none';
+                                    
+                                    // Exit the loop when complete
+                                    return;
+                            }
+                            
+                        } catch (error) {
+                            console.error('Error parsing scan event:', error);
+                        }
+                    }
+                }
+            }
+        } catch (error) {
+            console.error('Stream processing error:', error);
+            if (scanStatus) scanStatus.textContent = 'Error processing stream';
+        } finally {
+            // Clean up reader
+            reader.releaseLock();
+        }
+        
+    } catch (error) {
+        console.error('Error starting database scan:', error);
+        alert('Failed to start database scan: ' + error.message);
+    }
+}
+
+function cancelDatabaseScan() {
+    console.log('Cancelling database scan...');
+    
+    // Note: With fetch streams, we can't easily cancel mid-stream
+    // but we can reset the UI to let user know cancellation was requested
+    
+    // Reset UI
+    const scanBtn = document.getElementById('scan-geocode-btn');
+    const cancelBtn = document.getElementById('cancel-scan-btn');
+    const scanStatus = document.getElementById('scan-status');
+    
+    if (scanBtn) scanBtn.style.display = 'inline-block';
+    if (cancelBtn) cancelBtn.style.display = 'none';
+    if (scanStatus) scanStatus.textContent = 'Scan cancelled by user.';
+}
+
+function addScanResultToTable(result, status) {
+    const tbody = document.getElementById('scan-results-tbody');
+    if (!tbody) return;
+    
+    // Limit table to last 10 results
+    while (tbody.children.length >= 10) {
+        tbody.removeChild(tbody.firstChild);
+    }
+    
+    const row = document.createElement('tr');
+    row.className = `status-${status}`;
+    
+    const statusIcon = status === 'success' ? '✅' : 
+                      status === 'warning' ? '⚠️' : '❌';
+    
+    const coordinates = result.latitude && result.longitude ? 
+                       `${result.latitude}, ${result.longitude}` : 'Failed';
+    
+    const confidence = result.confidence_score !== undefined ? 
+                      `${result.confidence_score}%` : 'N/A';
+    
+    row.innerHTML = `
+        <td>${statusIcon} ${status.toUpperCase()}</td>
+        <td>${result.address || 'Unknown'}</td>
+        <td>${coordinates}</td>
+        <td>${confidence}</td>
+        <td>${result.provider || 'N/A'}</td>
+    `;
+    
+    tbody.appendChild(row);
+}
+
+async function downloadScanReport() {
+    const downloadBtn = document.getElementById('download-scan-report-btn');
+    const sessionId = downloadBtn?.dataset.sessionId;
+    
+    if (!sessionId) {
+        alert('No processing session available for report generation');
+        return;
+    }
+    
+    try {
+        const response = await fetch(`/api/admin/data-convert/download-report/${sessionId}`);
+        
+        if (!response.ok) {
+            throw new Error(`HTTP error! status: ${response.status}`);
+        }
+        
+        // Get the filename from the response headers
+        const filename = response.headers.get('content-disposition')
+            ?.split('filename=')[1]
+            ?.replace(/"/g, '') || `scan-geocoding-report-${sessionId}.txt`;
+        
+        // Create blob and download
+        const blob = await response.blob();
+        const url = window.URL.createObjectURL(blob);
+        const a = document.createElement('a');
+        a.href = url;
+        a.download = filename;
+        document.body.appendChild(a);
+        a.click();
+        window.URL.revokeObjectURL(url);
+        document.body.removeChild(a);
+        
+        // Show success message in scan status area
+        const scanStatus = document.getElementById('scan-status');
+        if (scanStatus) {
+            scanStatus.innerHTML = '✅ Report downloaded successfully';
+        }
+        
+    } catch (error) {
+        console.error('Download scan report error:', error);
+        // Show error message in scan status area
+        const scanStatus = document.getElementById('scan-status');
+        if (scanStatus) {
+            scanStatus.innerHTML = `❌ Failed to download report: ${error.message}`;
+        } else {
+            alert(`Failed to download report: ${error.message}`);
+        }
+    }
+}
+
+function resetScanInterface() {
+    // Reset to initial state
+    const scanBtn = document.getElementById('scan-geocode-btn');
+    const cancelBtn = document.getElementById('cancel-scan-btn');
+    const processingSection = document.getElementById('scan-processing-section');
+    
+    if (scanBtn) scanBtn.style.display = 'inline-block';
+    if (cancelBtn) cancelBtn.style.display = 'none';
+    if (processingSection) processingSection.style.display = 'none';
+}

map/app/routes/dataConvert.js

@@ -27,4 +27,7 @@ router.post('/save-geocoded', dataConvertController.saveGeocodedData);
 // Download processing report
 router.get('/download-report/:sessionId', dataConvertController.downloadReport);
 
+// Scan database for records missing geo-location data and geocode them
+router.post('/scan-and-geocode', dataConvertController.scanAndGeocode);
+
 module.exports = router;

map/build-nocodb.sh

@@ -917,6 +917,32 @@ create_locations_table() {
                 "title": "last_updated_by_user",
                 "uidt": "SingleLineText",
                 "rqd": false
+            },
+            {
+                "column_name": "geocode_confidence",
+                "title": "Geocode Confidence",
+                "uidt": "Number",
+                "rqd": false,
+                "meta": {
+                    "min": 0,
+                    "max": 100
+                }
+            },
+            {
+                "column_name": "geocode_provider",
+                "title": "Geocode Provider",
+                "uidt": "SingleSelect",
+                "rqd": false,
+                "colOptions": {
+                    "options": [
+                        {"title": "Mapbox", "color": "#2563eb"},
+                        {"title": "Nominatim", "color": "#059669"},
+                        {"title": "Photon", "color": "#dc2626"},
+                        {"title": "LocationIQ", "color": "#7c3aed"},
+                        {"title": "ArcGIS", "color": "#ea580c"},
+                        {"title": "Unknown", "color": "#6b7280"}
+                    ]
+                }
             }
         ]
     }'