diff --git a/map/README.md b/map/README.md index d88c17b..cdb6113 100644 --- a/map/README.md +++ b/map/README.md @@ -855,13 +855,35 @@ When connected to a MkDocs documentation site: ### Address Search For geographic location search: -- **Geocoding Integration**: Powered by Nominatim/OpenStreetMap +- **Multi-Provider Geocoding**: Uses multiple geocoding providers for maximum accuracy +- **Mapbox Integration**: Premium provider with structured address input (when API key provided) +- **Fallback Providers**: Free providers (Nominatim, Photon, ArcGIS, LocationIQ) for comprehensive coverage - **Multiple Results**: Returns up to 5 address matches - **Map Integration**: Click results to view location on map - **Temporary Markers**: Visual markers for search results - **Quick Actions**: Add locations directly from search results - **Coordinate Display**: Shows precise latitude/longitude coordinates +#### Geocoding Configuration + +For improved geocoding accuracy, you can configure premium providers: + +```env +# Mapbox API Key (Recommended for best accuracy) +MAPBOX_ACCESS_TOKEN=your_mapbox_access_token_here + +# LocationIQ API Key (Alternative premium option) +LOCATIONIQ_API_KEY=your_locationiq_api_key_here +``` + +**Mapbox Setup:** +1. Sign up at [mapbox.com](https://www.mapbox.com/) +2. Get your access token from the dashboard +3. Add `MAPBOX_ACCESS_TOKEN=your_token_here` to your `.env` file +4. Restart the application + +When Mapbox is configured, it will be used as the primary geocoding provider with automatic fallback to free providers if needed. + ### Database Search For searching through loaded location data: - **Full-text Search**: Search through names, addresses, emails, phone numbers, and notes diff --git a/map/app/controllers/dataConvertController.js b/map/app/controllers/dataConvertController.js index f391e0f..f55a87e 100644 --- a/map/app/controllers/dataConvertController.js +++ b/map/app/controllers/dataConvertController.js @@ -75,14 +75,71 @@ class DataConvertController { // Process each address with progress updates for (let i = 0; i < results.length; i++) { const row = results[i]; - const address = row.address || row.Address || row.ADDRESS; + + // Extract address - with better validation + const addressField = row.address || row.Address || row.ADDRESS || + row.street_address || row['Street Address'] || + row.full_address || row['Full Address']; + + // Extract unit number if available + const unitField = row.unit || row.Unit || row.UNIT || + row.unit_number || row['Unit Number'] || row.unit_no; + + if (!addressField || addressField.trim() === '') { + logger.warn(`Row ${i + 1}: Empty or missing address field`); + + const errorRow = { + ...row, + latitude: '', + longitude: '', + 'Geo-Location': '', + geocoded_address: '', + geocode_success: false, + geocode_status: 'FAILED', + geocode_error: 'Missing address field', + csv_filename: originalFilename, + row_number: i + 1 + }; + + allResults.push(errorRow); + errors.push({ + index: i, + address: 'No address provided', + error: 'Missing address field' + }); + + // Send progress update + res.write(`data: ${JSON.stringify({ + type: 'progress', + current: i + 1, + total: total, + currentAddress: 'No address - skipping', + status: 'failed' + })}\n\n`); + res.flush && res.flush(); + + continue; // Skip to next row + } - // Send progress update + // Construct full address with unit if available + let address = addressField.trim(); + if (unitField && unitField.toString().trim()) { + const unit = unitField.toString().trim(); + // Add unit prefix if it doesn't already exist + if (!unit.toLowerCase().startsWith('unit') && + !unit.toLowerCase().startsWith('apt') && + !unit.toLowerCase().startsWith('#')) { + address = `Unit ${unit}, ${address}`; + } else { + address = `${unit}, ${address}`; + } + } // Send progress update res.write(`data: ${JSON.stringify({ type: 'progress', current: i + 1, total: total, - address: address + currentAddress: address, + status: 'processing' })}\n\n`); res.flush && res.flush(); @@ -93,6 +150,11 @@ class DataConvertController { const geocodeResult = await forwardGeocode(address); if (geocodeResult && geocodeResult.coordinates) { + // Check if result is malformed + const isMalformed = geocodeResult.validation && geocodeResult.validation.isMalformed; + const confidence = geocodeResult.validation ? geocodeResult.validation.confidence : 100; + const warnings = geocodeResult.validation ? geocodeResult.validation.warnings : []; + const processedRow = { ...row, latitude: geocodeResult.coordinates.lat, @@ -100,30 +162,38 @@ class DataConvertController { 'Geo-Location': `${geocodeResult.coordinates.lat};${geocodeResult.coordinates.lng}`, geocoded_address: geocodeResult.formattedAddress || address, geocode_success: true, - geocode_status: 'SUCCESS', + geocode_status: isMalformed ? 'WARNING' : 'SUCCESS', geocode_error: '', - csv_filename: originalFilename // Include filename for notes + confidence_score: confidence, + warnings: warnings.join('; '), + is_malformed: isMalformed, + provider: geocodeResult.provider || 'Unknown', + csv_filename: originalFilename, + row_number: i + 1 }; processedData.push(processedRow); - allResults.push(processedRow); // Add to full results for report + allResults.push(processedRow); - // Send success update + // Send success update with status const successMessage = { type: 'geocoded', data: processedRow, - index: i + index: i, + status: isMalformed ? 'warning' : 'success', + confidence: confidence, + warnings: warnings }; const successJson = JSON.stringify(successMessage); - logger.debug(`Sending geocoded update: ${successJson.length} chars`); + logger.info(`Successfully geocoded: ${address} (Confidence: ${confidence}%)`); res.write(`data: ${successJson}\n\n`); - res.flush && res.flush(); // Ensure data is sent immediately + res.flush && res.flush(); } else { throw new Error('Geocoding failed - no coordinates returned'); } } catch (error) { - logger.error(`Failed to geocode address: ${address}`, error); + logger.error(`Failed to geocode address: ${address}`, error.message); // Create error row with original data plus error info const errorRow = { @@ -135,10 +205,14 @@ class DataConvertController { geocode_success: false, geocode_status: 'FAILED', geocode_error: error.message, - csv_filename: originalFilename + confidence_score: 0, + warnings: '', + is_malformed: false, + csv_filename: originalFilename, + row_number: i + 1 }; - allResults.push(errorRow); // Add to full results for report + allResults.push(errorRow); const errorData = { index: i, @@ -163,14 +237,21 @@ class DataConvertController { } // Store processing results for report generation + const successful = processedData.filter(r => r.geocode_status === 'SUCCESS').length; + const warnings = processedData.filter(r => r.geocode_status === 'WARNING').length; + const failed = errors.length; + const malformed = processedData.filter(r => r.is_malformed).length; + processingResults.set(sessionId, { filename: originalFilename, timestamp: new Date().toISOString(), allResults: allResults, summary: { total: total, - successful: processedData.length, - failed: errors.length + successful: successful, + warnings: warnings, + failed: failed, + malformed: malformed } }); @@ -178,7 +259,10 @@ class DataConvertController { const completeMessage = { type: 'complete', processed: processedData.length, + successful: successful, + warnings: warnings, errors: errors.length, + malformed: malformed, total: total, sessionId: sessionId // Include session ID for report download }; @@ -382,18 +466,18 @@ class DataConvertController { const results = processingResults.get(sessionId); const { filename, timestamp, allResults, summary } = results; - // Convert results to CSV format - const csvContent = this.generateReportCSV(allResults, filename, timestamp, summary); + // Generate comprehensive report content + const reportContent = this.generateComprehensiveReport(allResults, filename, timestamp, summary); - // Set headers for CSV download - const reportFilename = `geocoding-report-${sessionId}.csv`; - res.setHeader('Content-Type', 'text/csv'); + // Set headers for text download + const reportFilename = `geocoding-report-${sessionId}.txt`; + res.setHeader('Content-Type', 'text/plain'); res.setHeader('Content-Disposition', `attachment; filename="${reportFilename}"`); res.setHeader('Cache-Control', 'no-cache'); - logger.info(`Generating report for session ${sessionId}: ${allResults.length} records`); + logger.info(`Generating comprehensive report for session ${sessionId}: ${allResults.length} records`); - res.send(csvContent); + res.send(reportContent); // Clean up stored results after download (optional) setTimeout(() => { @@ -410,6 +494,101 @@ class DataConvertController { } } + // Generate comprehensive text report + generateComprehensiveReport(results, originalFilename, timestamp, summary) { + let report = `Geocoding Processing Report\n`; + report += `Generated: ${timestamp}\n`; + report += `Original File: ${originalFilename}\n`; + report += `================================\n\n`; + + report += `Summary:\n`; + report += `- Total Addresses: ${summary.total}\n`; + report += `- Successfully Geocoded: ${summary.successful}\n`; + report += `- Warnings (Low Confidence): ${summary.warnings}\n`; + report += `- Failed: ${summary.failed}\n`; + report += `- Potentially Malformed: ${summary.malformed}\n\n`; + + // Section for malformed addresses requiring review + const malformedResults = results.filter(r => r.is_malformed); + if (malformedResults.length > 0) { + report += `ADDRESSES REQUIRING REVIEW (Potentially Malformed):\n`; + report += `================================================\n`; + malformedResults.forEach((result, index) => { + const originalAddress = result.address || result.Address || result.ADDRESS || 'N/A'; + report += `\n${index + 1}. Original: ${originalAddress}\n`; + report += ` Result: ${result.geocoded_address || 'N/A'}\n`; + report += ` Confidence: ${result.confidence_score || 0}%\n`; + if (result.warnings) { + report += ` Warnings: ${result.warnings}\n`; + } + report += ` Coordinates: ${result.latitude || 'N/A'}, ${result.longitude || 'N/A'}\n`; + report += ` Row: ${result.row_number}\n`; + }); + report += `\n`; + } + + // Failed addresses section + const failedResults = results.filter(r => r.geocode_status === 'FAILED'); + if (failedResults.length > 0) { + report += `FAILED GEOCODING ATTEMPTS:\n`; + report += `========================\n`; + failedResults.forEach((result, index) => { + const originalAddress = result.address || result.Address || result.ADDRESS || 'N/A'; + report += `\n${index + 1}. Address: ${originalAddress}\n`; + report += ` Error: ${result.geocode_error}\n`; + report += ` Row: ${result.row_number}\n`; + }); + report += `\n`; + } + + // Successful geocoding with low confidence + const lowConfidenceResults = results.filter(r => + r.geocode_status === 'SUCCESS' && + r.confidence_score && + r.confidence_score < 75 + ); + if (lowConfidenceResults.length > 0) { + report += `LOW CONFIDENCE SUCCESSFUL GEOCODING:\n`; + report += `==================================\n`; + lowConfidenceResults.forEach((result, index) => { + const originalAddress = result.address || result.Address || result.ADDRESS || 'N/A'; + report += `\n${index + 1}. Original: ${originalAddress}\n`; + report += ` Result: ${result.geocoded_address}\n`; + report += ` Confidence: ${result.confidence_score}%\n`; + if (result.warnings) { + report += ` Warnings: ${result.warnings}\n`; + } + report += ` Row: ${result.row_number}\n`; + }); + report += `\n`; + } + + // Summary statistics + report += `DETAILED STATISTICS:\n`; + report += `==================\n`; + report += `Success Rate: ${((summary.successful / summary.total) * 100).toFixed(1)}%\n`; + report += `Warning Rate: ${((summary.warnings / summary.total) * 100).toFixed(1)}%\n`; + report += `Failure Rate: ${((summary.failed / summary.total) * 100).toFixed(1)}%\n`; + report += `Malformed Rate: ${((summary.malformed / summary.total) * 100).toFixed(1)}%\n\n`; + + // Recommendations + report += `RECOMMENDATIONS:\n`; + report += `===============\n`; + if (summary.malformed > 0) { + report += `- Review ${summary.malformed} addresses marked as potentially malformed\n`; + } + if (summary.failed > 0) { + report += `- Check ${summary.failed} failed addresses for formatting issues\n`; + } + if (summary.warnings > 0) { + report += `- Verify ${summary.warnings} low confidence results manually\n`; + } + report += `- Consider using more specific address formats for better results\n`; + report += `- Ensure addresses include proper directional indicators (NW, SW, etc.)\n`; + + return report; + } + // Generate CSV content for the report generateReportCSV(allResults, originalFilename, timestamp, summary) { if (!allResults || allResults.length === 0) { diff --git a/map/app/public/admin.html b/map/app/public/admin.html index d636959..623e538 100644 --- a/map/app/public/admin.html +++ b/map/app/public/admin.html @@ -1066,6 +1066,16 @@

Convert Data

Upload a CSV file containing addresses to geocode and import into the map.

+ +
+
+

Geocoding Provider Status

+
+ Checking provider availability... +
+
+
+

CSV Upload

@@ -1227,6 +1237,7 @@ Original Address Geocoded Address Coordinates + Provider diff --git a/map/app/public/css/admin/data-convert.css b/map/app/public/css/admin/data-convert.css index f80b488..705f063 100644 --- a/map/app/public/css/admin/data-convert.css +++ b/map/app/public/css/admin/data-convert.css @@ -1,6 +1,72 @@ /* Data Conversion Interface Styles */ /* CSV upload, processing, and results preview components */ +/* Geocoding Provider Status */ +.geocoding-status-container { + background: #f8f9fa; + border: 1px solid #e9ecef; + border-radius: 6px; + padding: 15px; +} + +.provider-list { + display: flex; + flex-wrap: wrap; + gap: 10px; + margin-bottom: 10px; +} + +.provider-item { + display: flex; + align-items: center; + gap: 5px; + padding: 5px 10px; + border-radius: 4px; + font-size: 14px; + background: white; + border: 1px solid #e9ecef; +} + +.provider-item.provider-available { + border-color: #28a745; + background: #f8fff9; +} + +.provider-item.provider-unavailable { + border-color: #dc3545; + background: #fff8f8; + opacity: 0.7; +} + +.provider-premium { + background: #ffc107; + color: #212529; + font-size: 11px; + padding: 1px 4px; + border-radius: 3px; + font-weight: bold; +} + +.provider-note { + margin-top: 10px; + padding: 10px; + background: #e8f4fd; + border: 1px solid #b8daff; + border-radius: 4px; + font-size: 13px; +} + +.provider-note ul { + margin: 5px 0 0 20px; +} + +.provider-note code { + background: #f8f9fa; + padding: 2px 4px; + border-radius: 3px; + font-family: monospace; +} + .data-convert-container { display: flex; flex-direction: column; @@ -192,11 +258,55 @@ animation: slideIn 0.3s ease-out; } +.result-warning { + background: #fff3cd; + border-left: 4px solid #ffc107; + animation: slideIn 0.3s ease-out; +} + .result-error { background: #f8d7da; animation: slideIn 0.3s ease-out; } +/* Status icons */ +.status-icon { + font-weight: bold; + padding: 2px 6px; + border-radius: 3px; + font-size: 12px; +} + +.status-icon.success { + background: #d1e7dd; + color: #0f5132; +} + +.status-icon.warning { + background: #fff3cd; + color: #664d03; +} + +.status-icon.error { + background: #f8d7da; + color: #721c24; +} + +/* Custom markers for warnings */ +.custom-marker { + display: flex; + align-items: center; + justify-content: center; +} + +.warning-marker { + filter: drop-shadow(0 2px 4px rgba(255, 193, 7, 0.4)); +} + +.success-marker { + filter: drop-shadow(0 2px 4px rgba(40, 167, 69, 0.4)); +} + .error-message { color: #721c24; font-style: italic; diff --git a/map/app/public/js/data-convert.js b/map/app/public/js/data-convert.js index ee96114..922a366 100644 --- a/map/app/public/js/data-convert.js +++ b/map/app/public/js/data-convert.js @@ -4,6 +4,83 @@ let resultsMap = null; let markers = []; let eventListenersInitialized = false; +// Check and display geocoding provider status +async function checkGeocodingProviders() { + const statusElement = document.getElementById('provider-status'); + if (!statusElement) return; + + try { + const response = await fetch('/api/geocode/provider-status', { + method: 'GET', + credentials: 'same-origin', + headers: { + 'Accept': 'application/json', + 'Content-Type': 'application/json' + } + }); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + const data = await response.json(); + + if (data.success) { + const providers = data.providers; + let statusHTML = '
'; + + providers.forEach(provider => { + const icon = provider.available ? '✅' : '❌'; + const status = provider.available ? 'Available' : 'Not configured'; + const className = provider.available ? 'provider-available' : 'provider-unavailable'; + + statusHTML += ` +
+ ${icon} + ${provider.name}: ${status} + ${provider.name === 'Mapbox' && provider.available ? + '🌟 Premium' : ''} +
+ `; + }); + + statusHTML += '
'; + + // Add configuration note if no premium providers + const hasPremium = providers.some(p => p.available && ['Mapbox', 'LocationIQ'].includes(p.name)); + if (!hasPremium) { + statusHTML += ` +
+

💡 Tip: For better geocoding accuracy, configure a premium provider:

+
    +
  • Mapbox: Add MAPBOX_ACCESS_TOKEN to your .env file
    • +
  • LocationIQ: Add LOCATIONIQ_API_KEY to your .env file
    • +
+
+ `; + } else if (providers.find(p => p.name === 'Mapbox' && p.available)) { + statusHTML += ` +
+

🌟 Mapbox Configured! Using premium geocoding for better accuracy.

+
+ `; + } + + statusElement.innerHTML = statusHTML; + } else { + statusElement.innerHTML = '❌ Failed to check provider status'; + } + } catch (error) { + console.error('Failed to check geocoding providers:', error); + statusElement.innerHTML = ` +
+ ⚠️ Unable to check provider status + Using fallback providers: Nominatim, Photon, ArcGIS +
+ `; + } +} + // Utility function to show status messages function showDataConvertStatus(message, type = 'info') { // Try to use the global showStatus from admin.js if available @@ -49,6 +126,9 @@ function setupDataConvertEventListeners() { return; } + // Check geocoding provider status when section loads + checkGeocodingProviders(); + const fileInput = document.getElementById('csv-file-input'); const browseBtn = document.getElementById('browse-btn'); const uploadArea = document.getElementById('upload-area'); @@ -290,15 +370,36 @@ function handleProcessingUpdate(data) { case 'progress': updateProgress(data.current, data.total); - document.getElementById('current-address').textContent = `Processing: ${data.address}`; + + // Show current address with status + if (data.status === 'failed') { + document.getElementById('current-address').innerHTML = `✗ ${data.currentAddress || data.address}`; + } else if (data.status === 'processing') { + document.getElementById('current-address').innerHTML = `⟳ Processing: ${data.currentAddress || data.address}`; + } else { + document.getElementById('current-address').innerHTML = `✓ ${data.currentAddress || data.address}`; + } break; case 'geocoded': - // Mark as successful and add to processing data - const successData = { ...data.data, geocode_success: true }; + // Check if result has warnings + const isWarning = data.status === 'warning' || (data.data && data.data.is_malformed); + const confidence = data.confidence || (data.data && data.data.confidence_score) || 100; + const warnings = data.warnings || (data.data && data.data.warnings) || []; + + // Mark data with appropriate status and add to processing data + const successData = { + ...data.data, + geocode_success: true, + confidence_score: confidence, + warnings: Array.isArray(warnings) ? warnings.join('; ') : warnings + }; processingData.push(successData); - addResultToTable(successData, 'success'); - addMarkerToMap(successData); + + // Add to table with appropriate status + const resultStatus = isWarning ? 'warning' : 'success'; + addResultToTable(successData, resultStatus, confidence, warnings); + addMarkerToMap(successData, isWarning); updateProgress(data.index + 1, data.total); break; @@ -312,6 +413,20 @@ function handleProcessingUpdate(data) { case 'complete': console.log('Received complete event:', data); currentSessionId = data.sessionId; // Store session ID for report download + + // Show comprehensive completion message + let completionMessage = `Complete! Processed ${data.total} addresses:\n`; + completionMessage += `✓ ${data.successful || 0} successful\n`; + if (data.warnings > 0) { + completionMessage += `⚠ ${data.warnings} with warnings (low confidence)\n`; + } + if (data.malformed > 0) { + completionMessage += `🔍 ${data.malformed} potentially malformed (need review)\n`; + } + completionMessage += `✗ ${data.errors || data.failed || 0} failed`; + + document.getElementById('current-address').innerHTML = completionMessage.replace(/\n/g, '
'); + onProcessingComplete(data); break; @@ -351,38 +466,85 @@ function initializeResultsMap() { }, 100); } -function addResultToTable(data, status) { +function addResultToTable(data, status, confidence = null, warnings = []) { const tbody = document.getElementById('results-tbody'); const row = document.createElement('tr'); - row.className = status === 'success' ? 'result-success' : 'result-error'; + // Set row class based on status if (status === 'success') { + row.className = 'result-success'; + } else if (status === 'warning') { + row.className = 'result-warning'; + } else { + row.className = 'result-error'; + } + + if (status === 'success' || status === 'warning') { + // Add confidence indicator for successful geocoding + let statusIcon = status === 'warning' ? + `⚠` : + `✓`; + + if (confidence !== null && confidence < 100) { + statusIcon += ` (${Math.round(confidence)}%)`; + } + + let addressCell = escapeHtml(data.geocoded_address || ''); + if (warnings && warnings.length > 0) { + const warningText = Array.isArray(warnings) ? warnings.join(', ') : warnings; + addressCell += `
⚠ ${escapeHtml(warningText)}`; + } + row.innerHTML = ` - ✓ + ${statusIcon} ${escapeHtml(data.address || data.Address || '')} - ${escapeHtml(data.geocoded_address || '')} + ${addressCell} ${data.latitude.toFixed(6)}, ${data.longitude.toFixed(6)} + ${data.provider ? escapeHtml(data.provider) : 'N/A'} `; } else { row.innerHTML = ` ✗ - ${escapeHtml(data.address || '')} - ${escapeHtml(data.error || 'Geocoding failed')} + ${escapeHtml(data.address || data.Address || '')} + ${escapeHtml(data.geocode_error || data.error || 'Geocoding failed')} `; } tbody.appendChild(row); } -function addMarkerToMap(data) { +function addMarkerToMap(data, isWarning = false) { if (!resultsMap || !data.latitude || !data.longitude) return; - const marker = L.marker([data.latitude, data.longitude]) - .bindPopup(` - ${escapeHtml(data.geocoded_address || data.address)}
- ${data.latitude.toFixed(6)}, ${data.longitude.toFixed(6)} - `); + // Choose marker color based on status + const markerColor = isWarning ? 'orange' : 'green'; + const marker = L.marker([data.latitude, data.longitude], { + icon: L.divIcon({ + className: `custom-marker ${isWarning ? 'warning-marker' : 'success-marker'}`, + html: `
`, + iconSize: [16, 16], + iconAnchor: [8, 8] + }) + }); + + // Create popup content with warning information + let popupContent = `${escapeHtml(data.geocoded_address || data.address)}
`; + popupContent += `${data.latitude.toFixed(6)}, ${data.longitude.toFixed(6)}`; + + if (data.provider) { + popupContent += `
Provider: ${data.provider}`; + } + + if (isWarning && data.confidence_score) { + popupContent += `
⚠ Confidence: ${Math.round(data.confidence_score)}%`; + } + + if (isWarning && data.warnings) { + popupContent += `
${escapeHtml(data.warnings)}`; + } + + marker.bindPopup(popupContent); marker.addTo(resultsMap); markers.push(marker); @@ -396,11 +558,9 @@ function addMarkerToMap(data) { function onProcessingComplete(data) { console.log('Processing complete called with data:', data); - document.getElementById('current-address').textContent = - `Complete! Processed ${data.processed || data.success || 0} addresses successfully, ${data.errors || data.failed || 0} errors.`; - - const processedCount = data.processed || data.success || processingData.filter(item => item.geocode_success !== false).length; + const processedCount = data.processed || data.successful || processingData.filter(item => item.geocode_success !== false).length; + // Show comprehensive processing actions with download option if (processedCount > 0) { console.log('Showing processing actions for', processedCount, 'successful items'); const actionsDiv = document.getElementById('processing-actions'); @@ -412,17 +572,105 @@ function onProcessingComplete(data) { } } - // Show download report button if we have a session ID - if (currentSessionId && data.total > 0) { - showDownloadReportButton(data.total, data.errors || data.failed || 0); + // Add download report button if session ID is available + if (currentSessionId) { + showDownloadReportButton(data); } - const errorCount = data.errors || data.failed || 0; - showDataConvertStatus(`Processing complete: ${processedCount} successful, ${errorCount} errors`, - errorCount > 0 ? 'warning' : 'success'); + // Update final message with detailed statistics + let finalMessage = `Geocoding Complete!\n\n`; + finalMessage += `📊 Summary:\n`; + finalMessage += `• Total Processed: ${data.total || 0}\n`; + finalMessage += `• ✅ Successful: ${data.successful || 0}\n`; + + if (data.warnings > 0) { + finalMessage += `• ⚠️ Warnings: ${data.warnings} (low confidence, review recommended)\n`; + } + + if (data.malformed > 0) { + finalMessage += `• 🔍 Potentially Malformed: ${data.malformed} (need manual review)\n`; + } + + finalMessage += `• ❌ Failed: ${data.errors || data.failed || 0}\n\n`; + + if (data.warnings > 0 || data.malformed > 0) { + finalMessage += `⚠️ Note: ${(data.warnings || 0) + (data.malformed || 0)} addresses may need manual verification.\n`; + finalMessage += `Please download the detailed report for review.`; + } else if ((data.successful || 0) > 0) { + finalMessage += `✅ All geocoded addresses appear to have high confidence results!`; + } + + showDataConvertStatus(finalMessage, (data.errors || data.failed || 0) > 0 ? 'warning' : 'success'); } -// Enhanced save function with better feedback +// Add download report button function +function showDownloadReportButton(data) { + const processingActions = document.getElementById('processing-actions'); + if (!processingActions) return; + + // Check if download button already exists + let downloadBtn = document.getElementById('download-report-btn'); + if (!downloadBtn) { + downloadBtn = document.createElement('button'); + downloadBtn.id = 'download-report-btn'; + downloadBtn.className = 'btn btn-info'; + downloadBtn.innerHTML = '📄 Download Detailed Report'; + downloadBtn.addEventListener('click', downloadProcessingReport); + + // Insert download button before save results button + const saveBtn = document.getElementById('save-results-btn'); + if (saveBtn && saveBtn.parentNode === processingActions) { + processingActions.insertBefore(downloadBtn, saveBtn); + } else { + processingActions.appendChild(downloadBtn); + } + } + + // Update button text with statistics + const warningsCount = (data.warnings || 0) + (data.malformed || 0); + if (warningsCount > 0) { + downloadBtn.innerHTML = `📄 Download Report (${warningsCount} need review)`; + downloadBtn.className = 'btn btn-warning'; + } +} + +// Download processing report function +async function downloadProcessingReport() { + if (!currentSessionId) { + showDataConvertStatus('No report available to download', 'error'); + return; + } + + try { + const response = await fetch(`/api/admin/data-convert/download-report/${currentSessionId}`); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + // Get the filename from the response headers + const filename = response.headers.get('content-disposition') + ?.split('filename=')[1] + ?.replace(/"/g, '') || `geocoding-report-${currentSessionId}.txt`; + + // Create blob and download + const blob = await response.blob(); + const url = window.URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = filename; + document.body.appendChild(a); + a.click(); + window.URL.revokeObjectURL(url); + document.body.removeChild(a); + + showDataConvertStatus('Report downloaded successfully', 'success'); + + } catch (error) { + console.error('Download report error:', error); + showDataConvertStatus('Failed to download report: ' + error.message, 'error'); + } +} async function saveGeocodedResults() { const successfulData = processingData.filter(item => item.geocode_success !== false && item.latitude && item.longitude); diff --git a/map/app/routes/geocoding.js b/map/app/routes/geocoding.js index e67b8d6..731101c 100644 --- a/map/app/routes/geocoding.js +++ b/map/app/routes/geocoding.js @@ -160,6 +160,54 @@ router.get('/search', geocodeLimiter, async (req, res) => { } }); +/** + * Get geocoding provider status + * GET /api/geocode/provider-status + */ +router.get('/provider-status', (req, res) => { + try { + const providers = [ + { + name: 'Mapbox', + available: !!(process.env.MAPBOX_ACCESS_TOKEN || process.env.MAPBOX_API_KEY), + type: 'premium' + }, + { + name: 'Nominatim', + available: true, + type: 'free' + }, + { + name: 'Photon', + available: true, + type: 'free' + }, + { + name: 'LocationIQ', + available: !!process.env.LOCATIONIQ_API_KEY, + type: 'premium' + }, + { + name: 'ArcGIS', + available: true, + type: 'free' + } + ]; + + res.json({ + success: true, + providers: providers, + hasPremium: providers.some(p => p.available && p.type === 'premium') + }); + } catch (error) { + console.error('Error checking provider status:', error); + res.status(500).json({ + success: false, + error: 'Failed to check provider status' + }); + } +}); + /** * Get geocoding cache statistics (admin endpoint) * GET /api/geocode/cache/stats diff --git a/map/app/services/geocoding.js b/map/app/services/geocoding.js index fc2c70a..cfac6a6 100644 --- a/map/app/services/geocoding.js +++ b/map/app/services/geocoding.js @@ -30,11 +30,586 @@ setInterval(() => { }, 60 * 60 * 1000); // Run every hour /** - * Reverse geocode coordinates to get address - * @param {number} lat - Latitude - * @param {number} lng - Longitude - * @returns {Promise} Geocoding result + * Multi-provider geocoding configuration + * Providers are tried in order until one succeeds with good confidence */ +// Provider configuration - order matters (higher quality providers first) +const GEOCODING_PROVIDERS = [ + // Premium provider (when API key is available) + { + name: 'Mapbox', + func: geocodeWithMapbox, + enabled: () => !!(process.env.MAPBOX_ACCESS_TOKEN || process.env.MAPBOX_API_KEY), + options: { timeout: 8000, delay: 0 } + }, + // Free providers (fallbacks) + { + name: 'Nominatim', + func: geocodeWithNominatim, + enabled: () => true, + options: { timeout: 5000, delay: 1000 } + }, + { + name: 'Photon', + func: geocodeWithPhoton, + enabled: () => true, + options: { timeout: 5000, delay: 500 } + }, + { + name: 'LocationIQ', + func: geocodeWithLocationIQ, + enabled: () => !!process.env.LOCATIONIQ_API_KEY, + options: { timeout: 5000, delay: 0 } + }, + { + name: 'ArcGIS', + func: geocodeWithArcGIS, + enabled: () => true, + options: { timeout: 8000, delay: 500 } + } +]; + +/** + * Geocode with Nominatim (OpenStreetMap) + */ +// Geocoding provider functions +async function geocodeWithMapbox(address, options = {}) { + const { timeout = 5000, delay = 0 } = options; + const apiKey = process.env.MAPBOX_ACCESS_TOKEN || process.env.MAPBOX_API_KEY; + + if (!apiKey) { + throw new Error('Mapbox API key not configured'); + } + + // Rate limiting (Mapbox has generous rate limits) + if (delay > 0) { + await new Promise(resolve => setTimeout(resolve, delay)); + } + + // Parse address components for structured input + const addressComponents = parseAddressString(address); + let url; + + if (addressComponents.hasComponents) { + // Use structured input for better accuracy + const params = new URLSearchParams({ + access_token: apiKey, + limit: 1, + country: addressComponents.country || 'ca' // Default to Canada + }); + + if (addressComponents.address_number) params.append('address_number', addressComponents.address_number); + if (addressComponents.street) params.append('street', addressComponents.street); + if (addressComponents.place) params.append('place', addressComponents.place); + if (addressComponents.region) params.append('region', addressComponents.region); + if (addressComponents.postcode) params.append('postcode', addressComponents.postcode); + + url = `https://api.mapbox.com/search/geocode/v6/forward?${params.toString()}`; + } else { + // Fallback to simple search + url = `https://api.mapbox.com/geocoding/v5/mapbox.places/${encodeURIComponent(address)}.json?access_token=${apiKey}&limit=1&country=ca`; + } + + logger.info(`Geocoding with Mapbox: ${address}`); + + try { + const response = await axios.get(url, { timeout }); + const data = response.data; + + // Debug: Log the API response structure + logger.debug(`Mapbox response structure:`, { + hasFeatures: !!(data.features && data.features.length > 0), + hasData: !!(data.data && data.data.length > 0), + featuresLength: data.features?.length || 0, + dataLength: data.data?.length || 0, + firstFeature: data.features?.[0] ? Object.keys(data.features[0]) : null, + firstData: data.data?.[0] ? Object.keys(data.data[0]) : null + }); + + let result; + + // Handle direct feature response (newer format) + if (data.geometry && data.properties) { + result = data; + } else if (data.features && data.features.length > 0) { + // v5 API response format (legacy) + result = data.features[0]; + } else if (data.data && data.data.length > 0) { + // v6 API response format (structured) + result = data.data[0]; + } + + if (!result) { + logger.info(`Mapbox returned no results for address: ${address}`); + return null; + } + + // Extract coordinates - try multiple possible locations + let latitude, longitude; + + if (result.properties?.coordinates?.latitude && result.properties?.coordinates?.longitude) { + // New format: properties.coordinates object + latitude = result.properties.coordinates.latitude; + longitude = result.properties.coordinates.longitude; + } else if (result.geometry?.coordinates && Array.isArray(result.geometry.coordinates) && result.geometry.coordinates.length >= 2) { + // GeoJSON format: geometry.coordinates [lng, lat] + longitude = result.geometry.coordinates[0]; + latitude = result.geometry.coordinates[1]; + } else if (result.center && Array.isArray(result.center) && result.center.length >= 2) { + // v5 format: center [lng, lat] + longitude = result.center[0]; + latitude = result.center[1]; + } else { + logger.error(`Mapbox result missing valid coordinates:`, { + hasPropsCoords: !!(result.properties?.coordinates), + hasGeomCoords: !!(result.geometry?.coordinates), + hasCenter: !!result.center, + result: result + }); + return null; + } + + // Extract formatted address + let formattedAddress = result.properties?.full_address || + result.properties?.name_preferred || + result.properties?.name || + result.place_name || + 'Unknown Address'; + + // Calculate confidence from match_code if available + let confidence = 100; + if (result.properties?.match_code) { + const matchCode = result.properties.match_code; + if (matchCode.confidence === 'exact') confidence = 100; + else if (matchCode.confidence === 'high') confidence = 90; + else if (matchCode.confidence === 'medium') confidence = 70; + else if (matchCode.confidence === 'low') confidence = 50; + else confidence = (result.relevance || 1) * 100; + } else { + confidence = (result.relevance || 1) * 100; + } + + return { + latitude: latitude, + longitude: longitude, + formattedAddress: formattedAddress, + provider: 'Mapbox', + confidence: confidence, + components: extractMapboxComponents(result), + raw: result + }; + + // No results found + logger.info(`Mapbox returned no results for address: ${address}`); + return null; + } catch (error) { + logger.error('Mapbox geocoding error:', error.message); + throw error; + } +} + +async function geocodeWithNominatim(address, options = {}) { + const { timeout = 5000, delay = 1000 } = options; + + // Rate limiting for Nominatim (1 request per second) + if (delay > 0) { + await new Promise(resolve => setTimeout(resolve, delay)); + } + + const url = `https://nominatim.openstreetmap.org/search?format=json&q=${encodeURIComponent(address)}&limit=1&addressdetails=1`; + + logger.info(`Geocoding with Nominatim: ${address}`); + + try { + const response = await axios.get(url, { + timeout, + headers: { + 'User-Agent': 'MapApp/1.0' + } + }); + + const data = response.data; + if (!data || data.length === 0) { + return null; + } + + const result = data[0]; + return { + latitude: parseFloat(result.lat), + longitude: parseFloat(result.lon), + formattedAddress: result.display_name, + provider: 'Nominatim', + confidence: calculateNominatimConfidence(result), + components: extractAddressComponents(result.address || {}), + raw: result + }; + } catch (error) { + logger.error('Nominatim geocoding error:', error.message); + throw error; + } +} + +/** + * Geocode with Photon (OpenStreetMap-based) + */ +async function geocodeWithPhoton(address, options = {}) { + const { timeout = 15000, delay = 500 } = options; + + // Rate limiting + if (delay > 0) { + await new Promise(resolve => setTimeout(resolve, delay)); + } + + const response = await axios.get('https://photon.komoot.io/api/', { + params: { + q: address, + limit: 1, + lang: 'en' + }, + timeout + }); + + if (!response.data?.features || response.data.features.length === 0) { + return null; + } + + const feature = response.data.features[0]; + const coords = feature.geometry.coordinates; + const props = feature.properties; + + return { + provider: 'photon', + latitude: coords[1], + longitude: coords[0], + formattedAddress: buildFormattedAddressFromPhoton(props), + components: extractPhotonComponents(props), + confidence: calculatePhotonConfidence(feature), + raw: feature + }; +} + +/** + * Geocode with LocationIQ (fallback) + */ +async function geocodeWithLocationIQ(address, options = {}) { + const { timeout = 15000, delay = 0 } = options; + const apiKey = process.env.LOCATIONIQ_API_KEY; + + // Rate limiting + if (delay > 0) { + await new Promise(resolve => setTimeout(resolve, delay)); + } + + // LocationIQ can work without API key but has rate limits + const params = { + format: 'json', + q: address, + limit: 1, + addressdetails: 1, + countrycodes: 'ca' + }; + + // Add API key if available for better rate limits + if (apiKey) { + params.key = apiKey; + } + + const response = await axios.get('https://us1.locationiq.com/v1/search.php', { + params, + headers: { + 'User-Agent': 'NocoDB Map Viewer 1.0' + }, + timeout + }); + + if (!response.data || response.data.length === 0) { + return null; + } + + const data = response.data[0]; + return { + provider: 'locationiq', + latitude: parseFloat(data.lat), + longitude: parseFloat(data.lon), + formattedAddress: buildFormattedAddress(data.address), + components: extractAddressComponents(data.address), + confidence: calculateNominatimConfidence(data), // Similar format to Nominatim + boundingBox: data.boundingbox, + raw: data + }; +} + +/** + * Geocode with ArcGIS World Geocoding Service + */ +async function geocodeWithArcGIS(address, options = {}) { + const { timeout = 15000, delay = 500 } = options; + + // Rate limiting + if (delay > 0) { + await new Promise(resolve => setTimeout(resolve, delay)); + } + + const response = await axios.get('https://geocode-api.arcgis.com/arcgis/rest/services/World/GeocodeServer/findAddressCandidates', { + params: { + SingleLine: address, + f: 'json', + outFields: '*', + maxLocations: 1, + countryCode: 'CA' + }, + timeout + }); + + if (!response.data?.candidates || response.data.candidates.length === 0) { + return null; + } + + const candidate = response.data.candidates[0]; + const location = candidate.location; + const attributes = candidate.attributes; + + return { + provider: 'arcgis', + latitude: location.y, + longitude: location.x, + formattedAddress: attributes.LongLabel || candidate.address, + components: extractArcGISComponents(attributes), + confidence: candidate.score || 50, // ArcGIS provides score 0-100 + raw: candidate + }; +} + +/** + * Helper functions for address processing + */ +function buildFormattedAddress(addressComponents) { + if (!addressComponents) return ''; + + let formattedAddress = ''; + if (addressComponents.house_number) formattedAddress += addressComponents.house_number + ' '; + if (addressComponents.road) formattedAddress += addressComponents.road + ', '; + if (addressComponents.suburb) formattedAddress += addressComponents.suburb + ', '; + if (addressComponents.city) formattedAddress += addressComponents.city + ', '; + if (addressComponents.state) formattedAddress += addressComponents.state + ' '; + if (addressComponents.postcode) formattedAddress += addressComponents.postcode; + + return formattedAddress.trim().replace(/,$/, ''); +} + +function buildFormattedAddressFromPhoton(props) { + let address = ''; + if (props.housenumber) address += props.housenumber + ' '; + if (props.street) address += props.street + ', '; + if (props.district) address += props.district + ', '; + if (props.city) address += props.city + ', '; + if (props.state) address += props.state + ' '; + if (props.postcode) address += props.postcode; + + return address.trim().replace(/,$/, ''); +} + +function extractAddressComponents(address) { + // If address is a string, parse it + if (typeof address === 'string') { + return parseAddressString(address); + } + + // If address is already an object (from Nominatim/LocationIQ response) + return { + house_number: address?.house_number || '', + road: address?.road || '', + suburb: address?.suburb || address?.neighbourhood || '', + city: address?.city || address?.town || address?.village || '', + state: address?.state || address?.province || '', + postcode: address?.postcode || '', + country: address?.country || '' + }; +} + +function parseAddressString(addressStr) { + // Parse a string address into components for Mapbox structured input + const components = { + address_number: '', + street: '', + place: '', + region: '', + postcode: '', + country: 'ca', + hasComponents: false + }; + + if (!addressStr || typeof addressStr !== 'string') { + return components; + } + + // Clean up the address string + const cleanAddress = addressStr.trim(); + + // Basic regex patterns for Canadian addresses + const patterns = { + // Match house number at the start + houseNumber: /^(\d+[A-Za-z]?)\s+(.+)/, + // Match postal code (Canadian format: A1A 1A1) + postalCode: /([A-Za-z]\d[A-Za-z]\s?\d[A-Za-z]\d)$/i, + // Match province abbreviations + province: /,?\s*(AB|BC|MB|NB|NL|NT|NS|NU|ON|PE|QC|SK|YT|Alberta|British Columbia|Manitoba|New Brunswick|Newfoundland|Northwest Territories|Nova Scotia|Nunavut|Ontario|Prince Edward Island|Quebec|Saskatchewan|Yukon)\s*,?\s*CA(?:NADA)?$/i + }; + + let workingAddress = cleanAddress; + + // Extract postal code + const postalMatch = workingAddress.match(patterns.postalCode); + if (postalMatch) { + components.postcode = postalMatch[1].replace(/\s/g, ''); + workingAddress = workingAddress.replace(patterns.postalCode, '').trim(); + } + + // Extract province/region + const provinceMatch = workingAddress.match(patterns.province); + if (provinceMatch) { + components.region = provinceMatch[1]; + workingAddress = workingAddress.replace(patterns.province, '').trim(); + } + + // Extract house number and street + const houseMatch = workingAddress.match(patterns.houseNumber); + if (houseMatch) { + components.address_number = houseMatch[1]; + workingAddress = houseMatch[2]; + components.hasComponents = true; + } + + // The remaining part is likely the street and city + const parts = workingAddress.split(',').map(p => p.trim()).filter(p => p); + + if (parts.length > 0) { + components.street = parts[0]; + components.hasComponents = true; + } + + if (parts.length > 1) { + components.place = parts[1]; + } + + return components; +} + +function extractPhotonComponents(props) { + return { + house_number: props.housenumber || '', + road: props.street || '', + suburb: props.district || '', + city: props.city || '', + state: props.state || '', + postcode: props.postcode || '', + country: props.country || '' + }; +} + +function extractArcGISComponents(attributes) { + return { + house_number: attributes.AddNum || '', + road: attributes.StName || '', + suburb: attributes.District || '', + city: attributes.City || '', + state: attributes.Region || '', + postcode: attributes.Postal || '', + country: attributes.Country || '' + }; +} + +function extractMapboxComponents(result) { + // Universal Mapbox component extractor for multiple API formats + const components = { + house_number: '', + road: '', + suburb: '', + city: '', + state: '', + postcode: '', + country: '' + }; + + // New format: properties.context object + if (result.properties?.context) { + const ctx = result.properties.context; + + components.house_number = ctx.address?.address_number || ''; + components.road = ctx.address?.street_name || ctx.street?.name || ''; + components.suburb = ctx.neighborhood?.name || ''; + components.city = ctx.place?.name || ''; + components.state = ctx.region?.name || ''; + components.postcode = ctx.postcode?.name || ''; + components.country = ctx.country?.name || ''; + + return components; + } + + // Legacy v6 format: properties object + if (result.properties) { + const props = result.properties; + components.house_number = props.address_number || ''; + components.road = props.street || ''; + components.suburb = props.neighborhood || ''; + components.city = props.place || ''; + components.state = props.region || ''; + components.postcode = props.postcode || ''; + components.country = props.country || ''; + + return components; + } + + // Legacy v5 format: context array + if (result.context && Array.isArray(result.context)) { + result.context.forEach(item => { + const id = item.id || ''; + if (id.startsWith('postcode.')) components.postcode = item.text; + else if (id.startsWith('place.')) components.city = item.text; + else if (id.startsWith('region.')) components.state = item.text; + else if (id.startsWith('country.')) components.country = item.text; + else if (id.startsWith('neighborhood.')) components.suburb = item.text; + }); + + // Extract house number from place name (first part) + if (result.place_name) { + const match = result.place_name.match(/^(\d+[A-Za-z]?)\s+/); + if (match) components.house_number = match[1]; + + // Extract road from place name or address + if (components.house_number) { + const addressPart = result.place_name.replace(new RegExp(`^${components.house_number}\\s+`), '').split(',')[0]; + components.road = addressPart.trim(); + } + } + } + + return components; +} + +function calculateNominatimConfidence(data) { + // Basic confidence calculation for Nominatim results + let confidence = 100; + + if (!data.address?.house_number) confidence -= 20; + if (!data.address?.road) confidence -= 30; + if (data.type === 'administrative') confidence -= 25; + if (data.class === 'place' && data.type === 'suburb') confidence -= 20; + + return Math.max(confidence, 10); +} + +function calculatePhotonConfidence(feature) { + // Basic confidence for Photon results + let confidence = 100; + const props = feature.properties; + + if (!props.housenumber) confidence -= 20; + if (!props.street) confidence -= 30; + if (props.osm_type === 'relation') confidence -= 15; + + return Math.max(confidence, 10); +} async function reverseGeocode(lat, lng) { // Create cache key - use full precision const cacheKey = `${lat},${lng}`; @@ -168,11 +743,106 @@ async function forwardGeocodeSearch(address, limit = 5) { } /** - * Forward geocode address to get coordinates + * Validate geocoding result and calculate confidence score + * @param {string} originalAddress - The original address searched + * @param {Object} geocodeResult - The geocoding result + * @returns {Object} Validation result with confidence score + */ +function validateGeocodeResult(originalAddress, geocodeResult) { + const validation = { + isValid: true, + confidence: 100, + warnings: [], + isMalformed: false + }; + + if (!geocodeResult || !geocodeResult.formattedAddress) { + validation.isValid = false; + validation.confidence = 0; + return validation; + } + + // Extract key components from original address + const originalLower = originalAddress.toLowerCase(); + const resultLower = geocodeResult.formattedAddress.toLowerCase(); + + // Check for street number presence + const streetNumberMatch = originalAddress.match(/^\d+/); + const resultStreetNumber = geocodeResult.components.house_number; + + if (streetNumberMatch && !resultStreetNumber) { + validation.warnings.push('Street number not found in result'); + validation.confidence -= 30; + validation.isMalformed = true; + } else if (streetNumberMatch && resultStreetNumber) { + if (streetNumberMatch[0] !== resultStreetNumber) { + validation.warnings.push('Street number mismatch'); + validation.confidence -= 40; + validation.isMalformed = true; + } + } + + // Check for street name presence + const streetNameWords = originalLower + .replace(/^\d+\s*/, '') // Remove leading number + .replace(/,.*$/, '') // Remove everything after first comma + .trim() + .split(/\s+/) + .filter(word => !['nw', 'ne', 'sw', 'se', 'street', 'st', 'avenue', 'ave', 'road', 'rd', 'crescent', 'close'].includes(word)); + + let matchedWords = 0; + streetNameWords.forEach(word => { + if (resultLower.includes(word)) { + matchedWords++; + } + }); + + const matchPercentage = streetNameWords.length > 0 ? (matchedWords / streetNameWords.length) * 100 : 0; + + if (matchPercentage < 50) { + validation.warnings.push('Poor street name match'); + validation.confidence -= 30; + validation.isMalformed = true; + } else if (matchPercentage < 75) { + validation.warnings.push('Partial street name match'); + validation.confidence -= 15; + } + + // Check for generic/fallback results (often indicates geocoding failure) + const genericIndicators = ['castle downs', 'clover bar', 'downtown', 'city centre']; + const hasGenericResult = genericIndicators.some(indicator => + resultLower.includes(indicator) && !originalLower.includes(indicator) + ); + + if (hasGenericResult) { + validation.warnings.push('Result appears to be generic area, not specific address'); + validation.confidence -= 25; + validation.isMalformed = true; + } + + // Final validation + validation.isValid = validation.confidence >= 50; + + return validation; +} + +/** + * Multi-provider forward geocode with fallback support * @param {string} address - Address to geocode - * @returns {Promise} Geocoding result + * @returns {Promise} Geocoding result with provider info */ async function forwardGeocode(address) { + // Input validation + if (!address || typeof address !== 'string') { + logger.warn(`Invalid address provided for geocoding: ${address}`); + throw new Error('Invalid address: address must be a non-empty string'); + } + + address = address.trim(); + if (address.length === 0) { + throw new Error('Invalid address: address cannot be empty'); + } + // Create cache key const cacheKey = `addr:${address.toLowerCase()}`; @@ -183,58 +853,138 @@ async function forwardGeocode(address) { return cached.data; } - try { - // Add delay to respect rate limits - increase delay for batch processing - await new Promise(resolve => setTimeout(resolve, 1500)); + // Try different address format variations + const addressVariations = [ + address, // Original + address.replace(/\s+NW\s*$/i, ' Northwest'), // Expand NW + address.replace(/\s+Northwest\s*$/i, ' NW'), // Contract Northwest + address.replace(/,\s*CA\s*$/i, ', Canada'), // Expand CA to Canada + address.replace(/\s+Street\s+/i, ' St '), // Abbreviate Street + address.replace(/\s+St\s+/i, ' Street '), // Expand St + address.replace(/\s+Avenue\s+/i, ' Ave '), // Abbreviate Avenue + address.replace(/\s+Ave\s+/i, ' Avenue '), // Expand Ave + ]; + + // Provider functions mapping + const providerFunctions = { + 'Mapbox': geocodeWithMapbox, + 'Nominatim': geocodeWithNominatim, + 'Photon': geocodeWithPhoton, + 'LocationIQ': geocodeWithLocationIQ, + 'ArcGIS': geocodeWithArcGIS + }; + + let bestResult = null; + let bestConfidence = 0; + const allErrors = []; + + // Try each provider with each address variation + for (const provider of GEOCODING_PROVIDERS) { + logger.info(`Trying provider: ${provider.name}`); - logger.info(`Forward geocoding: ${address}`); - - const response = await axios.get('https://nominatim.openstreetmap.org/search', { - params: { - format: 'json', - q: address, - limit: 1, - addressdetails: 1, - 'accept-language': 'en' - }, - headers: { - 'User-Agent': 'NocoDB Map Viewer 1.0 (contact@example.com)' - }, - timeout: 15000 // Increase timeout to 15 seconds - }); - - if (!response.data || response.data.length === 0) { - throw new Error('No results found'); + for (let varIndex = 0; varIndex < addressVariations.length; varIndex++) { + const addressVariation = addressVariations[varIndex]; + + try { + logger.info(`${provider.name} - attempt ${varIndex + 1}/${addressVariations.length}: ${addressVariation}`); + + const providerResult = await providerFunctions[provider.name](addressVariation, provider.options); + + if (!providerResult) { + continue; // No results from this provider/variation + } + + // Convert to standard format + const result = { + fullAddress: providerResult.formattedAddress, + formattedAddress: providerResult.formattedAddress, + components: providerResult.components, + coordinates: { + lat: providerResult.latitude, + lng: providerResult.longitude + }, + latitude: providerResult.latitude, + longitude: providerResult.longitude, + provider: providerResult.provider, + providerConfidence: providerResult.confidence, + addressVariation: addressVariation, + variationIndex: varIndex, + boundingBox: providerResult.boundingBox || null, + raw: providerResult.raw + }; + + // Validate the result + const validation = validateGeocodeResult(address, result); + result.validation = validation; + + // Calculate combined confidence (provider confidence + validation confidence) + const combinedConfidence = Math.round((providerResult.confidence + validation.confidence) / 2); + result.combinedConfidence = combinedConfidence; + + logger.info(`${provider.name} result - Provider: ${providerResult.confidence}%, Validation: ${validation.confidence}%, Combined: ${combinedConfidence}%`); + + // If this is a very high confidence result, use it immediately + if (combinedConfidence >= 90 && validation.confidence >= 80) { + logger.info(`High confidence result found with ${provider.name}, using immediately`); + + // Cache and return the result + geocodeCache.set(cacheKey, { + data: result, + timestamp: Date.now() + }); + + return result; + } + + // Keep track of the best result so far + if (combinedConfidence > bestConfidence) { + bestResult = result; + bestConfidence = combinedConfidence; + } + + // If we have a decent result from the first variation, don't try more variations for this provider + if (varIndex === 0 && combinedConfidence >= 70) { + logger.info(`Good result from ${provider.name} with original address, skipping variations`); + break; + } + + } catch (error) { + logger.error(`${provider.name} error with "${addressVariation}": ${error.message}`); + allErrors.push(`${provider.name}: ${error.message}`); + + // If rate limited, wait extra time + if (error.response?.status === 429) { + await new Promise(resolve => setTimeout(resolve, 5000)); + } + + continue; // Try next variation or provider + } } - // Process the first result - const result = processGeocodeResponse(response.data[0]); + // If we found a good result with this provider, we can stop trying other providers + if (bestConfidence >= 75) { + logger.info(`Acceptable result found with ${provider.name} (${bestConfidence}%), stopping provider search`); + break; + } + } + + // If we have any result, return the best one + if (bestResult) { + logger.info(`Returning best result from ${bestResult.provider} with ${bestConfidence}% confidence`); // Cache the result geocodeCache.set(cacheKey, { - data: result, + data: bestResult, timestamp: Date.now() }); - return result; - - } catch (error) { - logger.error('Forward geocoding error:', error.message); - - if (error.response?.status === 429) { - throw new Error('Rate limit exceeded. Please try again later.'); - } else if (error.response?.status === 403) { - throw new Error('Access denied by geocoding service'); - } else if (error.response?.status === 500) { - throw new Error('Geocoding service internal error'); - } else if (error.code === 'ECONNABORTED') { - throw new Error('Geocoding request timeout'); - } else if (error.code === 'ENOTFOUND' || error.code === 'ECONNREFUSED') { - throw new Error('Cannot connect to geocoding service'); - } else { - throw new Error(`Geocoding failed: ${error.message}`); - } + return bestResult; } + + // All providers failed + const errorMessage = `All geocoding providers failed: ${allErrors.join('; ')}`; + logger.error(errorMessage); + throw new Error('Geocoding failed: No providers could locate this address'); } /**