New update for the geo-coding system to include system to automatically scan the nocodb locations to build geo-locations
This commit is contained in:
parent
44298834ef
commit
9aaefd149e
@ -15,6 +15,7 @@ class DataConvertController {
|
||||
this.parseCSV = this.parseCSV.bind(this);
|
||||
this.saveGeocodedData = this.saveGeocodedData.bind(this);
|
||||
this.downloadReport = this.downloadReport.bind(this);
|
||||
this.scanAndGeocode = this.scanAndGeocode.bind(this);
|
||||
}
|
||||
|
||||
// Process CSV upload and geocode addresses with SSE progress updates
|
||||
@ -152,7 +153,10 @@ class DataConvertController {
|
||||
if (geocodeResult && geocodeResult.coordinates) {
|
||||
// Check if result is malformed
|
||||
const isMalformed = geocodeResult.validation && geocodeResult.validation.isMalformed;
|
||||
const confidence = geocodeResult.validation ? geocodeResult.validation.confidence : 100;
|
||||
// Use combined confidence for best overall assessment
|
||||
const confidence = geocodeResult.combinedConfidence !== undefined ?
|
||||
geocodeResult.combinedConfidence :
|
||||
(geocodeResult.validation ? geocodeResult.validation.confidence : 100);
|
||||
const warnings = geocodeResult.validation ? geocodeResult.validation.warnings : [];
|
||||
|
||||
const processedRow = {
|
||||
@ -165,6 +169,8 @@ class DataConvertController {
|
||||
geocode_status: isMalformed ? 'WARNING' : 'SUCCESS',
|
||||
geocode_error: '',
|
||||
confidence_score: confidence,
|
||||
provider_confidence: geocodeResult.providerConfidence || null,
|
||||
validation_confidence: geocodeResult.validation ? geocodeResult.validation.confidence : null,
|
||||
warnings: warnings.join('; '),
|
||||
is_malformed: isMalformed,
|
||||
provider: geocodeResult.provider || 'Unknown',
|
||||
@ -333,6 +339,8 @@ class DataConvertController {
|
||||
latitude: parseFloat(location.latitude),
|
||||
longitude: parseFloat(location.longitude),
|
||||
Address: originalAddress, // Always use the original address from CSV
|
||||
'Geocode Confidence': location.confidence_score || null, // Add confidence score
|
||||
'Geocode Provider': location.provider || null, // Add provider name
|
||||
created_by_user: req.session.userEmail || 'csv_import',
|
||||
last_updated_by_user: req.session.userEmail || 'csv_import'
|
||||
};
|
||||
@ -367,7 +375,7 @@ class DataConvertController {
|
||||
const lowerKey = key.toLowerCase();
|
||||
|
||||
// Skip already processed fields
|
||||
if (['latitude', 'longitude', 'geo-location', 'geocoded_address', 'geocode_success', 'address', 'csv_filename'].includes(lowerKey)) {
|
||||
if (['latitude', 'longitude', 'geo-location', 'geocoded_address', 'geocode_success', 'address', 'csv_filename', 'confidence_score', 'provider_confidence', 'validation_confidence', 'warnings', 'is_malformed', 'provider', 'row_number', 'geocode_status', 'geocode_error'].includes(lowerKey)) {
|
||||
return;
|
||||
}
|
||||
|
||||
@ -402,6 +410,21 @@ class DataConvertController {
|
||||
noteParts.push(`Geocoded as: ${geocodedAddress}`);
|
||||
}
|
||||
|
||||
// Add confidence information if available
|
||||
if (location.confidence_score !== undefined && location.confidence_score !== null) {
|
||||
noteParts.push(`Geocode confidence: ${location.confidence_score}%`);
|
||||
}
|
||||
|
||||
// Add provider information if available
|
||||
if (location.provider) {
|
||||
noteParts.push(`Provider: ${location.provider}`);
|
||||
}
|
||||
|
||||
// Add warnings if present
|
||||
if (location.warnings && location.warnings.trim()) {
|
||||
noteParts.push(`Warnings: ${location.warnings}`);
|
||||
}
|
||||
|
||||
locationData[targetField] = noteParts.join(' | ');
|
||||
} else {
|
||||
locationData[targetField] = location[key];
|
||||
@ -418,6 +441,21 @@ class DataConvertController {
|
||||
noteParts.push(`Geocoded as: ${geocodedAddress}`);
|
||||
}
|
||||
|
||||
// Add confidence information if available
|
||||
if (location.confidence_score !== undefined && location.confidence_score !== null) {
|
||||
noteParts.push(`Geocode confidence: ${location.confidence_score}%`);
|
||||
}
|
||||
|
||||
// Add provider information if available
|
||||
if (location.provider) {
|
||||
noteParts.push(`Provider: ${location.provider}`);
|
||||
}
|
||||
|
||||
// Add warnings if present
|
||||
if (location.warnings && location.warnings.trim()) {
|
||||
noteParts.push(`Warnings: ${location.warnings}`);
|
||||
}
|
||||
|
||||
locationData['Notes'] = noteParts.join(' | ');
|
||||
}
|
||||
|
||||
@ -455,6 +493,7 @@ class DataConvertController {
|
||||
async downloadReport(req, res) {
|
||||
try {
|
||||
const { sessionId } = req.params;
|
||||
const format = req.query.format || 'csv'; // Default to CSV, support 'txt' for backward compatibility
|
||||
|
||||
if (!sessionId || !processingResults.has(sessionId)) {
|
||||
return res.status(404).json({
|
||||
@ -466,16 +505,27 @@ class DataConvertController {
|
||||
const results = processingResults.get(sessionId);
|
||||
const { filename, timestamp, allResults, summary } = results;
|
||||
|
||||
// Generate comprehensive report content
|
||||
const reportContent = this.generateComprehensiveReport(allResults, filename, timestamp, summary);
|
||||
let reportContent, contentType, fileExtension;
|
||||
|
||||
// Set headers for text download
|
||||
const reportFilename = `geocoding-report-${sessionId}.txt`;
|
||||
res.setHeader('Content-Type', 'text/plain');
|
||||
if (format === 'csv') {
|
||||
// Generate CSV report
|
||||
reportContent = this.generateReportCSV(allResults, filename, timestamp, summary);
|
||||
contentType = 'text/csv';
|
||||
fileExtension = 'csv';
|
||||
} else {
|
||||
// Generate text report (backward compatibility)
|
||||
reportContent = this.generateComprehensiveReport(allResults, filename, timestamp, summary);
|
||||
contentType = 'text/plain';
|
||||
fileExtension = 'txt';
|
||||
}
|
||||
|
||||
// Set headers for download
|
||||
const reportFilename = `geocoding-report-${sessionId}.${fileExtension}`;
|
||||
res.setHeader('Content-Type', contentType);
|
||||
res.setHeader('Content-Disposition', `attachment; filename="${reportFilename}"`);
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
|
||||
logger.info(`Generating comprehensive report for session ${sessionId}: ${allResults.length} records`);
|
||||
logger.info(`Generating ${format.toUpperCase()} report for session ${sessionId}: ${allResults.length} records`);
|
||||
|
||||
res.send(reportContent);
|
||||
|
||||
@ -651,6 +701,290 @@ class DataConvertController {
|
||||
|
||||
return stringField;
|
||||
}
|
||||
|
||||
// Scan NocoDB database for records missing geo-location data and geocode them
|
||||
async scanAndGeocode(req, res) {
|
||||
try {
|
||||
const sessionId = Date.now().toString();
|
||||
|
||||
// Set up SSE headers
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
'Connection': 'keep-alive',
|
||||
'X-Accel-Buffering': 'no'
|
||||
});
|
||||
|
||||
logger.info(`Starting database scan for missing geo-location data (session: ${sessionId})`);
|
||||
|
||||
// Send initial status
|
||||
res.write(`data: ${JSON.stringify({
|
||||
type: 'status',
|
||||
message: 'Scanning database for records missing geo-location data...',
|
||||
sessionId: sessionId
|
||||
})}\n\n`);
|
||||
res.flush && res.flush();
|
||||
|
||||
// Fetch all records from NocoDB
|
||||
let allRecords = [];
|
||||
let offset = 0;
|
||||
const limit = 100; // Process in batches
|
||||
let hasMoreRecords = true;
|
||||
|
||||
while (hasMoreRecords) {
|
||||
try {
|
||||
const response = await nocodbService.getAll(config.nocodb.tableId, { limit, offset });
|
||||
|
||||
if (response && response.list && response.list.length > 0) {
|
||||
allRecords.push(...response.list);
|
||||
offset += limit;
|
||||
|
||||
// Send progress update
|
||||
res.write(`data: ${JSON.stringify({
|
||||
type: 'scanning',
|
||||
message: `Fetched ${allRecords.length} records from database...`,
|
||||
count: allRecords.length
|
||||
})}\n\n`);
|
||||
res.flush && res.flush();
|
||||
|
||||
// Check if we've fetched all records
|
||||
hasMoreRecords = response.list.length === limit;
|
||||
} else {
|
||||
hasMoreRecords = false;
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Error fetching records from NocoDB:', error);
|
||||
res.write(`data: ${JSON.stringify({
|
||||
type: 'error',
|
||||
message: `Error fetching records: ${error.message}`
|
||||
})}\n\n`);
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(`Database scan complete: found ${allRecords.length} total records`);
|
||||
|
||||
// Filter records that need geocoding
|
||||
const recordsNeedingGeocode = allRecords.filter(record => {
|
||||
// Check if record is missing geo-location data
|
||||
const hasGeoLocation = record['Geo-Location'] &&
|
||||
record['Geo-Location'].trim() !== '' &&
|
||||
record['Geo-Location'] !== 'null';
|
||||
const hasCoordinates = (record.latitude && record.longitude) ||
|
||||
(record.Latitude && record.Longitude);
|
||||
const hasAddress = record.Address || record.address || record.ADDRESS;
|
||||
|
||||
return !hasGeoLocation && !hasCoordinates && hasAddress;
|
||||
});
|
||||
|
||||
const totalToGeocode = recordsNeedingGeocode.length;
|
||||
|
||||
logger.info(`Found ${totalToGeocode} records needing geocoding`);
|
||||
|
||||
// Send summary
|
||||
res.write(`data: ${JSON.stringify({
|
||||
type: 'scan_complete',
|
||||
message: `Scan complete: ${totalToGeocode} records need geocoding`,
|
||||
total: allRecords.length,
|
||||
needingGeocode: totalToGeocode
|
||||
})}\n\n`);
|
||||
res.flush && res.flush();
|
||||
|
||||
if (totalToGeocode === 0) {
|
||||
res.write(`data: ${JSON.stringify({
|
||||
type: 'complete',
|
||||
message: 'No records found that need geocoding. All records already have location data!',
|
||||
results: { success: 0, failed: 0, skipped: allRecords.length }
|
||||
})}\n\n`);
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
|
||||
// Process geocoding
|
||||
const results = {
|
||||
success: 0,
|
||||
failed: 0,
|
||||
errors: [],
|
||||
sessionId: sessionId
|
||||
};
|
||||
|
||||
const allResults = [];
|
||||
let processedCount = 0;
|
||||
|
||||
for (const record of recordsNeedingGeocode) {
|
||||
try {
|
||||
processedCount++;
|
||||
const address = record.Address || record.address || record.ADDRESS;
|
||||
|
||||
// Send progress update
|
||||
res.write(`data: ${JSON.stringify({
|
||||
type: 'progress',
|
||||
current: processedCount,
|
||||
total: totalToGeocode,
|
||||
currentAddress: address,
|
||||
status: 'processing'
|
||||
})}\n\n`);
|
||||
res.flush && res.flush();
|
||||
|
||||
logger.info(`Geocoding ${processedCount}/${totalToGeocode}: ${address} (Record ID: ${record.id || record.Id || record.ID})`);
|
||||
|
||||
// Geocode the address
|
||||
const geocodeResult = await forwardGeocode(address);
|
||||
|
||||
if (geocodeResult && geocodeResult.coordinates) {
|
||||
// Check if result is malformed
|
||||
const isMalformed = geocodeResult.validation && geocodeResult.validation.isMalformed;
|
||||
// Use combined confidence for best overall assessment
|
||||
const confidence = geocodeResult.combinedConfidence !== undefined ?
|
||||
geocodeResult.combinedConfidence :
|
||||
(geocodeResult.validation ? geocodeResult.validation.confidence : 100);
|
||||
const warnings = geocodeResult.validation ? geocodeResult.validation.warnings : [];
|
||||
|
||||
// Update the record in NocoDB
|
||||
const updateData = {
|
||||
'Geo-Location': `${geocodeResult.coordinates.lat};${geocodeResult.coordinates.lng}`,
|
||||
latitude: geocodeResult.coordinates.lat,
|
||||
longitude: geocodeResult.coordinates.lng,
|
||||
'Geocode Confidence': confidence,
|
||||
'Geocode Provider': geocodeResult.provider || 'Unknown',
|
||||
last_updated_by_user: req.session?.userEmail || 'scan_geocode'
|
||||
};
|
||||
|
||||
// Update the record in NocoDB
|
||||
await nocodbService.update(config.nocodb.tableId, record.id || record.Id || record.ID, updateData);
|
||||
|
||||
const processedRecord = {
|
||||
id: record.id || record.Id || record.ID,
|
||||
address: address,
|
||||
latitude: geocodeResult.coordinates.lat,
|
||||
longitude: geocodeResult.coordinates.lng,
|
||||
confidence_score: confidence,
|
||||
provider: geocodeResult.provider || 'Unknown',
|
||||
status: isMalformed ? 'WARNING' : 'SUCCESS',
|
||||
warnings: warnings.join('; ')
|
||||
};
|
||||
|
||||
allResults.push(processedRecord);
|
||||
results.success++;
|
||||
|
||||
// Send success update
|
||||
const successMessage = {
|
||||
type: 'geocoded',
|
||||
data: processedRecord,
|
||||
index: processedCount - 1,
|
||||
status: isMalformed ? 'warning' : 'success',
|
||||
confidence: confidence,
|
||||
warnings: warnings
|
||||
};
|
||||
|
||||
logger.info(`✓ Successfully geocoded and updated: ${address} (Confidence: ${confidence}%)`);
|
||||
res.write(`data: ${JSON.stringify(successMessage)}\n\n`);
|
||||
res.flush && res.flush();
|
||||
|
||||
} else {
|
||||
throw new Error('Geocoding failed - no coordinates returned');
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
logger.error(`Failed to geocode record ${processedCount}/${totalToGeocode}:`, error.message);
|
||||
|
||||
const errorRecord = {
|
||||
id: record.id || record.Id || record.ID,
|
||||
address: record.Address || record.address || record.ADDRESS,
|
||||
error: error.message,
|
||||
status: 'ERROR'
|
||||
};
|
||||
|
||||
allResults.push(errorRecord);
|
||||
results.failed++;
|
||||
results.errors.push({
|
||||
address: errorRecord.address,
|
||||
error: error.message
|
||||
});
|
||||
|
||||
// Send error update
|
||||
res.write(`data: ${JSON.stringify({
|
||||
type: 'error',
|
||||
data: errorRecord,
|
||||
index: processedCount - 1,
|
||||
message: `Failed to geocode: ${errorRecord.address}`
|
||||
})}\n\n`);
|
||||
res.flush && res.flush();
|
||||
}
|
||||
|
||||
// Rate limiting to be nice to geocoding APIs
|
||||
if (processedCount < totalToGeocode) {
|
||||
await new Promise(resolve => setTimeout(resolve, 500)); // 0.5 second delay between requests
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate summary statistics for report
|
||||
const successful = allResults.filter(r => r.status === 'SUCCESS').length;
|
||||
const warnings = allResults.filter(r => r.status === 'WARNING').length;
|
||||
const failed = allResults.filter(r => r.status === 'ERROR').length;
|
||||
const malformed = allResults.filter(r => r.warnings && r.warnings.includes('malformed')).length;
|
||||
const total = successful + warnings + failed;
|
||||
|
||||
// Transform scan results to match CSV processing format for report generation
|
||||
const transformedResults = allResults.map(result => ({
|
||||
// Original format fields
|
||||
address: result.address,
|
||||
Address: result.address,
|
||||
geocoded_address: result.address, // For scan, this is the same
|
||||
latitude: result.latitude,
|
||||
longitude: result.longitude,
|
||||
'Geo-Location': result.latitude && result.longitude ? `${result.latitude};${result.longitude}` : '',
|
||||
confidence_score: result.confidence_score,
|
||||
provider: result.provider,
|
||||
|
||||
// Status mapping
|
||||
geocode_success: result.status !== 'ERROR',
|
||||
geocode_status: result.status,
|
||||
geocode_error: result.error || '',
|
||||
is_malformed: result.warnings && result.warnings.includes('malformed'),
|
||||
warnings: result.warnings || '',
|
||||
|
||||
// Scan-specific fields
|
||||
record_id: result.id,
|
||||
source: 'database_scan',
|
||||
row_number: result.id // Use record ID as row number for scan
|
||||
}));
|
||||
|
||||
// Store results for potential report download
|
||||
processingResults.set(sessionId, {
|
||||
filename: 'database_scan',
|
||||
timestamp: new Date().toISOString(),
|
||||
allResults: transformedResults,
|
||||
summary: {
|
||||
total: total,
|
||||
successful: successful,
|
||||
warnings: warnings,
|
||||
failed: failed,
|
||||
malformed: malformed
|
||||
}
|
||||
});
|
||||
|
||||
// Send completion message
|
||||
logger.info(`Database scan and geocoding completed: ${results.success} successful, ${results.failed} failed`);
|
||||
|
||||
res.write(`data: ${JSON.stringify({
|
||||
type: 'complete',
|
||||
message: `Scan and geocode completed! Successfully updated ${results.success} records, ${results.failed} failed.`,
|
||||
results: results,
|
||||
sessionId: sessionId
|
||||
})}\n\n`);
|
||||
res.end();
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Database scan error:', error);
|
||||
res.write(`data: ${JSON.stringify({
|
||||
type: 'error',
|
||||
message: `Database scan failed: ${error.message}`
|
||||
})}\n\n`);
|
||||
res.end();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = new DataConvertController();
|
||||
|
||||
@ -1255,6 +1255,116 @@
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Scan & Geocode Existing Records -->
|
||||
<div class="scan-geocode-container" style="margin-top: 30px;">
|
||||
<div class="info-box">
|
||||
<h3>🔍 Scan & Geocode Database</h3>
|
||||
<p>Scan your existing database for records that are missing location data and automatically geocode them.</p>
|
||||
|
||||
<div class="scan-info">
|
||||
<h4>What this does:</h4>
|
||||
<ul>
|
||||
<li>Scans all records in your database</li>
|
||||
<li>Finds records with addresses but no geo-location data</li>
|
||||
<li>Automatically geocodes those addresses using the same multi-provider system</li>
|
||||
<li>Updates records with coordinates, confidence scores, and provider information</li>
|
||||
<li>Provides detailed progress tracking and error reporting</li>
|
||||
</ul>
|
||||
|
||||
<div class="scan-warning">
|
||||
<h4>⚠️ Important Notes:</h4>
|
||||
<ul>
|
||||
<li>This will modify existing records in your database</li>
|
||||
<li>Only processes records that have an address but no coordinates</li>
|
||||
<li>Rate limited to be respectful to geocoding APIs (0.5 seconds between requests)</li>
|
||||
<li>You can download a detailed report when completed</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="scan-actions">
|
||||
<button type="button" class="btn btn-primary" id="scan-geocode-btn">
|
||||
🔍 Start Database Scan & Geocode
|
||||
</button>
|
||||
<button type="button" class="btn btn-secondary" id="cancel-scan-btn" style="display: none;">
|
||||
⏹️ Cancel Scan
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="scan-processing-section" id="scan-processing-section" style="display: none;">
|
||||
<h3>Database Scan Progress</h3>
|
||||
|
||||
<div class="scan-phase" id="scan-phase">
|
||||
<h4>Phase 1: Database Scan</h4>
|
||||
<p id="scan-status">Scanning database for records...</p>
|
||||
</div>
|
||||
|
||||
<div class="progress-bar-container" id="scan-progress-container" style="display: none;">
|
||||
<div class="progress-bar">
|
||||
<div class="progress-bar-fill" id="scan-progress-bar-fill"></div>
|
||||
</div>
|
||||
<p class="progress-text">
|
||||
<span id="scan-progress-current">0</span> /
|
||||
<span id="scan-progress-total">0</span> addresses geocoded
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="processing-status" id="scan-processing-status">
|
||||
<p id="scan-current-address"></p>
|
||||
</div>
|
||||
|
||||
<div class="scan-summary" id="scan-summary" style="display: none;">
|
||||
<h4>Scan Summary</h4>
|
||||
<div class="summary-stats">
|
||||
<div class="stat-item">
|
||||
<span class="stat-label">Total Records:</span>
|
||||
<span class="stat-value" id="total-records">0</span>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<span class="stat-label">Need Geocoding:</span>
|
||||
<span class="stat-value" id="need-geocoding">0</span>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<span class="stat-label">Successfully Geocoded:</span>
|
||||
<span class="stat-value" id="successfully-geocoded">0</span>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<span class="stat-label">Failed:</span>
|
||||
<span class="stat-value" id="failed-geocoded">0</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="scan-results-preview" id="scan-results-preview" style="display: none;">
|
||||
<h4>Recent Results</h4>
|
||||
<div class="scan-results-table-container">
|
||||
<table class="results-table" id="scan-results-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Status</th>
|
||||
<th>Address</th>
|
||||
<th>Coordinates</th>
|
||||
<th>Confidence</th>
|
||||
<th>Provider</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="scan-results-tbody"></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="scan-actions-completed" id="scan-actions-completed" style="display: none;">
|
||||
<button type="button" class="btn btn-primary" id="download-scan-report-btn">
|
||||
📄 Download Detailed Report
|
||||
</button>
|
||||
<button type="button" class="btn btn-secondary" id="new-scan-btn">
|
||||
🔍 Start New Scan
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Email Lists (Listmonk) Section -->
|
||||
|
||||
@ -189,6 +189,29 @@ function setupDataConvertEventListeners() {
|
||||
newUploadBtn.addEventListener('click', resetToUpload);
|
||||
}
|
||||
|
||||
// Scan & Geocode Database button
|
||||
const scanGeocodeBtn = document.getElementById('scan-geocode-btn');
|
||||
const cancelScanBtn = document.getElementById('cancel-scan-btn');
|
||||
const downloadScanReportBtn = document.getElementById('download-scan-report-btn');
|
||||
const newScanBtn = document.getElementById('new-scan-btn');
|
||||
|
||||
if (scanGeocodeBtn) {
|
||||
scanGeocodeBtn.addEventListener('click', startDatabaseScan);
|
||||
console.log('Scan geocode button event listener attached');
|
||||
}
|
||||
|
||||
if (cancelScanBtn) {
|
||||
cancelScanBtn.addEventListener('click', cancelDatabaseScan);
|
||||
}
|
||||
|
||||
if (downloadScanReportBtn) {
|
||||
downloadScanReportBtn.addEventListener('click', downloadScanReport);
|
||||
}
|
||||
|
||||
if (newScanBtn) {
|
||||
newScanBtn.addEventListener('click', resetScanInterface);
|
||||
}
|
||||
|
||||
// Mark as initialized
|
||||
eventListenersInitialized = true;
|
||||
console.log('Data convert event listeners initialized successfully');
|
||||
@ -937,3 +960,296 @@ async function downloadProcessingReport() {
|
||||
downloadBtn.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
// === DATABASE SCAN & GEOCODE FUNCTIONALITY ===
|
||||
|
||||
async function startDatabaseScan() {
|
||||
try {
|
||||
const scanBtn = document.getElementById('scan-geocode-btn');
|
||||
const cancelBtn = document.getElementById('cancel-scan-btn');
|
||||
const processingSection = document.getElementById('scan-processing-section');
|
||||
const scanPhase = document.getElementById('scan-phase');
|
||||
const scanStatus = document.getElementById('scan-status');
|
||||
const progressContainer = document.getElementById('scan-progress-container');
|
||||
const summary = document.getElementById('scan-summary');
|
||||
const resultsPreview = document.getElementById('scan-results-preview');
|
||||
const actionsCompleted = document.getElementById('scan-actions-completed');
|
||||
|
||||
// Show processing UI and hide scan button
|
||||
if (scanBtn) scanBtn.style.display = 'none';
|
||||
if (cancelBtn) cancelBtn.style.display = 'inline-block';
|
||||
if (processingSection) processingSection.style.display = 'block';
|
||||
|
||||
// Reset UI state
|
||||
if (progressContainer) progressContainer.style.display = 'none';
|
||||
if (summary) summary.style.display = 'none';
|
||||
if (resultsPreview) resultsPreview.style.display = 'none';
|
||||
if (actionsCompleted) actionsCompleted.style.display = 'none';
|
||||
|
||||
// Clear previous results
|
||||
const scanResultsTable = document.getElementById('scan-results-tbody');
|
||||
if (scanResultsTable) scanResultsTable.innerHTML = '';
|
||||
|
||||
console.log('Starting database scan and geocode...');
|
||||
|
||||
// Make POST request for Server-Sent Events
|
||||
const response = await fetch('/api/admin/data-convert/scan-and-geocode', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Accept': 'text/event-stream'
|
||||
}
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||||
}
|
||||
|
||||
// Process the stream manually
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
|
||||
let scanSessionId = null;
|
||||
let scanResults = [];
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
const chunk = decoder.decode(value);
|
||||
const lines = chunk.split('\n');
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.startsWith('data: ') && line.trim() !== 'data: ') {
|
||||
try {
|
||||
const data = JSON.parse(line.substring(6));
|
||||
console.log('Scan event received:', data);
|
||||
|
||||
switch (data.type) {
|
||||
case 'status':
|
||||
if (scanStatus) scanStatus.textContent = data.message;
|
||||
if (data.sessionId) {
|
||||
scanSessionId = data.sessionId;
|
||||
console.log('Initial scan session ID set:', scanSessionId);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'scanning':
|
||||
if (scanStatus) scanStatus.textContent = data.message;
|
||||
break;
|
||||
|
||||
case 'scan_complete':
|
||||
if (scanStatus) scanStatus.textContent = data.message;
|
||||
|
||||
// Update scan summary
|
||||
document.getElementById('total-records').textContent = data.total || 0;
|
||||
document.getElementById('need-geocoding').textContent = data.needingGeocode || 0;
|
||||
document.getElementById('successfully-geocoded').textContent = '0';
|
||||
document.getElementById('failed-geocoded').textContent = '0';
|
||||
|
||||
if (summary) summary.style.display = 'block';
|
||||
|
||||
if (data.needingGeocode > 0) {
|
||||
// Show progress bar and update phase
|
||||
if (progressContainer) progressContainer.style.display = 'block';
|
||||
if (scanPhase) {
|
||||
scanPhase.innerHTML = '<h4>Phase 2: Geocoding Addresses</h4>';
|
||||
}
|
||||
|
||||
// Update progress totals
|
||||
document.getElementById('scan-progress-total').textContent = data.needingGeocode;
|
||||
}
|
||||
break;
|
||||
|
||||
case 'progress':
|
||||
// Update progress bar
|
||||
const progressPercent = (data.current / data.total) * 100;
|
||||
const progressBar = document.getElementById('scan-progress-bar-fill');
|
||||
if (progressBar) progressBar.style.width = `${progressPercent}%`;
|
||||
|
||||
document.getElementById('scan-progress-current').textContent = data.current;
|
||||
document.getElementById('scan-current-address').textContent = `Processing: ${data.currentAddress}`;
|
||||
break;
|
||||
|
||||
case 'geocoded':
|
||||
// Update success counter
|
||||
const successCount = parseInt(document.getElementById('successfully-geocoded').textContent) + 1;
|
||||
document.getElementById('successfully-geocoded').textContent = successCount;
|
||||
|
||||
// Add to results preview
|
||||
addScanResultToTable(data.data, data.status);
|
||||
scanResults.push(data.data);
|
||||
|
||||
// Show results preview if not already visible
|
||||
if (resultsPreview) resultsPreview.style.display = 'block';
|
||||
break;
|
||||
|
||||
case 'error':
|
||||
if (data.data) {
|
||||
// Update failed counter
|
||||
const failedCount = parseInt(document.getElementById('failed-geocoded').textContent) + 1;
|
||||
document.getElementById('failed-geocoded').textContent = failedCount;
|
||||
|
||||
// Add to results preview
|
||||
addScanResultToTable(data.data, 'error');
|
||||
scanResults.push(data.data);
|
||||
} else {
|
||||
// General error
|
||||
if (scanStatus) scanStatus.textContent = `Error: ${data.message}`;
|
||||
}
|
||||
break;
|
||||
|
||||
case 'complete':
|
||||
if (scanStatus) scanStatus.textContent = data.message;
|
||||
|
||||
// Show completed actions
|
||||
if (actionsCompleted) actionsCompleted.style.display = 'block';
|
||||
|
||||
// Store session ID for report download from the completion message
|
||||
const finalSessionId = data.sessionId || data.results?.sessionId || scanSessionId;
|
||||
if (finalSessionId) {
|
||||
const downloadBtn = document.getElementById('download-scan-report-btn');
|
||||
if (downloadBtn) {
|
||||
downloadBtn.dataset.sessionId = finalSessionId;
|
||||
console.log('Stored scan session ID for report download:', finalSessionId);
|
||||
}
|
||||
} else {
|
||||
console.warn('No session ID available for scan report download');
|
||||
}
|
||||
|
||||
// Hide cancel button
|
||||
if (cancelBtn) cancelBtn.style.display = 'none';
|
||||
|
||||
// Exit the loop when complete
|
||||
return;
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error parsing scan event:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Stream processing error:', error);
|
||||
if (scanStatus) scanStatus.textContent = 'Error processing stream';
|
||||
} finally {
|
||||
// Clean up reader
|
||||
reader.releaseLock();
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error starting database scan:', error);
|
||||
alert('Failed to start database scan: ' + error.message);
|
||||
}
|
||||
}
|
||||
|
||||
function cancelDatabaseScan() {
|
||||
console.log('Cancelling database scan...');
|
||||
|
||||
// Note: With fetch streams, we can't easily cancel mid-stream
|
||||
// but we can reset the UI to let user know cancellation was requested
|
||||
|
||||
// Reset UI
|
||||
const scanBtn = document.getElementById('scan-geocode-btn');
|
||||
const cancelBtn = document.getElementById('cancel-scan-btn');
|
||||
const scanStatus = document.getElementById('scan-status');
|
||||
|
||||
if (scanBtn) scanBtn.style.display = 'inline-block';
|
||||
if (cancelBtn) cancelBtn.style.display = 'none';
|
||||
if (scanStatus) scanStatus.textContent = 'Scan cancelled by user.';
|
||||
}
|
||||
|
||||
function addScanResultToTable(result, status) {
|
||||
const tbody = document.getElementById('scan-results-tbody');
|
||||
if (!tbody) return;
|
||||
|
||||
// Limit table to last 10 results
|
||||
while (tbody.children.length >= 10) {
|
||||
tbody.removeChild(tbody.firstChild);
|
||||
}
|
||||
|
||||
const row = document.createElement('tr');
|
||||
row.className = `status-${status}`;
|
||||
|
||||
const statusIcon = status === 'success' ? '✅' :
|
||||
status === 'warning' ? '⚠️' : '❌';
|
||||
|
||||
const coordinates = result.latitude && result.longitude ?
|
||||
`${result.latitude}, ${result.longitude}` : 'Failed';
|
||||
|
||||
const confidence = result.confidence_score !== undefined ?
|
||||
`${result.confidence_score}%` : 'N/A';
|
||||
|
||||
row.innerHTML = `
|
||||
<td>${statusIcon} ${status.toUpperCase()}</td>
|
||||
<td>${result.address || 'Unknown'}</td>
|
||||
<td>${coordinates}</td>
|
||||
<td>${confidence}</td>
|
||||
<td>${result.provider || 'N/A'}</td>
|
||||
`;
|
||||
|
||||
tbody.appendChild(row);
|
||||
}
|
||||
|
||||
async function downloadScanReport() {
|
||||
const downloadBtn = document.getElementById('download-scan-report-btn');
|
||||
const sessionId = downloadBtn?.dataset.sessionId;
|
||||
|
||||
if (!sessionId) {
|
||||
alert('No processing session available for report generation');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(`/api/admin/data-convert/download-report/${sessionId}`);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
|
||||
// Get the filename from the response headers
|
||||
const filename = response.headers.get('content-disposition')
|
||||
?.split('filename=')[1]
|
||||
?.replace(/"/g, '') || `scan-geocoding-report-${sessionId}.txt`;
|
||||
|
||||
// Create blob and download
|
||||
const blob = await response.blob();
|
||||
const url = window.URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = url;
|
||||
a.download = filename;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
window.URL.revokeObjectURL(url);
|
||||
document.body.removeChild(a);
|
||||
|
||||
// Show success message in scan status area
|
||||
const scanStatus = document.getElementById('scan-status');
|
||||
if (scanStatus) {
|
||||
scanStatus.innerHTML = '✅ Report downloaded successfully';
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('Download scan report error:', error);
|
||||
// Show error message in scan status area
|
||||
const scanStatus = document.getElementById('scan-status');
|
||||
if (scanStatus) {
|
||||
scanStatus.innerHTML = `❌ Failed to download report: ${error.message}`;
|
||||
} else {
|
||||
alert(`Failed to download report: ${error.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function resetScanInterface() {
|
||||
// Reset to initial state
|
||||
const scanBtn = document.getElementById('scan-geocode-btn');
|
||||
const cancelBtn = document.getElementById('cancel-scan-btn');
|
||||
const processingSection = document.getElementById('scan-processing-section');
|
||||
|
||||
if (scanBtn) scanBtn.style.display = 'inline-block';
|
||||
if (cancelBtn) cancelBtn.style.display = 'none';
|
||||
if (processingSection) processingSection.style.display = 'none';
|
||||
}
|
||||
|
||||
@ -27,4 +27,7 @@ router.post('/save-geocoded', dataConvertController.saveGeocodedData);
|
||||
// Download processing report
|
||||
router.get('/download-report/:sessionId', dataConvertController.downloadReport);
|
||||
|
||||
// Scan database for records missing geo-location data and geocode them
|
||||
router.post('/scan-and-geocode', dataConvertController.scanAndGeocode);
|
||||
|
||||
module.exports = router;
|
||||
|
||||
@ -917,6 +917,32 @@ create_locations_table() {
|
||||
"title": "last_updated_by_user",
|
||||
"uidt": "SingleLineText",
|
||||
"rqd": false
|
||||
},
|
||||
{
|
||||
"column_name": "geocode_confidence",
|
||||
"title": "Geocode Confidence",
|
||||
"uidt": "Number",
|
||||
"rqd": false,
|
||||
"meta": {
|
||||
"min": 0,
|
||||
"max": 100
|
||||
}
|
||||
},
|
||||
{
|
||||
"column_name": "geocode_provider",
|
||||
"title": "Geocode Provider",
|
||||
"uidt": "SingleSelect",
|
||||
"rqd": false,
|
||||
"colOptions": {
|
||||
"options": [
|
||||
{"title": "Mapbox", "color": "#2563eb"},
|
||||
{"title": "Nominatim", "color": "#059669"},
|
||||
{"title": "Photon", "color": "#dc2626"},
|
||||
{"title": "LocationIQ", "color": "#7c3aed"},
|
||||
{"title": "ArcGIS", "color": "#ea580c"},
|
||||
{"title": "Unknown", "color": "#6b7280"}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}'
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user