diff --git a/config.sh b/config.sh index f5a1cbc..655aef7 100755 --- a/config.sh +++ b/config.sh @@ -633,7 +633,7 @@ EMAIL_FROM_ADDRESS=changeme@$new_domain APP_NAME="$new_domain Map" # Listmonk Configuration -LISTMONK_API_URL=changeme +LISTMONK_API_URL=http://listmonk_app:9000/api LISTMONK_USERNAME=changeme LISTMONK_PASSWORD=changeme LISTMONK_SYNC_ENABLED=true @@ -763,8 +763,7 @@ EMAIL_FROM_ADDRESS=changeme@$new_domain APP_NAME="$new_domain Map" # Listmonk Configuration -LISTMONK_API_URL=changeme -LISTMONK_USERNAME=changeme +LISTMONK_API_URL=http://listmonk_app:9000/api LISTMONK_PASSWORD=changeme LISTMONK_SYNC_ENABLED=true LISTMONK_INITIAL_SYNC=true # Set to true only for first run to sync existing data diff --git a/map/README.md b/map/README.md index ada0aa7..d5f676e 100644 --- a/map/README.md +++ b/map/README.md @@ -40,7 +40,7 @@ - οΏ½πŸ” Role-based access control (Admin vs User permissions) - ⏰ Temporary user accounts with automatic expiration - πŸ“§ Email notifications and password recovery via SMTP -- πŸ“Š CSV data import with batch geocoding and visual progress tracking +- πŸ“Š CSV data import with batch geocoding, visual progress tracking, and downloadable error reports - βœ‚οΈ **Cut feature for geographic overlays** - Admin-drawn polygons for map regions - πŸ—ΊοΈ Interactive polygon drawing with click-to-add-points system - 🎨 Customizable cut properties (color, opacity, category, visibility) diff --git a/map/app/controllers/dataConvertController.js b/map/app/controllers/dataConvertController.js index 2ce4bbd..426b9ca 100644 --- a/map/app/controllers/dataConvertController.js +++ b/map/app/controllers/dataConvertController.js @@ -5,12 +5,16 @@ const { forwardGeocode } = require('../services/geocoding'); const logger = require('../utils/logger'); const config = require('../config'); +// In-memory storage for processing results (in production, use Redis or database) +const processingResults = new Map(); + class DataConvertController { constructor() { // Bind methods to preserve 'this' context this.processCSV = this.processCSV.bind(this); this.parseCSV = this.parseCSV.bind(this); this.saveGeocodedData = this.saveGeocodedData.bind(this); + this.downloadReport = this.downloadReport.bind(this); } // Process CSV upload and geocode addresses with SSE progress updates @@ -25,6 +29,7 @@ class DataConvertController { // Store the filename for later use in notes const originalFilename = req.file.originalname; + const sessionId = Date.now().toString(); // Simple session ID for storing results // Set up SSE headers res.writeHead(200, { @@ -63,6 +68,7 @@ class DataConvertController { // Process all addresses const processedData = []; + const allResults = []; // Store ALL results for report generation const errors = []; const total = results.length; @@ -94,10 +100,13 @@ class DataConvertController { 'Geo-Location': `${geocodeResult.coordinates.lat};${geocodeResult.coordinates.lng}`, geocoded_address: geocodeResult.formattedAddress || address, geocode_success: true, + geocode_status: 'SUCCESS', + geocode_error: '', csv_filename: originalFilename // Include filename for notes }; processedData.push(processedRow); + allResults.push(processedRow); // Add to full results for report // Send success update const successMessage = { @@ -115,6 +124,22 @@ class DataConvertController { } catch (error) { logger.error(`Failed to geocode address: ${address}`, error); + + // Create error row with original data plus error info + const errorRow = { + ...row, + latitude: '', + longitude: '', + 'Geo-Location': '', + geocoded_address: '', + geocode_success: false, + geocode_status: 'FAILED', + geocode_error: error.message, + csv_filename: originalFilename + }; + + allResults.push(errorRow); // Add to full results for report + const errorData = { index: i, address: address, @@ -137,12 +162,25 @@ class DataConvertController { await new Promise(resolve => setTimeout(resolve, 2000)); } + // Store processing results for report generation + processingResults.set(sessionId, { + filename: originalFilename, + timestamp: new Date().toISOString(), + allResults: allResults, + summary: { + total: total, + successful: processedData.length, + failed: errors.length + } + }); + // Send completion const completeMessage = { type: 'complete', processed: processedData.length, errors: errors.length, - total: total + total: total, + sessionId: sessionId // Include session ID for report download }; const completeJson = JSON.stringify(completeMessage); logger.info(`Sending completion message: ${completeJson.length} chars`); @@ -302,6 +340,112 @@ class DataConvertController { }); } } + + // Generate and download processing report + async downloadReport(req, res) { + try { + const { sessionId } = req.params; + + if (!sessionId || !processingResults.has(sessionId)) { + return res.status(404).json({ + success: false, + error: 'Processing results not found or expired' + }); + } + + const results = processingResults.get(sessionId); + const { filename, timestamp, allResults, summary } = results; + + // Convert results to CSV format + const csvContent = this.generateReportCSV(allResults, filename, timestamp, summary); + + // Set headers for CSV download + const reportFilename = `geocoding-report-${sessionId}.csv`; + res.setHeader('Content-Type', 'text/csv'); + res.setHeader('Content-Disposition', `attachment; filename="${reportFilename}"`); + res.setHeader('Cache-Control', 'no-cache'); + + logger.info(`Generating report for session ${sessionId}: ${allResults.length} records`); + + res.send(csvContent); + + // Clean up stored results after download (optional) + setTimeout(() => { + processingResults.delete(sessionId); + logger.info(`Cleaned up processing results for session ${sessionId}`); + }, 60000); // Delete after 1 minute + + } catch (error) { + logger.error('Download report error:', error); + res.status(500).json({ + success: false, + error: 'Failed to generate report' + }); + } + } + + // Generate CSV content for the report + generateReportCSV(allResults, originalFilename, timestamp, summary) { + if (!allResults || allResults.length === 0) { + return 'No data available for report generation'; + } + + // Get all unique field names from the results + const allFields = new Set(); + allResults.forEach(row => { + Object.keys(row).forEach(field => allFields.add(field)); + }); + + // Define the header order - put important fields first + const priorityHeaders = [ + 'geocode_status', 'geocode_error', 'address', 'Address', + 'geocoded_address', 'latitude', 'longitude', 'Geo-Location' + ]; + + const otherHeaders = Array.from(allFields).filter(field => + !priorityHeaders.includes(field) && + !['geocode_success', 'csv_filename'].includes(field) + ).sort(); + + const headers = [...priorityHeaders.filter(h => allFields.has(h)), ...otherHeaders]; + + // Generate CSV header with metadata + let csvContent = `# Geocoding Processing Report\n`; + csvContent += `# Original File: ${originalFilename}\n`; + csvContent += `# Processed: ${timestamp}\n`; + csvContent += `# Total Records: ${summary.total}\n`; + csvContent += `# Successful: ${summary.successful}\n`; + csvContent += `# Failed: ${summary.failed}\n`; + csvContent += `# \n`; + + // Add CSV headers + csvContent += headers.map(header => this.escapeCSVField(header)).join(',') + '\n'; + + // Add data rows + allResults.forEach(row => { + const values = headers.map(header => { + const value = row[header]; + return this.escapeCSVField(value !== undefined && value !== null ? String(value) : ''); + }); + csvContent += values.join(',') + '\n'; + }); + + return csvContent; + } + + // Escape CSV fields properly + escapeCSVField(field) { + if (field === null || field === undefined) return ''; + + const stringField = String(field); + + // If field contains comma, quote, or newline, wrap in quotes and escape quotes + if (stringField.includes(',') || stringField.includes('"') || stringField.includes('\n') || stringField.includes('\r')) { + return '"' + stringField.replace(/"/g, '""') + '"'; + } + + return stringField; + } } module.exports = new DataConvertController(); diff --git a/map/app/public/js/data-convert.js b/map/app/public/js/data-convert.js index 3ff5056..ee96114 100644 --- a/map/app/public/js/data-convert.js +++ b/map/app/public/js/data-convert.js @@ -1,4 +1,5 @@ let processingData = []; +let currentSessionId = null; // Store session ID for report download let resultsMap = null; let markers = []; let eventListenersInitialized = false; @@ -310,6 +311,7 @@ function handleProcessingUpdate(data) { case 'complete': console.log('Received complete event:', data); + currentSessionId = data.sessionId; // Store session ID for report download onProcessingComplete(data); break; @@ -410,6 +412,11 @@ function onProcessingComplete(data) { } } + // Show download report button if we have a session ID + if (currentSessionId && data.total > 0) { + showDownloadReportButton(data.total, data.errors || data.failed || 0); + } + const errorCount = data.errors || data.failed || 0; showDataConvertStatus(`Processing complete: ${processedCount} successful, ${errorCount} errors`, errorCount > 0 ? 'warning' : 'success'); @@ -551,6 +558,13 @@ function resetToUpload() { // Clear any existing data processingData = []; + currentSessionId = null; // Reset session ID + + // Remove download report button if it exists + const downloadBtn = document.getElementById('download-report-btn'); + if (downloadBtn) { + downloadBtn.remove(); + } if (markers && markers.length > 0) { markers.forEach(marker => { if (resultsMap && marker) { @@ -594,3 +608,84 @@ function resetToUpload() { // Reset form clearUpload(); } + +// Show download report button +function showDownloadReportButton(totalRecords, errorCount) { + const actionsDiv = document.getElementById('processing-actions'); + if (!actionsDiv) return; + + // Check if button already exists + let downloadBtn = document.getElementById('download-report-btn'); + if (!downloadBtn) { + downloadBtn = document.createElement('button'); + downloadBtn.id = 'download-report-btn'; + downloadBtn.type = 'button'; + downloadBtn.className = 'btn btn-info'; + downloadBtn.addEventListener('click', downloadProcessingReport); + + // Insert before the save results button + const saveBtn = document.getElementById('save-results-btn'); + if (saveBtn) { + actionsDiv.insertBefore(downloadBtn, saveBtn); + } else { + actionsDiv.appendChild(downloadBtn); + } + } + + // Update button text with error info + if (errorCount > 0) { + downloadBtn.textContent = `πŸ“„ Download Full Report (${totalRecords} records, ${errorCount} errors)`; + downloadBtn.title = `Download complete processing report including ${errorCount} error records for review`; + } else { + downloadBtn.textContent = `πŸ“„ Download Processing Report (${totalRecords} records)`; + downloadBtn.title = 'Download complete processing report'; + } + + downloadBtn.style.marginRight = '10px'; // Add some spacing +} + +// Download processing report +async function downloadProcessingReport() { + if (!currentSessionId) { + showDataConvertStatus('No processing session available for report generation', 'error'); + return; + } + + const downloadBtn = document.getElementById('download-report-btn'); + const originalText = downloadBtn.textContent; + + try { + downloadBtn.disabled = true; + downloadBtn.textContent = 'πŸ“„ Generating Report...'; + + showDataConvertStatus('Generating processing report...', 'info'); + + // Create download link + const downloadUrl = `/api/admin/data-convert/download-report/${currentSessionId}`; + + // Create a temporary anchor element and trigger download + const link = document.createElement('a'); + link.href = downloadUrl; + link.download = `geocoding-report-${currentSessionId}.csv`; + link.style.display = 'none'; + + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + + showDataConvertStatus('Processing report downloaded successfully', 'success'); + + // Update button text + downloadBtn.textContent = 'πŸ“„ Report Downloaded'; + setTimeout(() => { + downloadBtn.textContent = originalText; + downloadBtn.disabled = false; + }, 3000); + + } catch (error) { + console.error('Download report error:', error); + showDataConvertStatus('Failed to download processing report: ' + error.message, 'error'); + downloadBtn.textContent = originalText; + downloadBtn.disabled = false; + } +} diff --git a/map/app/routes/dataConvert.js b/map/app/routes/dataConvert.js index 14f37d1..e963a22 100644 --- a/map/app/routes/dataConvert.js +++ b/map/app/routes/dataConvert.js @@ -24,4 +24,7 @@ router.post('/process-csv', upload.single('csvFile'), dataConvertController.proc // Save geocoded data router.post('/save-geocoded', dataConvertController.saveGeocodedData); +// Download processing report +router.get('/download-report/:sessionId', dataConvertController.downloadReport); + module.exports = router; diff --git a/map/files-explainer.md b/map/files-explainer.md index d5063ee..10a3fa6 100644 --- a/map/files-explainer.md +++ b/map/files-explainer.md @@ -74,7 +74,7 @@ Controller for handling shift management, signup/cancellation, and admin operati # app/controllers/dataConvertController.js -Controller for handling CSV upload and batch geocoding of addresses. Parses CSV files, validates address data, uses the geocoding service to get coordinates, and provides real-time progress updates via Server-Sent Events (SSE). +Controller for handling CSV upload and batch geocoding of addresses. Parses CSV files, validates address data, uses the geocoding service to get coordinates, and provides real-time progress updates via Server-Sent Events (SSE). Enhanced with comprehensive error logging and downloadable processing reports that include both successful and failed geocoding attempts for review and debugging. # app/controllers/dashboardController.js