download data import csv report

config.sh

@@ -633,7 +633,7 @@ EMAIL_FROM_ADDRESS=changeme@$new_domain
 APP_NAME="$new_domain Map"
 
 # Listmonk Configuration
-LISTMONK_API_URL=changeme
+LISTMONK_API_URL=http://listmonk_app:9000/api
 LISTMONK_USERNAME=changeme
 LISTMONK_PASSWORD=changeme
 LISTMONK_SYNC_ENABLED=true
@@ -763,8 +763,7 @@ EMAIL_FROM_ADDRESS=changeme@$new_domain
 APP_NAME="$new_domain Map"
 
 # Listmonk Configuration
-LISTMONK_API_URL=changeme
-LISTMONK_USERNAME=changeme
+LISTMONK_API_URL=http://listmonk_app:9000/api
 LISTMONK_PASSWORD=changeme
 LISTMONK_SYNC_ENABLED=true
 LISTMONK_INITIAL_SYNC=true  # Set to true only for first run to sync existing data

map/README.md

@@ -40,7 +40,7 @@
 - <20>🔐 Role-based access control (Admin vs User permissions)
 - ⏰ Temporary user accounts with automatic expiration
 - 📧 Email notifications and password recovery via SMTP
-- 📊 CSV data import with batch geocoding and visual progress tracking
+- 📊 CSV data import with batch geocoding, visual progress tracking, and downloadable error reports
 - ✂️ **Cut feature for geographic overlays** - Admin-drawn polygons for map regions
 - 🗺️ Interactive polygon drawing with click-to-add-points system
 - 🎨 Customizable cut properties (color, opacity, category, visibility)

map/app/controllers/dataConvertController.js

@@ -5,12 +5,16 @@ const { forwardGeocode } = require('../services/geocoding');
 const logger = require('../utils/logger');
 const config = require('../config');
 
+// In-memory storage for processing results (in production, use Redis or database)
+const processingResults = new Map();
+
 class DataConvertController {
     constructor() {
         // Bind methods to preserve 'this' context
         this.processCSV = this.processCSV.bind(this);
         this.parseCSV = this.parseCSV.bind(this);
         this.saveGeocodedData = this.saveGeocodedData.bind(this);
+        this.downloadReport = this.downloadReport.bind(this);
     }
 
     // Process CSV upload and geocode addresses with SSE progress updates
@@ -25,6 +29,7 @@ class DataConvertController {
 
             // Store the filename for later use in notes
             const originalFilename = req.file.originalname;
+            const sessionId = Date.now().toString(); // Simple session ID for storing results
 
             // Set up SSE headers
             res.writeHead(200, {
@@ -63,6 +68,7 @@ class DataConvertController {
 
             // Process all addresses
             const processedData = [];
+            const allResults = []; // Store ALL results for report generation
             const errors = [];
             const total = results.length;
 
@@ -94,10 +100,13 @@ class DataConvertController {
                             'Geo-Location': `${geocodeResult.coordinates.lat};${geocodeResult.coordinates.lng}`,
                             geocoded_address: geocodeResult.formattedAddress || address,
                             geocode_success: true,
+                            geocode_status: 'SUCCESS',
+                            geocode_error: '',
                             csv_filename: originalFilename // Include filename for notes
                         };
                         
                         processedData.push(processedRow);
+                        allResults.push(processedRow); // Add to full results for report
 
                         // Send success update
                         const successMessage = { 
@@ -115,6 +124,22 @@ class DataConvertController {
                     
                 } catch (error) {
                     logger.error(`Failed to geocode address: ${address}`, error);
+                    
+                    // Create error row with original data plus error info
+                    const errorRow = {
+                        ...row,
+                        latitude: '',
+                        longitude: '',
+                        'Geo-Location': '',
+                        geocoded_address: '',
+                        geocode_success: false,
+                        geocode_status: 'FAILED',
+                        geocode_error: error.message,
+                        csv_filename: originalFilename
+                    };
+                    
+                    allResults.push(errorRow); // Add to full results for report
+                    
                     const errorData = {
                         index: i,
                         address: address,
@@ -137,12 +162,25 @@ class DataConvertController {
                 await new Promise(resolve => setTimeout(resolve, 2000));
             }
 
+            // Store processing results for report generation
+            processingResults.set(sessionId, {
+                filename: originalFilename,
+                timestamp: new Date().toISOString(),
+                allResults: allResults,
+                summary: {
+                    total: total,
+                    successful: processedData.length,
+                    failed: errors.length
+                }
+            });
+
             // Send completion
             const completeMessage = { 
                 type: 'complete',
                 processed: processedData.length,
                 errors: errors.length,
-                total: total
+                total: total,
+                sessionId: sessionId // Include session ID for report download
             };
             const completeJson = JSON.stringify(completeMessage);
             logger.info(`Sending completion message: ${completeJson.length} chars`);
@@ -302,6 +340,112 @@ class DataConvertController {
             });
         }
     }
+
+    // Generate and download processing report
+    async downloadReport(req, res) {
+        try {
+            const { sessionId } = req.params;
+            
+            if (!sessionId || !processingResults.has(sessionId)) {
+                return res.status(404).json({
+                    success: false,
+                    error: 'Processing results not found or expired'
+                });
+            }
+
+            const results = processingResults.get(sessionId);
+            const { filename, timestamp, allResults, summary } = results;
+
+            // Convert results to CSV format
+            const csvContent = this.generateReportCSV(allResults, filename, timestamp, summary);
+            
+            // Set headers for CSV download
+            const reportFilename = `geocoding-report-${sessionId}.csv`;
+            res.setHeader('Content-Type', 'text/csv');
+            res.setHeader('Content-Disposition', `attachment; filename="${reportFilename}"`);
+            res.setHeader('Cache-Control', 'no-cache');
+
+            logger.info(`Generating report for session ${sessionId}: ${allResults.length} records`);
+
+            res.send(csvContent);
+
+            // Clean up stored results after download (optional)
+            setTimeout(() => {
+                processingResults.delete(sessionId);
+                logger.info(`Cleaned up processing results for session ${sessionId}`);
+            }, 60000); // Delete after 1 minute
+
+        } catch (error) {
+            logger.error('Download report error:', error);
+            res.status(500).json({
+                success: false,
+                error: 'Failed to generate report'
+            });
+        }
+    }
+
+    // Generate CSV content for the report
+    generateReportCSV(allResults, originalFilename, timestamp, summary) {
+        if (!allResults || allResults.length === 0) {
+            return 'No data available for report generation';
+        }
+
+        // Get all unique field names from the results
+        const allFields = new Set();
+        allResults.forEach(row => {
+            Object.keys(row).forEach(field => allFields.add(field));
+        });
+
+        // Define the header order - put important fields first
+        const priorityHeaders = [
+            'geocode_status', 'geocode_error', 'address', 'Address', 
+            'geocoded_address', 'latitude', 'longitude', 'Geo-Location'
+        ];
+        
+        const otherHeaders = Array.from(allFields).filter(field => 
+            !priorityHeaders.includes(field) && 
+            !['geocode_success', 'csv_filename'].includes(field)
+        ).sort();
+
+        const headers = [...priorityHeaders.filter(h => allFields.has(h)), ...otherHeaders];
+
+        // Generate CSV header with metadata
+        let csvContent = `# Geocoding Processing Report\n`;
+        csvContent += `# Original File: ${originalFilename}\n`;
+        csvContent += `# Processed: ${timestamp}\n`;
+        csvContent += `# Total Records: ${summary.total}\n`;
+        csvContent += `# Successful: ${summary.successful}\n`;
+        csvContent += `# Failed: ${summary.failed}\n`;
+        csvContent += `# \n`;
+        
+        // Add CSV headers
+        csvContent += headers.map(header => this.escapeCSVField(header)).join(',') + '\n';
+
+        // Add data rows
+        allResults.forEach(row => {
+            const values = headers.map(header => {
+                const value = row[header];
+                return this.escapeCSVField(value !== undefined && value !== null ? String(value) : '');
+            });
+            csvContent += values.join(',') + '\n';
+        });
+
+        return csvContent;
+    }
+
+    // Escape CSV fields properly
+    escapeCSVField(field) {
+        if (field === null || field === undefined) return '';
+        
+        const stringField = String(field);
+        
+        // If field contains comma, quote, or newline, wrap in quotes and escape quotes
+        if (stringField.includes(',') || stringField.includes('"') || stringField.includes('\n') || stringField.includes('\r')) {
+            return '"' + stringField.replace(/"/g, '""') + '"';
+        }
+        
+        return stringField;
+    }
 }
 
 module.exports = new DataConvertController();

map/app/public/js/data-convert.js

@@ -1,4 +1,5 @@
 let processingData = [];
+let currentSessionId = null; // Store session ID for report download
 let resultsMap = null;
 let markers = [];
 let eventListenersInitialized = false;
@@ -310,6 +311,7 @@ function handleProcessingUpdate(data) {
             
         case 'complete':
             console.log('Received complete event:', data);
+            currentSessionId = data.sessionId; // Store session ID for report download
             onProcessingComplete(data);
             break;
             
@@ -410,6 +412,11 @@ function onProcessingComplete(data) {
         }
     }
     
+    // Show download report button if we have a session ID
+    if (currentSessionId && data.total > 0) {
+        showDownloadReportButton(data.total, data.errors || data.failed || 0);
+    }
+    
     const errorCount = data.errors || data.failed || 0;
     showDataConvertStatus(`Processing complete: ${processedCount} successful, ${errorCount} errors`, 
                errorCount > 0 ? 'warning' : 'success');
@@ -551,6 +558,13 @@ function resetToUpload() {
     
     // Clear any existing data
     processingData = [];
+    currentSessionId = null; // Reset session ID
+    
+    // Remove download report button if it exists
+    const downloadBtn = document.getElementById('download-report-btn');
+    if (downloadBtn) {
+        downloadBtn.remove();
+    }
     if (markers && markers.length > 0) {
         markers.forEach(marker => {
             if (resultsMap && marker) {
@@ -594,3 +608,84 @@ function resetToUpload() {
     // Reset form
     clearUpload();
 }
+
+// Show download report button
+function showDownloadReportButton(totalRecords, errorCount) {
+    const actionsDiv = document.getElementById('processing-actions');
+    if (!actionsDiv) return;
+    
+    // Check if button already exists
+    let downloadBtn = document.getElementById('download-report-btn');
+    if (!downloadBtn) {
+        downloadBtn = document.createElement('button');
+        downloadBtn.id = 'download-report-btn';
+        downloadBtn.type = 'button';
+        downloadBtn.className = 'btn btn-info';
+        downloadBtn.addEventListener('click', downloadProcessingReport);
+        
+        // Insert before the save results button
+        const saveBtn = document.getElementById('save-results-btn');
+        if (saveBtn) {
+            actionsDiv.insertBefore(downloadBtn, saveBtn);
+        } else {
+            actionsDiv.appendChild(downloadBtn);
+        }
+    }
+    
+    // Update button text with error info
+    if (errorCount > 0) {
+        downloadBtn.textContent = `📄 Download Full Report (${totalRecords} records, ${errorCount} errors)`;
+        downloadBtn.title = `Download complete processing report including ${errorCount} error records for review`;
+    } else {
+        downloadBtn.textContent = `📄 Download Processing Report (${totalRecords} records)`;
+        downloadBtn.title = 'Download complete processing report';
+    }
+    
+    downloadBtn.style.marginRight = '10px'; // Add some spacing
+}
+
+// Download processing report
+async function downloadProcessingReport() {
+    if (!currentSessionId) {
+        showDataConvertStatus('No processing session available for report generation', 'error');
+        return;
+    }
+    
+    const downloadBtn = document.getElementById('download-report-btn');
+    const originalText = downloadBtn.textContent;
+    
+    try {
+        downloadBtn.disabled = true;
+        downloadBtn.textContent = '📄 Generating Report...';
+        
+        showDataConvertStatus('Generating processing report...', 'info');
+        
+        // Create download link
+        const downloadUrl = `/api/admin/data-convert/download-report/${currentSessionId}`;
+        
+        // Create a temporary anchor element and trigger download
+        const link = document.createElement('a');
+        link.href = downloadUrl;
+        link.download = `geocoding-report-${currentSessionId}.csv`;
+        link.style.display = 'none';
+        
+        document.body.appendChild(link);
+        link.click();
+        document.body.removeChild(link);
+        
+        showDataConvertStatus('Processing report downloaded successfully', 'success');
+        
+        // Update button text
+        downloadBtn.textContent = '📄 Report Downloaded';
+        setTimeout(() => {
+            downloadBtn.textContent = originalText;
+            downloadBtn.disabled = false;
+        }, 3000);
+        
+    } catch (error) {
+        console.error('Download report error:', error);
+        showDataConvertStatus('Failed to download processing report: ' + error.message, 'error');
+        downloadBtn.textContent = originalText;
+        downloadBtn.disabled = false;
+    }
+}

map/app/routes/dataConvert.js

@@ -24,4 +24,7 @@ router.post('/process-csv', upload.single('csvFile'), dataConvertController.proc
 // Save geocoded data
 router.post('/save-geocoded', dataConvertController.saveGeocodedData);
 
+// Download processing report
+router.get('/download-report/:sessionId', dataConvertController.downloadReport);
+
 module.exports = router;

map/files-explainer.md

@@ -74,7 +74,7 @@ Controller for handling shift management, signup/cancellation, and admin operati
 
 # app/controllers/dataConvertController.js
 
-Controller for handling CSV upload and batch geocoding of addresses. Parses CSV files, validates address data, uses the geocoding service to get coordinates, and provides real-time progress updates via Server-Sent Events (SSE).
+Controller for handling CSV upload and batch geocoding of addresses. Parses CSV files, validates address data, uses the geocoding service to get coordinates, and provides real-time progress updates via Server-Sent Events (SSE). Enhanced with comprehensive error logging and downloadable processing reports that include both successful and failed geocoding attempts for review and debugging.
 
 # app/controllers/dashboardController.js