download data import csv report
This commit is contained in:
parent
2a30d3857c
commit
94600839f0
@ -633,7 +633,7 @@ EMAIL_FROM_ADDRESS=changeme@$new_domain
|
||||
APP_NAME="$new_domain Map"
|
||||
|
||||
# Listmonk Configuration
|
||||
LISTMONK_API_URL=changeme
|
||||
LISTMONK_API_URL=http://listmonk_app:9000/api
|
||||
LISTMONK_USERNAME=changeme
|
||||
LISTMONK_PASSWORD=changeme
|
||||
LISTMONK_SYNC_ENABLED=true
|
||||
@ -763,8 +763,7 @@ EMAIL_FROM_ADDRESS=changeme@$new_domain
|
||||
APP_NAME="$new_domain Map"
|
||||
|
||||
# Listmonk Configuration
|
||||
LISTMONK_API_URL=changeme
|
||||
LISTMONK_USERNAME=changeme
|
||||
LISTMONK_API_URL=http://listmonk_app:9000/api
|
||||
LISTMONK_PASSWORD=changeme
|
||||
LISTMONK_SYNC_ENABLED=true
|
||||
LISTMONK_INITIAL_SYNC=true # Set to true only for first run to sync existing data
|
||||
|
||||
@ -40,7 +40,7 @@
|
||||
- <20>🔐 Role-based access control (Admin vs User permissions)
|
||||
- ⏰ Temporary user accounts with automatic expiration
|
||||
- 📧 Email notifications and password recovery via SMTP
|
||||
- 📊 CSV data import with batch geocoding and visual progress tracking
|
||||
- 📊 CSV data import with batch geocoding, visual progress tracking, and downloadable error reports
|
||||
- ✂️ **Cut feature for geographic overlays** - Admin-drawn polygons for map regions
|
||||
- 🗺️ Interactive polygon drawing with click-to-add-points system
|
||||
- 🎨 Customizable cut properties (color, opacity, category, visibility)
|
||||
|
||||
@ -5,12 +5,16 @@ const { forwardGeocode } = require('../services/geocoding');
|
||||
const logger = require('../utils/logger');
|
||||
const config = require('../config');
|
||||
|
||||
// In-memory storage for processing results (in production, use Redis or database)
|
||||
const processingResults = new Map();
|
||||
|
||||
class DataConvertController {
|
||||
constructor() {
|
||||
// Bind methods to preserve 'this' context
|
||||
this.processCSV = this.processCSV.bind(this);
|
||||
this.parseCSV = this.parseCSV.bind(this);
|
||||
this.saveGeocodedData = this.saveGeocodedData.bind(this);
|
||||
this.downloadReport = this.downloadReport.bind(this);
|
||||
}
|
||||
|
||||
// Process CSV upload and geocode addresses with SSE progress updates
|
||||
@ -25,6 +29,7 @@ class DataConvertController {
|
||||
|
||||
// Store the filename for later use in notes
|
||||
const originalFilename = req.file.originalname;
|
||||
const sessionId = Date.now().toString(); // Simple session ID for storing results
|
||||
|
||||
// Set up SSE headers
|
||||
res.writeHead(200, {
|
||||
@ -63,6 +68,7 @@ class DataConvertController {
|
||||
|
||||
// Process all addresses
|
||||
const processedData = [];
|
||||
const allResults = []; // Store ALL results for report generation
|
||||
const errors = [];
|
||||
const total = results.length;
|
||||
|
||||
@ -94,10 +100,13 @@ class DataConvertController {
|
||||
'Geo-Location': `${geocodeResult.coordinates.lat};${geocodeResult.coordinates.lng}`,
|
||||
geocoded_address: geocodeResult.formattedAddress || address,
|
||||
geocode_success: true,
|
||||
geocode_status: 'SUCCESS',
|
||||
geocode_error: '',
|
||||
csv_filename: originalFilename // Include filename for notes
|
||||
};
|
||||
|
||||
processedData.push(processedRow);
|
||||
allResults.push(processedRow); // Add to full results for report
|
||||
|
||||
// Send success update
|
||||
const successMessage = {
|
||||
@ -115,6 +124,22 @@ class DataConvertController {
|
||||
|
||||
} catch (error) {
|
||||
logger.error(`Failed to geocode address: ${address}`, error);
|
||||
|
||||
// Create error row with original data plus error info
|
||||
const errorRow = {
|
||||
...row,
|
||||
latitude: '',
|
||||
longitude: '',
|
||||
'Geo-Location': '',
|
||||
geocoded_address: '',
|
||||
geocode_success: false,
|
||||
geocode_status: 'FAILED',
|
||||
geocode_error: error.message,
|
||||
csv_filename: originalFilename
|
||||
};
|
||||
|
||||
allResults.push(errorRow); // Add to full results for report
|
||||
|
||||
const errorData = {
|
||||
index: i,
|
||||
address: address,
|
||||
@ -137,12 +162,25 @@ class DataConvertController {
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
}
|
||||
|
||||
// Store processing results for report generation
|
||||
processingResults.set(sessionId, {
|
||||
filename: originalFilename,
|
||||
timestamp: new Date().toISOString(),
|
||||
allResults: allResults,
|
||||
summary: {
|
||||
total: total,
|
||||
successful: processedData.length,
|
||||
failed: errors.length
|
||||
}
|
||||
});
|
||||
|
||||
// Send completion
|
||||
const completeMessage = {
|
||||
type: 'complete',
|
||||
processed: processedData.length,
|
||||
errors: errors.length,
|
||||
total: total
|
||||
total: total,
|
||||
sessionId: sessionId // Include session ID for report download
|
||||
};
|
||||
const completeJson = JSON.stringify(completeMessage);
|
||||
logger.info(`Sending completion message: ${completeJson.length} chars`);
|
||||
@ -302,6 +340,112 @@ class DataConvertController {
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Generate and download processing report
|
||||
async downloadReport(req, res) {
|
||||
try {
|
||||
const { sessionId } = req.params;
|
||||
|
||||
if (!sessionId || !processingResults.has(sessionId)) {
|
||||
return res.status(404).json({
|
||||
success: false,
|
||||
error: 'Processing results not found or expired'
|
||||
});
|
||||
}
|
||||
|
||||
const results = processingResults.get(sessionId);
|
||||
const { filename, timestamp, allResults, summary } = results;
|
||||
|
||||
// Convert results to CSV format
|
||||
const csvContent = this.generateReportCSV(allResults, filename, timestamp, summary);
|
||||
|
||||
// Set headers for CSV download
|
||||
const reportFilename = `geocoding-report-${sessionId}.csv`;
|
||||
res.setHeader('Content-Type', 'text/csv');
|
||||
res.setHeader('Content-Disposition', `attachment; filename="${reportFilename}"`);
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
|
||||
logger.info(`Generating report for session ${sessionId}: ${allResults.length} records`);
|
||||
|
||||
res.send(csvContent);
|
||||
|
||||
// Clean up stored results after download (optional)
|
||||
setTimeout(() => {
|
||||
processingResults.delete(sessionId);
|
||||
logger.info(`Cleaned up processing results for session ${sessionId}`);
|
||||
}, 60000); // Delete after 1 minute
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Download report error:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: 'Failed to generate report'
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Generate CSV content for the report
|
||||
generateReportCSV(allResults, originalFilename, timestamp, summary) {
|
||||
if (!allResults || allResults.length === 0) {
|
||||
return 'No data available for report generation';
|
||||
}
|
||||
|
||||
// Get all unique field names from the results
|
||||
const allFields = new Set();
|
||||
allResults.forEach(row => {
|
||||
Object.keys(row).forEach(field => allFields.add(field));
|
||||
});
|
||||
|
||||
// Define the header order - put important fields first
|
||||
const priorityHeaders = [
|
||||
'geocode_status', 'geocode_error', 'address', 'Address',
|
||||
'geocoded_address', 'latitude', 'longitude', 'Geo-Location'
|
||||
];
|
||||
|
||||
const otherHeaders = Array.from(allFields).filter(field =>
|
||||
!priorityHeaders.includes(field) &&
|
||||
!['geocode_success', 'csv_filename'].includes(field)
|
||||
).sort();
|
||||
|
||||
const headers = [...priorityHeaders.filter(h => allFields.has(h)), ...otherHeaders];
|
||||
|
||||
// Generate CSV header with metadata
|
||||
let csvContent = `# Geocoding Processing Report\n`;
|
||||
csvContent += `# Original File: ${originalFilename}\n`;
|
||||
csvContent += `# Processed: ${timestamp}\n`;
|
||||
csvContent += `# Total Records: ${summary.total}\n`;
|
||||
csvContent += `# Successful: ${summary.successful}\n`;
|
||||
csvContent += `# Failed: ${summary.failed}\n`;
|
||||
csvContent += `# \n`;
|
||||
|
||||
// Add CSV headers
|
||||
csvContent += headers.map(header => this.escapeCSVField(header)).join(',') + '\n';
|
||||
|
||||
// Add data rows
|
||||
allResults.forEach(row => {
|
||||
const values = headers.map(header => {
|
||||
const value = row[header];
|
||||
return this.escapeCSVField(value !== undefined && value !== null ? String(value) : '');
|
||||
});
|
||||
csvContent += values.join(',') + '\n';
|
||||
});
|
||||
|
||||
return csvContent;
|
||||
}
|
||||
|
||||
// Escape CSV fields properly
|
||||
escapeCSVField(field) {
|
||||
if (field === null || field === undefined) return '';
|
||||
|
||||
const stringField = String(field);
|
||||
|
||||
// If field contains comma, quote, or newline, wrap in quotes and escape quotes
|
||||
if (stringField.includes(',') || stringField.includes('"') || stringField.includes('\n') || stringField.includes('\r')) {
|
||||
return '"' + stringField.replace(/"/g, '""') + '"';
|
||||
}
|
||||
|
||||
return stringField;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = new DataConvertController();
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
let processingData = [];
|
||||
let currentSessionId = null; // Store session ID for report download
|
||||
let resultsMap = null;
|
||||
let markers = [];
|
||||
let eventListenersInitialized = false;
|
||||
@ -310,6 +311,7 @@ function handleProcessingUpdate(data) {
|
||||
|
||||
case 'complete':
|
||||
console.log('Received complete event:', data);
|
||||
currentSessionId = data.sessionId; // Store session ID for report download
|
||||
onProcessingComplete(data);
|
||||
break;
|
||||
|
||||
@ -410,6 +412,11 @@ function onProcessingComplete(data) {
|
||||
}
|
||||
}
|
||||
|
||||
// Show download report button if we have a session ID
|
||||
if (currentSessionId && data.total > 0) {
|
||||
showDownloadReportButton(data.total, data.errors || data.failed || 0);
|
||||
}
|
||||
|
||||
const errorCount = data.errors || data.failed || 0;
|
||||
showDataConvertStatus(`Processing complete: ${processedCount} successful, ${errorCount} errors`,
|
||||
errorCount > 0 ? 'warning' : 'success');
|
||||
@ -551,6 +558,13 @@ function resetToUpload() {
|
||||
|
||||
// Clear any existing data
|
||||
processingData = [];
|
||||
currentSessionId = null; // Reset session ID
|
||||
|
||||
// Remove download report button if it exists
|
||||
const downloadBtn = document.getElementById('download-report-btn');
|
||||
if (downloadBtn) {
|
||||
downloadBtn.remove();
|
||||
}
|
||||
if (markers && markers.length > 0) {
|
||||
markers.forEach(marker => {
|
||||
if (resultsMap && marker) {
|
||||
@ -594,3 +608,84 @@ function resetToUpload() {
|
||||
// Reset form
|
||||
clearUpload();
|
||||
}
|
||||
|
||||
// Show download report button
|
||||
function showDownloadReportButton(totalRecords, errorCount) {
|
||||
const actionsDiv = document.getElementById('processing-actions');
|
||||
if (!actionsDiv) return;
|
||||
|
||||
// Check if button already exists
|
||||
let downloadBtn = document.getElementById('download-report-btn');
|
||||
if (!downloadBtn) {
|
||||
downloadBtn = document.createElement('button');
|
||||
downloadBtn.id = 'download-report-btn';
|
||||
downloadBtn.type = 'button';
|
||||
downloadBtn.className = 'btn btn-info';
|
||||
downloadBtn.addEventListener('click', downloadProcessingReport);
|
||||
|
||||
// Insert before the save results button
|
||||
const saveBtn = document.getElementById('save-results-btn');
|
||||
if (saveBtn) {
|
||||
actionsDiv.insertBefore(downloadBtn, saveBtn);
|
||||
} else {
|
||||
actionsDiv.appendChild(downloadBtn);
|
||||
}
|
||||
}
|
||||
|
||||
// Update button text with error info
|
||||
if (errorCount > 0) {
|
||||
downloadBtn.textContent = `📄 Download Full Report (${totalRecords} records, ${errorCount} errors)`;
|
||||
downloadBtn.title = `Download complete processing report including ${errorCount} error records for review`;
|
||||
} else {
|
||||
downloadBtn.textContent = `📄 Download Processing Report (${totalRecords} records)`;
|
||||
downloadBtn.title = 'Download complete processing report';
|
||||
}
|
||||
|
||||
downloadBtn.style.marginRight = '10px'; // Add some spacing
|
||||
}
|
||||
|
||||
// Download processing report
|
||||
async function downloadProcessingReport() {
|
||||
if (!currentSessionId) {
|
||||
showDataConvertStatus('No processing session available for report generation', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
const downloadBtn = document.getElementById('download-report-btn');
|
||||
const originalText = downloadBtn.textContent;
|
||||
|
||||
try {
|
||||
downloadBtn.disabled = true;
|
||||
downloadBtn.textContent = '📄 Generating Report...';
|
||||
|
||||
showDataConvertStatus('Generating processing report...', 'info');
|
||||
|
||||
// Create download link
|
||||
const downloadUrl = `/api/admin/data-convert/download-report/${currentSessionId}`;
|
||||
|
||||
// Create a temporary anchor element and trigger download
|
||||
const link = document.createElement('a');
|
||||
link.href = downloadUrl;
|
||||
link.download = `geocoding-report-${currentSessionId}.csv`;
|
||||
link.style.display = 'none';
|
||||
|
||||
document.body.appendChild(link);
|
||||
link.click();
|
||||
document.body.removeChild(link);
|
||||
|
||||
showDataConvertStatus('Processing report downloaded successfully', 'success');
|
||||
|
||||
// Update button text
|
||||
downloadBtn.textContent = '📄 Report Downloaded';
|
||||
setTimeout(() => {
|
||||
downloadBtn.textContent = originalText;
|
||||
downloadBtn.disabled = false;
|
||||
}, 3000);
|
||||
|
||||
} catch (error) {
|
||||
console.error('Download report error:', error);
|
||||
showDataConvertStatus('Failed to download processing report: ' + error.message, 'error');
|
||||
downloadBtn.textContent = originalText;
|
||||
downloadBtn.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
@ -24,4 +24,7 @@ router.post('/process-csv', upload.single('csvFile'), dataConvertController.proc
|
||||
// Save geocoded data
|
||||
router.post('/save-geocoded', dataConvertController.saveGeocodedData);
|
||||
|
||||
// Download processing report
|
||||
router.get('/download-report/:sessionId', dataConvertController.downloadReport);
|
||||
|
||||
module.exports = router;
|
||||
|
||||
@ -74,7 +74,7 @@ Controller for handling shift management, signup/cancellation, and admin operati
|
||||
|
||||
# app/controllers/dataConvertController.js
|
||||
|
||||
Controller for handling CSV upload and batch geocoding of addresses. Parses CSV files, validates address data, uses the geocoding service to get coordinates, and provides real-time progress updates via Server-Sent Events (SSE).
|
||||
Controller for handling CSV upload and batch geocoding of addresses. Parses CSV files, validates address data, uses the geocoding service to get coordinates, and provides real-time progress updates via Server-Sent Events (SSE). Enhanced with comprehensive error logging and downloadable processing reports that include both successful and failed geocoding attempts for review and debugging.
|
||||
|
||||
# app/controllers/dashboardController.js
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user