freealberta/map/app/public/js/data-convert.js

692 lines
24 KiB
JavaScript

let processingData = [];
let currentSessionId = null; // Store session ID for report download
let resultsMap = null;
let markers = [];
let eventListenersInitialized = false;
// Utility function to show status messages
function showDataConvertStatus(message, type = 'info') {
// Try to use the global showStatus from admin.js if available
if (typeof window.showStatus === 'function') {
return window.showStatus(message, type);
}
// Fallback to console
console.log(`[${type.toUpperCase()}] ${message}`);
// Try to display in status container if it exists
const statusContainer = document.getElementById('status-container');
if (statusContainer) {
const statusEl = document.createElement('div');
statusEl.className = `status-message status-${type}`;
statusEl.textContent = message;
statusContainer.appendChild(statusEl);
// Auto-remove after 5 seconds
setTimeout(() => {
if (statusEl.parentNode) {
statusEl.parentNode.removeChild(statusEl);
}
}, 5000);
}
}
// Initialize when DOM is loaded
document.addEventListener('DOMContentLoaded', () => {
console.log('Data convert JS loaded');
// Don't auto-initialize, wait for section to be activated
// Make setupDataConvertEventListeners available globally for admin.js
window.setupDataConvertEventListeners = setupDataConvertEventListeners;
});
function setupDataConvertEventListeners() {
console.log('Setting up data convert event listeners...');
// Prevent duplicate event listeners
if (eventListenersInitialized) {
console.log('Event listeners already initialized, skipping...');
return;
}
const fileInput = document.getElementById('csv-file-input');
const browseBtn = document.getElementById('browse-btn');
const uploadArea = document.getElementById('upload-area');
const uploadForm = document.getElementById('csv-upload-form');
const clearBtn = document.getElementById('clear-upload-btn');
const saveResultsBtn = document.getElementById('save-results-btn');
const newUploadBtn = document.getElementById('new-upload-btn');
console.log('Elements found:', {
fileInput: !!fileInput,
browseBtn: !!browseBtn,
uploadArea: !!uploadArea,
uploadForm: !!uploadForm,
clearBtn: !!clearBtn,
saveResultsBtn: !!saveResultsBtn,
newUploadBtn: !!newUploadBtn
});
// File input change
if (fileInput) {
fileInput.addEventListener('change', handleFileSelect);
}
// Browse button
if (browseBtn) {
browseBtn.addEventListener('click', () => {
console.log('Browse button clicked');
fileInput?.click();
});
}
// Drag and drop
if (uploadArea) {
uploadArea.addEventListener('dragover', handleDragOver);
uploadArea.addEventListener('dragleave', handleDragLeave);
uploadArea.addEventListener('drop', handleDrop);
uploadArea.addEventListener('click', () => fileInput?.click());
}
// Form submission
if (uploadForm) {
uploadForm.addEventListener('submit', handleCSVUpload);
}
// Clear button
if (clearBtn) {
clearBtn.addEventListener('click', clearUpload);
}
// Save results button - ADD DATA TO MAP
if (saveResultsBtn) {
saveResultsBtn.addEventListener('click', saveGeocodedResults);
console.log('Save results button event listener attached');
}
// New upload button
if (newUploadBtn) {
newUploadBtn.addEventListener('click', resetToUpload);
}
// Mark as initialized
eventListenersInitialized = true;
console.log('Data convert event listeners initialized successfully');
}
function handleFileSelect(e) {
const file = e.target.files[0];
if (file) {
displayFileInfo(file);
}
}
function handleDragOver(e) {
e.preventDefault();
e.currentTarget.classList.add('drag-over');
}
function handleDragLeave(e) {
e.preventDefault();
e.currentTarget.classList.remove('drag-over');
}
function handleDrop(e) {
e.preventDefault();
e.currentTarget.classList.remove('drag-over');
const files = e.dataTransfer.files;
if (files.length > 0) {
const file = files[0];
if (file.type === 'text/csv' || file.name.endsWith('.csv')) {
document.getElementById('csv-file-input').files = files;
displayFileInfo(file);
} else {
showDataConvertStatus('Please upload a CSV file', 'error');
}
}
}
function displayFileInfo(file) {
document.getElementById('file-info').style.display = 'block';
document.getElementById('file-name').textContent = file.name;
document.getElementById('file-size').textContent = formatFileSize(file.size);
document.getElementById('process-csv-btn').disabled = false;
}
function formatFileSize(bytes) {
if (bytes === 0) return '0 Bytes';
const k = 1024;
const sizes = ['Bytes', 'KB', 'MB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
}
async function handleCSVUpload(e) {
e.preventDefault();
const fileInput = document.getElementById('csv-file-input');
const file = fileInput?.files[0];
if (!file) {
showDataConvertStatus('Please select a CSV file', 'error');
return;
}
// Reset processing data
processingData = [];
// Show processing section
const uploadSection = document.getElementById('upload-section');
const processingSection = document.getElementById('processing-section');
if (!uploadSection || !processingSection) {
console.error('Required DOM elements not found');
showDataConvertStatus('Interface error: required elements not found', 'error');
return;
}
uploadSection.style.display = 'none';
processingSection.style.display = 'block';
// Initialize results map
initializeResultsMap();
// Create form data
const formData = new FormData();
formData.append('csvFile', file);
try {
// Send the file and handle SSE response
const response = await fetch('/api/admin/data-convert/process-csv', {
method: 'POST',
body: formData
});
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
// Read the response as a stream for SSE
const reader = response.body.getReader();
const decoder = new TextDecoder();
let buffer = ''; // Buffer to accumulate partial data
while (true) {
const { done, value } = await reader.read();
if (done) break;
// Decode the chunk and add to buffer
buffer += decoder.decode(value, { stream: true });
// Split buffer by lines and process complete lines
const lines = buffer.split('\n');
// Keep the last potentially incomplete line in the buffer
buffer = lines.pop() || '';
for (const line of lines) {
if (line.startsWith('data: ')) {
try {
const jsonData = line.substring(6).trim(); // Remove 'data: ' prefix and trim
if (jsonData && jsonData !== '') {
const data = JSON.parse(jsonData);
handleProcessingUpdate(data);
}
} catch (parseError) {
console.warn('Failed to parse SSE data:', parseError);
console.warn('Problematic line:', line);
console.warn('JSON data:', line.substring(6));
}
} else if (line.trim() === '') {
// Empty line, ignore
continue;
} else if (line.trim() !== '' && !line.startsWith('event:') && !line.startsWith('id:')) {
// Unexpected line format
console.warn('Unexpected SSE line format:', line);
}
}
}
// Process any remaining data in buffer
if (buffer.trim()) {
const line = buffer;
if (line.startsWith('data: ')) {
try {
const jsonData = line.substring(6).trim();
if (jsonData && jsonData !== '') {
const data = JSON.parse(jsonData);
handleProcessingUpdate(data);
}
} catch (parseError) {
console.warn('Failed to parse final SSE data:', parseError);
console.warn('Final buffer content:', buffer);
}
}
}
} catch (error) {
console.error('CSV processing error:', error);
showDataConvertStatus(error.message || 'Failed to process CSV', 'error');
resetToUpload();
}
}
// Enhanced processing update handler
function handleProcessingUpdate(data) {
console.log('Processing update:', data);
// Validate data structure
if (!data || typeof data !== 'object') {
console.warn('Invalid data received:', data);
return;
}
switch (data.type) {
case 'start':
updateProgress(0, data.total);
document.getElementById('current-address').textContent = 'Starting geocoding process...';
break;
case 'progress':
updateProgress(data.current, data.total);
document.getElementById('current-address').textContent = `Processing: ${data.address}`;
break;
case 'geocoded':
// Mark as successful and add to processing data
const successData = { ...data.data, geocode_success: true };
processingData.push(successData);
addResultToTable(successData, 'success');
addMarkerToMap(successData);
updateProgress(data.index + 1, data.total);
break;
case 'error':
// Mark as failed and add to processing data
const errorData = { ...data.data, geocode_success: false };
processingData.push(errorData);
addResultToTable(errorData, 'error');
break;
case 'complete':
console.log('Received complete event:', data);
currentSessionId = data.sessionId; // Store session ID for report download
onProcessingComplete(data);
break;
case 'fatal_error':
showDataConvertStatus(data.message, 'error');
resetToUpload();
break;
default:
console.warn('Unknown data type received:', data.type);
}
}
function updateProgress(current, total) {
const percentage = (current / total) * 100;
document.getElementById('progress-bar-fill').style.width = percentage + '%';
document.getElementById('progress-current').textContent = current;
document.getElementById('progress-total').textContent = total;
}
function initializeResultsMap() {
const mapContainer = document.getElementById('results-map');
if (!mapContainer) return;
// Initialize map
resultsMap = L.map('results-map').setView([53.5461, -113.4938], 11);
L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', {
attribution: '© OpenStreetMap contributors'
}).addTo(resultsMap);
document.getElementById('results-preview').style.display = 'block';
// Fix map sizing
setTimeout(() => {
resultsMap.invalidateSize();
}, 100);
}
function addResultToTable(data, status) {
const tbody = document.getElementById('results-tbody');
const row = document.createElement('tr');
row.className = status === 'success' ? 'result-success' : 'result-error';
if (status === 'success') {
row.innerHTML = `
<td><span class="status-icon success">✓</span></td>
<td>${escapeHtml(data.address || data.Address || '')}</td>
<td>${escapeHtml(data.geocoded_address || '')}</td>
<td>${data.latitude.toFixed(6)}, ${data.longitude.toFixed(6)}</td>
`;
} else {
row.innerHTML = `
<td><span class="status-icon error">✗</span></td>
<td>${escapeHtml(data.address || '')}</td>
<td colspan="2" class="error-message">${escapeHtml(data.error || 'Geocoding failed')}</td>
`;
}
tbody.appendChild(row);
}
function addMarkerToMap(data) {
if (!resultsMap || !data.latitude || !data.longitude) return;
const marker = L.marker([data.latitude, data.longitude])
.bindPopup(`
<strong>${escapeHtml(data.geocoded_address || data.address)}</strong><br>
${data.latitude.toFixed(6)}, ${data.longitude.toFixed(6)}
`);
marker.addTo(resultsMap);
markers.push(marker);
// Adjust map bounds to show all markers
if (markers.length > 0) {
const group = new L.featureGroup(markers);
resultsMap.fitBounds(group.getBounds().pad(0.1));
}
}
function onProcessingComplete(data) {
console.log('Processing complete called with data:', data);
document.getElementById('current-address').textContent =
`Complete! Processed ${data.processed || data.success || 0} addresses successfully, ${data.errors || data.failed || 0} errors.`;
const processedCount = data.processed || data.success || processingData.filter(item => item.geocode_success !== false).length;
if (processedCount > 0) {
console.log('Showing processing actions for', processedCount, 'successful items');
const actionsDiv = document.getElementById('processing-actions');
if (actionsDiv) {
actionsDiv.style.display = 'block';
console.log('Processing actions div is now visible');
} else {
console.error('processing-actions div not found!');
}
}
// Show download report button if we have a session ID
if (currentSessionId && data.total > 0) {
showDownloadReportButton(data.total, data.errors || data.failed || 0);
}
const errorCount = data.errors || data.failed || 0;
showDataConvertStatus(`Processing complete: ${processedCount} successful, ${errorCount} errors`,
errorCount > 0 ? 'warning' : 'success');
}
// Enhanced save function with better feedback
async function saveGeocodedResults() {
const successfulData = processingData.filter(item => item.geocode_success !== false && item.latitude && item.longitude);
if (successfulData.length === 0) {
showDataConvertStatus('No successfully geocoded data to save', 'error');
return;
}
// Disable save button and show progress
const saveBtn = document.getElementById('save-results-btn');
const originalText = saveBtn.textContent;
saveBtn.disabled = true;
saveBtn.textContent = 'Adding to map...';
// Show saving status
showDataConvertStatus(`Adding ${successfulData.length} locations to map...`, 'info');
let successCount = 0;
let failedCount = 0;
const errors = [];
try {
// Use the bulk save endpoint instead of individual location creation
// This is more efficient and avoids rate limiting issues
const response = await fetch('/api/admin/data-convert/save-geocoded', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({ data: successfulData })
});
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
const result = await response.json();
if (result.success) {
successCount = result.results.success;
failedCount = result.results.failed;
// Show detailed errors if any
if (result.results.errors && result.results.errors.length > 0) {
result.results.errors.forEach((error, index) => {
errors.push(`Location ${index + 1}: ${error.error}`);
});
}
console.log(`Bulk save completed: ${successCount} successful, ${failedCount} failed`);
} else {
throw new Error(result.error || 'Bulk save failed');
}
// Show final results
if (successCount > 0) {
const message = `Successfully added ${successCount} locations to map.` +
(failedCount > 0 ? ` ${failedCount} failed.` : '');
showDataConvertStatus(message, failedCount > 0 ? 'warning' : 'success');
// Update UI to show completion
saveBtn.textContent = `Added ${successCount} locations!`;
setTimeout(() => {
saveBtn.style.display = 'none';
document.getElementById('new-upload-btn').style.display = 'inline-block';
}, 3000);
} else {
throw new Error('Failed to add any locations to the map');
}
// Log errors if any
if (errors.length > 0) {
console.error('Import errors:', errors);
}
} catch (error) {
console.error('Save error:', error);
showDataConvertStatus('Failed to add data to map: ' + error.message, 'error');
saveBtn.disabled = false;
saveBtn.textContent = originalText;
}
}
// Utility function to escape HTML
function escapeHtml(text) {
const map = {
'&': '&amp;',
'<': '&lt;',
'>': '&gt;',
'"': '&quot;',
"'": '&#039;'
};
return text ? text.replace(/[&<>"']/g, function(m) { return map[m]; }) : '';
}
// Clear upload and reset form
function clearUpload() {
const fileInput = document.getElementById('csv-file-input');
const fileInfo = document.getElementById('file-info');
const processBtn = document.getElementById('process-csv-btn');
if (fileInput) fileInput.value = '';
if (fileInfo) fileInfo.style.display = 'none';
if (processBtn) processBtn.disabled = true;
}
// Reset to upload state
function resetToUpload() {
console.log('Resetting to upload state');
const uploadSection = document.getElementById('upload-section');
const processingSection = document.getElementById('processing-section');
const actionsDiv = document.getElementById('processing-actions');
const resultsPreview = document.getElementById('results-preview');
const saveBtn = document.getElementById('save-results-btn');
const newUploadBtn = document.getElementById('new-upload-btn');
if (uploadSection) uploadSection.style.display = 'block';
if (processingSection) processingSection.style.display = 'none';
if (actionsDiv) actionsDiv.style.display = 'none';
if (resultsPreview) resultsPreview.style.display = 'none';
// Reset buttons
if (saveBtn) {
saveBtn.style.display = 'inline-block';
saveBtn.disabled = false;
saveBtn.textContent = 'Add Data to Map';
}
if (newUploadBtn) {
newUploadBtn.style.display = 'none';
}
// Clear any existing data
processingData = [];
currentSessionId = null; // Reset session ID
// Remove download report button if it exists
const downloadBtn = document.getElementById('download-report-btn');
if (downloadBtn) {
downloadBtn.remove();
}
if (markers && markers.length > 0) {
markers.forEach(marker => {
if (resultsMap && marker) {
try {
resultsMap.removeLayer(marker);
} catch (e) {
console.warn('Error removing marker:', e);
}
}
});
}
markers = [];
// Reset results table
const tbody = document.getElementById('results-tbody');
if (tbody) {
tbody.innerHTML = '';
}
// Reset progress
const progressFill = document.getElementById('progress-bar-fill');
const progressCurrent = document.getElementById('progress-current');
const progressTotal = document.getElementById('progress-total');
const currentAddress = document.getElementById('current-address');
if (progressFill) progressFill.style.width = '0%';
if (progressCurrent) progressCurrent.textContent = '0';
if (progressTotal) progressTotal.textContent = '0';
if (currentAddress) currentAddress.textContent = '';
// Destroy existing map if it exists
if (resultsMap) {
try {
resultsMap.remove();
resultsMap = null;
} catch (e) {
console.warn('Error removing map:', e);
}
}
// Reset form
clearUpload();
}
// Show download report button
function showDownloadReportButton(totalRecords, errorCount) {
const actionsDiv = document.getElementById('processing-actions');
if (!actionsDiv) return;
// Check if button already exists
let downloadBtn = document.getElementById('download-report-btn');
if (!downloadBtn) {
downloadBtn = document.createElement('button');
downloadBtn.id = 'download-report-btn';
downloadBtn.type = 'button';
downloadBtn.className = 'btn btn-info';
downloadBtn.addEventListener('click', downloadProcessingReport);
// Insert before the save results button
const saveBtn = document.getElementById('save-results-btn');
if (saveBtn) {
actionsDiv.insertBefore(downloadBtn, saveBtn);
} else {
actionsDiv.appendChild(downloadBtn);
}
}
// Update button text with error info
if (errorCount > 0) {
downloadBtn.textContent = `📄 Download Full Report (${totalRecords} records, ${errorCount} errors)`;
downloadBtn.title = `Download complete processing report including ${errorCount} error records for review`;
} else {
downloadBtn.textContent = `📄 Download Processing Report (${totalRecords} records)`;
downloadBtn.title = 'Download complete processing report';
}
downloadBtn.style.marginRight = '10px'; // Add some spacing
}
// Download processing report
async function downloadProcessingReport() {
if (!currentSessionId) {
showDataConvertStatus('No processing session available for report generation', 'error');
return;
}
const downloadBtn = document.getElementById('download-report-btn');
const originalText = downloadBtn.textContent;
try {
downloadBtn.disabled = true;
downloadBtn.textContent = '📄 Generating Report...';
showDataConvertStatus('Generating processing report...', 'info');
// Create download link
const downloadUrl = `/api/admin/data-convert/download-report/${currentSessionId}`;
// Create a temporary anchor element and trigger download
const link = document.createElement('a');
link.href = downloadUrl;
link.download = `geocoding-report-${currentSessionId}.csv`;
link.style.display = 'none';
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
showDataConvertStatus('Processing report downloaded successfully', 'success');
// Update button text
downloadBtn.textContent = '📄 Report Downloaded';
setTimeout(() => {
downloadBtn.textContent = originalText;
downloadBtn.disabled = false;
}, 3000);
} catch (error) {
console.error('Download report error:', error);
showDataConvertStatus('Failed to download processing report: ' + error.message, 'error');
downloadBtn.textContent = originalText;
downloadBtn.disabled = false;
}
}