242 lines
9.4 KiB
JavaScript
Executable File
242 lines
9.4 KiB
JavaScript
Executable File
#!/usr/bin/env node
|
|
|
|
const fs = require('fs');
|
|
const Papa = require('papaparse');
|
|
const path = require('path');
|
|
|
|
// Input and output file paths
|
|
const INPUT_FILE = '/mnt/storagessd1tb/changemaker.lite.dev/changemaker.lite/map/data/City of Edmonton Ward Boundary and Council Composition_Current_20250807.geojson';
|
|
const OUTPUT_FILE = path.join(path.dirname(INPUT_FILE), 'edmonton_ward_boundaries_optimized.csv');
|
|
|
|
// Function to optimize GeoJSON precision
|
|
function optimizeGeoJSONPrecision(geoJSON, precision = 7) {
|
|
if (!geoJSON) return null;
|
|
|
|
function roundCoords(coords, precision) {
|
|
const multiplier = Math.pow(10, precision);
|
|
|
|
if (Array.isArray(coords[0]) && Array.isArray(coords[0][0])) {
|
|
// Multi-dimensional array (Polygon/MultiPolygon)
|
|
return coords.map(coord => roundCoords(coord, precision));
|
|
} else if (Array.isArray(coords[0]) && typeof coords[0][0] === 'number') {
|
|
// Array of coordinate pairs
|
|
return coords.map(([lon, lat]) => [
|
|
Math.round(lon * multiplier) / multiplier,
|
|
Math.round(lat * multiplier) / multiplier
|
|
]);
|
|
} else if (typeof coords[0] === 'number' && typeof coords[1] === 'number') {
|
|
// Single coordinate pair
|
|
return [
|
|
Math.round(coords[0] * multiplier) / multiplier,
|
|
Math.round(coords[1] * multiplier) / multiplier
|
|
];
|
|
}
|
|
return coords;
|
|
}
|
|
|
|
const optimized = JSON.parse(JSON.stringify(geoJSON));
|
|
optimized.coordinates = roundCoords(optimized.coordinates, precision);
|
|
return optimized;
|
|
}
|
|
|
|
// Smart polygon simplification - only remove redundant points
|
|
function smartSimplify(coordinates, maxPoints = 500) {
|
|
return coordinates.map(ring => {
|
|
if (ring.length <= maxPoints) return ring;
|
|
|
|
// Calculate step to evenly sample points
|
|
const step = ring.length / maxPoints;
|
|
const simplified = [];
|
|
|
|
for (let i = 0; i < maxPoints - 1; i++) {
|
|
const index = Math.floor(i * step);
|
|
simplified.push(ring[index]);
|
|
}
|
|
|
|
// Always include the last point to close the polygon
|
|
simplified.push(ring[ring.length - 1]);
|
|
|
|
return simplified;
|
|
});
|
|
}
|
|
|
|
// Function to calculate bounds from GeoJSON
|
|
function calculateBounds(geoJSON) {
|
|
if (!geoJSON) return null;
|
|
|
|
let minLat = Infinity, maxLat = -Infinity;
|
|
let minLon = Infinity, maxLon = -Infinity;
|
|
|
|
function processCoords(coords) {
|
|
if (Array.isArray(coords[0]) && Array.isArray(coords[0][0])) {
|
|
coords.forEach(processCoords);
|
|
} else if (Array.isArray(coords[0]) && typeof coords[0][0] === 'number') {
|
|
coords.forEach(([lon, lat]) => {
|
|
minLat = Math.min(minLat, lat);
|
|
maxLat = Math.max(maxLat, lat);
|
|
minLon = Math.min(minLon, lon);
|
|
maxLon = Math.max(maxLon, lon);
|
|
});
|
|
}
|
|
}
|
|
|
|
processCoords(geoJSON.coordinates);
|
|
return `[[${minLat.toFixed(7)},${minLon.toFixed(7)}],[${maxLat.toFixed(7)},${maxLon.toFixed(7)}]]`;
|
|
}
|
|
|
|
// Compact JSON stringify (no extra whitespace)
|
|
function compactStringify(obj) {
|
|
return JSON.stringify(obj, null, 0);
|
|
}
|
|
|
|
// Main conversion function
|
|
function convertGeoJSON() {
|
|
console.log('📁 Reading GeoJSON file:', INPUT_FILE);
|
|
|
|
// Read the GeoJSON file
|
|
const geoJSONData = fs.readFileSync(INPUT_FILE, 'utf8');
|
|
|
|
console.log('📊 Parsing GeoJSON...');
|
|
|
|
// Parse the GeoJSON
|
|
const parsed = JSON.parse(geoJSONData);
|
|
|
|
if (!parsed.features || !Array.isArray(parsed.features)) {
|
|
console.error('❌ Invalid GeoJSON format - no features array found');
|
|
return;
|
|
}
|
|
|
|
console.log(`✅ Found ${parsed.features.length} ward boundaries to convert`);
|
|
console.log('🎯 Optimizing precision for NocoDB (100k char limit)...\n');
|
|
|
|
// Get current timestamp
|
|
const now = new Date();
|
|
const timestamp = now.toISOString().replace('T', ' ').replace('Z', '+00:00').slice(0, -7) + '+00:00';
|
|
|
|
const stats = {
|
|
totalRows: 0,
|
|
optimizedRows: 0,
|
|
simplifiedRows: 0,
|
|
precision7: 0,
|
|
precision6: 0,
|
|
precision5: 0
|
|
};
|
|
|
|
// Convert each feature
|
|
const convertedData = parsed.features.map((feature, index) => {
|
|
if ((index + 1) % 5 === 0) {
|
|
console.log(` Converting... ${index + 1}/${parsed.features.length}`);
|
|
}
|
|
|
|
stats.totalRows++;
|
|
|
|
const properties = feature.properties || {};
|
|
const geometry = feature.geometry;
|
|
|
|
if (!geometry) {
|
|
console.warn(`⚠️ Feature ${index + 1} has no geometry, skipping`);
|
|
return null;
|
|
}
|
|
|
|
// Start with 7 decimal precision (~1cm accuracy)
|
|
let geoJSON = optimizeGeoJSONPrecision(geometry, 7);
|
|
let geoJSONString = geoJSON ? compactStringify(geoJSON) : '';
|
|
|
|
// If still too large, try 6 decimals (~10cm accuracy)
|
|
if (geoJSONString.length > 99000) {
|
|
stats.precision7++;
|
|
geoJSON = optimizeGeoJSONPrecision(geometry, 6);
|
|
geoJSONString = compactStringify(geoJSON);
|
|
|
|
// If still too large, try 5 decimals (~1m accuracy)
|
|
if (geoJSONString.length > 99000) {
|
|
stats.precision6++;
|
|
geoJSON = optimizeGeoJSONPrecision(geometry, 5);
|
|
geoJSONString = compactStringify(geoJSON);
|
|
|
|
// If STILL too large, simplify points
|
|
if (geoJSONString.length > 99000) {
|
|
stats.precision5++;
|
|
const originalPoints = JSON.stringify(geoJSON).length;
|
|
|
|
if (geoJSON.type === 'Polygon') {
|
|
geoJSON.coordinates = smartSimplify(geoJSON.coordinates);
|
|
} else if (geoJSON.type === 'MultiPolygon') {
|
|
geoJSON.coordinates = geoJSON.coordinates.map(polygon =>
|
|
smartSimplify(polygon)
|
|
);
|
|
}
|
|
|
|
geoJSONString = compactStringify(geoJSON);
|
|
stats.simplifiedRows++;
|
|
|
|
console.log(` ⚠️ ${properties.name_1 || properties.name_2 || `Ward ${index + 1}`}: Simplified from ${originalPoints} to ${geoJSONString.length} chars`);
|
|
}
|
|
}
|
|
} else {
|
|
stats.optimizedRows++;
|
|
}
|
|
|
|
// Create ward name from available properties
|
|
const wardName = properties.name_1 || properties.name_2 || `Ward ${index + 1}`;
|
|
const fullWardName = properties.name_2 || (properties.name_1 ? `${properties.name_1} Ward` : `Ward ${index + 1}`);
|
|
|
|
// Create description with available information
|
|
let description = fullWardName;
|
|
if (properties.councillor) {
|
|
description += ` - Councillor: ${properties.councillor}`;
|
|
}
|
|
if (properties.councillor2) {
|
|
description += ` & ${properties.councillor2}`;
|
|
}
|
|
if (properties.effective_start_date) {
|
|
const startDate = new Date(properties.effective_start_date).toLocaleDateString();
|
|
description += ` (Effective: ${startDate})`;
|
|
}
|
|
|
|
return {
|
|
'ID': index + 1,
|
|
'CreatedAt': timestamp,
|
|
'UpdatedAt': timestamp,
|
|
'Name': wardName,
|
|
'Description': description.substring(0, 5000),
|
|
'Color': '#ff6b35', // Orange color for ward boundaries
|
|
'Opacity': 0.3,
|
|
'Category': 'Ward',
|
|
'Public Visibility': true,
|
|
'Official Cut': true,
|
|
'GeoJSON Data': geoJSONString,
|
|
'Bounds': calculateBounds(geoJSON),
|
|
'Created By': 'City of Edmonton',
|
|
'Created At': properties.effective_start_date || null,
|
|
'Updated At': timestamp
|
|
};
|
|
}).filter(row => row !== null); // Remove any null entries
|
|
|
|
console.log('\n📝 Converting to CSV format...');
|
|
|
|
// Convert back to CSV
|
|
const outputCSV = Papa.unparse(convertedData);
|
|
|
|
// Write the file
|
|
fs.writeFileSync(OUTPUT_FILE, outputCSV);
|
|
|
|
const fileSizeMB = (Buffer.byteLength(outputCSV) / 1024 / 1024).toFixed(2);
|
|
|
|
console.log('\n✨ Conversion complete!');
|
|
console.log('══════════════════════════════════════════════════════════');
|
|
console.log(`📊 Total ward boundaries: ${stats.totalRows}`);
|
|
console.log(`✅ Optimized with 7 decimals (1cm): ${stats.optimizedRows}`);
|
|
console.log(`📉 Reduced to 6 decimals (10cm): ${stats.precision7}`);
|
|
console.log(`📉 Reduced to 5 decimals (1m): ${stats.precision6}`);
|
|
console.log(`⚠️ Point reduction needed: ${stats.simplifiedRows}`);
|
|
console.log(`💾 File size: ${fileSizeMB} MB`);
|
|
console.log(`📁 Output saved to: ${OUTPUT_FILE}`);
|
|
console.log('══════════════════════════════════════════════════════════');
|
|
console.log('\n✅ All geometries are under 100,000 characters!');
|
|
console.log(' You can import this directly to NocoDB without changing column types.');
|
|
}
|
|
|
|
// Run the conversion
|
|
convertGeoJSON();
|