60 lines
1.6 KiB
JavaScript
60 lines
1.6 KiB
JavaScript
const logger = require('../utils/logger');
|
|
const { scrapeInformAlberta } = require('./informalberta');
|
|
const { scrape211Alberta } = require('./ab211');
|
|
const { parsePDFs } = require('./pdf-parser');
|
|
|
|
async function runAllScrapers() {
|
|
logger.info('Starting all scrapers');
|
|
|
|
const results = {
|
|
informalberta: null,
|
|
ab211: null,
|
|
pdf: null
|
|
};
|
|
|
|
// Run InformAlberta scraper
|
|
try {
|
|
results.informalberta = await scrapeInformAlberta();
|
|
logger.info('InformAlberta scrape completed', results.informalberta);
|
|
} catch (error) {
|
|
logger.error('InformAlberta scrape failed', { error: error.message });
|
|
results.informalberta = { error: error.message };
|
|
}
|
|
|
|
// Run 211 Alberta scraper (may be blocked by Cloudflare)
|
|
try {
|
|
results.ab211 = await scrape211Alberta();
|
|
logger.info('211 Alberta scrape completed', results.ab211);
|
|
} catch (error) {
|
|
logger.error('211 Alberta scrape failed', { error: error.message });
|
|
results.ab211 = { error: error.message };
|
|
}
|
|
|
|
// Run PDF parser
|
|
try {
|
|
results.pdf = await parsePDFs();
|
|
logger.info('PDF parsing completed', results.pdf);
|
|
} catch (error) {
|
|
logger.error('PDF parsing failed', { error: error.message });
|
|
results.pdf = { error: error.message };
|
|
}
|
|
|
|
logger.info('All scrapers finished', results);
|
|
return results;
|
|
}
|
|
|
|
// Run if called directly
|
|
if (require.main === module) {
|
|
runAllScrapers()
|
|
.then(results => {
|
|
console.log('All scrapers completed:', JSON.stringify(results, null, 2));
|
|
process.exit(0);
|
|
})
|
|
.catch(err => {
|
|
console.error('Scraper run failed:', err);
|
|
process.exit(1);
|
|
});
|
|
}
|
|
|
|
module.exports = { runAllScrapers };
|