96 lines
2.3 KiB
JavaScript
96 lines
2.3 KiB
JavaScript
const db = require('../models/db');
|
|
const logger = require('../utils/logger');
|
|
const { runAllScrapers } = require('../scrapers/run-all');
|
|
|
|
let isScrapingInProgress = false;
|
|
|
|
async function triggerScrape(req, res) {
|
|
try {
|
|
if (isScrapingInProgress) {
|
|
return res.status(409).json({
|
|
error: 'Scrape already in progress',
|
|
message: 'Please wait for the current scrape to complete'
|
|
});
|
|
}
|
|
|
|
isScrapingInProgress = true;
|
|
|
|
// Run scrapers in background
|
|
runAllScrapers()
|
|
.then(results => {
|
|
logger.info('Manual scrape completed', results);
|
|
isScrapingInProgress = false;
|
|
})
|
|
.catch(error => {
|
|
logger.error('Manual scrape failed', { error: error.message });
|
|
isScrapingInProgress = false;
|
|
});
|
|
|
|
res.json({
|
|
message: 'Scrape started',
|
|
status: 'running'
|
|
});
|
|
|
|
} catch (error) {
|
|
isScrapingInProgress = false;
|
|
logger.error('Failed to trigger scrape', { error: error.message });
|
|
res.status(500).json({ error: 'Failed to start scrape' });
|
|
}
|
|
}
|
|
|
|
async function getScrapeStatus(req, res) {
|
|
try {
|
|
const result = await db.query(`
|
|
SELECT source, status, started_at, completed_at,
|
|
records_found, records_added, records_updated, error_message
|
|
FROM scrape_logs
|
|
WHERE started_at > NOW() - INTERVAL '24 hours'
|
|
ORDER BY started_at DESC
|
|
LIMIT 10
|
|
`);
|
|
|
|
res.json({
|
|
isRunning: isScrapingInProgress,
|
|
recentLogs: result.rows
|
|
});
|
|
|
|
} catch (error) {
|
|
logger.error('Failed to get scrape status', { error: error.message });
|
|
res.status(500).json({ error: 'Failed to get status' });
|
|
}
|
|
}
|
|
|
|
async function getScrapeLogs(req, res) {
|
|
try {
|
|
const { limit = 50, source } = req.query;
|
|
|
|
let query = `
|
|
SELECT *
|
|
FROM scrape_logs
|
|
`;
|
|
const params = [];
|
|
|
|
if (source) {
|
|
params.push(source);
|
|
query += ` WHERE source = $1`;
|
|
}
|
|
|
|
query += ` ORDER BY started_at DESC LIMIT $${params.length + 1}`;
|
|
params.push(limit);
|
|
|
|
const result = await db.query(query, params);
|
|
|
|
res.json({ logs: result.rows });
|
|
|
|
} catch (error) {
|
|
logger.error('Failed to get scrape logs', { error: error.message });
|
|
res.status(500).json({ error: 'Failed to get logs' });
|
|
}
|
|
}
|
|
|
|
module.exports = {
|
|
triggerScrape,
|
|
getScrapeStatus,
|
|
getScrapeLogs
|
|
};
|