const db = require('../models/db'); const logger = require('../utils/logger'); const { runAllScrapers } = require('../scrapers/run-all'); let isScrapingInProgress = false; async function triggerScrape(req, res) { try { if (isScrapingInProgress) { return res.status(409).json({ error: 'Scrape already in progress', message: 'Please wait for the current scrape to complete' }); } isScrapingInProgress = true; // Run scrapers in background runAllScrapers() .then(results => { logger.info('Manual scrape completed', results); isScrapingInProgress = false; }) .catch(error => { logger.error('Manual scrape failed', { error: error.message }); isScrapingInProgress = false; }); res.json({ message: 'Scrape started', status: 'running' }); } catch (error) { isScrapingInProgress = false; logger.error('Failed to trigger scrape', { error: error.message }); res.status(500).json({ error: 'Failed to start scrape' }); } } async function getScrapeStatus(req, res) { try { const result = await db.query(` SELECT source, status, started_at, completed_at, records_found, records_added, records_updated, error_message FROM scrape_logs WHERE started_at > NOW() - INTERVAL '24 hours' ORDER BY started_at DESC LIMIT 10 `); res.json({ isRunning: isScrapingInProgress, recentLogs: result.rows }); } catch (error) { logger.error('Failed to get scrape status', { error: error.message }); res.status(500).json({ error: 'Failed to get status' }); } } async function getScrapeLogs(req, res) { try { const { limit = 50, source } = req.query; let query = ` SELECT * FROM scrape_logs `; const params = []; if (source) { params.push(source); query += ` WHERE source = $1`; } query += ` ORDER BY started_at DESC LIMIT $${params.length + 1}`; params.push(limit); const result = await db.query(query, params); res.json({ logs: result.rows }); } catch (error) { logger.error('Failed to get scrape logs', { error: error.message }); res.status(500).json({ error: 'Failed to get logs' }); } } module.exports = { triggerScrape, getScrapeStatus, getScrapeLogs };