freealberta/influence/app/utils/health-check.js
admin e5c32ad25a Add health check utility, logger, metrics, backup, and SMTP toggle scripts
- Implemented a comprehensive health check utility to monitor system dependencies including NocoDB, SMTP, Represent API, disk space, and memory usage.
- Created a logger utility using Winston for structured logging with daily rotation and various log levels.
- Developed a metrics utility using Prometheus client to track application performance metrics such as email sends, HTTP requests, and user activity.
- Added a backup script for automated backups of NocoDB data, uploaded files, and environment configurations with optional S3 support.
- Introduced a toggle script to switch between development (MailHog) and production (ProtonMail) SMTP configurations.
2025-10-23 11:33:00 -06:00

381 lines
11 KiB
JavaScript

const axios = require('axios');
const os = require('os');
const fs = require('fs').promises;
const path = require('path');
const nodemailer = require('nodemailer');
const logger = require('./logger');
/**
* Health check utility for monitoring all system dependencies
*/
class HealthCheck {
constructor() {
this.services = {
nocodb: { name: 'NocoDB', healthy: false, lastCheck: null, details: {} },
smtp: { name: 'SMTP Server', healthy: false, lastCheck: null, details: {} },
representAPI: { name: 'Represent API', healthy: false, lastCheck: null, details: {} },
disk: { name: 'Disk Space', healthy: false, lastCheck: null, details: {} },
memory: { name: 'Memory', healthy: false, lastCheck: null, details: {} },
};
}
/**
* Check NocoDB connectivity and API health
*/
async checkNocoDB() {
const start = Date.now();
try {
const response = await axios.get(`${process.env.NOCODB_API_URL}/db/meta/projects`, {
headers: {
'xc-token': process.env.NOCODB_API_TOKEN
},
timeout: 5000
});
const duration = Date.now() - start;
this.services.nocodb = {
name: 'NocoDB',
healthy: response.status === 200,
lastCheck: new Date().toISOString(),
details: {
status: response.status,
responseTime: `${duration}ms`,
url: process.env.NOCODB_API_URL
}
};
logger.logHealthCheck('nocodb', 'healthy', { responseTime: duration });
return this.services.nocodb;
} catch (error) {
this.services.nocodb = {
name: 'NocoDB',
healthy: false,
lastCheck: new Date().toISOString(),
details: {
error: error.message,
url: process.env.NOCODB_API_URL
}
};
logger.logHealthCheck('nocodb', 'unhealthy', { error: error.message });
return this.services.nocodb;
}
}
/**
* Check SMTP server connectivity
*/
async checkSMTP() {
const start = Date.now();
// Skip SMTP check if in test mode
if (process.env.EMAIL_TEST_MODE === 'true') {
this.services.smtp = {
name: 'SMTP Server',
healthy: true,
lastCheck: new Date().toISOString(),
details: {
mode: 'test',
message: 'SMTP test mode enabled - emails logged only'
}
};
return this.services.smtp;
}
try {
const transporter = nodemailer.createTransport({
host: process.env.SMTP_HOST,
port: parseInt(process.env.SMTP_PORT || '587'),
secure: process.env.SMTP_SECURE === 'true',
auth: {
user: process.env.SMTP_USER,
pass: process.env.SMTP_PASS
},
connectionTimeout: 5000
});
await transporter.verify();
const duration = Date.now() - start;
this.services.smtp = {
name: 'SMTP Server',
healthy: true,
lastCheck: new Date().toISOString(),
details: {
host: process.env.SMTP_HOST,
port: process.env.SMTP_PORT,
responseTime: `${duration}ms`
}
};
logger.logHealthCheck('smtp', 'healthy', { responseTime: duration });
return this.services.smtp;
} catch (error) {
this.services.smtp = {
name: 'SMTP Server',
healthy: false,
lastCheck: new Date().toISOString(),
details: {
error: error.message,
host: process.env.SMTP_HOST
}
};
logger.logHealthCheck('smtp', 'unhealthy', { error: error.message });
return this.services.smtp;
}
}
/**
* Check Represent API availability
*/
async checkRepresentAPI() {
const start = Date.now();
const testUrl = `${process.env.REPRESENT_API_BASE || 'https://represent.opennorth.ca'}/representatives/house-of-commons/`;
try {
const response = await axios.get(testUrl, {
timeout: 5000
});
const duration = Date.now() - start;
this.services.representAPI = {
name: 'Represent API',
healthy: response.status === 200,
lastCheck: new Date().toISOString(),
details: {
status: response.status,
responseTime: `${duration}ms`,
url: testUrl
}
};
logger.logHealthCheck('represent-api', 'healthy', { responseTime: duration });
return this.services.representAPI;
} catch (error) {
this.services.representAPI = {
name: 'Represent API',
healthy: false,
lastCheck: new Date().toISOString(),
details: {
error: error.message,
url: testUrl
}
};
logger.logHealthCheck('represent-api', 'unhealthy', { error: error.message });
return this.services.representAPI;
}
}
/**
* Check disk space for uploads directory
*/
async checkDiskSpace() {
try {
const uploadsDir = path.join(__dirname, '../public/uploads');
// Ensure directory exists
try {
await fs.access(uploadsDir);
} catch {
await fs.mkdir(uploadsDir, { recursive: true });
}
// Get directory size
const directorySize = await this.getDirectorySize(uploadsDir);
// Get system disk usage (platform-specific approximation)
const freeSpace = os.freemem();
const totalSpace = os.totalmem();
const usedPercentage = ((totalSpace - freeSpace) / totalSpace) * 100;
// Consider healthy if less than 90% full
const healthy = usedPercentage < 90;
this.services.disk = {
name: 'Disk Space',
healthy,
lastCheck: new Date().toISOString(),
details: {
uploadsSize: this.formatBytes(directorySize),
systemUsedPercentage: `${usedPercentage.toFixed(2)}%`,
freeSpace: this.formatBytes(freeSpace),
totalSpace: this.formatBytes(totalSpace),
path: uploadsDir
}
};
logger.logHealthCheck('disk', healthy ? 'healthy' : 'unhealthy', this.services.disk.details);
return this.services.disk;
} catch (error) {
this.services.disk = {
name: 'Disk Space',
healthy: false,
lastCheck: new Date().toISOString(),
details: {
error: error.message
}
};
logger.logHealthCheck('disk', 'unhealthy', { error: error.message });
return this.services.disk;
}
}
/**
* Check memory usage
*/
async checkMemory() {
try {
const totalMemory = os.totalmem();
const freeMemory = os.freemem();
const usedMemory = totalMemory - freeMemory;
const usedPercentage = (usedMemory / totalMemory) * 100;
// Get Node.js process memory
const processMemory = process.memoryUsage();
// Consider healthy if less than 85% used
const healthy = usedPercentage < 85;
this.services.memory = {
name: 'Memory',
healthy,
lastCheck: new Date().toISOString(),
details: {
system: {
total: this.formatBytes(totalMemory),
used: this.formatBytes(usedMemory),
free: this.formatBytes(freeMemory),
usedPercentage: `${usedPercentage.toFixed(2)}%`
},
process: {
rss: this.formatBytes(processMemory.rss),
heapTotal: this.formatBytes(processMemory.heapTotal),
heapUsed: this.formatBytes(processMemory.heapUsed),
external: this.formatBytes(processMemory.external)
}
}
};
logger.logHealthCheck('memory', healthy ? 'healthy' : 'unhealthy', this.services.memory.details);
return this.services.memory;
} catch (error) {
this.services.memory = {
name: 'Memory',
healthy: false,
lastCheck: new Date().toISOString(),
details: {
error: error.message
}
};
logger.logHealthCheck('memory', 'unhealthy', { error: error.message });
return this.services.memory;
}
}
/**
* Run all health checks
*/
async checkAll() {
const results = await Promise.allSettled([
this.checkNocoDB(),
this.checkSMTP(),
this.checkRepresentAPI(),
this.checkDiskSpace(),
this.checkMemory()
]);
const overall = {
status: Object.values(this.services).every(s => s.healthy) ? 'healthy' : 'degraded',
timestamp: new Date().toISOString(),
services: this.services,
uptime: process.uptime(),
version: process.env.npm_package_version || '1.0.0',
environment: process.env.NODE_ENV || 'development'
};
// Check if any critical services are down
// Note: In development, NocoDB might not be available yet
// Only mark as unhealthy if multiple critical services fail
const criticalServices = ['nocodb'];
const criticalDown = criticalServices.filter(name => !this.services[name].healthy);
// Allow degraded state if only external services (NocoDB, Represent API) are down
// Only fail health check if core services (disk, memory) are unhealthy
const coreServicesHealthy = this.services.disk.healthy && this.services.memory.healthy;
if (!coreServicesHealthy) {
overall.status = 'unhealthy';
} else if (criticalDown.length > 0) {
// Mark as degraded (not unhealthy) if external services are down
// This allows the container to start even if NocoDB isn't ready
overall.status = 'degraded';
}
return overall;
}
/**
* Get health check for a specific service
*/
async checkService(serviceName) {
const checkMethods = {
nocodb: () => this.checkNocoDB(),
smtp: () => this.checkSMTP(),
representAPI: () => this.checkRepresentAPI(),
disk: () => this.checkDiskSpace(),
memory: () => this.checkMemory()
};
if (checkMethods[serviceName]) {
return await checkMethods[serviceName]();
}
throw new Error(`Unknown service: ${serviceName}`);
}
/**
* Helper: Get directory size recursively
*/
async getDirectorySize(directory) {
let size = 0;
try {
const files = await fs.readdir(directory);
for (const file of files) {
const filePath = path.join(directory, file);
const stats = await fs.stat(filePath);
if (stats.isDirectory()) {
size += await this.getDirectorySize(filePath);
} else {
size += stats.size;
}
}
} catch (error) {
logger.warn(`Error calculating directory size: ${error.message}`);
}
return size;
}
/**
* Helper: Format bytes to human readable
*/
formatBytes(bytes) {
if (bytes === 0) return '0 Bytes';
const k = 1024;
const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
}
}
module.exports = new HealthCheck();