- Implemented a comprehensive health check utility to monitor system dependencies including NocoDB, SMTP, Represent API, disk space, and memory usage. - Created a logger utility using Winston for structured logging with daily rotation and various log levels. - Developed a metrics utility using Prometheus client to track application performance metrics such as email sends, HTTP requests, and user activity. - Added a backup script for automated backups of NocoDB data, uploaded files, and environment configurations with optional S3 support. - Introduced a toggle script to switch between development (MailHog) and production (ProtonMail) SMTP configurations.
381 lines
11 KiB
JavaScript
381 lines
11 KiB
JavaScript
const axios = require('axios');
|
|
const os = require('os');
|
|
const fs = require('fs').promises;
|
|
const path = require('path');
|
|
const nodemailer = require('nodemailer');
|
|
const logger = require('./logger');
|
|
|
|
/**
|
|
* Health check utility for monitoring all system dependencies
|
|
*/
|
|
class HealthCheck {
|
|
constructor() {
|
|
this.services = {
|
|
nocodb: { name: 'NocoDB', healthy: false, lastCheck: null, details: {} },
|
|
smtp: { name: 'SMTP Server', healthy: false, lastCheck: null, details: {} },
|
|
representAPI: { name: 'Represent API', healthy: false, lastCheck: null, details: {} },
|
|
disk: { name: 'Disk Space', healthy: false, lastCheck: null, details: {} },
|
|
memory: { name: 'Memory', healthy: false, lastCheck: null, details: {} },
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Check NocoDB connectivity and API health
|
|
*/
|
|
async checkNocoDB() {
|
|
const start = Date.now();
|
|
try {
|
|
const response = await axios.get(`${process.env.NOCODB_API_URL}/db/meta/projects`, {
|
|
headers: {
|
|
'xc-token': process.env.NOCODB_API_TOKEN
|
|
},
|
|
timeout: 5000
|
|
});
|
|
|
|
const duration = Date.now() - start;
|
|
this.services.nocodb = {
|
|
name: 'NocoDB',
|
|
healthy: response.status === 200,
|
|
lastCheck: new Date().toISOString(),
|
|
details: {
|
|
status: response.status,
|
|
responseTime: `${duration}ms`,
|
|
url: process.env.NOCODB_API_URL
|
|
}
|
|
};
|
|
|
|
logger.logHealthCheck('nocodb', 'healthy', { responseTime: duration });
|
|
return this.services.nocodb;
|
|
} catch (error) {
|
|
this.services.nocodb = {
|
|
name: 'NocoDB',
|
|
healthy: false,
|
|
lastCheck: new Date().toISOString(),
|
|
details: {
|
|
error: error.message,
|
|
url: process.env.NOCODB_API_URL
|
|
}
|
|
};
|
|
|
|
logger.logHealthCheck('nocodb', 'unhealthy', { error: error.message });
|
|
return this.services.nocodb;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check SMTP server connectivity
|
|
*/
|
|
async checkSMTP() {
|
|
const start = Date.now();
|
|
|
|
// Skip SMTP check if in test mode
|
|
if (process.env.EMAIL_TEST_MODE === 'true') {
|
|
this.services.smtp = {
|
|
name: 'SMTP Server',
|
|
healthy: true,
|
|
lastCheck: new Date().toISOString(),
|
|
details: {
|
|
mode: 'test',
|
|
message: 'SMTP test mode enabled - emails logged only'
|
|
}
|
|
};
|
|
return this.services.smtp;
|
|
}
|
|
|
|
try {
|
|
const transporter = nodemailer.createTransport({
|
|
host: process.env.SMTP_HOST,
|
|
port: parseInt(process.env.SMTP_PORT || '587'),
|
|
secure: process.env.SMTP_SECURE === 'true',
|
|
auth: {
|
|
user: process.env.SMTP_USER,
|
|
pass: process.env.SMTP_PASS
|
|
},
|
|
connectionTimeout: 5000
|
|
});
|
|
|
|
await transporter.verify();
|
|
const duration = Date.now() - start;
|
|
|
|
this.services.smtp = {
|
|
name: 'SMTP Server',
|
|
healthy: true,
|
|
lastCheck: new Date().toISOString(),
|
|
details: {
|
|
host: process.env.SMTP_HOST,
|
|
port: process.env.SMTP_PORT,
|
|
responseTime: `${duration}ms`
|
|
}
|
|
};
|
|
|
|
logger.logHealthCheck('smtp', 'healthy', { responseTime: duration });
|
|
return this.services.smtp;
|
|
} catch (error) {
|
|
this.services.smtp = {
|
|
name: 'SMTP Server',
|
|
healthy: false,
|
|
lastCheck: new Date().toISOString(),
|
|
details: {
|
|
error: error.message,
|
|
host: process.env.SMTP_HOST
|
|
}
|
|
};
|
|
|
|
logger.logHealthCheck('smtp', 'unhealthy', { error: error.message });
|
|
return this.services.smtp;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check Represent API availability
|
|
*/
|
|
async checkRepresentAPI() {
|
|
const start = Date.now();
|
|
const testUrl = `${process.env.REPRESENT_API_BASE || 'https://represent.opennorth.ca'}/representatives/house-of-commons/`;
|
|
|
|
try {
|
|
const response = await axios.get(testUrl, {
|
|
timeout: 5000
|
|
});
|
|
|
|
const duration = Date.now() - start;
|
|
this.services.representAPI = {
|
|
name: 'Represent API',
|
|
healthy: response.status === 200,
|
|
lastCheck: new Date().toISOString(),
|
|
details: {
|
|
status: response.status,
|
|
responseTime: `${duration}ms`,
|
|
url: testUrl
|
|
}
|
|
};
|
|
|
|
logger.logHealthCheck('represent-api', 'healthy', { responseTime: duration });
|
|
return this.services.representAPI;
|
|
} catch (error) {
|
|
this.services.representAPI = {
|
|
name: 'Represent API',
|
|
healthy: false,
|
|
lastCheck: new Date().toISOString(),
|
|
details: {
|
|
error: error.message,
|
|
url: testUrl
|
|
}
|
|
};
|
|
|
|
logger.logHealthCheck('represent-api', 'unhealthy', { error: error.message });
|
|
return this.services.representAPI;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check disk space for uploads directory
|
|
*/
|
|
async checkDiskSpace() {
|
|
try {
|
|
const uploadsDir = path.join(__dirname, '../public/uploads');
|
|
|
|
// Ensure directory exists
|
|
try {
|
|
await fs.access(uploadsDir);
|
|
} catch {
|
|
await fs.mkdir(uploadsDir, { recursive: true });
|
|
}
|
|
|
|
// Get directory size
|
|
const directorySize = await this.getDirectorySize(uploadsDir);
|
|
|
|
// Get system disk usage (platform-specific approximation)
|
|
const freeSpace = os.freemem();
|
|
const totalSpace = os.totalmem();
|
|
const usedPercentage = ((totalSpace - freeSpace) / totalSpace) * 100;
|
|
|
|
// Consider healthy if less than 90% full
|
|
const healthy = usedPercentage < 90;
|
|
|
|
this.services.disk = {
|
|
name: 'Disk Space',
|
|
healthy,
|
|
lastCheck: new Date().toISOString(),
|
|
details: {
|
|
uploadsSize: this.formatBytes(directorySize),
|
|
systemUsedPercentage: `${usedPercentage.toFixed(2)}%`,
|
|
freeSpace: this.formatBytes(freeSpace),
|
|
totalSpace: this.formatBytes(totalSpace),
|
|
path: uploadsDir
|
|
}
|
|
};
|
|
|
|
logger.logHealthCheck('disk', healthy ? 'healthy' : 'unhealthy', this.services.disk.details);
|
|
return this.services.disk;
|
|
} catch (error) {
|
|
this.services.disk = {
|
|
name: 'Disk Space',
|
|
healthy: false,
|
|
lastCheck: new Date().toISOString(),
|
|
details: {
|
|
error: error.message
|
|
}
|
|
};
|
|
|
|
logger.logHealthCheck('disk', 'unhealthy', { error: error.message });
|
|
return this.services.disk;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check memory usage
|
|
*/
|
|
async checkMemory() {
|
|
try {
|
|
const totalMemory = os.totalmem();
|
|
const freeMemory = os.freemem();
|
|
const usedMemory = totalMemory - freeMemory;
|
|
const usedPercentage = (usedMemory / totalMemory) * 100;
|
|
|
|
// Get Node.js process memory
|
|
const processMemory = process.memoryUsage();
|
|
|
|
// Consider healthy if less than 85% used
|
|
const healthy = usedPercentage < 85;
|
|
|
|
this.services.memory = {
|
|
name: 'Memory',
|
|
healthy,
|
|
lastCheck: new Date().toISOString(),
|
|
details: {
|
|
system: {
|
|
total: this.formatBytes(totalMemory),
|
|
used: this.formatBytes(usedMemory),
|
|
free: this.formatBytes(freeMemory),
|
|
usedPercentage: `${usedPercentage.toFixed(2)}%`
|
|
},
|
|
process: {
|
|
rss: this.formatBytes(processMemory.rss),
|
|
heapTotal: this.formatBytes(processMemory.heapTotal),
|
|
heapUsed: this.formatBytes(processMemory.heapUsed),
|
|
external: this.formatBytes(processMemory.external)
|
|
}
|
|
}
|
|
};
|
|
|
|
logger.logHealthCheck('memory', healthy ? 'healthy' : 'unhealthy', this.services.memory.details);
|
|
return this.services.memory;
|
|
} catch (error) {
|
|
this.services.memory = {
|
|
name: 'Memory',
|
|
healthy: false,
|
|
lastCheck: new Date().toISOString(),
|
|
details: {
|
|
error: error.message
|
|
}
|
|
};
|
|
|
|
logger.logHealthCheck('memory', 'unhealthy', { error: error.message });
|
|
return this.services.memory;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Run all health checks
|
|
*/
|
|
async checkAll() {
|
|
const results = await Promise.allSettled([
|
|
this.checkNocoDB(),
|
|
this.checkSMTP(),
|
|
this.checkRepresentAPI(),
|
|
this.checkDiskSpace(),
|
|
this.checkMemory()
|
|
]);
|
|
|
|
const overall = {
|
|
status: Object.values(this.services).every(s => s.healthy) ? 'healthy' : 'degraded',
|
|
timestamp: new Date().toISOString(),
|
|
services: this.services,
|
|
uptime: process.uptime(),
|
|
version: process.env.npm_package_version || '1.0.0',
|
|
environment: process.env.NODE_ENV || 'development'
|
|
};
|
|
|
|
// Check if any critical services are down
|
|
// Note: In development, NocoDB might not be available yet
|
|
// Only mark as unhealthy if multiple critical services fail
|
|
const criticalServices = ['nocodb'];
|
|
const criticalDown = criticalServices.filter(name => !this.services[name].healthy);
|
|
|
|
// Allow degraded state if only external services (NocoDB, Represent API) are down
|
|
// Only fail health check if core services (disk, memory) are unhealthy
|
|
const coreServicesHealthy = this.services.disk.healthy && this.services.memory.healthy;
|
|
|
|
if (!coreServicesHealthy) {
|
|
overall.status = 'unhealthy';
|
|
} else if (criticalDown.length > 0) {
|
|
// Mark as degraded (not unhealthy) if external services are down
|
|
// This allows the container to start even if NocoDB isn't ready
|
|
overall.status = 'degraded';
|
|
}
|
|
|
|
return overall;
|
|
}
|
|
|
|
/**
|
|
* Get health check for a specific service
|
|
*/
|
|
async checkService(serviceName) {
|
|
const checkMethods = {
|
|
nocodb: () => this.checkNocoDB(),
|
|
smtp: () => this.checkSMTP(),
|
|
representAPI: () => this.checkRepresentAPI(),
|
|
disk: () => this.checkDiskSpace(),
|
|
memory: () => this.checkMemory()
|
|
};
|
|
|
|
if (checkMethods[serviceName]) {
|
|
return await checkMethods[serviceName]();
|
|
}
|
|
|
|
throw new Error(`Unknown service: ${serviceName}`);
|
|
}
|
|
|
|
/**
|
|
* Helper: Get directory size recursively
|
|
*/
|
|
async getDirectorySize(directory) {
|
|
let size = 0;
|
|
|
|
try {
|
|
const files = await fs.readdir(directory);
|
|
|
|
for (const file of files) {
|
|
const filePath = path.join(directory, file);
|
|
const stats = await fs.stat(filePath);
|
|
|
|
if (stats.isDirectory()) {
|
|
size += await this.getDirectorySize(filePath);
|
|
} else {
|
|
size += stats.size;
|
|
}
|
|
}
|
|
} catch (error) {
|
|
logger.warn(`Error calculating directory size: ${error.message}`);
|
|
}
|
|
|
|
return size;
|
|
}
|
|
|
|
/**
|
|
* Helper: Format bytes to human readable
|
|
*/
|
|
formatBytes(bytes) {
|
|
if (bytes === 0) return '0 Bytes';
|
|
|
|
const k = 1024;
|
|
const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
|
|
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
|
|
|
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
|
|
}
|
|
}
|
|
|
|
module.exports = new HealthCheck();
|