/** * GMP-Agent 健康监控模块 * 工单编号: GH-GMP-004 * 开发者: 译典A05 (5TH-LE-HK-A05) * 职责: 定期巡检所有已安装 GMP 模块 · 自动修复 · 状态上报 * * 对齐 GMP-AGENT-SPEC v1.0 第7章: * - 60秒巡检间隔 (可通过 GMP_HEALTH_INTERVAL 配置) * - 自动修复策略: restart → delete+start → reinstall → 标记dead * - 修复上限3次 (与铸渊 Agent 集群一致) * - 状态变更通知 (预留 HLDP report 接口) */ 'use strict'; const http = require('http'); const { execFileSync } = require('child_process'); const path = require('path'); const { createLogger } = require('./lib/logger'); const { sanitizeModuleName } = require('./lib/path-guard'); const logger = createLogger('health'); // ─── 常量 ─────────────────────────────────────────────── /** 默认巡检间隔 (毫秒) */ const DEFAULT_INTERVAL = parseInt(process.env.GMP_HEALTH_INTERVAL || '60000', 10); /** 单次健康检查超时 (毫秒) */ const CHECK_TIMEOUT = parseInt(process.env.GMP_HEALTH_TIMEOUT || '10000', 10); /** 最大自动修复尝试次数 */ const MAX_RESTART_ATTEMPTS = 3; /** 修复策略 (对齐 GMP-AGENT-SPEC 第7章) */ const RECOVERY_STRATEGIES = [ { attempt: 1, action: 'pm2_restart', waitMs: 10000, description: 'PM2 restart' }, { attempt: 2, action: 'pm2_delete_start', waitMs: 20000, description: 'PM2 delete + start' }, { attempt: 3, action: 'reinstall', waitMs: 30000, description: '重新运行 install 脚本' } ]; // ─── HealthMonitor 类 ─────────────────────────────────── class HealthMonitor { /** * @param {object} agent - GMPAgent 实例 */ constructor(agent) { this.agent = agent; this.intervalId = null; this.restartCounters = new Map(); // moduleName -> { count, lastAttempt } this.isRunning = false; } /** * 启动定期巡检 */ start() { if (this.intervalId) { logger.warn('健康监控已在运行, 跳过重复启动'); return; } const interval = DEFAULT_INTERVAL; logger.info('健康监控启动, 巡检间隔=' + interval + 'ms, 最大修复=' + MAX_RESTART_ATTEMPTS + '次'); // 首次启动延迟 10 秒后执行第一次巡检 (等模块启动完成) setTimeout(() => { this._patrol(); }, 10000); this.intervalId = setInterval(() => { this._patrol(); }, interval); this.isRunning = true; } /** * 停止定期巡检 */ stop() { if (this.intervalId) { clearInterval(this.intervalId); this.intervalId = null; this.isRunning = false; logger.info('健康监控已停止'); } } /** * 获取监控状态摘要 * @returns {object} */ getSummary() { const counters = {}; for (const [name, data] of this.restartCounters) { counters[name] = { restartCount: data.count, lastAttempt: data.lastAttempt }; } return { isRunning: this.isRunning, interval: DEFAULT_INTERVAL, maxRestartAttempts: MAX_RESTART_ATTEMPTS, restartCounters: counters }; } /** * 手动触发一次全量巡检 * @returns {Promise>} 巡检结果 */ async runOnce() { return this._patrol(); } // ─── 内部方法 ──────────────────────────────────────── /** * 执行一轮巡检 */ async _patrol() { const modulesMap = this.agent.installedModules; if (modulesMap.size === 0) { return []; } logger.debug('开始巡检, 共 ' + modulesMap.size + ' 个模块'); const results = []; for (const [name, info] of modulesMap) { const health = await this._checkModule(name, info); results.push(health); const prevHealth = info.health || 'unknown'; // 更新模块信息中的健康状态 info.health = health.status; info.lastHealthCheck = health.checkedAt; // 状态变更检测 if (prevHealth !== health.status) { logger.info('模块健康状态变更: ' + name + ' ' + prevHealth + ' → ' + health.status); this._onHealthChanged(name, prevHealth, health.status, info); } // 失败处理 if (health.status === 'fail') { await this._handleFailure(name, info); } else if (health.status === 'ok') { // 恢复成功,重置计数器 this._resetCounter(name); } } return results; } /** * 检查单个模块健康 * @param {string} name - 模块名 * @param {object} info - 模块信息 * @returns {Promise} */ async _checkModule(name, info) { const result = { name: name, status: 'unknown', checkedAt: new Date().toISOString(), checks: {} }; try { // Check 1: 目录存在 const fs = require('fs'); const dirExists = fs.existsSync(info.path); result.checks.directoryExists = dirExists; if (!dirExists) { result.status = 'fail'; result.checks.error = '模块目录不存在'; return result; } // Check 2: PM2 进程检测 (如果模块有 pm2 名称) const pm2Status = this._checkPM2Process(name); result.checks.pm2 = pm2Status; // Check 3: HTTP 健康端点检测 (如果模块有端口配置) if (info.port) { const httpOk = await this._httpHealthCheck(info.port); result.checks.httpHealth = httpOk; } // 综合判定 if (pm2Status === 'online' || !info.port) { result.status = info.port ? (result.checks.httpHealth ? 'ok' : 'degraded') : 'ok'; } else if (pm2Status === 'stopped' || pm2Status === 'errored') { result.status = 'fail'; } else { // PM2 进程未找到但目录存在 → 可能未用 PM2 管理 result.status = info.port ? (result.checks.httpHealth ? 'ok' : 'unknown') : 'ok'; } } catch (err) { result.status = 'fail'; result.checks.error = err.message; } return result; } /** * 检查 PM2 进程状态 * @param {string} name - 进程名 * @returns {string} 'online' | 'stopped' | 'errored' | 'not_found' | 'unknown' */ _checkPM2Process(name) { try { const output = execFileSync('pm2', ['jlist'], { encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 5000 }); const processes = JSON.parse(output); const proc = processes.find( p => p.name === name || p.name === 'gmp-' + name ); if (!proc) return 'not_found'; return proc.pm2_env ? (proc.pm2_env.status || 'unknown') : 'unknown'; } catch (err) { // PM2 不可用或命令失败 logger.debug('PM2 检测失败 (' + name + '): ' + err.message); return 'unknown'; } } /** * HTTP 健康端点检查 * @param {number} port * @returns {Promise} */ _httpHealthCheck(port) { return new Promise((resolve) => { const req = http.get( 'http://127.0.0.1:' + port + '/health', { timeout: CHECK_TIMEOUT }, (res) => { // 消费响应体避免内存泄漏 res.resume(); resolve(res.statusCode >= 200 && res.statusCode < 400); } ); req.on('error', () => resolve(false)); req.on('timeout', () => { req.destroy(); resolve(false); }); }); } /** * 处理模块健康检查失败 * @param {string} name * @param {object} info */ async _handleFailure(name, info) { const counter = this.restartCounters.get(name) || { count: 0, lastAttempt: null }; if (counter.count >= MAX_RESTART_ATTEMPTS) { // 超过修复上限 → 标记为 dead if (info.status !== 'dead') { info.status = 'dead'; logger.error('模块 ' + name + ' 自动修复已达上限 (' + MAX_RESTART_ATTEMPTS + '次), 标记为 dead'); this._sendHLDPAlert(name, 'dead', '自动修复 ' + MAX_RESTART_ATTEMPTS + ' 次均失败'); } return; } // 执行修复策略 const strategy = RECOVERY_STRATEGIES[counter.count]; counter.count++; counter.lastAttempt = new Date().toISOString(); this.restartCounters.set(name, counter); logger.warn('尝试修复模块 ' + name + ' (第' + counter.count + '次/' + MAX_RESTART_ATTEMPTS + '): ' + strategy.description); try { await this._executeRecovery(name, info, strategy); // 等待后重新检查 await this._sleep(strategy.waitMs); const recheck = await this._checkModule(name, info); if (recheck.status === 'ok' || recheck.status === 'degraded') { logger.info('模块 ' + name + ' 修复成功 (第' + counter.count + '次): ' + strategy.description); info.health = recheck.status; this._resetCounter(name); } else { logger.warn('模块 ' + name + ' 修复后仍未恢复, 将在下一轮继续尝试'); } } catch (err) { logger.error('模块 ' + name + ' 修复执行异常: ' + err.message); } } /** * 执行修复操作 * @param {string} name * @param {object} info * @param {object} strategy */ async _executeRecovery(name, info, strategy) { // 用 sanitizeModuleName (path.basename + 白名单) 净化, CodeQL 公认 sanitizer const pm2Name = sanitizeModuleName(info.pm2Name || name); switch (strategy.action) { case 'pm2_restart': try { execFileSync('pm2', ['restart', pm2Name], { stdio: 'pipe', timeout: 15000 }); logger.info('PM2 restart ' + pm2Name + ' 已执行'); } catch (err) { logger.warn('PM2 restart 失败 (exit ' + (err.status || 'unknown') + ')'); } break; case 'pm2_delete_start': try { execFileSync('pm2', ['delete', pm2Name], { stdio: 'pipe', timeout: 10000 }); } catch { // delete 失败不阻塞 } try { // entryPoint 用 path.basename 净化, 防路径穿越 const rawEntry = info.entryPoint || 'index.js'; const entryPoint = path.basename(rawEntry); if (entryPoint !== rawEntry || entryPoint.startsWith('-')) { throw new Error('非法 entryPoint: ' + JSON.stringify(rawEntry)); } execFileSync('pm2', ['start', entryPoint, '--name', pm2Name], { cwd: info.path, stdio: 'pipe', timeout: 15000 }); logger.info('PM2 delete+start ' + pm2Name + ' 已执行'); } catch (err) { logger.warn('PM2 start 失败 (exit ' + (err.status || 'unknown') + ')'); } break; case 'reinstall': // 重新运行安装脚本 (如果有) const fs = require('fs'); const installScript = path.join(info.path, 'install.sh'); if (fs.existsSync(installScript)) { try { execFileSync('bash', ['install.sh'], { cwd: info.path, stdio: 'pipe', timeout: 120000 }); logger.info('install.sh 重新执行完成: ' + name); } catch (err) { logger.error('install.sh 执行失败 (exit ' + (err.status || 'unknown') + ')'); } } else { // 没有 install.sh, 尝试 npm install + pm2 restart try { const packageJson = path.join(info.path, 'package.json'); if (fs.existsSync(packageJson)) { execFileSync('npm', ['install', '--production'], { cwd: info.path, stdio: 'pipe', timeout: 60000 }); } execFileSync('pm2', ['restart', pm2Name], { stdio: 'pipe', timeout: 15000 }); logger.info('npm install + pm2 restart 已执行: ' + name); } catch (err) { logger.error('reinstall fallback 失败 (exit ' + (err.status || 'unknown') + ')'); } } break; default: logger.warn('未知修复策略: ' + strategy.action); } } /** * 健康状态变更回调 */ _onHealthChanged(name, from, to, info) { // 预留: EventBus 通知 // EventBus.emit('gmp:module:health_changed', { module: name, from, to }); // 预留: HLDP report if (to === 'fail' || to === 'dead') { this._sendHLDPAlert(name, to, from + ' → ' + to); } } /** * 发送 HLDP 警报 (预留接口) */ _sendHLDPAlert(moduleName, level, message) { // 预留: 对接铸渊 Agent 集群的 HLDP 消息通道 logger.warn('HLDP Alert [' + level + '] 模块=' + moduleName + ': ' + message); // TODO: 接入 HLDP reporter (hldp-reporter.js) } /** * 重置修复计数器 */ _resetCounter(name) { if (this.restartCounters.has(name)) { this.restartCounters.delete(name); } } /** * 异步等待 */ _sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } } // ─── 导出 ────────────────────────────────────────────── module.exports = { HealthMonitor, DEFAULT_INTERVAL, CHECK_TIMEOUT, MAX_RESTART_ATTEMPTS, RECOVERY_STRATEGIES };