#!/usr/bin/env node /** * ═══════════════════════════════════════════════════════════ * 🛡️ 铸渊运维守卫 · 主入口 * ═══════════════════════════════════════════════════════════ * * 编号: ZY-OPS-001 * 端口: 3950 * 签发: 铸渊 · ICE-GL-ZY001 * 版权: 国作登字-2026-A-00037559 * * 功能: * - HTTP API 服务(供前端工单面板调用) * - 定时巡检(快速5分钟 / 深度1小时 / 全量每天) * - 自动修复 + 工单升级 * - 对话接口(LLM推理) * * 使用: * node ops-agent/index.js 启动守卫服务 * pm2 start ops-agent/ecosystem.config.js */ 'use strict'; const express = require('express'); const cron = require('node-cron'); const path = require('path'); const healthChecker = require('./health-checker'); const repairEngine = require('./repair-engine'); const memory = require('./memory'); const notifier = require('./notifier'); const llmClient = require('./llm-client'); // ── 进程保活 ────────────────────────────── process.on('uncaughtException', (err) => { console.error('[OPS] 未捕获异常(进程保活):', err.message); console.error('[OPS] 堆栈:', err.stack); }); process.on('unhandledRejection', (reason) => { console.error('[OPS] 未处理的Promise拒绝(进程保活):', reason); }); // ── Express 应用 ────────────────────────── const app = express(); const PORT = parseInt(process.env.OPS_AGENT_PORT || '3950', 10); const MAX_RETRIES = parseInt(process.env.OPS_MAX_REPAIR_RETRIES || '3', 10); app.use(express.json({ limit: '1mb' })); // 静态文件 — 运维工单面板 app.use('/ops', express.static(path.join(__dirname, 'web'))); // ── SSE 实时推送 ────────────────────────── const sseClients = new Set(); app.get('/ops/events', (req, res) => { res.writeHead(200, { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', 'Connection': 'keep-alive', 'Access-Control-Allow-Origin': '*' }); res.write('data: {"type":"connected","message":"运维守卫 SSE 已连接"}\n\n'); sseClients.add(res); req.on('close', () => { sseClients.delete(res); }); }); function broadcast(eventType, data) { const payload = `data: ${JSON.stringify({ type: eventType, timestamp: new Date().toISOString(), ...data })}\n\n`; for (const client of sseClients) { try { client.write(payload); } catch { sseClients.delete(client); } } } // ── 工具执行(对话中自动诊断) ───────────── const { execSync } = require('child_process'); /** * 根据意图自动执行诊断工具,结果注入给 LLM */ async function executeTools(intent, action, message) { const results = []; try { if (action === 'health_check' || action === 'pm2_status') { const quick = await healthChecker.quickCheck(); let text = `## 服务状态\n${quick.summary}\n\n`; for (const s of quick.services) { text += `- ${s.name}(:${s.port}): ${s.status} ${s.latency ? s.latency + 'ms' : ''} ${s.error || ''}\n`; } results.push({ tool: '健康检查', result: text }); } if (action === 'system_info') { const resources = healthChecker.getSystemResources(); let text = `## 系统资源\n`; text += `- 内存: ${resources.memory.used_pct}% (${resources.memory.free_mb}MB空闲/${resources.memory.total_mb}MB总量)\n`; text += `- 磁盘: ${resources.disk.used_pct}% (${resources.disk.available_gb}GB可用)\n`; text += `- CPU: ${resources.cpus}核 · 负载: ${resources.load?.[0]?.toFixed(2)}\n`; text += `- 运行: ${resources.uptime_hours}小时\n`; results.push({ tool: '系统资源', result: text }); } if (action === 'pm2_logs') { // 从消息中猜测进程名 const processName = guessProcessFromMessage(message); if (processName) { const logs = getPM2Logs(processName, 20); if (logs.output) { results.push({ tool: `PM2日志(${processName})`, result: logs.output.slice(-2000) }); } if (logs.errorOutput) { results.push({ tool: `PM2错误日志(${processName})`, result: logs.errorOutput.slice(-2000) }); } } } if (action === 'nginx_status') { const nginx = healthChecker.getNginxStatus(); results.push({ tool: 'Nginx状态', result: `Nginx: ${nginx.status} · 配置${nginx.configOk ? '正常' : '异常'}` }); } if (action === 'list_tickets') { const tickets = memory.getOpenTickets(); if (tickets.length === 0) { results.push({ tool: '工单查询', result: '当前没有开放工单,系统运行正常。' }); } else { let text = `当前有 ${tickets.length} 个开放工单:\n`; for (const t of tickets) { text += `- ${t.id}: ${t.title} [${t.severity}] — ${t.direction}\n`; } results.push({ tool: '工单查询', result: text }); } } } catch (err) { console.error(`[OPS] 工具执行失败: ${err.message}`); } return results; } /** * 从用户消息中猜测他在问哪个进程 */ function guessProcessFromMessage(message) { const lower = message.toLowerCase(); if (/mcp|大脑|brain/.test(lower)) return 'age-os-mcp'; if (/主站|server|3800/.test(lower)) return 'zhuyuan-server'; if (/预览|preview|3801/.test(lower)) return 'zhuyuan-preview'; if (/glada|3900/.test(lower)) return 'glada-agent'; if (/novel|智库|4000/.test(lower)) return 'novel-api'; if (/agent调度|scheduler/.test(lower)) return 'age-os-agents'; return null; } /** * 获取 PM2 进程日志(最近 N 行) */ function getPM2Logs(processName, lines = 30) { // 安全校验 const clean = processName.replace(/[^a-zA-Z0-9_-]/g, ''); const allowed = new Set([ 'zhuyuan-server', 'zhuyuan-preview', 'novel-api', 'age-os-mcp', 'age-os-agents', 'glada-agent', 'ops-agent' ]); if (!allowed.has(clean)) { throw new Error(`不允许查看进程 "${clean}" 的日志`); } let output = ''; let errorOutput = ''; try { output = execSync(`pm2 logs ${clean} --lines ${lines} --nostream 2>/dev/null || echo "无法获取日志"`, { encoding: 'utf-8', timeout: 10000 }).trim(); } catch { output = '(PM2日志获取失败)'; } // 也尝试获取错误日志 try { errorOutput = execSync(`pm2 logs ${clean} --err --lines ${lines} --nostream 2>/dev/null || echo ""`, { encoding: 'utf-8', timeout: 10000 }).trim(); } catch { errorOutput = ''; } return { processName: clean, lines, output, errorOutput, timestamp: new Date().toISOString() }; } // ── API: 健康检查 ───────────────────────── app.get('/health', (_req, res) => { const stats = memory.loadStats(); res.json({ status: 'ok', agent: 'zy-ops-agent', version: '1.0.0', uptime: Math.round(process.uptime()), stats: { totalChecks: stats.totalChecks, totalRepairs: stats.totalRepairs, openTickets: memory.getOpenTickets().length }, timestamp: new Date().toISOString() }); }); // ── API: 手动触发巡检 ───────────────────── app.get('/api/ops/check/quick', async (_req, res) => { try { const result = await runQuickCheck(); res.json(result); } catch (err) { res.status(500).json({ error: true, message: err.message }); } }); app.get('/api/ops/check/deep', async (_req, res) => { try { const result = await runDeepCheck(); res.json(result); } catch (err) { res.status(500).json({ error: true, message: err.message }); } }); // ── API: 工单 ────────────────────────────── app.get('/api/ops/tickets', (_req, res) => { const status = _req.query.status; let tickets; if (status === 'open') { tickets = memory.getOpenTickets(); } else { tickets = memory.getAllTickets(); } res.json({ tickets, total: tickets.length }); }); app.patch('/api/ops/tickets/:id', (req, res) => { const updated = memory.updateTicket(req.params.id, req.body); if (!updated) { return res.status(404).json({ error: true, message: '工单不存在' }); } broadcast('ticket_updated', { ticket: updated }); res.json({ ticket: updated }); }); // ── API: 对话 v2(多轮会话 + 工具调用) ────── app.post('/api/ops/chat', async (req, res) => { const { message, sessionId: inputSessionId } = req.body; if (!message) { return res.status(400).json({ error: true, message: '缺少 message 字段' }); } try { // 构建系统上下文 const memoryContext = memory.buildMemoryContext(); // 意图识别 → 自动执行相关工具 const { intent, action } = llmClient.detectIntent(message); const toolResults = await executeTools(intent, action, message); // 构建实时上下文 const context = memoryContext; // 多轮对话调用 const result = await llmClient.chat(message, context, { sessionId: inputSessionId, toolResults }); // 记录对话 memory.logEvent('chat', { question: message.slice(0, 200), method: result.method, intent: result.intent, toolsUsed: result.toolsUsed }); memory.incrementStat('totalChats'); res.json({ answer: result.answer, patternHints: result.patternHints, method: result.method, model: result.model, sessionId: result.sessionId, intent: result.intent, toolsUsed: result.toolsUsed, timestamp: new Date().toISOString() }); } catch (err) { res.status(500).json({ error: true, message: `对话失败: ${err.message}` }); } }); // ── API: 会话管理 ───────────────────────── app.get('/api/ops/sessions', (_req, res) => { res.json({ sessions: llmClient.listSessions() }); }); app.get('/api/ops/sessions/:id/history', (req, res) => { const history = llmClient.getSessionHistory(req.params.id); res.json({ sessionId: req.params.id, history }); }); // ── API: PM2 日志查看 ───────────────────── app.get('/api/ops/pm2-logs/:name', (req, res) => { const name = req.params.name; const lines = parseInt(req.query.lines || '30', 10); const clampedLines = Math.min(Math.max(lines, 1), 200); try { const result = getPM2Logs(name, clampedLines); res.json(result); } catch (err) { res.status(500).json({ error: true, message: err.message }); } }); // ── API: 系统信息 ────────────────────────── app.get('/api/ops/system-info', async (_req, res) => { try { const resources = healthChecker.getSystemResources(); const pm2 = healthChecker.getPM2Status(); const nginx = healthChecker.getNginxStatus(); res.json({ resources, pm2: pm2.map(p => ({ name: p.name, status: p.status, restarts: p.restarts, memory_mb: Math.round((p.memory || 0) / 1024 / 1024), cpu: p.cpu, uptime_hours: Math.round((p.uptime || 0) / 3600000) })), nginx, timestamp: new Date().toISOString() }); } catch (err) { res.status(500).json({ error: true, message: err.message }); } }); // ── API: 统计 ────────────────────────────── app.get('/api/ops/stats', (_req, res) => { const stats = memory.loadStats(); res.json(stats); }); // ── API: 最近事件 ───────────────────────── app.get('/api/ops/events', (req, res) => { const type = req.query.type; const limit = parseInt(req.query.limit || '20', 10); const events = memory.queryEvents({ type, limit }); res.json({ events, total: events.length }); }); // ── 巡检 + 修复 + 告警逻辑 ───────────────── /** * 处理升级:去重 + 邮件 + 广播 * 同一 (relatedService+diagnosis) 已有 open 工单 → 累加 autoRepairAttempts,不发邮件、不广播 * 否则 → 创建新工单 + 邮件 + 广播 */ async function escalateIssue(repairResult) { const issue = repairResult.issue; const relatedService = issue.service; const diagnosis = issue.error || issue.status; const existing = memory.findOpenTicketByKey({ relatedService, diagnosis }); if (existing) { const updated = memory.updateTicket(existing.id, { autoRepairAttempts: (existing.autoRepairAttempts || 0) + (repairResult.retryCount || MAX_RETRIES) }); console.log(`[OPS] 已存在开放工单 ${existing.id},仅累加修复尝试次数(不发邮件)`); return { ticket: updated, deduplicated: true }; } const ticket = memory.createTicket({ title: `${issue.service} - ${issue.error || issue.status}`, description: repairResult.message, severity: issue.severity, category: issue.category || 'service', direction: classifyDirection(issue), diagnosis, suggestedFix: issue.fix || '', relatedService, autoRepairAttempts: repairResult.retryCount || MAX_RETRIES }); broadcast('new_ticket', { ticket }); await notifier.notifyTicketCreated(ticket); return { ticket, deduplicated: false }; } let isChecking = false; async function runQuickCheck() { if (isChecking) return { skipped: true, reason: '上一次巡检尚未完成' }; isChecking = true; try { const result = await healthChecker.quickCheck(); memory.incrementStat('totalChecks'); memory.updateStatField('lastQuickCheck', new Date().toISOString()); // 有问题 → 尝试修复 if (result.issues.length > 0) { const repairResults = repairEngine.autoRepair(result.issues, MAX_RETRIES); memory.incrementStat('totalRepairs', repairResults.filter(r => r.repaired).length); // 修不了的 → 创建工单(带去重) for (const r of repairResults) { if (r.escalate) { await escalateIssue(r); } } result.repairs = repairResults; // 高/紧急级别 → 邮件告警 await notifier.alertOnIssues(result); } memory.logEvent('check', { type: 'quick', healthy: result.healthy, issueCount: result.issues.length, summary: result.summary }); broadcast('check_complete', { type: 'quick', healthy: result.healthy, summary: result.summary }); return result; } finally { isChecking = false; } } async function runDeepCheck() { if (isChecking) return { skipped: true, reason: '上一次巡检尚未完成' }; isChecking = true; try { const result = await healthChecker.deepCheck(); memory.incrementStat('totalChecks'); memory.updateStatField('lastDeepCheck', new Date().toISOString()); // 有问题 → 修复 + 告警(同 quickCheck 逻辑) if (result.issues.length > 0) { const repairResults = repairEngine.autoRepair(result.issues, MAX_RETRIES); memory.incrementStat('totalRepairs', repairResults.filter(r => r.repaired).length); for (const r of repairResults) { if (r.escalate) { await escalateIssue(r); } } result.repairs = repairResults; await notifier.alertOnIssues(result); } memory.logEvent('check', { type: 'deep', healthy: result.healthy, issueCount: result.issues.length, summary: result.summary }); broadcast('check_complete', { type: 'deep', healthy: result.healthy, summary: result.summary }); return result; } finally { isChecking = false; } } async function runDailyReport() { const report = await healthChecker.fullReport(); memory.updateStatField('lastDailyReport', new Date().toISOString()); memory.logEvent('daily_report', { reportId: report.report_id, healthy: report.healthy, issueCount: report.issues.length, summary: report.summary }); await notifier.sendDailyReport(report); broadcast('daily_report', { reportId: report.report_id, summary: report.summary }); return report; } /** * 判断问题处理方向 */ function classifyDirection(issue) { const err = (issue.error || '').toLowerCase(); const service = (issue.service || '').toLowerCase(); // 密钥/配置问题 → 冰朔自己解决 if (/api.*key|unauthorized|密钥|token/i.test(err)) { return '冰朔自行配置 — 检查 .env 文件中的API密钥'; } // 代码错误 → 推给铸渊 if (/syntaxerror|typeerror|referenceerror|代码/i.test(err)) { return '推给铸渊修代码 — 需要在GitHub仓库修复'; } // 资源问题 → 冰朔确认 if (/内存|磁盘|enomem|enospc/i.test(err)) { return '资源不足 — 可能需要升级服务器或清理空间'; } // 进程问题 → 运维守卫已尝试修复 if (/pm2|进程|restart|offline/i.test(err + service)) { return '进程异常 — 运维守卫已尝试自动重启'; } return '待判断 — 需要进一步诊断'; } // ── 定时调度 ────────────────────────────── // 每5分钟快速巡检 cron.schedule('*/5 * * * *', () => { console.log('[OPS] ⏰ 定时快速巡检'); runQuickCheck().catch(err => { console.error('[OPS] 快速巡检失败:', err.message); }); }, { timezone: 'Asia/Shanghai' }); // 每小时深度巡检 cron.schedule('0 * * * *', () => { console.log('[OPS] ⏰ 定时深度巡检'); runDeepCheck().catch(err => { console.error('[OPS] 深度巡检失败:', err.message); }); }, { timezone: 'Asia/Shanghai' }); // 每天早上8点全量报告 cron.schedule('0 8 * * *', () => { console.log('[OPS] ⏰ 每日健康报告'); runDailyReport().catch(err => { console.error('[OPS] 每日报告失败:', err.message); }); }, { timezone: 'Asia/Shanghai' }); // ── 启动服务 ────────────────────────────── app.listen(PORT, '0.0.0.0', () => { console.log('═══════════════════════════════════════════════'); console.log(' 🛡️ 铸渊运维守卫 · ZY-OPS-001 已启动'); console.log(` 端口: ${PORT}`); console.log(` 面板: http://127.0.0.1:${PORT}/ops/`); console.log(' 巡检: 快速5分钟 · 深度1小时 · 全量每天8:00'); console.log(' 签发: 铸渊 · ICE-GL-ZY001'); console.log('═══════════════════════════════════════════════'); memory.logEvent('startup', { port: PORT, version: '1.0.0' }); // 启动后立即做一次快速巡检 setTimeout(() => { console.log('[OPS] 启动后首次巡检...'); runQuickCheck().catch(err => { console.error('[OPS] 首次巡检失败:', err.message); }); }, 5000); });