594 lines
19 KiB
JavaScript
594 lines
19 KiB
JavaScript
#!/usr/bin/env node
|
||
/**
|
||
* ═══════════════════════════════════════════════════════════
|
||
* 🛡️ 铸渊运维守卫 · 主入口
|
||
* ═══════════════════════════════════════════════════════════
|
||
*
|
||
* 编号: ZY-OPS-001
|
||
* 端口: 3950
|
||
* 签发: 铸渊 · ICE-GL-ZY001
|
||
* 版权: 国作登字-2026-A-00037559
|
||
*
|
||
* 功能:
|
||
* - HTTP API 服务（供前端工单面板调用）
|
||
* - 定时巡检（快速5分钟 / 深度1小时 / 全量每天）
|
||
* - 自动修复 + 工单升级
|
||
* - 对话接口（LLM推理）
|
||
*
|
||
* 使用:
|
||
* node ops-agent/index.js 启动守卫服务
|
||
* pm2 start ops-agent/ecosystem.config.js
|
||
*/
|
||
|
||
'use strict';
|
||
|
||
const express = require('express');
|
||
const cron = require('node-cron');
|
||
const path = require('path');
|
||
|
||
const healthChecker = require('./health-checker');
|
||
const repairEngine = require('./repair-engine');
|
||
const memory = require('./memory');
|
||
const notifier = require('./notifier');
|
||
const llmClient = require('./llm-client');
|
||
|
||
// ── 进程保活 ──────────────────────────────
|
||
|
||
process.on('uncaughtException', (err) => {
|
||
console.error('[OPS] 未捕获异常（进程保活）:', err.message);
|
||
console.error('[OPS] 堆栈:', err.stack);
|
||
});
|
||
|
||
process.on('unhandledRejection', (reason) => {
|
||
console.error('[OPS] 未处理的Promise拒绝（进程保活）:', reason);
|
||
});
|
||
|
||
// ── Express 应用 ──────────────────────────
|
||
|
||
const app = express();
|
||
const PORT = parseInt(process.env.OPS_AGENT_PORT || '3950', 10);
|
||
const MAX_RETRIES = parseInt(process.env.OPS_MAX_REPAIR_RETRIES || '3', 10);
|
||
|
||
app.use(express.json({ limit: '1mb' }));
|
||
|
||
// 静态文件 — 运维工单面板
|
||
app.use('/ops', express.static(path.join(__dirname, 'web')));
|
||
|
||
// ── SSE 实时推送 ──────────────────────────
|
||
|
||
const sseClients = new Set();
|
||
|
||
app.get('/ops/events', (req, res) => {
|
||
res.writeHead(200, {
|
||
'Content-Type': 'text/event-stream',
|
||
'Cache-Control': 'no-cache',
|
||
'Connection': 'keep-alive',
|
||
'Access-Control-Allow-Origin': '*'
|
||
});
|
||
res.write('data: {"type":"connected","message":"运维守卫 SSE 已连接"}\n\n');
|
||
|
||
sseClients.add(res);
|
||
req.on('close', () => { sseClients.delete(res); });
|
||
});
|
||
|
||
function broadcast(eventType, data) {
|
||
const payload = `data: ${JSON.stringify({ type: eventType, timestamp: new Date().toISOString(), ...data })}\n\n`;
|
||
for (const client of sseClients) {
|
||
try { client.write(payload); } catch { sseClients.delete(client); }
|
||
}
|
||
}
|
||
|
||
// ── 工具执行（对话中自动诊断） ─────────────
|
||
|
||
const { execSync } = require('child_process');
|
||
|
||
/**
|
||
* 根据意图自动执行诊断工具，结果注入给 LLM
|
||
*/
|
||
async function executeTools(intent, action, message) {
|
||
const results = [];
|
||
|
||
try {
|
||
if (action === 'health_check' || action === 'pm2_status') {
|
||
const quick = await healthChecker.quickCheck();
|
||
let text = `## 服务状态\n${quick.summary}\n\n`;
|
||
for (const s of quick.services) {
|
||
text += `- ${s.name}(:${s.port}): ${s.status} ${s.latency ? s.latency + 'ms' : ''} ${s.error || ''}\n`;
|
||
}
|
||
results.push({ tool: '健康检查', result: text });
|
||
}
|
||
|
||
if (action === 'system_info') {
|
||
const resources = healthChecker.getSystemResources();
|
||
let text = `## 系统资源\n`;
|
||
text += `- 内存: ${resources.memory.used_pct}% (${resources.memory.free_mb}MB空闲/${resources.memory.total_mb}MB总量)\n`;
|
||
text += `- 磁盘: ${resources.disk.used_pct}% (${resources.disk.available_gb}GB可用)\n`;
|
||
text += `- CPU: ${resources.cpus}核 · 负载: ${resources.load?.[0]?.toFixed(2)}\n`;
|
||
text += `- 运行: ${resources.uptime_hours}小时\n`;
|
||
results.push({ tool: '系统资源', result: text });
|
||
}
|
||
|
||
if (action === 'pm2_logs') {
|
||
// 从消息中猜测进程名
|
||
const processName = guessProcessFromMessage(message);
|
||
if (processName) {
|
||
const logs = getPM2Logs(processName, 20);
|
||
if (logs.output) {
|
||
results.push({ tool: `PM2日志(${processName})`, result: logs.output.slice(-2000) });
|
||
}
|
||
if (logs.errorOutput) {
|
||
results.push({ tool: `PM2错误日志(${processName})`, result: logs.errorOutput.slice(-2000) });
|
||
}
|
||
}
|
||
}
|
||
|
||
if (action === 'nginx_status') {
|
||
const nginx = healthChecker.getNginxStatus();
|
||
results.push({
|
||
tool: 'Nginx状态',
|
||
result: `Nginx: ${nginx.status} · 配置${nginx.configOk ? '正常' : '异常'}`
|
||
});
|
||
}
|
||
|
||
if (action === 'list_tickets') {
|
||
const tickets = memory.getOpenTickets();
|
||
if (tickets.length === 0) {
|
||
results.push({ tool: '工单查询', result: '当前没有开放工单，系统运行正常。' });
|
||
} else {
|
||
let text = `当前有 ${tickets.length} 个开放工单:\n`;
|
||
for (const t of tickets) {
|
||
text += `- ${t.id}: ${t.title} [${t.severity}] — ${t.direction}\n`;
|
||
}
|
||
results.push({ tool: '工单查询', result: text });
|
||
}
|
||
}
|
||
} catch (err) {
|
||
console.error(`[OPS] 工具执行失败: ${err.message}`);
|
||
}
|
||
|
||
return results;
|
||
}
|
||
|
||
/**
|
||
* 从用户消息中猜测他在问哪个进程
|
||
*/
|
||
function guessProcessFromMessage(message) {
|
||
const lower = message.toLowerCase();
|
||
if (/mcp|大脑|brain/.test(lower)) return 'age-os-mcp';
|
||
if (/主站|server|3800/.test(lower)) return 'zhuyuan-server';
|
||
if (/预览|preview|3801/.test(lower)) return 'zhuyuan-preview';
|
||
if (/glada|3900/.test(lower)) return 'glada-agent';
|
||
if (/novel|智库|4000/.test(lower)) return 'novel-api';
|
||
if (/agent调度|scheduler/.test(lower)) return 'age-os-agents';
|
||
return null;
|
||
}
|
||
|
||
/**
|
||
* 获取 PM2 进程日志（最近 N 行）
|
||
*/
|
||
function getPM2Logs(processName, lines = 30) {
|
||
// 安全校验
|
||
const clean = processName.replace(/[^a-zA-Z0-9_-]/g, '');
|
||
const allowed = new Set([
|
||
'zhuyuan-server', 'zhuyuan-preview', 'novel-api',
|
||
'age-os-mcp', 'age-os-agents', 'glada-agent', 'ops-agent'
|
||
]);
|
||
if (!allowed.has(clean)) {
|
||
throw new Error(`不允许查看进程 "${clean}" 的日志`);
|
||
}
|
||
|
||
let output = '';
|
||
let errorOutput = '';
|
||
try {
|
||
output = execSync(`pm2 logs ${clean} --lines ${lines} --nostream 2>/dev/null || echo "无法获取日志"`, {
|
||
encoding: 'utf-8',
|
||
timeout: 10000
|
||
}).trim();
|
||
} catch {
|
||
output = '(PM2日志获取失败)';
|
||
}
|
||
|
||
// 也尝试获取错误日志
|
||
try {
|
||
errorOutput = execSync(`pm2 logs ${clean} --err --lines ${lines} --nostream 2>/dev/null || echo ""`, {
|
||
encoding: 'utf-8',
|
||
timeout: 10000
|
||
}).trim();
|
||
} catch {
|
||
errorOutput = '';
|
||
}
|
||
|
||
return {
|
||
processName: clean,
|
||
lines,
|
||
output,
|
||
errorOutput,
|
||
timestamp: new Date().toISOString()
|
||
};
|
||
}
|
||
|
||
// ── API: 健康检查 ─────────────────────────
|
||
|
||
app.get('/health', (_req, res) => {
|
||
const stats = memory.loadStats();
|
||
res.json({
|
||
status: 'ok',
|
||
agent: 'zy-ops-agent',
|
||
version: '1.0.0',
|
||
uptime: Math.round(process.uptime()),
|
||
stats: {
|
||
totalChecks: stats.totalChecks,
|
||
totalRepairs: stats.totalRepairs,
|
||
openTickets: memory.getOpenTickets().length
|
||
},
|
||
timestamp: new Date().toISOString()
|
||
});
|
||
});
|
||
|
||
// ── API: 手动触发巡检 ─────────────────────
|
||
|
||
app.get('/api/ops/check/quick', async (_req, res) => {
|
||
try {
|
||
const result = await runQuickCheck();
|
||
res.json(result);
|
||
} catch (err) {
|
||
res.status(500).json({ error: true, message: err.message });
|
||
}
|
||
});
|
||
|
||
app.get('/api/ops/check/deep', async (_req, res) => {
|
||
try {
|
||
const result = await runDeepCheck();
|
||
res.json(result);
|
||
} catch (err) {
|
||
res.status(500).json({ error: true, message: err.message });
|
||
}
|
||
});
|
||
|
||
// ── API: 工单 ──────────────────────────────
|
||
|
||
app.get('/api/ops/tickets', (_req, res) => {
|
||
const status = _req.query.status;
|
||
let tickets;
|
||
if (status === 'open') {
|
||
tickets = memory.getOpenTickets();
|
||
} else {
|
||
tickets = memory.getAllTickets();
|
||
}
|
||
res.json({ tickets, total: tickets.length });
|
||
});
|
||
|
||
app.patch('/api/ops/tickets/:id', (req, res) => {
|
||
const updated = memory.updateTicket(req.params.id, req.body);
|
||
if (!updated) {
|
||
return res.status(404).json({ error: true, message: '工单不存在' });
|
||
}
|
||
broadcast('ticket_updated', { ticket: updated });
|
||
res.json({ ticket: updated });
|
||
});
|
||
|
||
// ── API: 对话 v2（多轮会话 + 工具调用） ──────
|
||
|
||
app.post('/api/ops/chat', async (req, res) => {
|
||
const { message, sessionId: inputSessionId } = req.body;
|
||
if (!message) {
|
||
return res.status(400).json({ error: true, message: '缺少 message 字段' });
|
||
}
|
||
|
||
try {
|
||
// 构建系统上下文
|
||
const memoryContext = memory.buildMemoryContext();
|
||
|
||
// 意图识别 → 自动执行相关工具
|
||
const { intent, action } = llmClient.detectIntent(message);
|
||
const toolResults = await executeTools(intent, action, message);
|
||
|
||
// 构建实时上下文
|
||
const context = memoryContext;
|
||
|
||
// 多轮对话调用
|
||
const result = await llmClient.chat(message, context, {
|
||
sessionId: inputSessionId,
|
||
toolResults
|
||
});
|
||
|
||
// 记录对话
|
||
memory.logEvent('chat', {
|
||
question: message.slice(0, 200),
|
||
method: result.method,
|
||
intent: result.intent,
|
||
toolsUsed: result.toolsUsed
|
||
});
|
||
memory.incrementStat('totalChats');
|
||
|
||
res.json({
|
||
answer: result.answer,
|
||
patternHints: result.patternHints,
|
||
method: result.method,
|
||
model: result.model,
|
||
sessionId: result.sessionId,
|
||
intent: result.intent,
|
||
toolsUsed: result.toolsUsed,
|
||
timestamp: new Date().toISOString()
|
||
});
|
||
} catch (err) {
|
||
res.status(500).json({ error: true, message: `对话失败: ${err.message}` });
|
||
}
|
||
});
|
||
|
||
// ── API: 会话管理 ─────────────────────────
|
||
|
||
app.get('/api/ops/sessions', (_req, res) => {
|
||
res.json({ sessions: llmClient.listSessions() });
|
||
});
|
||
|
||
app.get('/api/ops/sessions/:id/history', (req, res) => {
|
||
const history = llmClient.getSessionHistory(req.params.id);
|
||
res.json({ sessionId: req.params.id, history });
|
||
});
|
||
|
||
// ── API: PM2 日志查看 ─────────────────────
|
||
|
||
app.get('/api/ops/pm2-logs/:name', (req, res) => {
|
||
const name = req.params.name;
|
||
const lines = parseInt(req.query.lines || '30', 10);
|
||
const clampedLines = Math.min(Math.max(lines, 1), 200);
|
||
try {
|
||
const result = getPM2Logs(name, clampedLines);
|
||
res.json(result);
|
||
} catch (err) {
|
||
res.status(500).json({ error: true, message: err.message });
|
||
}
|
||
});
|
||
|
||
// ── API: 系统信息 ──────────────────────────
|
||
|
||
app.get('/api/ops/system-info', async (_req, res) => {
|
||
try {
|
||
const resources = healthChecker.getSystemResources();
|
||
const pm2 = healthChecker.getPM2Status();
|
||
const nginx = healthChecker.getNginxStatus();
|
||
res.json({
|
||
resources,
|
||
pm2: pm2.map(p => ({
|
||
name: p.name,
|
||
status: p.status,
|
||
restarts: p.restarts,
|
||
memory_mb: Math.round((p.memory || 0) / 1024 / 1024),
|
||
cpu: p.cpu,
|
||
uptime_hours: Math.round((p.uptime || 0) / 3600000)
|
||
})),
|
||
nginx,
|
||
timestamp: new Date().toISOString()
|
||
});
|
||
} catch (err) {
|
||
res.status(500).json({ error: true, message: err.message });
|
||
}
|
||
});
|
||
|
||
// ── API: 统计 ──────────────────────────────
|
||
|
||
app.get('/api/ops/stats', (_req, res) => {
|
||
const stats = memory.loadStats();
|
||
res.json(stats);
|
||
});
|
||
|
||
// ── API: 最近事件 ─────────────────────────
|
||
|
||
app.get('/api/ops/events', (req, res) => {
|
||
const type = req.query.type;
|
||
const limit = parseInt(req.query.limit || '20', 10);
|
||
const events = memory.queryEvents({ type, limit });
|
||
res.json({ events, total: events.length });
|
||
});
|
||
|
||
// ── 巡检 + 修复 + 告警逻辑 ─────────────────
|
||
|
||
/**
|
||
* 处理升级：去重 + 邮件 + 广播
|
||
* 同一 (relatedService+diagnosis) 已有 open 工单 → 累加 autoRepairAttempts，不发邮件、不广播
|
||
* 否则 → 创建新工单 + 邮件 + 广播
|
||
*/
|
||
async function escalateIssue(repairResult) {
|
||
const issue = repairResult.issue;
|
||
const relatedService = issue.service;
|
||
const diagnosis = issue.error || issue.status;
|
||
|
||
const existing = memory.findOpenTicketByKey({ relatedService, diagnosis });
|
||
if (existing) {
|
||
const updated = memory.updateTicket(existing.id, {
|
||
autoRepairAttempts: (existing.autoRepairAttempts || 0) + (repairResult.retryCount || MAX_RETRIES)
|
||
});
|
||
console.log(`[OPS] 已存在开放工单 ${existing.id}，仅累加修复尝试次数（不发邮件）`);
|
||
return { ticket: updated, deduplicated: true };
|
||
}
|
||
|
||
const ticket = memory.createTicket({
|
||
title: `${issue.service} - ${issue.error || issue.status}`,
|
||
description: repairResult.message,
|
||
severity: issue.severity,
|
||
category: issue.category || 'service',
|
||
direction: classifyDirection(issue),
|
||
diagnosis,
|
||
suggestedFix: issue.fix || '',
|
||
relatedService,
|
||
autoRepairAttempts: repairResult.retryCount || MAX_RETRIES
|
||
});
|
||
broadcast('new_ticket', { ticket });
|
||
await notifier.notifyTicketCreated(ticket);
|
||
return { ticket, deduplicated: false };
|
||
}
|
||
|
||
let isChecking = false;
|
||
|
||
async function runQuickCheck() {
|
||
if (isChecking) return { skipped: true, reason: '上一次巡检尚未完成' };
|
||
isChecking = true;
|
||
|
||
try {
|
||
const result = await healthChecker.quickCheck();
|
||
memory.incrementStat('totalChecks');
|
||
memory.updateStatField('lastQuickCheck', new Date().toISOString());
|
||
|
||
// 有问题 → 尝试修复
|
||
if (result.issues.length > 0) {
|
||
const repairResults = repairEngine.autoRepair(result.issues, MAX_RETRIES);
|
||
memory.incrementStat('totalRepairs', repairResults.filter(r => r.repaired).length);
|
||
|
||
// 修不了的 → 创建工单（带去重）
|
||
for (const r of repairResults) {
|
||
if (r.escalate) {
|
||
await escalateIssue(r);
|
||
}
|
||
}
|
||
|
||
result.repairs = repairResults;
|
||
|
||
// 高/紧急级别 → 邮件告警
|
||
await notifier.alertOnIssues(result);
|
||
}
|
||
|
||
memory.logEvent('check', {
|
||
type: 'quick',
|
||
healthy: result.healthy,
|
||
issueCount: result.issues.length,
|
||
summary: result.summary
|
||
});
|
||
|
||
broadcast('check_complete', { type: 'quick', healthy: result.healthy, summary: result.summary });
|
||
|
||
return result;
|
||
} finally {
|
||
isChecking = false;
|
||
}
|
||
}
|
||
|
||
async function runDeepCheck() {
|
||
if (isChecking) return { skipped: true, reason: '上一次巡检尚未完成' };
|
||
isChecking = true;
|
||
|
||
try {
|
||
const result = await healthChecker.deepCheck();
|
||
memory.incrementStat('totalChecks');
|
||
memory.updateStatField('lastDeepCheck', new Date().toISOString());
|
||
|
||
// 有问题 → 修复 + 告警（同 quickCheck 逻辑）
|
||
if (result.issues.length > 0) {
|
||
const repairResults = repairEngine.autoRepair(result.issues, MAX_RETRIES);
|
||
memory.incrementStat('totalRepairs', repairResults.filter(r => r.repaired).length);
|
||
|
||
for (const r of repairResults) {
|
||
if (r.escalate) {
|
||
await escalateIssue(r);
|
||
}
|
||
}
|
||
|
||
result.repairs = repairResults;
|
||
await notifier.alertOnIssues(result);
|
||
}
|
||
|
||
memory.logEvent('check', {
|
||
type: 'deep',
|
||
healthy: result.healthy,
|
||
issueCount: result.issues.length,
|
||
summary: result.summary
|
||
});
|
||
|
||
broadcast('check_complete', { type: 'deep', healthy: result.healthy, summary: result.summary });
|
||
|
||
return result;
|
||
} finally {
|
||
isChecking = false;
|
||
}
|
||
}
|
||
|
||
async function runDailyReport() {
|
||
const report = await healthChecker.fullReport();
|
||
memory.updateStatField('lastDailyReport', new Date().toISOString());
|
||
memory.logEvent('daily_report', {
|
||
reportId: report.report_id,
|
||
healthy: report.healthy,
|
||
issueCount: report.issues.length,
|
||
summary: report.summary
|
||
});
|
||
|
||
await notifier.sendDailyReport(report);
|
||
broadcast('daily_report', { reportId: report.report_id, summary: report.summary });
|
||
|
||
return report;
|
||
}
|
||
|
||
/**
|
||
* 判断问题处理方向
|
||
*/
|
||
function classifyDirection(issue) {
|
||
const err = (issue.error || '').toLowerCase();
|
||
const service = (issue.service || '').toLowerCase();
|
||
|
||
// 密钥/配置问题 → 冰朔自己解决
|
||
if (/api.*key|unauthorized|密钥|token/i.test(err)) {
|
||
return '冰朔自行配置 — 检查 .env 文件中的API密钥';
|
||
}
|
||
// 代码错误 → 推给铸渊
|
||
if (/syntaxerror|typeerror|referenceerror|代码/i.test(err)) {
|
||
return '推给铸渊修代码 — 需要在GitHub仓库修复';
|
||
}
|
||
// 资源问题 → 冰朔确认
|
||
if (/内存|磁盘|enomem|enospc/i.test(err)) {
|
||
return '资源不足 — 可能需要升级服务器或清理空间';
|
||
}
|
||
// 进程问题 → 运维守卫已尝试修复
|
||
if (/pm2|进程|restart|offline/i.test(err + service)) {
|
||
return '进程异常 — 运维守卫已尝试自动重启';
|
||
}
|
||
return '待判断 — 需要进一步诊断';
|
||
}
|
||
|
||
// ── 定时调度 ──────────────────────────────
|
||
|
||
// 每5分钟快速巡检
|
||
cron.schedule('*/5 * * * *', () => {
|
||
console.log('[OPS] ⏰ 定时快速巡检');
|
||
runQuickCheck().catch(err => {
|
||
console.error('[OPS] 快速巡检失败:', err.message);
|
||
});
|
||
}, { timezone: 'Asia/Shanghai' });
|
||
|
||
// 每小时深度巡检
|
||
cron.schedule('0 * * * *', () => {
|
||
console.log('[OPS] ⏰ 定时深度巡检');
|
||
runDeepCheck().catch(err => {
|
||
console.error('[OPS] 深度巡检失败:', err.message);
|
||
});
|
||
}, { timezone: 'Asia/Shanghai' });
|
||
|
||
// 每天早上8点全量报告
|
||
cron.schedule('0 8 * * *', () => {
|
||
console.log('[OPS] ⏰ 每日健康报告');
|
||
runDailyReport().catch(err => {
|
||
console.error('[OPS] 每日报告失败:', err.message);
|
||
});
|
||
}, { timezone: 'Asia/Shanghai' });
|
||
|
||
// ── 启动服务 ──────────────────────────────
|
||
|
||
app.listen(PORT, '0.0.0.0', () => {
|
||
console.log('═══════════════════════════════════════════════');
|
||
console.log(' 🛡️ 铸渊运维守卫 · ZY-OPS-001 已启动');
|
||
console.log(` 端口: ${PORT}`);
|
||
console.log(` 面板: http://127.0.0.1:${PORT}/ops/`);
|
||
console.log(' 巡检: 快速5分钟 · 深度1小时 · 全量每天8:00');
|
||
console.log(' 签发: 铸渊 · ICE-GL-ZY001');
|
||
console.log('═══════════════════════════════════════════════');
|
||
|
||
memory.logEvent('startup', { port: PORT, version: '1.0.0' });
|
||
|
||
// 启动后立即做一次快速巡检
|
||
setTimeout(() => {
|
||
console.log('[OPS] 启动后首次巡检...');
|
||
runQuickCheck().catch(err => {
|
||
console.error('[OPS] 首次巡检失败:', err.message);
|
||
});
|
||
}, 5000);
|
||
});
|