365 lines
12 KiB
JavaScript
365 lines
12 KiB
JavaScript
#!/usr/bin/env node
|
||
// ═══════════════════════════════════════════════
|
||
// 🔺 Sovereign: TCS-0002∞ | Root: SYS-GLW-0001
|
||
// 📜 Copyright: 国作登字-2026-A-00037559
|
||
// ═══════════════════════════════════════════════
|
||
// server/proxy/service/proxy-guardian.js
|
||
// 🛡️ 铸渊专线 · 代理守护Agent
|
||
//
|
||
// 自动处理代理服务的各种异常:
|
||
// - Xray进程监控 (崩溃自动重启)
|
||
// - 端口可达性检测
|
||
// - IP声誉检查
|
||
// - TLS指纹一致性
|
||
// - 异常流量模式检测
|
||
//
|
||
// 遇到无法自动解决的问题时:
|
||
// - 调用LLM API进行推理分析
|
||
// - 生成诊断报告
|
||
// - 通过邮件发送告警
|
||
//
|
||
// 运行方式: PM2 managed (zy-proxy-guardian)
|
||
// 检查间隔: 每10分钟
|
||
// ═══════════════════════════════════════════════
|
||
|
||
'use strict';
|
||
|
||
const { execSync, execFileSync } = require('child_process');
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
const https = require('https');
|
||
const http = require('http');
|
||
|
||
const DATA_DIR = process.env.ZY_PROXY_DATA_DIR || '/opt/zhuyuan-brain/proxy/data';
|
||
const LOG_DIR = process.env.ZY_PROXY_LOG_DIR || '/opt/zhuyuan-brain/proxy/logs';
|
||
const GUARDIAN_FILE = path.join(DATA_DIR, 'guardian-status.json');
|
||
const CHECK_INTERVAL = 10 * 60 * 1000; // 10分钟
|
||
|
||
// ── 检查项定义 ───────────────────────────────
|
||
const CHECKS = {
|
||
xray_process: '检查Xray进程是否运行',
|
||
port_443: '检查443端口是否可达',
|
||
subscription_service: '检查订阅服务是否响应',
|
||
disk_space: '检查磁盘空间',
|
||
memory_usage: '检查内存使用',
|
||
xray_errors: '检查Xray错误日志'
|
||
};
|
||
|
||
// ── 读取守护状态 ─────────────────────────────
|
||
function readGuardianStatus() {
|
||
try {
|
||
return JSON.parse(fs.readFileSync(GUARDIAN_FILE, 'utf8'));
|
||
} catch {
|
||
return {
|
||
last_check: null,
|
||
consecutive_failures: 0,
|
||
auto_fixes: 0,
|
||
llm_consultations: 0,
|
||
last_issue: null,
|
||
status: 'initializing'
|
||
};
|
||
}
|
||
}
|
||
|
||
// ── 保存守护状态 ─────────────────────────────
|
||
function saveGuardianStatus(status) {
|
||
if (!fs.existsSync(DATA_DIR)) {
|
||
fs.mkdirSync(DATA_DIR, { recursive: true });
|
||
}
|
||
fs.writeFileSync(GUARDIAN_FILE, JSON.stringify(status, null, 2));
|
||
}
|
||
|
||
// ── 执行命令 ─────────────────────────────────
|
||
function runCmd(cmd, timeout = 10000) {
|
||
try {
|
||
return { ok: true, output: execSync(cmd, { encoding: 'utf8', timeout }).trim() };
|
||
} catch (err) {
|
||
return { ok: false, output: err.message };
|
||
}
|
||
}
|
||
|
||
// ── 检查Xray进程 ────────────────────────────
|
||
function checkXrayProcess() {
|
||
const result = runCmd('pgrep -x xray');
|
||
if (result.ok && result.output) {
|
||
return { ok: true, detail: `PID: ${result.output}` };
|
||
}
|
||
return { ok: false, detail: 'Xray进程未运行', fixable: true };
|
||
}
|
||
|
||
// ── 检查端口 ─────────────────────────────────
|
||
function checkPort443() {
|
||
const result = runCmd('ss -tlnp | grep ":443 "');
|
||
if (result.ok && result.output.includes('443')) {
|
||
return { ok: true, detail: '443端口正在监听' };
|
||
}
|
||
return { ok: false, detail: '443端口未监听', fixable: true };
|
||
}
|
||
|
||
// ── 检查订阅服务 ─────────────────────────────
|
||
function checkSubscriptionService() {
|
||
return new Promise((resolve) => {
|
||
const req = http.get('http://127.0.0.1:3803/health', { timeout: 5000 }, (res) => {
|
||
let data = '';
|
||
res.on('data', (d) => { data += d; });
|
||
res.on('end', () => {
|
||
try {
|
||
const json = JSON.parse(data);
|
||
resolve({ ok: json.status === 'ok', detail: '订阅服务正常' });
|
||
} catch {
|
||
resolve({ ok: false, detail: '订阅服务响应异常', fixable: true });
|
||
}
|
||
});
|
||
});
|
||
req.on('error', () => {
|
||
resolve({ ok: false, detail: '订阅服务无法连接', fixable: true });
|
||
});
|
||
req.on('timeout', () => {
|
||
req.destroy();
|
||
resolve({ ok: false, detail: '订阅服务响应超时', fixable: true });
|
||
});
|
||
});
|
||
}
|
||
|
||
// ── 检查磁盘空间 ─────────────────────────────
|
||
function checkDiskSpace() {
|
||
const result = runCmd("df -h / | tail -1 | awk '{print $5}'");
|
||
if (result.ok) {
|
||
const usage = parseInt(result.output.replace('%', ''), 10);
|
||
if (usage < 90) {
|
||
return { ok: true, detail: `磁盘使用: ${usage}%` };
|
||
}
|
||
return { ok: false, detail: `磁盘使用过高: ${usage}%`, fixable: false };
|
||
}
|
||
return { ok: false, detail: '无法检查磁盘空间', fixable: false };
|
||
}
|
||
|
||
// ── 检查内存使用 ─────────────────────────────
|
||
function checkMemory() {
|
||
const result = runCmd("free -m | grep Mem | awk '{print int($3/$2*100)}'");
|
||
if (result.ok) {
|
||
const usage = parseInt(result.output, 10);
|
||
if (usage < 90) {
|
||
return { ok: true, detail: `内存使用: ${usage}%` };
|
||
}
|
||
return { ok: false, detail: `内存使用过高: ${usage}%`, fixable: false };
|
||
}
|
||
return { ok: false, detail: '无法检查内存', fixable: false };
|
||
}
|
||
|
||
// ── 检查Xray错误日志 ────────────────────────
|
||
function checkXrayErrors() {
|
||
const errorLog = path.join(LOG_DIR, 'error.log');
|
||
try {
|
||
if (!fs.existsSync(errorLog)) {
|
||
return { ok: true, detail: '无错误日志' };
|
||
}
|
||
const stat = fs.statSync(errorLog);
|
||
const fiveMinAgo = Date.now() - 5 * 60 * 1000;
|
||
|
||
if (stat.mtimeMs < fiveMinAgo) {
|
||
return { ok: true, detail: '最近5分钟无新错误' };
|
||
}
|
||
|
||
const lines = fs.readFileSync(errorLog, 'utf8').split('\n').slice(-10);
|
||
const criticalErrors = lines.filter(l =>
|
||
l.includes('failed') || l.includes('error') || l.includes('panic')
|
||
);
|
||
|
||
if (criticalErrors.length === 0) {
|
||
return { ok: true, detail: '无严重错误' };
|
||
}
|
||
|
||
return {
|
||
ok: false,
|
||
detail: `发现${criticalErrors.length}个错误: ${criticalErrors[0].slice(0, 100)}`,
|
||
fixable: false,
|
||
errors: criticalErrors
|
||
};
|
||
} catch {
|
||
return { ok: true, detail: '错误日志检查跳过' };
|
||
}
|
||
}
|
||
|
||
// ── 自动修复 ─────────────────────────────────
|
||
function autoFix(checkName) {
|
||
console.log(`[守护Agent] 尝试自动修复: ${checkName}`);
|
||
|
||
switch (checkName) {
|
||
case 'xray_process':
|
||
case 'port_443':
|
||
// 重启Xray
|
||
const restartResult = runCmd('systemctl restart xray', 30000);
|
||
if (restartResult.ok) {
|
||
console.log('[守护Agent] Xray已重启');
|
||
return true;
|
||
}
|
||
console.error('[守护Agent] Xray重启失败:', restartResult.output);
|
||
return false;
|
||
|
||
case 'subscription_service':
|
||
// 通过PM2重启订阅服务
|
||
const pmResult = runCmd('pm2 restart zy-proxy-v2-sub', 15000);
|
||
if (pmResult.ok) {
|
||
console.log('[守护Agent] 订阅服务已重启');
|
||
return true;
|
||
}
|
||
return false;
|
||
|
||
default:
|
||
return false;
|
||
}
|
||
}
|
||
|
||
// ── 调用LLM推理 (V3: 使用llm-router动态路由) ──
|
||
async function consultLLM(issue) {
|
||
let llmRouter;
|
||
try {
|
||
llmRouter = require('./llm-router');
|
||
} catch {
|
||
console.log('[守护Agent] LLM路由器未加载，跳过推理');
|
||
return null;
|
||
}
|
||
|
||
const prompt = `你是光湖语言世界VPN的守护Agent。以下是当前检测到的问题:
|
||
|
||
问题描述: ${issue.detail}
|
||
检查项: ${issue.checkName}
|
||
连续失败次数: ${issue.consecutiveFailures}
|
||
自动修复尝试: ${issue.autoFixAttempted ? '已尝试，失败' : '未尝试'}
|
||
|
||
请分析可能的原因并给出解决建议。回答要简洁明确。`;
|
||
|
||
const result = await llmRouter.callLLM(prompt, {
|
||
systemPrompt: '你是光湖语言世界VPN系统的守护Agent AI。负责诊断代理服务异常并给出修复建议。',
|
||
maxTokens: 500,
|
||
timeout: 30000
|
||
});
|
||
|
||
return result ? result.content : null;
|
||
}
|
||
|
||
// ── 发送告警邮件 ─────────────────────────────
|
||
function sendAlertEmail(subject, body) {
|
||
try {
|
||
const sendScript = path.join(__dirname, 'send-subscription.js');
|
||
// 使用execFileSync避免Shell命令注入 (不经过shell解释器)
|
||
execFileSync('node', [sendScript, 'alert', `${subject}\n\n${body}`], {
|
||
encoding: 'utf8',
|
||
timeout: 30000
|
||
});
|
||
console.log('[守护Agent] 告警邮件已发送');
|
||
} catch (err) {
|
||
console.error('[守护Agent] 邮件发送失败:', err.message);
|
||
}
|
||
}
|
||
|
||
// ── 主巡检 ───────────────────────────────────
|
||
async function patrol() {
|
||
console.log('[守护Agent] 开始巡检...');
|
||
|
||
const status = readGuardianStatus();
|
||
const results = {};
|
||
let hasIssue = false;
|
||
|
||
// 执行所有检查
|
||
results.xray_process = checkXrayProcess();
|
||
results.port_443 = checkPort443();
|
||
results.subscription_service = await checkSubscriptionService();
|
||
results.disk_space = checkDiskSpace();
|
||
results.memory_usage = checkMemory();
|
||
results.xray_errors = checkXrayErrors();
|
||
|
||
// 分析结果
|
||
const issues = [];
|
||
for (const [name, result] of Object.entries(results)) {
|
||
if (!result.ok) {
|
||
hasIssue = true;
|
||
issues.push({ checkName: name, ...result });
|
||
|
||
// 尝试自动修复
|
||
if (result.fixable) {
|
||
const fixed = autoFix(name);
|
||
if (fixed) {
|
||
status.auto_fixes++;
|
||
result.auto_fixed = true;
|
||
console.log(`[守护Agent] ✅ 自动修复成功: ${name}`);
|
||
} else {
|
||
result.auto_fixed = false;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 更新状态
|
||
status.last_check = new Date().toISOString();
|
||
status.checks = Object.fromEntries(
|
||
Object.entries(results).map(([k, v]) => [k, { ok: v.ok, detail: v.detail }])
|
||
);
|
||
|
||
if (hasIssue) {
|
||
status.consecutive_failures++;
|
||
status.last_issue = {
|
||
time: new Date().toISOString(),
|
||
issues: issues.map(i => ({ check: i.checkName, detail: i.detail }))
|
||
};
|
||
|
||
// 连续3次失败且有未修复的问题 → 调用LLM
|
||
const unfixedIssues = issues.filter(i => !i.auto_fixed && i.fixable !== false);
|
||
if (status.consecutive_failures >= 3 && unfixedIssues.length > 0) {
|
||
console.log('[守护Agent] 连续失败3次，调用LLM推理...');
|
||
for (const issue of unfixedIssues) {
|
||
const llmAdvice = await consultLLM({
|
||
...issue,
|
||
consecutiveFailures: status.consecutive_failures,
|
||
autoFixAttempted: true
|
||
});
|
||
if (llmAdvice) {
|
||
status.llm_consultations++;
|
||
console.log(`[守护Agent] LLM建议: ${llmAdvice.slice(0, 200)}`);
|
||
|
||
// 发送告警邮件（含LLM分析）
|
||
sendAlertEmail(
|
||
`🛡️ 铸渊专线告警: ${issue.checkName}`,
|
||
`问题: ${issue.detail}\n\n🤖 AI分析:\n${llmAdvice}`
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
// 首次失败也发送告警
|
||
if (status.consecutive_failures === 1) {
|
||
sendAlertEmail(
|
||
'🛡️ 铸渊专线异常检测',
|
||
issues.map(i => `- ${i.checkName}: ${i.detail}`).join('\n')
|
||
);
|
||
}
|
||
} else {
|
||
if (status.consecutive_failures > 0) {
|
||
console.log(`[守护Agent] ✅ 问题已恢复 (之前连续失败${status.consecutive_failures}次)`);
|
||
}
|
||
status.consecutive_failures = 0;
|
||
status.status = 'healthy';
|
||
}
|
||
|
||
saveGuardianStatus(status);
|
||
|
||
// 打印摘要
|
||
const okCount = Object.values(results).filter(r => r.ok).length;
|
||
const totalCount = Object.keys(results).length;
|
||
console.log(`[守护Agent] 巡检完成: ${okCount}/${totalCount} 通过`);
|
||
}
|
||
|
||
// ── 启动守护循环 ─────────────────────────────
|
||
console.log('🛡️ 光湖语言世界 · 守护Agent启动');
|
||
console.log(` 巡检间隔: ${CHECK_INTERVAL / 1000}秒`);
|
||
console.log(` LLM路由器: 动态多模型 (通过llm-router.js)`);
|
||
|
||
// 立即执行一次
|
||
patrol().catch(console.error);
|
||
|
||
// 定期执行
|
||
setInterval(() => {
|
||
patrol().catch(console.error);
|
||
}, CHECK_INTERVAL);
|