550 lines
18 KiB
JavaScript
550 lines
18 KiB
JavaScript
#!/usr/bin/env node
|
||
// ═══════════════════════════════════════════════
|
||
// 🔺 Sovereign: TCS-0002∞ | Root: SYS-GLW-0001
|
||
// 📜 Copyright: 国作登字-2026-A-00037559
|
||
// ═══════════════════════════════════════════════
|
||
// scripts/deputy-auto-repair.js
|
||
// 🔧 铸渊副将自动修复引擎 v1.0
|
||
//
|
||
// 核心理念 (冰朔第三十一次对话):
|
||
// 副将在铸渊休眠时自动处理系统报错
|
||
// 用铸渊配置的思维逻辑和推理能力
|
||
// 看到报错日志后自动运行修复
|
||
// 最多修复3次 · 仍失败则推送给人类干预
|
||
//
|
||
// 修复流程:
|
||
// 1. 读取部署日志 → 分析错误
|
||
// 2. 匹配已知修复方案 (经验数据库)
|
||
// 3. SSH到服务器执行修复命令
|
||
// 4. 验证修复结果 (健康检查)
|
||
// 5. 如需要 → 调用LLM深度推理
|
||
// 6. 记录修复过程到经验库
|
||
//
|
||
// 环境变量:
|
||
// GH_TOKEN — GitHub API
|
||
// ZY_SERVER_HOST, ZY_SERVER_USER — SSH访问
|
||
// ZY_LLM_API_KEY, ZY_LLM_BASE_URL — LLM深度推理
|
||
// LOG_FILE — 部署日志文件路径
|
||
// RUN_ID — 触发的工作流运行ID
|
||
// ═══════════════════════════════════════════════
|
||
|
||
'use strict';
|
||
|
||
const { execFileSync } = require('child_process');
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
const https = require('https');
|
||
const http = require('http');
|
||
|
||
const ROOT = path.resolve(__dirname, '..');
|
||
const LOGS_DIR = path.join(ROOT, 'data/deploy-logs');
|
||
const REPAIR_HISTORY = path.join(LOGS_DIR, 'repair-history.json');
|
||
const INDEX_FILE = path.join(LOGS_DIR, 'latest-index.json');
|
||
const EXPERIENCE_DB = path.join(ROOT, 'brain/dev-experience/experience-db.json');
|
||
|
||
const MAX_REPAIR_ATTEMPTS = 3;
|
||
|
||
// ── 辅助：判断是否需要人工干预 ─────────────────
|
||
function shouldRequestHuman(repairSuccess, attemptNumber) {
|
||
return !repairSuccess && attemptNumber >= MAX_REPAIR_ATTEMPTS;
|
||
}
|
||
|
||
// ── 修复策略库 (副将的思维逻辑) ────────────────
|
||
const REPAIR_STRATEGIES = [
|
||
{
|
||
id: 'REPAIR-PM2-RESTART',
|
||
name: 'PM2进程重启',
|
||
triggers: ['EADDRINUSE', 'ECONNREFUSED', 'HEALTH_FAIL', '502 Bad Gateway', 'pm2.*error'],
|
||
commands: [
|
||
'pm2 delete all 2>/dev/null || true',
|
||
'sleep 2',
|
||
'pm2 start /opt/zhuyuan/config/pm2/ecosystem.config.js',
|
||
'pm2 save',
|
||
'sleep 5'
|
||
],
|
||
verify: 'curl -sf http://localhost:3800/api/health || exit 1',
|
||
severity: 'medium',
|
||
success_rate: '高 · PM2重启通常能解决进程级问题'
|
||
},
|
||
{
|
||
id: 'REPAIR-NPM-INSTALL',
|
||
name: 'Node依赖重装',
|
||
triggers: ['MODULE_NOT_FOUND', 'npm ERR!', 'Cannot find module'],
|
||
commands: [
|
||
'cd /opt/zhuyuan/app',
|
||
'rm -rf node_modules',
|
||
'npm install --production 2>&1 | tail -20',
|
||
'pm2 restart all',
|
||
'sleep 5'
|
||
],
|
||
verify: 'curl -sf http://localhost:3800/api/health || exit 1',
|
||
severity: 'medium',
|
||
success_rate: '高 · 依赖问题通过重装通常能解决'
|
||
},
|
||
{
|
||
id: 'REPAIR-NGINX-RELOAD',
|
||
name: 'Nginx配置重载',
|
||
triggers: ['nginx.*failed', 'nginx.*error', '502 Bad Gateway'],
|
||
commands: [
|
||
'sudo nginx -t 2>&1',
|
||
'sudo systemctl reload nginx 2>&1 || sudo systemctl restart nginx 2>&1',
|
||
'sleep 3'
|
||
],
|
||
verify: 'curl -sf http://localhost:80/ -o /dev/null || exit 1',
|
||
severity: 'medium',
|
||
success_rate: '中 · 取决于配置文件是否正确'
|
||
},
|
||
{
|
||
id: 'REPAIR-DISK-CLEANUP',
|
||
name: '磁盘空间清理',
|
||
triggers: ['ENOSPC', '磁盘空间不足'],
|
||
commands: [
|
||
'sudo journalctl --vacuum-time=2d 2>/dev/null || true',
|
||
'pm2 flush 2>/dev/null || true',
|
||
'sudo find /tmp -type f -mtime +1 -delete 2>/dev/null || true',
|
||
'df -h /'
|
||
],
|
||
verify: 'USED=$(df / | tail -1 | awk \'{print $5}\' | tr -d \'%\'); [ "$USED" -lt 90 ] || exit 1',
|
||
severity: 'high',
|
||
success_rate: '中 · 取决于可清理的文件量'
|
||
},
|
||
{
|
||
id: 'REPAIR-MEMORY-FREE',
|
||
name: '内存释放',
|
||
triggers: ['ENOMEM', '内存不足'],
|
||
commands: [
|
||
'pm2 delete all 2>/dev/null || true',
|
||
'sync && echo 3 | sudo tee /proc/sys/vm/drop_caches',
|
||
'sleep 3',
|
||
'pm2 start /opt/zhuyuan/config/pm2/ecosystem.config.js',
|
||
'pm2 save',
|
||
'sleep 5',
|
||
'free -m'
|
||
],
|
||
verify: 'curl -sf http://localhost:3800/api/health || exit 1',
|
||
severity: 'high',
|
||
success_rate: '中 · 重启进程通常能释放泄漏的内存'
|
||
}
|
||
];
|
||
|
||
// ── SSH命令执行 (安全化·使用execFileSync避免shell注入) ────
|
||
function sshExec(command) {
|
||
const host = process.env.ZY_SERVER_HOST;
|
||
const user = process.env.ZY_SERVER_USER;
|
||
|
||
if (!host || !user) {
|
||
throw new Error('SSH配置缺失 (ZY_SERVER_HOST/ZY_SERVER_USER)');
|
||
}
|
||
|
||
// 安全检查：命令必须通过危险命令检测
|
||
if (isDangerousCommand(command)) {
|
||
return { ok: false, output: '', error: '⛔ 命令被安全策略拦截' };
|
||
}
|
||
|
||
const sshKeyPath = path.join(process.env.HOME || '/root', '.ssh', 'zy_key');
|
||
|
||
try {
|
||
// 使用execFileSync直接调用ssh二进制文件，避免shell解析
|
||
// command作为ssh的远程执行参数，由ssh在远端服务器上执行
|
||
const output = execFileSync('ssh', [
|
||
'-i', sshKeyPath,
|
||
'-o', 'ConnectTimeout=10',
|
||
`${user}@${host}`,
|
||
command
|
||
], {
|
||
timeout: 60000,
|
||
encoding: 'utf8',
|
||
stdio: ['pipe', 'pipe', 'pipe']
|
||
});
|
||
return { ok: true, output: output.trim() };
|
||
} catch (err) {
|
||
return {
|
||
ok: false,
|
||
output: err.stdout ? err.stdout.trim() : '',
|
||
error: err.stderr ? err.stderr.trim() : err.message
|
||
};
|
||
}
|
||
}
|
||
|
||
// ── LLM深度分析 ────────────────────────────
|
||
async function llmAnalysis(errorLog) {
|
||
const apiKey = process.env.ZY_LLM_API_KEY;
|
||
const baseUrl = process.env.ZY_LLM_BASE_URL;
|
||
|
||
if (!apiKey || !baseUrl) {
|
||
return { ok: false, analysis: 'LLM API未配置' };
|
||
}
|
||
|
||
const prompt = `你是铸渊副将(ZY-DEPUTY-001)，光湖系统的自动化运维代理。
|
||
以下是部署失败的日志，请分析根因并给出可在服务器上执行的修复命令。
|
||
|
||
## 日志:
|
||
\`\`\`
|
||
${errorLog.slice(0, 3000)}
|
||
\`\`\`
|
||
|
||
请用JSON格式回答:
|
||
{
|
||
"root_cause": "一句话根因",
|
||
"severity": "low/medium/high/critical",
|
||
"fix_commands": ["命令1", "命令2"],
|
||
"verify_command": "验证修复的命令",
|
||
"needs_human": true/false,
|
||
"explanation": "详细解释"
|
||
}`;
|
||
|
||
try {
|
||
const urlObj = new URL(`${baseUrl}/chat/completions`);
|
||
const isHttps = urlObj.protocol === 'https:';
|
||
const requestModule = isHttps ? https : http;
|
||
|
||
const body = JSON.stringify({
|
||
model: process.env.ZY_LLM_MODEL || 'deepseek-chat',
|
||
messages: [
|
||
{ role: 'system', content: '你是铸渊副将，精通Linux服务器运维、Node.js、Nginx、PM2。返回纯JSON格式。' },
|
||
{ role: 'user', content: prompt }
|
||
],
|
||
temperature: 0.2,
|
||
max_tokens: 1000
|
||
});
|
||
|
||
const result = await new Promise((resolve, reject) => {
|
||
const req = requestModule.request({
|
||
hostname: urlObj.hostname,
|
||
port: urlObj.port || (isHttps ? 443 : 80),
|
||
path: urlObj.pathname,
|
||
method: 'POST',
|
||
headers: {
|
||
'Content-Type': 'application/json',
|
||
'Authorization': `Bearer ${apiKey}`,
|
||
'Content-Length': Buffer.byteLength(body)
|
||
},
|
||
timeout: 30000
|
||
}, (res) => {
|
||
let data = '';
|
||
res.on('data', (d) => { data += d; });
|
||
res.on('end', () => {
|
||
try {
|
||
const json = JSON.parse(data);
|
||
const content = json.choices?.[0]?.message?.content || '';
|
||
// 尝试提取JSON部分
|
||
const jsonMatch = content.match(/\{[\s\S]*\}/);
|
||
if (jsonMatch) {
|
||
resolve(JSON.parse(jsonMatch[0]));
|
||
} else {
|
||
resolve({ root_cause: content.slice(0, 200), fix_commands: [], needs_human: true });
|
||
}
|
||
} catch {
|
||
resolve({ root_cause: 'LLM响应解析失败', fix_commands: [], needs_human: true });
|
||
}
|
||
});
|
||
});
|
||
req.on('error', reject);
|
||
req.on('timeout', () => { req.destroy(); reject(new Error('LLM API超时')); });
|
||
req.write(body);
|
||
req.end();
|
||
});
|
||
|
||
return { ok: true, analysis: result };
|
||
} catch (err) {
|
||
return { ok: false, analysis: `LLM调用失败: ${err.message}` };
|
||
}
|
||
}
|
||
|
||
// ── 修复主流程 ────────────────────────────
|
||
async function repair() {
|
||
console.log('🔧 铸渊副将自动修复引擎 v1.0');
|
||
console.log('═'.repeat(60));
|
||
|
||
const logFile = process.env.LOG_FILE;
|
||
const runId = process.env.RUN_ID;
|
||
const conclusion = process.env.DEPLOY_CONCLUSION;
|
||
const workflowName = process.env.WORKFLOW_NAME;
|
||
|
||
// §1 读取部署日志
|
||
let logData;
|
||
if (logFile) {
|
||
const logPath = path.join(ROOT, logFile);
|
||
try {
|
||
logData = JSON.parse(fs.readFileSync(logPath, 'utf8'));
|
||
console.log(`📋 读取日志: ${logFile}`);
|
||
} catch (err) {
|
||
console.error(`❌ 无法读取日志文件: ${err.message}`);
|
||
}
|
||
}
|
||
|
||
// §2 加载修复历史
|
||
let history;
|
||
try {
|
||
history = JSON.parse(fs.readFileSync(REPAIR_HISTORY, 'utf8'));
|
||
} catch {
|
||
history = { repairs: [] };
|
||
}
|
||
|
||
// 检查此运行ID的修复次数
|
||
const existingRepairs = history.repairs.filter(r => r.run_id === runId);
|
||
const attemptNumber = existingRepairs.length + 1;
|
||
|
||
if (attemptNumber > MAX_REPAIR_ATTEMPTS) {
|
||
console.log(`❌ 已达最大修复次数(${MAX_REPAIR_ATTEMPTS}) · 放弃修复 · 通知人类`);
|
||
setOutput('repair_success', 'false');
|
||
setOutput('repair_attempt', String(attemptNumber - 1));
|
||
setOutput('needs_human', 'true');
|
||
return;
|
||
}
|
||
|
||
console.log(`🔧 修复尝试 #${attemptNumber}/${MAX_REPAIR_ATTEMPTS}`);
|
||
console.log('');
|
||
|
||
// §3 分析错误并选择修复策略
|
||
const errorContent = logData
|
||
? logData.failed_jobs?.map(j => j.log_content).join('\n') || ''
|
||
: '';
|
||
|
||
let selectedStrategies = [];
|
||
|
||
// 先用模式匹配选择策略
|
||
for (const strategy of REPAIR_STRATEGIES) {
|
||
for (const trigger of strategy.triggers) {
|
||
const regex = new RegExp(trigger, 'i');
|
||
if (regex.test(errorContent) || regex.test(conclusion || '')) {
|
||
if (!selectedStrategies.find(s => s.id === strategy.id)) {
|
||
selectedStrategies.push(strategy);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 如果没匹配到策略，默认用PM2重启
|
||
if (selectedStrategies.length === 0) {
|
||
console.log('ℹ️ 未匹配到具体修复策略 · 使用默认修复(PM2重启)');
|
||
selectedStrategies = [REPAIR_STRATEGIES[0]]; // PM2重启是通用修复
|
||
}
|
||
|
||
console.log(`📋 选择 ${selectedStrategies.length} 个修复策略:`);
|
||
for (const s of selectedStrategies) {
|
||
console.log(` - ${s.name} (${s.id})`);
|
||
}
|
||
console.log('');
|
||
|
||
// §4 执行修复
|
||
let repairSuccess = false;
|
||
const repairRecord = {
|
||
run_id: runId,
|
||
workflow: workflowName,
|
||
attempt: attemptNumber,
|
||
timestamp: new Date().toISOString(),
|
||
strategies_applied: [],
|
||
results: [],
|
||
final_success: false
|
||
};
|
||
|
||
for (const strategy of selectedStrategies) {
|
||
console.log(`🔧 执行修复策略: ${strategy.name}`);
|
||
console.log('─'.repeat(40));
|
||
|
||
const strategyResult = {
|
||
strategy_id: strategy.id,
|
||
strategy_name: strategy.name,
|
||
commands_executed: [],
|
||
verify_result: null,
|
||
success: false
|
||
};
|
||
|
||
// 执行修复命令
|
||
let allCommandsOk = true;
|
||
for (const cmd of strategy.commands) {
|
||
console.log(` $ ${cmd}`);
|
||
const result = sshExec(cmd);
|
||
strategyResult.commands_executed.push({
|
||
command: cmd,
|
||
ok: result.ok,
|
||
output: result.output?.slice(0, 500),
|
||
error: result.error?.slice(0, 300)
|
||
});
|
||
|
||
if (result.output) {
|
||
console.log(` ${result.output.split('\n').slice(0, 3).join('\n ')}`);
|
||
}
|
||
if (!result.ok) {
|
||
console.log(` ⚠️ 命令执行异常: ${result.error?.slice(0, 100)}`);
|
||
// 不中断，继续执行其他命令
|
||
}
|
||
}
|
||
|
||
// 验证修复结果
|
||
console.log(` 🔍 验证: ${strategy.verify}`);
|
||
const verifyResult = sshExec(strategy.verify);
|
||
strategyResult.verify_result = {
|
||
ok: verifyResult.ok,
|
||
output: verifyResult.output?.slice(0, 300)
|
||
};
|
||
|
||
if (verifyResult.ok) {
|
||
console.log(' ✅ 验证通过 · 修复成功');
|
||
strategyResult.success = true;
|
||
repairSuccess = true;
|
||
} else {
|
||
console.log(' ❌ 验证失败 · 修复未成功');
|
||
}
|
||
|
||
repairRecord.strategies_applied.push(strategy.id);
|
||
repairRecord.results.push(strategyResult);
|
||
console.log('');
|
||
|
||
if (repairSuccess) break;
|
||
}
|
||
|
||
// §5 如果简单修复失败，尝试LLM深度分析
|
||
if (!repairSuccess && attemptNumber >= 2) {
|
||
console.log('🧠 简单修复失败 · 启动LLM深度推理...');
|
||
const llmResult = await llmAnalysis(errorContent);
|
||
|
||
if (llmResult.ok && llmResult.analysis) {
|
||
const analysis = llmResult.analysis;
|
||
console.log(` 📝 根因: ${analysis.root_cause || 'N/A'}`);
|
||
console.log(` ⚠️ 严重程度: ${analysis.severity || 'N/A'}`);
|
||
console.log(` 🔧 修复命令: ${(analysis.fix_commands || []).length} 条`);
|
||
console.log(` 👤 需要人工: ${analysis.needs_human ? '是' : '否'}`);
|
||
|
||
// 如果LLM建议的修复命令看起来安全，执行它们
|
||
if (!analysis.needs_human && analysis.fix_commands && analysis.fix_commands.length > 0) {
|
||
console.log('');
|
||
console.log('🤖 执行LLM建议的修复命令:');
|
||
for (const cmd of analysis.fix_commands) {
|
||
// 安全检查：不执行危险命令
|
||
if (isDangerousCommand(cmd)) {
|
||
console.log(` ⛔ 跳过危险命令: ${cmd}`);
|
||
continue;
|
||
}
|
||
console.log(` $ ${cmd}`);
|
||
const result = sshExec(cmd);
|
||
if (result.output) {
|
||
console.log(` ${result.output.split('\n').slice(0, 3).join('\n ')}`);
|
||
}
|
||
}
|
||
|
||
// 验证
|
||
if (analysis.verify_command && !isDangerousCommand(analysis.verify_command)) {
|
||
const verifyResult = sshExec(analysis.verify_command);
|
||
if (verifyResult.ok) {
|
||
console.log(' ✅ LLM修复方案验证通过');
|
||
repairSuccess = true;
|
||
}
|
||
}
|
||
}
|
||
|
||
repairRecord.llm_analysis = {
|
||
root_cause: analysis.root_cause,
|
||
severity: analysis.severity,
|
||
needs_human: analysis.needs_human,
|
||
commands_count: (analysis.fix_commands || []).length
|
||
};
|
||
} else {
|
||
console.log(` ⚠️ LLM分析失败: ${llmResult.analysis}`);
|
||
}
|
||
}
|
||
|
||
// §6 记录修复结果
|
||
repairRecord.final_success = repairSuccess;
|
||
history.repairs.push(repairRecord);
|
||
// 保留最近50条修复记录
|
||
history.repairs = history.repairs.slice(-50);
|
||
fs.writeFileSync(REPAIR_HISTORY, JSON.stringify(history, null, 2) + '\n');
|
||
|
||
// 更新日志索引
|
||
try {
|
||
const index = JSON.parse(fs.readFileSync(INDEX_FILE, 'utf8'));
|
||
index.stats.repair_attempted = (index.stats.repair_attempted || 0) + 1;
|
||
if (repairSuccess) {
|
||
index.stats.repair_success = (index.stats.repair_success || 0) + 1;
|
||
}
|
||
index.last_updated = new Date().toISOString();
|
||
fs.writeFileSync(INDEX_FILE, JSON.stringify(index, null, 2) + '\n');
|
||
} catch (err) {
|
||
console.log(`⚠️ 索引更新失败: ${err.message}`);
|
||
}
|
||
|
||
// §7 设置输出
|
||
const needsHuman = shouldRequestHuman(repairSuccess, attemptNumber);
|
||
setOutput('repair_success', repairSuccess ? 'true' : 'false');
|
||
setOutput('repair_attempt', String(attemptNumber));
|
||
setOutput('needs_human', needsHuman ? 'true' : 'false');
|
||
|
||
console.log('═'.repeat(60));
|
||
if (repairSuccess) {
|
||
console.log(`✅ 修复成功 · 第${attemptNumber}次尝试`);
|
||
} else if (needsHuman) {
|
||
console.log(`❌ ${MAX_REPAIR_ATTEMPTS}次修复均失败 · 需要人工干预`);
|
||
} else {
|
||
console.log(`❌ 第${attemptNumber}次修复失败 · 还有${MAX_REPAIR_ATTEMPTS - attemptNumber}次尝试机会 · 等待下次重试`);
|
||
}
|
||
}
|
||
|
||
// ── 危险命令检测 ────────────────────────────
|
||
function isDangerousCommand(cmd) {
|
||
const dangerous = [
|
||
/rm\s+-r[fF]*\s+(\/|~|\$HOME)/, // rm -rf / or ~ or $HOME
|
||
/mkfs/, // 格式化
|
||
/dd\s+if=/, // 磁盘写入
|
||
/shutdown/, // 关机
|
||
/reboot/, // 重启系统
|
||
/init\s+0/, // 关机
|
||
/:\(\)\{/, // fork bomb
|
||
/>\s*\/dev\/sd/, // 直接写磁盘设备
|
||
/chmod\s+-R\s+777\s+\//, // 全局777
|
||
/userdel/, // 删除用户
|
||
/passwd/, // 修改密码
|
||
/rm\s+-r[fF]*\s+\/\w/, // rm -rf /home etc
|
||
];
|
||
return dangerous.some(regex => regex.test(cmd));
|
||
}
|
||
|
||
// ── 工具函数 ────────────────────────────────
|
||
function setOutput(key, value) {
|
||
if (process.env.GITHUB_OUTPUT) {
|
||
fs.appendFileSync(process.env.GITHUB_OUTPUT, `${key}=${value}\n`);
|
||
}
|
||
}
|
||
|
||
function parseArgs() {
|
||
const args = process.argv.slice(2);
|
||
const cmd = args[0];
|
||
const params = {};
|
||
for (let i = 1; i < args.length; i++) {
|
||
if (args[i].startsWith('--')) {
|
||
params[args[i].slice(2)] = args[i + 1] || '';
|
||
i++;
|
||
}
|
||
}
|
||
return { cmd, ...params };
|
||
}
|
||
|
||
// ── 主入口 ──────────────────────────────────
|
||
async function main() {
|
||
const args = parseArgs();
|
||
|
||
switch (args.cmd) {
|
||
case 'repair':
|
||
await repair();
|
||
break;
|
||
default:
|
||
console.log('🔧 铸渊副将自动修复引擎 v1.0');
|
||
console.log('');
|
||
console.log('用法:');
|
||
console.log(' node deputy-auto-repair.js repair');
|
||
console.log('');
|
||
console.log('环境变量:');
|
||
console.log(' LOG_FILE — 部署日志文件路径');
|
||
console.log(' RUN_ID — 工作流运行ID');
|
||
console.log(' ZY_SERVER_HOST/ZY_SERVER_USER — SSH配置');
|
||
console.log(' ZY_LLM_API_KEY/ZY_LLM_BASE_URL — LLM API');
|
||
break;
|
||
}
|
||
}
|
||
|
||
main().catch(err => {
|
||
console.error(`❌ 自动修复引擎错误: ${err.message}`);
|
||
process.exit(1);
|
||
});
|