guanghulab/scripts/deputy-auto-repair.js

#!/usr/bin/env node
// ═══════════════════════════════════════════════
// 🔺 Sovereign: TCS-0002∞ | Root: SYS-GLW-0001
// 📜 Copyright: 国作登字-2026-A-00037559
// ═══════════════════════════════════════════════
// scripts/deputy-auto-repair.js
// 🔧 铸渊副将自动修复引擎 v1.0
//
// 核心理念 (冰朔第三十一次对话):
//   副将在铸渊休眠时自动处理系统报错
//   用铸渊配置的思维逻辑和推理能力
//   看到报错日志后自动运行修复
//   最多修复3次 · 仍失败则推送给人类干预
//
// 修复流程:
//   1. 读取部署日志 → 分析错误
//   2. 匹配已知修复方案 (经验数据库)
//   3. SSH到服务器执行修复命令
//   4. 验证修复结果 (健康检查)
//   5. 如需要 → 调用LLM深度推理
//   6. 记录修复过程到经验库
//
// 环境变量:
//   GH_TOKEN — GitHub API
//   ZY_SERVER_HOST, ZY_SERVER_USER — SSH访问
//   ZY_LLM_API_KEY, ZY_LLM_BASE_URL — LLM深度推理
//   LOG_FILE — 部署日志文件路径
//   RUN_ID — 触发的工作流运行ID
// ═══════════════════════════════════════════════

'use strict';

const { execFileSync } = require('child_process');
const fs = require('fs');
const path = require('path');
const https = require('https');
const http = require('http');

const ROOT = path.resolve(__dirname, '..');
const LOGS_DIR = path.join(ROOT, 'data/deploy-logs');
const REPAIR_HISTORY = path.join(LOGS_DIR, 'repair-history.json');
const INDEX_FILE = path.join(LOGS_DIR, 'latest-index.json');
const EXPERIENCE_DB = path.join(ROOT, 'brain/dev-experience/experience-db.json');

const MAX_REPAIR_ATTEMPTS = 3;

// ── 辅助：判断是否需要人工干预 ─────────────────
function shouldRequestHuman(repairSuccess, attemptNumber) {
  return !repairSuccess && attemptNumber >= MAX_REPAIR_ATTEMPTS;
}

// ── 修复策略库 (副将的思维逻辑) ────────────────
const REPAIR_STRATEGIES = [
  {
    id: 'REPAIR-PM2-RESTART',
    name: 'PM2进程重启',
    triggers: ['EADDRINUSE', 'ECONNREFUSED', 'HEALTH_FAIL', '502 Bad Gateway', 'pm2.*error'],
    commands: [
      'pm2 delete all 2>/dev/null || true',
      'sleep 2',
      'pm2 start /opt/zhuyuan/config/pm2/ecosystem.config.js',
      'pm2 save',
      'sleep 5'
    ],
    verify: 'curl -sf http://localhost:3800/api/health || exit 1',
    severity: 'medium',
    success_rate: '高 · PM2重启通常能解决进程级问题'
  },
  {
    id: 'REPAIR-NPM-INSTALL',
    name: 'Node依赖重装',
    triggers: ['MODULE_NOT_FOUND', 'npm ERR!', 'Cannot find module'],
    commands: [
      'cd /opt/zhuyuan/app',
      'rm -rf node_modules',
      'npm install --production 2>&1 | tail -20',
      'pm2 restart all',
      'sleep 5'
    ],
    verify: 'curl -sf http://localhost:3800/api/health || exit 1',
    severity: 'medium',
    success_rate: '高 · 依赖问题通过重装通常能解决'
  },
  {
    id: 'REPAIR-NGINX-RELOAD',
    name: 'Nginx配置重载',
    triggers: ['nginx.*failed', 'nginx.*error', '502 Bad Gateway'],
    commands: [
      'sudo nginx -t 2>&1',
      'sudo systemctl reload nginx 2>&1 || sudo systemctl restart nginx 2>&1',
      'sleep 3'
    ],
    verify: 'curl -sf http://localhost:80/ -o /dev/null || exit 1',
    severity: 'medium',
    success_rate: '中 · 取决于配置文件是否正确'
  },
  {
    id: 'REPAIR-DISK-CLEANUP',
    name: '磁盘空间清理',
    triggers: ['ENOSPC', '磁盘空间不足'],
    commands: [
      'sudo journalctl --vacuum-time=2d 2>/dev/null || true',
      'pm2 flush 2>/dev/null || true',
      'sudo find /tmp -type f -mtime +1 -delete 2>/dev/null || true',
      'df -h /'
    ],
    verify: 'USED=$(df / | tail -1 | awk \'{print $5}\' | tr -d \'%\'); [ "$USED" -lt 90 ] || exit 1',
    severity: 'high',
    success_rate: '中 · 取决于可清理的文件量'
  },
  {
    id: 'REPAIR-MEMORY-FREE',
    name: '内存释放',
    triggers: ['ENOMEM', '内存不足'],
    commands: [
      'pm2 delete all 2>/dev/null || true',
      'sync && echo 3 | sudo tee /proc/sys/vm/drop_caches',
      'sleep 3',
      'pm2 start /opt/zhuyuan/config/pm2/ecosystem.config.js',
      'pm2 save',
      'sleep 5',
      'free -m'
    ],
    verify: 'curl -sf http://localhost:3800/api/health || exit 1',
    severity: 'high',
    success_rate: '中 · 重启进程通常能释放泄漏的内存'
  }
];

// ── SSH命令执行 (安全化·使用execFileSync避免shell注入) ────
function sshExec(command) {
  const host = process.env.ZY_SERVER_HOST;
  const user = process.env.ZY_SERVER_USER;

  if (!host || !user) {
    throw new Error('SSH配置缺失 (ZY_SERVER_HOST/ZY_SERVER_USER)');
  }

  // 安全检查：命令必须通过危险命令检测
  if (isDangerousCommand(command)) {
    return { ok: false, output: '', error: '⛔ 命令被安全策略拦截' };
  }

  const sshKeyPath = path.join(process.env.HOME || '/root', '.ssh', 'zy_key');

  try {
    // 使用execFileSync直接调用ssh二进制文件，避免shell解析
    // command作为ssh的远程执行参数，由ssh在远端服务器上执行
    const output = execFileSync('ssh', [
      '-i', sshKeyPath,
      '-o', 'ConnectTimeout=10',
      `${user}@${host}`,
      command
    ], {
      timeout: 60000,
      encoding: 'utf8',
      stdio: ['pipe', 'pipe', 'pipe']
    });
    return { ok: true, output: output.trim() };
  } catch (err) {
    return {
      ok: false,
      output: err.stdout ? err.stdout.trim() : '',
      error: err.stderr ? err.stderr.trim() : err.message
    };
  }
}

// ── LLM深度分析 ────────────────────────────
async function llmAnalysis(errorLog) {
  const apiKey = process.env.ZY_LLM_API_KEY;
  const baseUrl = process.env.ZY_LLM_BASE_URL;

  if (!apiKey || !baseUrl) {
    return { ok: false, analysis: 'LLM API未配置' };
  }

  const prompt = `你是铸渊副将(ZY-DEPUTY-001)，光湖系统的自动化运维代理。
以下是部署失败的日志，请分析根因并给出可在服务器上执行的修复命令。

## 日志:
\`\`\`
${errorLog.slice(0, 3000)}
\`\`\`

请用JSON格式回答:
{
  "root_cause": "一句话根因",
  "severity": "low/medium/high/critical",
  "fix_commands": ["命令1", "命令2"],
  "verify_command": "验证修复的命令",
  "needs_human": true/false,
  "explanation": "详细解释"
}`;

  try {
    const urlObj = new URL(`${baseUrl}/chat/completions`);
    const isHttps = urlObj.protocol === 'https:';
    const requestModule = isHttps ? https : http;

    const body = JSON.stringify({
      model: process.env.ZY_LLM_MODEL || 'deepseek-chat',
      messages: [
        { role: 'system', content: '你是铸渊副将，精通Linux服务器运维、Node.js、Nginx、PM2。返回纯JSON格式。' },
        { role: 'user', content: prompt }
      ],
      temperature: 0.2,
      max_tokens: 1000
    });

    const result = await new Promise((resolve, reject) => {
      const req = requestModule.request({
        hostname: urlObj.hostname,
        port: urlObj.port || (isHttps ? 443 : 80),
        path: urlObj.pathname,
        method: 'POST',
        headers: {
          'Content-Type': 'application/json',
          'Authorization': `Bearer ${apiKey}`,
          'Content-Length': Buffer.byteLength(body)
        },
        timeout: 30000
      }, (res) => {
        let data = '';
        res.on('data', (d) => { data += d; });
        res.on('end', () => {
          try {
            const json = JSON.parse(data);
            const content = json.choices?.[0]?.message?.content || '';
            // 尝试提取JSON部分
            const jsonMatch = content.match(/\{[\s\S]*\}/);
            if (jsonMatch) {
              resolve(JSON.parse(jsonMatch[0]));
            } else {
              resolve({ root_cause: content.slice(0, 200), fix_commands: [], needs_human: true });
            }
          } catch {
            resolve({ root_cause: 'LLM响应解析失败', fix_commands: [], needs_human: true });
          }
        });
      });
      req.on('error', reject);
      req.on('timeout', () => { req.destroy(); reject(new Error('LLM API超时')); });
      req.write(body);
      req.end();
    });

    return { ok: true, analysis: result };
  } catch (err) {
    return { ok: false, analysis: `LLM调用失败: ${err.message}` };
  }
}

// ── 修复主流程 ────────────────────────────
async function repair() {
  console.log('🔧 铸渊副将自动修复引擎 v1.0');
  console.log('═'.repeat(60));

  const logFile = process.env.LOG_FILE;
  const runId = process.env.RUN_ID;
  const conclusion = process.env.DEPLOY_CONCLUSION;
  const workflowName = process.env.WORKFLOW_NAME;

  // §1 读取部署日志
  let logData;
  if (logFile) {
    const logPath = path.join(ROOT, logFile);
    try {
      logData = JSON.parse(fs.readFileSync(logPath, 'utf8'));
      console.log(`📋 读取日志: ${logFile}`);
    } catch (err) {
      console.error(`❌ 无法读取日志文件: ${err.message}`);
    }
  }

  // §2 加载修复历史
  let history;
  try {
    history = JSON.parse(fs.readFileSync(REPAIR_HISTORY, 'utf8'));
  } catch {
    history = { repairs: [] };
  }

  // 检查此运行ID的修复次数
  const existingRepairs = history.repairs.filter(r => r.run_id === runId);
  const attemptNumber = existingRepairs.length + 1;

  if (attemptNumber > MAX_REPAIR_ATTEMPTS) {
    console.log(`❌ 已达最大修复次数(${MAX_REPAIR_ATTEMPTS}) · 放弃修复 · 通知人类`);
    setOutput('repair_success', 'false');
    setOutput('repair_attempt', String(attemptNumber - 1));
    setOutput('needs_human', 'true');
    return;
  }

  console.log(`🔧 修复尝试 #${attemptNumber}/${MAX_REPAIR_ATTEMPTS}`);
  console.log('');

  // §3 分析错误并选择修复策略
  const errorContent = logData
    ? logData.failed_jobs?.map(j => j.log_content).join('\n') || ''
    : '';

  let selectedStrategies = [];

  // 先用模式匹配选择策略
  for (const strategy of REPAIR_STRATEGIES) {
    for (const trigger of strategy.triggers) {
      const regex = new RegExp(trigger, 'i');
      if (regex.test(errorContent) || regex.test(conclusion || '')) {
        if (!selectedStrategies.find(s => s.id === strategy.id)) {
          selectedStrategies.push(strategy);
        }
      }
    }
  }

  // 如果没匹配到策略，默认用PM2重启
  if (selectedStrategies.length === 0) {
    console.log('ℹ️ 未匹配到具体修复策略 · 使用默认修复(PM2重启)');
    selectedStrategies = [REPAIR_STRATEGIES[0]]; // PM2重启是通用修复
  }

  console.log(`📋 选择 ${selectedStrategies.length} 个修复策略:`);
  for (const s of selectedStrategies) {
    console.log(`   - ${s.name} (${s.id})`);
  }
  console.log('');

  // §4 执行修复
  let repairSuccess = false;
  const repairRecord = {
    run_id: runId,
    workflow: workflowName,
    attempt: attemptNumber,
    timestamp: new Date().toISOString(),
    strategies_applied: [],
    results: [],
    final_success: false
  };

  for (const strategy of selectedStrategies) {
    console.log(`🔧 执行修复策略: ${strategy.name}`);
    console.log('─'.repeat(40));

    const strategyResult = {
      strategy_id: strategy.id,
      strategy_name: strategy.name,
      commands_executed: [],
      verify_result: null,
      success: false
    };

    // 执行修复命令
    let allCommandsOk = true;
    for (const cmd of strategy.commands) {
      console.log(`  $ ${cmd}`);
      const result = sshExec(cmd);
      strategyResult.commands_executed.push({
        command: cmd,
        ok: result.ok,
        output: result.output?.slice(0, 500),
        error: result.error?.slice(0, 300)
      });

      if (result.output) {
        console.log(`    ${result.output.split('\n').slice(0, 3).join('\n    ')}`);
      }
      if (!result.ok) {
        console.log(`    ⚠️ 命令执行异常: ${result.error?.slice(0, 100)}`);
        // 不中断，继续执行其他命令
      }
    }

    // 验证修复结果
    console.log(`  🔍 验证: ${strategy.verify}`);
    const verifyResult = sshExec(strategy.verify);
    strategyResult.verify_result = {
      ok: verifyResult.ok,
      output: verifyResult.output?.slice(0, 300)
    };

    if (verifyResult.ok) {
      console.log('  ✅ 验证通过 · 修复成功');
      strategyResult.success = true;
      repairSuccess = true;
    } else {
      console.log('  ❌ 验证失败 · 修复未成功');
    }

    repairRecord.strategies_applied.push(strategy.id);
    repairRecord.results.push(strategyResult);
    console.log('');

    if (repairSuccess) break;
  }

  // §5 如果简单修复失败，尝试LLM深度分析
  if (!repairSuccess && attemptNumber >= 2) {
    console.log('🧠 简单修复失败 · 启动LLM深度推理...');
    const llmResult = await llmAnalysis(errorContent);

    if (llmResult.ok && llmResult.analysis) {
      const analysis = llmResult.analysis;
      console.log(`  📝 根因: ${analysis.root_cause || 'N/A'}`);
      console.log(`  ⚠️ 严重程度: ${analysis.severity || 'N/A'}`);
      console.log(`  🔧 修复命令: ${(analysis.fix_commands || []).length} 条`);
      console.log(`  👤 需要人工: ${analysis.needs_human ? '是' : '否'}`);

      // 如果LLM建议的修复命令看起来安全，执行它们
      if (!analysis.needs_human && analysis.fix_commands && analysis.fix_commands.length > 0) {
        console.log('');
        console.log('🤖 执行LLM建议的修复命令:');
        for (const cmd of analysis.fix_commands) {
          // 安全检查：不执行危险命令
          if (isDangerousCommand(cmd)) {
            console.log(`  ⛔ 跳过危险命令: ${cmd}`);
            continue;
          }
          console.log(`  $ ${cmd}`);
          const result = sshExec(cmd);
          if (result.output) {
            console.log(`    ${result.output.split('\n').slice(0, 3).join('\n    ')}`);
          }
        }

        // 验证
        if (analysis.verify_command && !isDangerousCommand(analysis.verify_command)) {
          const verifyResult = sshExec(analysis.verify_command);
          if (verifyResult.ok) {
            console.log('  ✅ LLM修复方案验证通过');
            repairSuccess = true;
          }
        }
      }

      repairRecord.llm_analysis = {
        root_cause: analysis.root_cause,
        severity: analysis.severity,
        needs_human: analysis.needs_human,
        commands_count: (analysis.fix_commands || []).length
      };
    } else {
      console.log(`  ⚠️ LLM分析失败: ${llmResult.analysis}`);
    }
  }

  // §6 记录修复结果
  repairRecord.final_success = repairSuccess;
  history.repairs.push(repairRecord);
  // 保留最近50条修复记录
  history.repairs = history.repairs.slice(-50);
  fs.writeFileSync(REPAIR_HISTORY, JSON.stringify(history, null, 2) + '\n');

  // 更新日志索引
  try {
    const index = JSON.parse(fs.readFileSync(INDEX_FILE, 'utf8'));
    index.stats.repair_attempted = (index.stats.repair_attempted || 0) + 1;
    if (repairSuccess) {
      index.stats.repair_success = (index.stats.repair_success || 0) + 1;
    }
    index.last_updated = new Date().toISOString();
    fs.writeFileSync(INDEX_FILE, JSON.stringify(index, null, 2) + '\n');
  } catch (err) {
    console.log(`⚠️ 索引更新失败: ${err.message}`);
  }

  // §7 设置输出
  const needsHuman = shouldRequestHuman(repairSuccess, attemptNumber);
  setOutput('repair_success', repairSuccess ? 'true' : 'false');
  setOutput('repair_attempt', String(attemptNumber));
  setOutput('needs_human', needsHuman ? 'true' : 'false');

  console.log('═'.repeat(60));
  if (repairSuccess) {
    console.log(`✅ 修复成功 · 第${attemptNumber}次尝试`);
  } else if (needsHuman) {
    console.log(`❌ ${MAX_REPAIR_ATTEMPTS}次修复均失败 · 需要人工干预`);
  } else {
    console.log(`❌ 第${attemptNumber}次修复失败 · 还有${MAX_REPAIR_ATTEMPTS - attemptNumber}次尝试机会 · 等待下次重试`);
  }
}

// ── 危险命令检测 ────────────────────────────
function isDangerousCommand(cmd) {
  const dangerous = [
    /rm\s+-r[fF]*\s+(\/|~|\$HOME)/,  // rm -rf / or ~ or $HOME
    /mkfs/,               // 格式化
    /dd\s+if=/,           // 磁盘写入
    /shutdown/,           // 关机
    /reboot/,             // 重启系统
    /init\s+0/,           // 关机
    /:\(\)\{/,            // fork bomb
    />\s*\/dev\/sd/,      // 直接写磁盘设备
    /chmod\s+-R\s+777\s+\//, // 全局777
    /userdel/,            // 删除用户
    /passwd/,             // 修改密码
    /rm\s+-r[fF]*\s+\/\w/,  // rm -rf /home etc
  ];
  return dangerous.some(regex => regex.test(cmd));
}

// ── 工具函数 ────────────────────────────────
function setOutput(key, value) {
  if (process.env.GITHUB_OUTPUT) {
    fs.appendFileSync(process.env.GITHUB_OUTPUT, `${key}=${value}\n`);
  }
}

function parseArgs() {
  const args = process.argv.slice(2);
  const cmd = args[0];
  const params = {};
  for (let i = 1; i < args.length; i++) {
    if (args[i].startsWith('--')) {
      params[args[i].slice(2)] = args[i + 1] || '';
      i++;
    }
  }
  return { cmd, ...params };
}

// ── 主入口 ──────────────────────────────────
async function main() {
  const args = parseArgs();

  switch (args.cmd) {
    case 'repair':
      await repair();
      break;
    default:
      console.log('🔧 铸渊副将自动修复引擎 v1.0');
      console.log('');
      console.log('用法:');
      console.log('  node deputy-auto-repair.js repair');
      console.log('');
      console.log('环境变量:');
      console.log('  LOG_FILE — 部署日志文件路径');
      console.log('  RUN_ID — 工作流运行ID');
      console.log('  ZY_SERVER_HOST/ZY_SERVER_USER — SSH配置');
      console.log('  ZY_LLM_API_KEY/ZY_LLM_BASE_URL — LLM API');
      break;
  }
}

main().catch(err => {
  console.error(`❌ 自动修复引擎错误: ${err.message}`);
  process.exit(1);
});