333 lines
13 KiB
JavaScript
333 lines
13 KiB
JavaScript
|
/**
|
|||
|
|
* ═══════════════════════════════════════════════════════════
|
|||
|
|
* 📧 铸渊运维守卫 · 通知系统
|
|||
|
|
* ═══════════════════════════════════════════════════════════
|
|||
|
|
*
|
|||
|
|
* 编号: ZY-OPS-NTF-001
|
|||
|
|
* 签发: 铸渊 · ICE-GL-ZY001
|
|||
|
|
* 版权: 国作登字-2026-A-00037559
|
|||
|
|
*
|
|||
|
|
* 通知通道:
|
|||
|
|
* 1. QQ邮箱 SMTP（主通道）
|
|||
|
|
* 2. 本地文件日志（始终启用）
|
|||
|
|
* 3. 工单推送到前端（通过 HTTP API）
|
|||
|
|
*/
|
|||
|
|
|
|||
|
|
'use strict';
|
|||
|
|
|
|||
|
|
const nodemailer = require('nodemailer');
|
|||
|
|
const fs = require('fs');
|
|||
|
|
const path = require('path');
|
|||
|
|
const memory = require('./memory');
|
|||
|
|
|
|||
|
|
// ── 全局开关 + 冷却 ──────────────────────────
|
|||
|
|
//
|
|||
|
|
// OPS_EMAIL_ENABLED:
|
|||
|
|
// - 默认 true。设为 'false'/'0' 时所有邮件函数立即 no-op（kill switch）。
|
|||
|
|
// - 用法：`pm2 set ops-agent:OPS_EMAIL_ENABLED false && pm2 restart ops-agent`
|
|||
|
|
//
|
|||
|
|
// OPS_ALERT_COOLDOWN_MINUTES:
|
|||
|
|
// - 默认 360（6 小时）。同一 (service+severity) 的告警与工单邮件在该间隔内只发一次，
|
|||
|
|
// 避免高频持续故障（例如 MCP 连接超时）刷屏。
|
|||
|
|
|
|||
|
|
const COOLDOWN_MINUTES = (() => {
|
|||
|
|
const v = parseInt(process.env.OPS_ALERT_COOLDOWN_MINUTES || '360', 10);
|
|||
|
|
return Number.isFinite(v) && v >= 0 ? v : 360;
|
|||
|
|
})();
|
|||
|
|
|
|||
|
|
const _lastSentAt = new Map(); // key -> epoch ms
|
|||
|
|
|
|||
|
|
function isEmailEnabled() {
|
|||
|
|
// ⚠️ D72 (2026-05-03) 冰朔决策：默认禁用所有邮件告警。
|
|||
|
|
// 服务器持续给冰朔发"大脑服务器异常"邮件，已成为噪音。
|
|||
|
|
// 如需重新启用：显式设置 OPS_EMAIL_ENABLED=true (大小写不敏感)。
|
|||
|
|
// 之前默认是 true，本次翻转默认为 false。
|
|||
|
|
const v = process.env.OPS_EMAIL_ENABLED;
|
|||
|
|
if (v === undefined || v === null || v === '') return false;
|
|||
|
|
const norm = String(v).trim().toLowerCase();
|
|||
|
|
return norm === 'true' || norm === '1' || norm === 'yes' || norm === 'on';
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function cooldownRemainingMs(key) {
|
|||
|
|
const last = _lastSentAt.get(key);
|
|||
|
|
if (!last) return 0;
|
|||
|
|
const elapsed = Date.now() - last;
|
|||
|
|
const windowMs = COOLDOWN_MINUTES * 60 * 1000;
|
|||
|
|
return elapsed >= windowMs ? 0 : windowMs - elapsed;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function markSent(key) {
|
|||
|
|
_lastSentAt.set(key, Date.now());
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ── SMTP 传输器（懒初始化） ──────────────
|
|||
|
|
|
|||
|
|
let _transporter = null;
|
|||
|
|
|
|||
|
|
function getSmtpTransporter() {
|
|||
|
|
if (_transporter) return _transporter;
|
|||
|
|
|
|||
|
|
const smtpUser = process.env.ZY_SMTP_USER;
|
|||
|
|
const smtpPass = process.env.ZY_SMTP_PASS;
|
|||
|
|
|
|||
|
|
if (!smtpUser || !smtpPass) {
|
|||
|
|
return null;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
_transporter = nodemailer.createTransport({
|
|||
|
|
host: 'smtp.qq.com',
|
|||
|
|
port: 465,
|
|||
|
|
secure: true,
|
|||
|
|
auth: {
|
|||
|
|
user: smtpUser,
|
|||
|
|
pass: smtpPass
|
|||
|
|
},
|
|||
|
|
tls: {
|
|||
|
|
rejectUnauthorized: true
|
|||
|
|
}
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
return _transporter;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ── 发送邮件告警 ────────────────────────────
|
|||
|
|
|
|||
|
|
async function sendAlertEmail(subject, htmlBody) {
|
|||
|
|
if (!isEmailEnabled()) {
|
|||
|
|
console.log('[Notifier] OPS_EMAIL_ENABLED=false，跳过邮件发送');
|
|||
|
|
return { sent: false, reason: 'disabled' };
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const transporter = getSmtpTransporter();
|
|||
|
|
if (!transporter) {
|
|||
|
|
console.log('[Notifier] SMTP 未配置，跳过邮件发送');
|
|||
|
|
return { sent: false, reason: 'smtp_not_configured' };
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const smtpUser = process.env.ZY_SMTP_USER;
|
|||
|
|
const notifyTo = process.env.OPS_NOTIFY_EMAIL || smtpUser;
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
const info = await transporter.sendMail({
|
|||
|
|
from: `"铸渊运维守卫" <${smtpUser}>`,
|
|||
|
|
to: notifyTo,
|
|||
|
|
subject: `🛡️ ${subject}`,
|
|||
|
|
html: htmlBody
|
|||
|
|
});
|
|||
|
|
console.log(`[Notifier] 邮件已发送: ${info.messageId}`);
|
|||
|
|
return { sent: true, messageId: info.messageId };
|
|||
|
|
} catch (err) {
|
|||
|
|
console.error(`[Notifier] 邮件发送失败: ${err.message}`);
|
|||
|
|
return { sent: false, reason: err.message };
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ── 生成告警邮件 HTML ──────────────────────
|
|||
|
|
|
|||
|
|
function buildAlertHTML(title, issues, checkResult) {
|
|||
|
|
const issueRows = issues.map(i => `
|
|||
|
|
<tr>
|
|||
|
|
<td style="padding:8px;border-bottom:1px solid #1a2332;color:${
|
|||
|
|
i.severity === 'critical' ? '#ff4444' :
|
|||
|
|
i.severity === 'high' ? '#ff8800' :
|
|||
|
|
'#ffcc00'
|
|||
|
|
}">${i.severity?.toUpperCase() || 'UNKNOWN'}</td>
|
|||
|
|
<td style="padding:8px;border-bottom:1px solid #1a2332;color:#e0e6ed">${i.service}</td>
|
|||
|
|
<td style="padding:8px;border-bottom:1px solid #1a2332;color:#8899aa">${i.error || i.status}</td>
|
|||
|
|
</tr>
|
|||
|
|
`).join('');
|
|||
|
|
|
|||
|
|
return `
|
|||
|
|
<!DOCTYPE html>
|
|||
|
|
<html>
|
|||
|
|
<body style="margin:0;padding:0;background:#050810;font-family:'Segoe UI',sans-serif">
|
|||
|
|
<div style="max-width:600px;margin:20px auto;background:#0a0f1a;border:1px solid #1a2332;border-radius:8px;overflow:hidden">
|
|||
|
|
<div style="background:linear-gradient(135deg,#0a1628,#1a0a28);padding:20px;text-align:center">
|
|||
|
|
<h1 style="color:#22d3ee;margin:0;font-size:18px">🛡️ 铸渊运维守卫 · 告警通知</h1>
|
|||
|
|
<p style="color:#667788;margin:5px 0 0;font-size:12px">${new Date().toLocaleString('zh-CN', { timeZone: 'Asia/Shanghai' })}</p>
|
|||
|
|
</div>
|
|||
|
|
<div style="padding:20px">
|
|||
|
|
<h2 style="color:#ff8800;font-size:16px;margin:0 0 15px">${title}</h2>
|
|||
|
|
<table style="width:100%;border-collapse:collapse">
|
|||
|
|
<thead>
|
|||
|
|
<tr style="border-bottom:2px solid #1a2332">
|
|||
|
|
<th style="padding:8px;text-align:left;color:#667788;font-size:12px">严重度</th>
|
|||
|
|
<th style="padding:8px;text-align:left;color:#667788;font-size:12px">服务</th>
|
|||
|
|
<th style="padding:8px;text-align:left;color:#667788;font-size:12px">问题</th>
|
|||
|
|
</tr>
|
|||
|
|
</thead>
|
|||
|
|
<tbody>
|
|||
|
|
${issueRows}
|
|||
|
|
</tbody>
|
|||
|
|
</table>
|
|||
|
|
${checkResult?.summary ? `<p style="color:#8899aa;margin:15px 0 0;font-size:13px">${checkResult.summary}</p>` : ''}
|
|||
|
|
</div>
|
|||
|
|
<div style="padding:15px 20px;background:#060a14;text-align:center">
|
|||
|
|
<p style="color:#445566;margin:0;font-size:11px">铸渊运维守卫 · ZY-OPS-001 · 光湖实验室</p>
|
|||
|
|
</div>
|
|||
|
|
</div>
|
|||
|
|
</body>
|
|||
|
|
</html>`;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ── 告警通知（巡检发现问题时调用） ──────────
|
|||
|
|
|
|||
|
|
async function alertOnIssues(checkResult) {
|
|||
|
|
if (!checkResult.issues || checkResult.issues.length === 0) return;
|
|||
|
|
|
|||
|
|
const criticalIssues = checkResult.issues.filter(i => i.severity === 'critical' || i.severity === 'high');
|
|||
|
|
if (criticalIssues.length === 0) return; // 只对高/紧急级别发邮件
|
|||
|
|
|
|||
|
|
// 冷却：剔除最近已通知过的 (service+severity)
|
|||
|
|
const freshIssues = [];
|
|||
|
|
const skipped = [];
|
|||
|
|
for (const i of criticalIssues) {
|
|||
|
|
const key = `alert:${i.service || 'unknown'}:${i.severity || 'unknown'}`;
|
|||
|
|
const remaining = cooldownRemainingMs(key);
|
|||
|
|
if (remaining > 0) {
|
|||
|
|
skipped.push({ key, remainingMinutes: Math.ceil(remaining / 60000) });
|
|||
|
|
} else {
|
|||
|
|
freshIssues.push({ issue: i, key });
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (skipped.length > 0) {
|
|||
|
|
console.log(`[Notifier] 冷却中跳过 ${skipped.length} 个告警: ${skipped.map(s => `${s.key}(${s.remainingMinutes}m)`).join(', ')}`);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (freshIssues.length === 0) {
|
|||
|
|
return { sent: false, reason: 'cooldown', skipped: skipped.length };
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const issuesToSend = freshIssues.map(f => f.issue);
|
|||
|
|
const title = `发现 ${issuesToSend.length} 个严重问题`;
|
|||
|
|
const html = buildAlertHTML(title, issuesToSend, checkResult);
|
|||
|
|
|
|||
|
|
// 发邮件
|
|||
|
|
const emailResult = await sendAlertEmail(title, html);
|
|||
|
|
|
|||
|
|
// 仅在真正发出去时记录冷却时间戳，避免 SMTP 失败导致永久静默
|
|||
|
|
if (emailResult && emailResult.sent) {
|
|||
|
|
for (const f of freshIssues) markSent(f.key);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 记录事件
|
|||
|
|
memory.logEvent('alert', {
|
|||
|
|
title,
|
|||
|
|
issueCount: issuesToSend.length,
|
|||
|
|
skippedByCooldown: skipped.length,
|
|||
|
|
emailSent: emailResult.sent,
|
|||
|
|
summary: checkResult.summary
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
return emailResult;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ── 工单升级通知 ────────────────────────────
|
|||
|
|
|
|||
|
|
async function notifyTicketCreated(ticket) {
|
|||
|
|
// 冷却兜底：同一 (service+severity) 在窗口内只发一次工单邮件
|
|||
|
|
const cooldownKey = `ticket:${ticket.relatedService || ticket.title || 'unknown'}:${ticket.severity || 'unknown'}`;
|
|||
|
|
const remaining = cooldownRemainingMs(cooldownKey);
|
|||
|
|
if (remaining > 0) {
|
|||
|
|
console.log(`[Notifier] 工单邮件冷却中(${Math.ceil(remaining / 60000)}分钟剩余)，跳过 ${ticket.id}`);
|
|||
|
|
return { sent: false, reason: 'cooldown' };
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const html = `
|
|||
|
|
<!DOCTYPE html>
|
|||
|
|
<html>
|
|||
|
|
<body style="margin:0;padding:0;background:#050810;font-family:'Segoe UI',sans-serif">
|
|||
|
|
<div style="max-width:600px;margin:20px auto;background:#0a0f1a;border:1px solid #1a2332;border-radius:8px;overflow:hidden">
|
|||
|
|
<div style="background:linear-gradient(135deg,#0a1628,#1a0a28);padding:20px;text-align:center">
|
|||
|
|
<h1 style="color:#22d3ee;margin:0;font-size:18px">🎫 运维工单 · ${ticket.id}</h1>
|
|||
|
|
</div>
|
|||
|
|
<div style="padding:20px">
|
|||
|
|
<h2 style="color:#e0e6ed;font-size:16px;margin:0 0 10px">${ticket.title}</h2>
|
|||
|
|
<p style="color:#8899aa;margin:5px 0"><b>严重度:</b> <span style="color:${
|
|||
|
|
ticket.severity === 'critical' ? '#ff4444' :
|
|||
|
|
ticket.severity === 'high' ? '#ff8800' : '#ffcc00'
|
|||
|
|
}">${ticket.severity}</span></p>
|
|||
|
|
<p style="color:#8899aa;margin:5px 0"><b>处理方向:</b> ${ticket.direction}</p>
|
|||
|
|
<p style="color:#8899aa;margin:5px 0"><b>诊断:</b> ${ticket.diagnosis}</p>
|
|||
|
|
${ticket.suggestedFix ? `<p style="color:#8899aa;margin:5px 0"><b>建议:</b> ${ticket.suggestedFix}</p>` : ''}
|
|||
|
|
<p style="color:#8899aa;margin:5px 0"><b>自动修复尝试:</b> ${ticket.autoRepairAttempts}次（均未成功）</p>
|
|||
|
|
</div>
|
|||
|
|
<div style="padding:15px 20px;background:#060a14;text-align:center">
|
|||
|
|
<p style="color:#445566;margin:0;font-size:11px">铸渊运维守卫 · 自动生成工单</p>
|
|||
|
|
</div>
|
|||
|
|
</div>
|
|||
|
|
</body>
|
|||
|
|
</html>`;
|
|||
|
|
|
|||
|
|
const result = await sendAlertEmail(`新工单 ${ticket.id}: ${ticket.title}`, html);
|
|||
|
|
if (result && result.sent) markSent(cooldownKey);
|
|||
|
|
return result;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ── 每日报告邮件 ────────────────────────────
|
|||
|
|
|
|||
|
|
async function sendDailyReport(report) {
|
|||
|
|
const serviceRows = (report.services || []).map(s => `
|
|||
|
|
<tr>
|
|||
|
|
<td style="padding:6px 8px;border-bottom:1px solid #1a2332;color:#e0e6ed">${s.name}</td>
|
|||
|
|
<td style="padding:6px 8px;border-bottom:1px solid #1a2332;color:${s.status === 'online' ? '#22d3ee' : '#ff4444'}">${s.status}</td>
|
|||
|
|
<td style="padding:6px 8px;border-bottom:1px solid #1a2332;color:#8899aa">${s.latency || '-'}ms</td>
|
|||
|
|
</tr>
|
|||
|
|
`).join('');
|
|||
|
|
|
|||
|
|
const html = `
|
|||
|
|
<!DOCTYPE html>
|
|||
|
|
<html>
|
|||
|
|
<body style="margin:0;padding:0;background:#050810;font-family:'Segoe UI',sans-serif">
|
|||
|
|
<div style="max-width:600px;margin:20px auto;background:#0a0f1a;border:1px solid #1a2332;border-radius:8px;overflow:hidden">
|
|||
|
|
<div style="background:linear-gradient(135deg,#0a1628,#1a0a28);padding:20px;text-align:center">
|
|||
|
|
<h1 style="color:#22d3ee;margin:0;font-size:18px">📊 光湖每日健康报告</h1>
|
|||
|
|
<p style="color:#667788;margin:5px 0 0;font-size:12px">${report.report_id}</p>
|
|||
|
|
</div>
|
|||
|
|
<div style="padding:20px">
|
|||
|
|
<h3 style="color:#e0e6ed;margin:0 0 10px">服务状态</h3>
|
|||
|
|
<table style="width:100%;border-collapse:collapse;font-size:13px">
|
|||
|
|
<thead><tr style="border-bottom:2px solid #1a2332">
|
|||
|
|
<th style="padding:6px 8px;text-align:left;color:#667788">服务</th>
|
|||
|
|
<th style="padding:6px 8px;text-align:left;color:#667788">状态</th>
|
|||
|
|
<th style="padding:6px 8px;text-align:left;color:#667788">延迟</th>
|
|||
|
|
</tr></thead>
|
|||
|
|
<tbody>${serviceRows}</tbody>
|
|||
|
|
</table>
|
|||
|
|
${report.resources ? `
|
|||
|
|
<h3 style="color:#e0e6ed;margin:15px 0 10px">系统资源</h3>
|
|||
|
|
<p style="color:#8899aa;font-size:13px">
|
|||
|
|
内存: ${report.resources.memory.used_pct}% ·
|
|||
|
|
磁盘: ${report.resources.disk.used_pct}% ·
|
|||
|
|
负载: ${report.resources.load?.[0]?.toFixed(2) || '-'} ·
|
|||
|
|
运行: ${report.resources.uptime_hours}h
|
|||
|
|
</p>` : ''}
|
|||
|
|
<p style="color:#22d3ee;margin:15px 0 0;font-size:14px">${report.summary}</p>
|
|||
|
|
${report.recommendations ? `
|
|||
|
|
<h3 style="color:#e0e6ed;margin:15px 0 10px">建议</h3>
|
|||
|
|
<ul style="color:#8899aa;font-size:13px;padding-left:20px">
|
|||
|
|
${report.recommendations.map(r => `<li>${r}</li>`).join('')}
|
|||
|
|
</ul>` : ''}
|
|||
|
|
</div>
|
|||
|
|
<div style="padding:15px 20px;background:#060a14;text-align:center">
|
|||
|
|
<p style="color:#445566;margin:0;font-size:11px">铸渊运维守卫 · 每日自动报告</p>
|
|||
|
|
</div>
|
|||
|
|
</div>
|
|||
|
|
</body>
|
|||
|
|
</html>`;
|
|||
|
|
|
|||
|
|
return sendAlertEmail(`每日报告 ${report.report_id}`, html);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
module.exports = {
|
|||
|
|
sendAlertEmail,
|
|||
|
|
alertOnIssues,
|
|||
|
|
notifyTicketCreated,
|
|||
|
|
sendDailyReport,
|
|||
|
|
getSmtpTransporter,
|
|||
|
|
isEmailEnabled,
|
|||
|
|
// 供测试使用
|
|||
|
|
_resetCooldownForTests: () => _lastSentAt.clear()
|
|||
|
|
};
|