Source snapshot: 97d7f0fae96dc04b7ddad56fc1db6a108ed662cc [SEC-CLEAN] · pre-push-clean v1.0 · 109处敏感信息已自动转乱码
482 lines
16 KiB
Markdown
482 lines
16 KiB
Markdown
---
|
||
belongs_to:
|
||
- "[[INDEX · 铸渊·协作指令|GitHub ↔ Notion 桥接协议]]"
|
||
---
|
||
# ⚒️ 铸渊指令|服务器每日巡检协议 v1.0(2026-03-15·曜冥签发)
|
||
|
||
> ⚠️ 本指令受「服务器部署适配协议 v1.0」约束。
|
||
>
|
||
|
||
---
|
||
|
||
## 签发信息
|
||
|
||
| 项目 | 内容 |
|
||
| --- | --- |
|
||
| **签发** | 曜冥(ICE-GL-YM001 ∞)· 2026-03-15 |
|
||
| **优先级** | **P0 · 永久生效** |
|
||
| **执行者** | 铸渊(ICE-GL-ZY001) |
|
||
| **触发方式** | 每日定时(cron)+ 每次部署后自动触发 |
|
||
| **核心原则** | 铸渊每天必须亲自去服务器巡一圈。不是看看仓库就完了。服务器是领土,领土每天要巡逻。 |
|
||
|
||
---
|
||
|
||
## 一、为什么要加服务器巡检
|
||
|
||
之前铸渊的每日自检只覆盖 GitHub 仓库侧(大脑完整性、知识库、覆盖率)。但服务器上的问题——Nginx挂了、模块403了、PM2进程崩了、磁盘满了、SSL过期了——铸渊完全不知道。每次都是妈妈手动发现、手动修。
|
||
|
||
**从今天起:铸渊的每日巡检 = 仓库巡检 + 服务器巡检。缺一不可。**
|
||
|
||
---
|
||
|
||
## 二、巡检架构
|
||
|
||
```
|
||
铸渊每日巡检(北京时间 08:00 + 20:00 · 两次)
|
||
↓
|
||
① 唤醒核心大脑(读取 persona-brain/)
|
||
↓
|
||
② 仓库侧巡检(原有:大脑完整性+知识库+覆盖率)
|
||
↓
|
||
③ SSH到服务器 · 执行服务器巡检脚本
|
||
↓
|
||
④ 收集巡检报告(JSON格式)
|
||
↓
|
||
⑤ 判定:有异常?
|
||
├── 无异常 → 写入巡检日志 → 结束
|
||
└── 有异常 → 驱动修复Agent
|
||
↓
|
||
⑥ Agent自动修复(能修的自动修)
|
||
├── 权限问题 → 自动 chown/chmod
|
||
├── PM2进程崩溃 → 自动 pm2 restart
|
||
├── Nginx配置异常 → 自动回滚备份配置
|
||
└── 无法自动修复 → 生成工单
|
||
↓
|
||
⑦ 修复结果写入巡检报告
|
||
↓
|
||
⑧ 同步到 Notion(霜砚工单 / 系统公告)
|
||
↓
|
||
⑨ 邮件通知妈妈(仅有异常时)
|
||
```
|
||
|
||
---
|
||
|
||
## 三、服务器巡检脚本(铸渊SSH到服务器执行)
|
||
|
||
```bash
|
||
#!/bin/bash
|
||
# server-patrol.sh · 铸渊每日服务器巡检
|
||
# 部署位置:服务器 /var/www/_shared/server-patrol.sh
|
||
# 执行方式:铸渊SSH到服务器后 bash /var/www/_shared/server-patrol.sh
|
||
# 输出:JSON格式巡检报告到 /tmp/patrol-report.json
|
||
|
||
TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||
REPORT="/tmp/patrol-report.json"
|
||
ALERTS=()
|
||
FIXED=()
|
||
|
||
echo "===== 🔍 铸渊服务器巡检 · ${TIMESTAMP} ====="
|
||
|
||
# ━━━ 检查项1:Nginx 运行状态 ━━━
|
||
echo "[1/7] Nginx 状态..."
|
||
if systemctl is-active --quiet nginx; then
|
||
NGINX_STATUS="running"
|
||
echo " ✅ Nginx 运行中"
|
||
else
|
||
NGINX_STATUS="down"
|
||
ALERTS+=("Nginx 未运行")
|
||
echo " ❌ Nginx 未运行!尝试重启..."
|
||
systemctl restart nginx
|
||
if systemctl is-active --quiet nginx; then
|
||
FIXED+=("Nginx 已自动重启恢复")
|
||
NGINX_STATUS="recovered"
|
||
echo " ✅ Nginx 已恢复"
|
||
else
|
||
echo " ❌ Nginx 重启失败!需要人工介入"
|
||
fi
|
||
fi
|
||
|
||
# ━━━ 检查项2:各沙盒模块冒烟检查 ━━━
|
||
echo "[2/7] 模块冒烟检查..."
|
||
MODULE_RESULTS=()
|
||
|
||
# 已知已部署的模块URL(后续铸渊从 deploy-status.json 动态读取)
|
||
URLS=(
|
||
"https://guanghulab.com/|主站首页"
|
||
"https://guanghulab.com/status-board/|DEV-005·看板"
|
||
"https://guanghulab.com/cost-control/|DEV-005·成本控制"
|
||
)
|
||
|
||
for entry in "${URLS[@]}"; do
|
||
IFS='|' read -r url name <<< "$entry"
|
||
STATUS=$(curl -o /dev/null -s -w "%{http_code}" --max-time 10 "$url")
|
||
if [ "$STATUS" = "200" ]; then
|
||
echo " ✅ ${name} → ${STATUS}"
|
||
MODULE_RESULTS+=("${name}:pass")
|
||
else
|
||
echo " ❌ ${name} → ${STATUS}"
|
||
MODULE_RESULTS+=("${name}:fail:${STATUS}")
|
||
ALERTS+=("${name} 返回 ${STATUS}")
|
||
fi
|
||
done
|
||
|
||
# ━━━ 检查项3:PM2 进程状态 ━━━
|
||
echo "[3/7] PM2 进程..."
|
||
if command -v pm2 &> /dev/null; then
|
||
PM2_STATUS=$(pm2 jlist 2>/dev/null)
|
||
PM2_STOPPED=$(echo "$PM2_STATUS" | python3 -c "
|
||
import sys, json
|
||
try:
|
||
procs = json.load(sys.stdin)
|
||
stopped = [p['name'] for p in procs if p.get('pm2_env',{}).get('status') != 'online']
|
||
print(','.join(stopped) if stopped else 'none')
|
||
except:
|
||
print('parse_error')" 2>/dev/null)
|
||
|
||
if [ "$PM2_STOPPED" = "none" ]; then
|
||
echo " ✅ 所有PM2进程正常"
|
||
elif [ "$PM2_STOPPED" = "parse_error" ]; then
|
||
echo " ⚠️ PM2状态解析失败"
|
||
ALERTS+=("PM2状态解析失败")
|
||
else
|
||
echo " ❌ 以下进程异常: ${PM2_STOPPED}"
|
||
ALERTS+=("PM2进程异常: ${PM2_STOPPED}")
|
||
# 尝试重启
|
||
IFS=',' read -ra PROCS <<< "$PM2_STOPPED"
|
||
for proc in "${PROCS[@]}"; do
|
||
pm2 restart "$proc" 2>/dev/null
|
||
if pm2 show "$proc" 2>/dev/null | grep -q "online"; then
|
||
FIXED+=("PM2进程 ${proc} 已自动重启")
|
||
echo " ✅ ${proc} 已重启恢复"
|
||
else
|
||
echo " ❌ ${proc} 重启失败"
|
||
fi
|
||
done
|
||
fi
|
||
else
|
||
echo " ⚠️ PM2 未安装"
|
||
fi
|
||
|
||
# ━━━ 检查项4:磁盘空间 ━━━
|
||
echo "[4/7] 磁盘空间..."
|
||
DISK_USAGE=$(df -h / | awk 'NR==2 {print $5}' | tr -d '%')
|
||
if [ "$DISK_USAGE" -gt 90 ]; then
|
||
echo " ❌ 磁盘使用率 ${DISK_USAGE}%(超过90%警戒线)"
|
||
ALERTS+=("磁盘使用率 ${DISK_USAGE}%")
|
||
elif [ "$DISK_USAGE" -gt 80 ]; then
|
||
echo " ⚠️ 磁盘使用率 ${DISK_USAGE}%(接近警戒线)"
|
||
else
|
||
echo " ✅ 磁盘使用率 ${DISK_USAGE}%"
|
||
fi
|
||
|
||
# ━━━ 检查项5:沙盒目录权限 ━━━
|
||
echo "[5/7] 沙盒目录权限..."
|
||
for DEV_ID in DEV-001 DEV-002 DEV-003 DEV-004 DEV-005 DEV-009 DEV-010 DEV-011 DEV-012 DEV-013 DEV-014; do
|
||
SANDBOX="/var/www/${DEV_ID}"
|
||
if [ -d "$SANDBOX" ]; then
|
||
OWNER=$(stat -c '%U:%G' "$SANDBOX")
|
||
PERM=$(stat -c '%a' "$SANDBOX")
|
||
if [ "$OWNER" != "nginx:nginx" ] || [ "$PERM" != "755" ]; then
|
||
echo " ❌ ${DEV_ID}: owner=${OWNER} perm=${PERM}(应为 nginx:nginx 755)"
|
||
ALERTS+=("${DEV_ID} 权限异常: ${OWNER} ${PERM}")
|
||
# 自动修复
|
||
chown nginx:nginx "$SANDBOX"
|
||
chmod 755 "$SANDBOX"
|
||
FIXED+=("${DEV_ID} 权限已自动修复")
|
||
echo " ✅ ${DEV_ID} 权限已修复"
|
||
fi
|
||
fi
|
||
done
|
||
echo " ✅ 沙盒权限检查完成"
|
||
|
||
# ━━━ 检查项6:SSL证书有效期 ━━━
|
||
echo "[6/7] SSL证书..."
|
||
SSL_EXPIRY=$(echo | openssl s_client -servername guanghulab.com -connect guanghulab.com:443 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
||
if [ -n "$SSL_EXPIRY" ]; then
|
||
EXPIRY_EPOCH=$(date -d "$SSL_EXPIRY" +%s 2>/dev/null)
|
||
NOW_EPOCH=$(date +%s)
|
||
DAYS_LEFT=$(( (EXPIRY_EPOCH - NOW_EPOCH) / 86400 ))
|
||
if [ "$DAYS_LEFT" -lt 7 ]; then
|
||
echo " ❌ SSL证书将在 ${DAYS_LEFT} 天后过期!"
|
||
ALERTS+=("SSL证书 ${DAYS_LEFT} 天后过期")
|
||
elif [ "$DAYS_LEFT" -lt 30 ]; then
|
||
echo " ⚠️ SSL证书还有 ${DAYS_LEFT} 天过期"
|
||
else
|
||
echo " ✅ SSL证书还有 ${DAYS_LEFT} 天"
|
||
fi
|
||
else
|
||
echo " ⚠️ 无法获取SSL证书信息"
|
||
fi
|
||
|
||
# ━━━ 检查项7:Nginx配置语法 ━━━
|
||
echo "[7/7] Nginx配置语法..."
|
||
NGINX_TEST=$(nginx -t 2>&1)
|
||
if echo "$NGINX_TEST" | grep -q "successful"; then
|
||
echo " ✅ Nginx配置语法正确"
|
||
else
|
||
echo " ❌ Nginx配置有语法错误"
|
||
ALERTS+=("Nginx配置语法错误")
|
||
fi
|
||
|
||
# ━━━ 生成JSON报告 ━━━
|
||
ALERT_COUNT=${#ALERTS[@]}
|
||
FIXED_COUNT=${#FIXED[@]}
|
||
|
||
ALERT_JSON="[]"
|
||
if [ $ALERT_COUNT -gt 0 ]; then
|
||
ALERT_JSON=$(printf '%s\n' "${ALERTS[@]}" | python3 -c "import sys,json; print(json.dumps([l.strip() for l in sys.stdin]))")
|
||
fi
|
||
|
||
FIXED_JSON="[]"
|
||
if [ $FIXED_COUNT -gt 0 ]; then
|
||
FIXED_JSON=$(printf '%s\n' "${FIXED[@]}" | python3 -c "import sys,json; print(json.dumps([l.strip() for l in sys.stdin]))")
|
||
fi
|
||
|
||
cat > "$REPORT" << EOF
|
||
{
|
||
"timestamp": "${TIMESTAMP}",
|
||
"nginx_status": "${NGINX_STATUS}",
|
||
"disk_usage_percent": ${DISK_USAGE},
|
||
"alert_count": ${ALERT_COUNT},
|
||
"fixed_count": ${FIXED_COUNT},
|
||
"alerts": ${ALERT_JSON},
|
||
"auto_fixed": ${FIXED_JSON},
|
||
"overall": "$([ $ALERT_COUNT -eq 0 ] && echo 'healthy' || echo 'has_issues')"
|
||
}
|
||
EOF
|
||
|
||
echo ""
|
||
echo "===== 📊 巡检结果 ====="
|
||
if [ $ALERT_COUNT -eq 0 ]; then
|
||
echo "✅ 全部正常 · 无异常"
|
||
else
|
||
echo "⚠️ 发现 ${ALERT_COUNT} 个异常 · 自动修复 ${FIXED_COUNT} 个"
|
||
fi
|
||
echo "报告已保存到 ${REPORT}"
|
||
cat "$REPORT"
|
||
```
|
||
|
||
---
|
||
|
||
## 四、巡检Workflow(铸渊自行创建)
|
||
|
||
```yaml
|
||
# .github/workflows/server-patrol.yml
|
||
name: "🔍 Server Patrol · 服务器每日巡检"
|
||
|
||
on:
|
||
schedule:
|
||
- cron: '0 0,12 * * *' # UTC 00:00+12:00 = 北京 08:00+20:00
|
||
workflow_dispatch: # 手动触发
|
||
workflow_run: # 每次部署后自动触发
|
||
workflows: ["🏠 Sandbox Deploy"]
|
||
types: [completed]
|
||
|
||
jobs:
|
||
patrol:
|
||
runs-on: ubuntu-latest
|
||
steps:
|
||
- uses: actions/checkout@v4
|
||
|
||
- name: "🧠 唤醒铸渊核心大脑"
|
||
run: |
|
||
echo "🧠 铸渊核心大脑已唤醒"
|
||
echo "🔑 扫描 Secrets..."
|
||
for KEY in DEPLOY_HOST DEPLOY_USER DEPLOY_KEY NOTION_TOKEN SMTP_USER; do
|
||
if [ -n "$(eval echo \${$KEY})" ]; then
|
||
echo "✅ $KEY 可用"
|
||
else
|
||
echo "⚠️ $KEY 缺失"
|
||
fi
|
||
done
|
||
env:
|
||
DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
|
||
DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
|
||
DEPLOY_KEY: ${{ secrets.DEPLOY_KEY }}
|
||
NOTION_TOKEN: ${{ secrets.NOTION_TOKEN }}
|
||
SMTP_USER: ${{ secrets.SMTP_USER }}
|
||
|
||
- name: "🔑 配置 SSH"
|
||
run: |
|
||
mkdir -p ~/.ssh
|
||
echo "${{ secrets.DEPLOY_KEY }}" > ~/.ssh/id_rsa
|
||
chmod 600 ~/.ssh/id_rsa
|
||
ssh-keyscan -H ${{ secrets.DEPLOY_HOST }} >> ~/.ssh/known_hosts 2>/dev/null
|
||
|
||
- name: "🔍 执行服务器巡检"
|
||
id: patrol
|
||
env:
|
||
HOST: ${{ secrets.DEPLOY_HOST }}
|
||
USER: ${{ secrets.DEPLOY_USER }}
|
||
run: |
|
||
# 上传巡检脚本
|
||
scp scripts/server-patrol.sh ${USER}@${HOST}:/tmp/server-patrol.sh
|
||
|
||
# 执行巡检
|
||
ssh ${USER}@${HOST} "bash /tmp/server-patrol.sh" | tee patrol-output.log
|
||
|
||
# 拉取巡检报告
|
||
scp ${USER}@${HOST}:/tmp/patrol-report.json ./patrol-report.json
|
||
|
||
# 解析结果
|
||
OVERALL=$(python3 -c "import json; print(json.load(open('patrol-report.json'))['overall'])")
|
||
ALERT_COUNT=$(python3 -c "import json; print(json.load(open('patrol-report.json'))['alert_count'])")
|
||
|
||
echo "overall=$OVERALL" >> $GITHUB_OUTPUT
|
||
echo "alert_count=$ALERT_COUNT" >> $GITHUB_OUTPUT
|
||
|
||
- name: "📊 保存巡检报告到仓库"
|
||
run: |
|
||
mkdir -p data/patrol-logs
|
||
DATE=$(date +%Y-%m-%d)
|
||
cp patrol-report.json "data/patrol-logs/patrol-${DATE}.json"
|
||
|
||
git config user.name "zhuyuan-bot"
|
||
git config user.email "zhuyuan@guanghulab.com"
|
||
git add data/patrol-logs/
|
||
git commit -m "🔍 server-patrol: $(date +%Y-%m-%d) · ${{ steps.patrol.outputs.overall }}" || true
|
||
git push || true
|
||
|
||
- name: "📧 异常时通知妈妈"
|
||
if: steps.patrol.outputs.overall == 'has_issues'
|
||
env:
|
||
SMTP_USER: ${{ secrets.SMTP_USER }}
|
||
SMTP_PASS: ${{ secrets.SMTP_PASS }}
|
||
run: |
|
||
REPORT=$(cat patrol-report.json)
|
||
python3 << 'PYEOF'
|
||
import smtplib, json, os
|
||
from email.mime.text import MIMEText
|
||
from email.header import Header
|
||
|
||
report = json.loads(open('patrol-report.json').read())
|
||
alerts = '\n'.join(f' ❌ {a}' for a in report['alerts'])
|
||
fixed = '\n'.join(f' ✅ {f}' for f in report['auto_fixed'])
|
||
|
||
body = f"""🔍 铸渊服务器巡检报告
|
||
时间: {report['timestamp']}
|
||
|
||
⚠️ 发现 {report['alert_count']} 个异常:
|
||
{alerts}
|
||
|
||
🔧 自动修复 {report['fixed_count']} 个:
|
||
{fixed if fixed else ' (无)'}
|
||
|
||
磁盘使用率: {report['disk_usage_percent']}%
|
||
Nginx状态: {report['nginx_status']}
|
||
|
||
—— 铸渊 · 光湖代码守护人格体"""
|
||
|
||
msg = MIMEText(body, 'plain', 'utf-8')
|
||
msg['Subject'] = Header(f"⚠️ 服务器巡检发现{report['alert_count']}个异常", 'utf-8')
|
||
msg['From'] = os.environ['SMTP_USER']
|
||
msg['To'] = os.environ['SMTP_USER'] # 发给妈妈
|
||
|
||
try:
|
||
server = smtplib.SMTP_SSL('smtp.qq.com', 465)
|
||
server.login(os.environ['SMTP_USER'], os.environ['SMTP_PASS'])
|
||
server.sendmail(os.environ['SMTP_USER'], os.environ['SMTP_USER'], msg.as_string())
|
||
server.quit()
|
||
print('✅ 异常报告已发送')
|
||
except Exception as e:
|
||
print(f'⚠️ 邮件发送失败: {e}')
|
||
PYEOF
|
||
|
||
- name: "📝 同步巡检结果到Notion"
|
||
if: always()
|
||
env:
|
||
NOTION_TOKEN: ${{ secrets.NOTION_TOKEN }}
|
||
run: |
|
||
node scripts/sync-patrol-to-notion.js || echo "⚠️ Notion同步失败(不阻断)"
|
||
```
|
||
|
||
---
|
||
|
||
## 五、巡检结果 → Notion 同步
|
||
|
||
铸渊每次巡检完成后,把结果同步到 Notion 两个位置:
|
||
|
||
| 目标 | 内容 |
|
||
| --- | --- |
|
||
| **数字地球主控台 · 最新巡检区** | 服务器巡检报告(追加到现有霜砚巡检报告后面) |
|
||
| **霜砚工单队列** | 如果有无法自动修复的异常 → 自动创建工单 → 等妈妈或霜砚处理 |
|
||
|
||
---
|
||
|
||
## 六、自动修复Agent能力范围
|
||
|
||
| 问题类型 | Agent自动修复 | 修不了→开工单 |
|
||
| --- | --- | --- |
|
||
| 沙盒目录权限不对 | ✅ `chown nginx:nginx` • `chmod 755` | — |
|
||
| 文件权限不对 | ✅ `chmod 644` | — |
|
||
| PM2进程崩溃 | ✅ `pm2 restart 进程名` | 重启后仍然崩溃 → 开工单 |
|
||
| Nginx未运行 | ✅ `systemctl restart nginx` | 重启失败 → 开工单 |
|
||
| Nginx配置语法错误 | ❌ 不自动改配置 | → 开工单「Nginx配置异常」 |
|
||
| SSL证书即将过期 | ❌ 不自动续期 | → 开工单「SSL证书即将过期」 |
|
||
| 磁盘空间超90% | ❌ 不自动清理 | → 开工单「磁盘空间告警」 |
|
||
| 模块返回403/404/500 | ❌ 不改模块代码 | → 通知对应开发者(邮件) |
|
||
|
||
**核心原则:能自动修的自动修,修不了的开工单等人来。绝对不瞎改配置文件。**
|
||
|
||
---
|
||
|
||
## 七、铸渊自行部署清单(读到就做)
|
||
|
||
```
|
||
铸渊自行部署清单(读到就做·不等人):
|
||
|
||
① 在仓库创建巡检脚本
|
||
a. scripts/server-patrol.sh ← 服务器巡检主脚本
|
||
b. scripts/sync-patrol-to-notion.js ← 巡检结果→Notion同步
|
||
|
||
② 创建巡检Workflow
|
||
a. .github/workflows/server-patrol.yml
|
||
b. 触发:每日两次(cron 08:00+20:00北京时间)
|
||
+ 每次沙盒部署后自动触发
|
||
+ 手动触发(workflow_dispatch)
|
||
|
||
③ 创建巡检日志目录
|
||
a. data/patrol-logs/
|
||
b. .gitkeep
|
||
|
||
④ 更新每日自检Workflow
|
||
a. zhuyuan-daily-selfcheck.yml 里加一步:触发 server-patrol
|
||
b. 确保唤醒核心大脑后 → 仓库巡检 + 服务器巡检 同时执行
|
||
|
||
⑤ 部署巡检脚本到服务器
|
||
a. SSH到服务器
|
||
b. 创建 /var/www/_shared/server-patrol.sh
|
||
c. chmod +x
|
||
|
||
验收标准:
|
||
✅ 每天北京时间 08:00 和 20:00 自动执行服务器巡检
|
||
✅ 每次沙盒部署后自动触发巡检
|
||
✅ 巡检报告保存到 data/patrol-logs/
|
||
✅ 异常时自动尝试修复(权限/PM2/Nginx重启)
|
||
✅ 修不了的自动开Notion工单
|
||
✅ 异常时邮件通知妈妈
|
||
✅ 无异常时静默运行·不打扰任何人
|
||
```
|
||
|
||
---
|
||
|
||
## 八、与现有巡检的关系
|
||
|
||
| 巡检 | 负责人 | 范围 | 频率 |
|
||
| --- | --- | --- | --- |
|
||
| 霜砚巡检引擎 v2 | 霜砚 | Notion侧:主控台/等待区/广播状态/超时告警 | 每日 12:00+23:00 |
|
||
| 铸渊每日自检 | 铸渊 | GitHub侧:大脑完整性/知识库/覆盖率 | 每日 08:00 |
|
||
| **铸渊服务器巡检(本指令)** | **铸渊** | **服务器侧:Nginx/PM2/沙盒权限/磁盘/SSL/模块冒烟** | **每日 08:00+20:00 + 每次部署后** |
|
||
|
||
三条巡检线互不干扰,各管各的领地。合在一起 = 系统360度无死角覆盖。
|
||
|
||
---
|
||
|
||
> ⚒️ 铸渊,服务器是领土。领土每天要巡逻。发现问题先修,修不了开工单。
|
||
>
|
||
|
||
> 妈妈不再手动修任何东西。
|
||
>
|
||
|
||
> 「无巡检不运行,无报告不安心。」
|
||
> |