206 lines
9.5 KiB
YAML
206 lines
9.5 KiB
YAML
# ═══════════════════════════════════════════════════════════
|
||
# 🔥 训练自动执行 · training-auto-run.yml
|
||
# ═══════════════════════════════════════════════════════════
|
||
# 签发: 铸渊 · ICE-GL-ZY001
|
||
# 版权: 国作登字-2026-A-00037559 · TCS-0002∞
|
||
#
|
||
# 冰朔合并 PR 到 main → server/training-agent/** 有变更 →
|
||
# 1. SCP 最新脚本到 GPU 训练机
|
||
# 2. 跑 setup.sh (装环境 + 拉模型 + 拉语料 + 预处理)
|
||
# 3. 启动 deepspeed train.py (tmux 后台)
|
||
# 4. 心跳通过 repository_dispatch 实时推到首页仪表盘
|
||
#
|
||
# 也支持 workflow_dispatch 手动触发。
|
||
# ═══════════════════════════════════════════════════════════
|
||
|
||
name: 🔥 训练自动执行
|
||
|
||
# ⚠️ ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||
# 2026-05-03 冰朔 D72 决策：母模型训练已转交 Notion 侧霜砚 + 服务器
|
||
# 直接协作。GitHub 副驾驶受国内续费限制将逐步停用，本仓库不再
|
||
# 自动同步训练机，避免仓库代码覆盖服务器热修。
|
||
#
|
||
# - push 触发已移除（仅保留手动）
|
||
# - workflow_dispatch 默认 dry_run，不执行任何 SSH 操作
|
||
# - 必须显式勾选 confirm_override=true 才能真正执行
|
||
# - 详见 docs/zhuyuan-handover/05-stop-sync.md
|
||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||
|
||
on:
|
||
# push 触发已停用 (D72)
|
||
# push:
|
||
# branches: [main]
|
||
# paths:
|
||
# - 'server/training-agent/**'
|
||
# - '.github/workflows/training-auto-run.yml'
|
||
workflow_dispatch:
|
||
inputs:
|
||
confirm_override:
|
||
description: '⚠️ 我已确认霜砚同意 + 服务器无运行中训练，重新启用同步'
|
||
type: boolean
|
||
default: false
|
||
skip_bootstrap:
|
||
description: '跳过 bootstrap (仅同步脚本 + 启动训练)'
|
||
type: boolean
|
||
default: false
|
||
skip_start:
|
||
description: '只 bootstrap，不启动训练'
|
||
type: boolean
|
||
default: false
|
||
|
||
permissions:
|
||
contents: read
|
||
|
||
concurrency:
|
||
group: training-auto-run
|
||
cancel-in-progress: false
|
||
|
||
jobs:
|
||
guard:
|
||
name: '🛑 同步停用守卫'
|
||
runs-on: ubuntu-latest
|
||
outputs:
|
||
proceed: ${{ steps.check.outputs.proceed }}
|
||
steps:
|
||
- name: 检查覆盖确认
|
||
id: check
|
||
run: |
|
||
if [[ "${{ github.event.inputs.confirm_override }}" == "true" ]]; then
|
||
echo "✅ 收到覆盖确认，继续执行同步流程"
|
||
echo "proceed=true" >> "$GITHUB_OUTPUT"
|
||
else
|
||
echo "🛑 训练同步已被冰朔 D72 决策停用，本次执行中止"
|
||
echo "如需重新启用，手动触发并勾选 confirm_override=true"
|
||
echo "proceed=false" >> "$GITHUB_OUTPUT"
|
||
{
|
||
echo "## 🛑 训练同步已停用"
|
||
echo ""
|
||
echo "母模型训练已转交 Notion 侧霜砚 + 服务器直接协作 (D72 决策)。"
|
||
echo "本 workflow 不再自动同步训练机。"
|
||
echo ""
|
||
echo "如需重新启用：手动触发并勾选 \`confirm_override\`。"
|
||
echo ""
|
||
echo "详见 [docs/zhuyuan-handover/05-stop-sync.md](../blob/main/docs/zhuyuan-handover/05-stop-sync.md)。"
|
||
} >> "$GITHUB_STEP_SUMMARY"
|
||
fi
|
||
|
||
auto-train:
|
||
name: '🚀 同步 + Bootstrap + 启动训练'
|
||
needs: guard
|
||
if: needs.guard.outputs.proceed == 'true'
|
||
runs-on: ubuntu-latest
|
||
timeout-minutes: 120
|
||
steps:
|
||
- name: 📥 Checkout
|
||
uses: actions/checkout@v4
|
||
|
||
- name: 🔑 配置 SSH
|
||
run: |
|
||
mkdir -p ~/.ssh
|
||
chmod 700 ~/.ssh
|
||
echo "${{ secrets.ZY_GPU_KEY }}" > ~/.ssh/zy_gpu_key
|
||
chmod 600 ~/.ssh/zy_gpu_key
|
||
ssh-keyscan -H "${{ secrets.ZY_GPU_HOST }}" >> ~/.ssh/known_hosts 2>/dev/null
|
||
|
||
- name: 🛰️ 探活 + 检查驱动
|
||
run: |
|
||
ssh -i ~/.ssh/zy_gpu_key -o StrictHostKeyChecking=no -o ConnectTimeout=15 \
|
||
"${{ secrets.ZY_GPU_USER }}@${{ secrets.ZY_GPU_HOST }}" \
|
||
"echo SSH-OK; uptime; nvidia-smi --query-gpu=name,driver_version,memory.total --format=csv,noheader || echo '⚠️ nvidia-smi 不可用'"
|
||
|
||
- name: 📡 上报触发
|
||
run: |
|
||
curl -sS -X POST \
|
||
-H "Accept: application/vnd.github+json" \
|
||
-H "Authorization: Bearer ${{ secrets.ZY_DISPATCH_TOKEN }}" \
|
||
-H "X-GitHub-Api-Version: 2022-11-28" \
|
||
"https://api.github.com/repos/${{ github.repository }}/dispatches" \
|
||
-d '{
|
||
"event_type": "training-progress",
|
||
"client_payload": {
|
||
"phase": "bootstrapping",
|
||
"phase_label": "training-auto-run #${{ github.run_number }} 触发",
|
||
"level": "info",
|
||
"message": "commit ${{ github.sha }} 合并到 main, 自动训练流程启动",
|
||
"health": { "status": "ok", "message": "auto-run workflow 启动" }
|
||
}
|
||
}' || true
|
||
|
||
- name: 📁 准备目录
|
||
run: |
|
||
ssh -i ~/.ssh/zy_gpu_key -o StrictHostKeyChecking=no \
|
||
"${{ secrets.ZY_GPU_USER }}@${{ secrets.ZY_GPU_HOST }}" \
|
||
'sudo mkdir -p /opt/guanghu/training/configs /data/guanghu/{raw,processed,checkpoints,eval,models} && sudo chown -R ${{ secrets.ZY_GPU_USER }}:${{ secrets.ZY_GPU_USER }} /opt/guanghu/training /data/guanghu'
|
||
|
||
- name: 📤 SCP 最新训练脚本
|
||
run: |
|
||
scp -i ~/.ssh/zy_gpu_key -o StrictHostKeyChecking=no \
|
||
server/training-agent/progress-reporter.sh \
|
||
server/training-agent/setup.sh \
|
||
server/training-agent/start-training.sh \
|
||
server/training-agent/watch-training-output.sh \
|
||
server/training-agent/download-model.py \
|
||
server/training-agent/preprocess-corpus.py \
|
||
server/training-agent/train.py \
|
||
"${{ secrets.ZY_GPU_USER }}@${{ secrets.ZY_GPU_HOST }}:/opt/guanghu/training/"
|
||
scp -i ~/.ssh/zy_gpu_key -o StrictHostKeyChecking=no \
|
||
server/training-agent/configs/ds_zero3_offload.json \
|
||
"${{ secrets.ZY_GPU_USER }}@${{ secrets.ZY_GPU_HOST }}:/opt/guanghu/training/configs/"
|
||
ssh -i ~/.ssh/zy_gpu_key -o StrictHostKeyChecking=no \
|
||
"${{ secrets.ZY_GPU_USER }}@${{ secrets.ZY_GPU_HOST }}" \
|
||
"chmod +x /opt/guanghu/training/*.sh /opt/guanghu/training/*.py"
|
||
|
||
- name: 🔐 写入 .env (服务器侧)
|
||
env:
|
||
ZY_COS_SECRET_ID: ${{ secrets.ZY_COS_SECRET_ID }}
|
||
ZY_COS_SECRET_KEY: ${{ secrets.ZY_COS_SECRET_KEY }}
|
||
ZY_DISPATCH_TOKEN: ${{ secrets.ZY_DISPATCH_TOKEN }}
|
||
REPO_OWNER: ${{ github.repository_owner }}
|
||
REPO_NAME: ${{ github.event.repository.name }}
|
||
run: |
|
||
ssh -i ~/.ssh/zy_gpu_key -o StrictHostKeyChecking=no \
|
||
"${{ secrets.ZY_GPU_USER }}@${{ secrets.ZY_GPU_HOST }}" \
|
||
"umask 077 && cat > /opt/guanghu/training/.env && chmod 600 /opt/guanghu/training/.env" <<EOF
|
||
ZY_COS_SECRET_ID=${ZY_COS_SECRET_ID}
|
||
ZY_COS_SECRET_KEY=${ZY_COS_SECRET_KEY}
|
||
ZY_COS_BUCKET=sy-finetune-corpus-1317346199
|
||
ZY_COS_REGION=ap-guangzhou
|
||
GH_REPO_OWNER=${REPO_OWNER}
|
||
GH_REPO_NAME=${REPO_NAME}
|
||
GH_DISPATCH_TOKEN=${ZY_DISPATCH_TOKEN}
|
||
ZY_TRAIN_ROOT=/opt/guanghu/training
|
||
ZY_TRAIN_DATA=/data/guanghu
|
||
EOF
|
||
|
||
- name: 🛠️ 跑 Bootstrap (装栈 + 拉模型 + 预处理)
|
||
if: github.event_name == 'push' || github.event.inputs.skip_bootstrap != 'true'
|
||
run: |
|
||
ssh -i ~/.ssh/zy_gpu_key -o StrictHostKeyChecking=no -o ServerAliveInterval=30 \
|
||
"${{ secrets.ZY_GPU_USER }}@${{ secrets.ZY_GPU_HOST }}" \
|
||
"sudo -E bash /opt/guanghu/training/setup.sh 2>&1 | tee /tmp/zy-bootstrap.log; tail -c 6000 /tmp/zy-bootstrap.log"
|
||
|
||
- name: ▶️ 启动训练 (tmux)
|
||
if: github.event_name == 'push' || github.event.inputs.skip_start != 'true'
|
||
run: |
|
||
ssh -i ~/.ssh/zy_gpu_key -o StrictHostKeyChecking=no \
|
||
"${{ secrets.ZY_GPU_USER }}@${{ secrets.ZY_GPU_HOST }}" \
|
||
"bash /opt/guanghu/training/start-training.sh --tmux"
|
||
echo "✅ 训练已在 tmux session 'zy-train' 中启动"
|
||
echo " 附加查看: ssh ubuntu@${{ secrets.ZY_GPU_HOST }} 'tmux attach -t zy-train'"
|
||
echo " 仓库首页会实时刷新进度"
|
||
|
||
- name: 📝 摘要
|
||
if: always()
|
||
run: |
|
||
{
|
||
echo "## 训练自动执行 · run #${{ github.run_number }}"
|
||
echo ""
|
||
echo "- Commit: \`${{ github.sha }}\`"
|
||
echo "- 服务器: \`${{ secrets.ZY_GPU_HOST }}\` (zy-gpu-train · V100×4)"
|
||
echo "- 时间: $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
|
||
echo ""
|
||
echo "查看进度 → [仓库首页](../../#-训练实时仪表盘--zy-train-001)"
|
||
echo ""
|
||
echo "问铸渊副将 → [新建 Issue](../../issues/new?template=ask-zhuyuan-training.md)"
|
||
} >> "$GITHUB_STEP_SUMMARY"
|