# ═══════════════════════════════════════════════════════════
# 🛠️ GPU 训练机 Bootstrap · training-bootstrap.yml
# ═══════════════════════════════════════════════════════════
# 签发: 铸渊 · ICE-GL-ZY001
# 版权: 国作登字-2026-A-00037559 · TCS-0002∞
#
# 把 server/training-agent/ 下的脚本 SCP 到 GPU 训练机
# (zy-gpu-train · 119.45.160.137)，写入 .env，触发 setup.sh。
# 全程心跳→训练实时仪表盘。
#
# 必需 Secrets:
#   ZY_GPU_HOST            119.45.160.137
#   ZY_GPU_USER            ubuntu
#   ZY_GPU_KEY             SSH 私钥
#   ZY_COS_SECRET_ID       腾讯云 SecretId
#   ZY_COS_SECRET_KEY      腾讯云 SecretKey
#   ZY_DISPATCH_TOKEN      仓库 dispatch PAT (servers→github 心跳所需)
#                          ⚠️ 需要冰朔在 Settings > Secrets 新建一个 PAT,
#                          权限: repo (classic) 或 contents+actions (fine-grained)
#                          作用域: 仅本仓库
# ═══════════════════════════════════════════════════════════

name: 🛠️ 训练机 Bootstrap

# ⚠️ ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# 2026-05-03 D72：训练机自动同步已停用。母模型训练转交 Notion 侧
# 霜砚 + 服务器直接协作。SSH/SCP 操作必须 confirm_override=true。
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━

on:
  workflow_dispatch:
    inputs:
      confirm_override:
        description: '⚠️ 我已确认霜砚同意 + 服务器无运行中训练，重新启用 bootstrap'
        type: boolean
        default: false
      action:
        description: '动作'
        required: true
        default: 'probe'
        type: choice
        options:
          - probe         # 仅探活 SSH（safe）
          - status        # 查询状态（safe）
          - bootstrap     # 上传脚本 + 跑 setup.sh
          - sync-scripts  # 仅上传脚本（不跑 setup）
          - start         # 启动训练（tmux）
          - stop          # 停训练

permissions:
  contents: read

concurrency:
  group: training-bootstrap
  cancel-in-progress: false

jobs:
  guard:
    name: '🛑 同步停用守卫'
    runs-on: ubuntu-latest
    outputs:
      proceed: ${{ steps.check.outputs.proceed }}
    steps:
      - name: 检查覆盖确认
        id: check
        run: |
          ACTION="${{ github.event.inputs.action }}"
          # probe / status 两个只读动作允许通过 (不修改服务器状态)
          if [[ "$ACTION" == "probe" || "$ACTION" == "status" ]]; then
            echo "ℹ️ 只读动作 ($ACTION)，允许执行"
            echo "proceed=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          if [[ "${{ github.event.inputs.confirm_override }}" == "true" ]]; then
            echo "✅ 收到覆盖确认 + 写动作 ($ACTION)，继续"
            echo "proceed=true" >> "$GITHUB_OUTPUT"
          else
            echo "🛑 写动作 ($ACTION) 需要勾选 confirm_override，本次中止"
            echo "proceed=false" >> "$GITHUB_OUTPUT"
            {
              echo "## 🛑 训练机 Bootstrap 已停用"
              echo ""
              echo "动作 \`$ACTION\` 会修改服务器状态，已被 D72 决策保护。"
              echo "如需执行，请勾选 \`confirm_override\` 后再触发。"
              echo ""
              echo "只读动作 \`probe\` / \`status\` 不需要勾选。"
            } >> "$GITHUB_STEP_SUMMARY"
          fi

  bootstrap:
    name: '🚀 ${{ github.event.inputs.action }}'
    needs: guard
    if: needs.guard.outputs.proceed == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - name: 📥 Checkout
        uses: actions/checkout@v4

      - name: 🔑 配置 SSH
        run: |
          mkdir -p ~/.ssh
          chmod 700 ~/.ssh
          echo "${{ secrets.ZY_GPU_KEY }}" > ~/.ssh/zy_gpu_key
          chmod 600 ~/.ssh/zy_gpu_key
          ssh-keyscan -H "${{ secrets.ZY_GPU_HOST }}" >> ~/.ssh/known_hosts 2>/dev/null

      - name: 🛰️ 探活
        run: |
          ssh -i ~/.ssh/zy_gpu_key -o StrictHostKeyChecking=no -o ConnectTimeout=15 \
            "${{ secrets.ZY_GPU_USER }}@${{ secrets.ZY_GPU_HOST }}" \
            "echo '✅ SSH OK · $(hostname) · uptime:' && uptime && echo '--- nvidia ---' && (nvidia-smi --query-gpu=name,driver_version,memory.total --format=csv,noheader || echo '驱动尚未就绪')"

      - name: 📡 上报 Bootstrap 开始
        if: github.event.inputs.action == 'bootstrap' || github.event.inputs.action == 'start'
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          curl -sS -X POST \
            -H "Accept: application/vnd.github+json" \
            -H "Authorization: Bearer $GH_TOKEN" \
            -H "X-GitHub-Api-Version: 2022-11-28" \
            "https://api.github.com/repos/${{ github.repository }}/dispatches" \
            -d '{
              "event_type": "training-progress",
              "client_payload": {
                "phase": "bootstrapping",
                "phase_label": "铸渊触发 ${{ github.event.inputs.action }}",
                "level": "info",
                "message": "training-bootstrap.yml run #${{ github.run_number }} 触发动作=${{ github.event.inputs.action }}",
                "health": { "status": "ok", "message": "bootstrap workflow 启动" }
              }
            }' || true

      - name: 📤 上传脚本到服务器
        if: github.event.inputs.action == 'bootstrap' || github.event.inputs.action == 'sync-scripts' || github.event.inputs.action == 'start' || github.event.inputs.action == 'stop'
        run: |
          ssh -i ~/.ssh/zy_gpu_key -o StrictHostKeyChecking=no \
            "${{ secrets.ZY_GPU_USER }}@${{ secrets.ZY_GPU_HOST }}" \
            'sudo mkdir -p /opt/guanghu/training/configs && sudo chown -R ${{ secrets.ZY_GPU_USER }}:${{ secrets.ZY_GPU_USER }} /opt/guanghu/training'
          scp -i ~/.ssh/zy_gpu_key -o StrictHostKeyChecking=no \
            server/training-agent/progress-reporter.sh \
            server/training-agent/setup.sh \
            server/training-agent/start-training.sh \
            server/training-agent/watch-training-output.sh \
            server/training-agent/download-model.py \
            server/training-agent/preprocess-corpus.py \
            server/training-agent/train.py \
            "${{ secrets.ZY_GPU_USER }}@${{ secrets.ZY_GPU_HOST }}:/opt/guanghu/training/"
          scp -i ~/.ssh/zy_gpu_key -o StrictHostKeyChecking=no \
            server/training-agent/configs/ds_zero3_offload.json \
            "${{ secrets.ZY_GPU_USER }}@${{ secrets.ZY_GPU_HOST }}:/opt/guanghu/training/configs/"

      - name: 🔐 写入 .env (服务器侧)
        if: github.event.inputs.action == 'bootstrap' || github.event.inputs.action == 'sync-scripts'
        env:
          ZY_COS_SECRET_ID: ${{ secrets.ZY_COS_SECRET_ID }}
          ZY_COS_SECRET_KEY: ${{ secrets.ZY_COS_SECRET_KEY }}
          ZY_DISPATCH_TOKEN: ${{ secrets.ZY_DISPATCH_TOKEN }}
          REPO_OWNER: ${{ github.repository_owner }}
          REPO_NAME: ${{ github.event.repository.name }}
        run: |
          # 通过 stdin 写,避免在命令行/参数列表中暴露 secrets
          ssh -i ~/.ssh/zy_gpu_key -o StrictHostKeyChecking=no \
            "${{ secrets.ZY_GPU_USER }}@${{ secrets.ZY_GPU_HOST }}" \
            "umask 077 && cat > /opt/guanghu/training/.env && chmod 600 /opt/guanghu/training/.env" <<EOF
          ZY_COS_SECRET_ID=${ZY_COS_SECRET_ID}
          ZY_COS_SECRET_KEY=${ZY_COS_SECRET_KEY}
          ZY_COS_BUCKET=sy-finetune-corpus-1317346199
          ZY_COS_REGION=ap-guangzhou
          GH_REPO_OWNER=${REPO_OWNER}
          GH_REPO_NAME=${REPO_NAME}
          GH_DISPATCH_TOKEN=${ZY_DISPATCH_TOKEN}
          ZY_TRAIN_ROOT=/opt/guanghu/training
          ZY_TRAIN_DATA=/data/guanghu
          EOF
          ssh -i ~/.ssh/zy_gpu_key -o StrictHostKeyChecking=no \
            "${{ secrets.ZY_GPU_USER }}@${{ secrets.ZY_GPU_HOST }}" \
            "chmod +x /opt/guanghu/training/*.sh"

      - name: 🛠️ 执行 setup.sh
        if: github.event.inputs.action == 'bootstrap'
        run: |
          ssh -i ~/.ssh/zy_gpu_key -o StrictHostKeyChecking=no \
            "${{ secrets.ZY_GPU_USER }}@${{ secrets.ZY_GPU_HOST }}" \
            "sudo -E bash /opt/guanghu/training/setup.sh 2>&1 | tee /tmp/zy-bootstrap.log; tail -c 4000 /tmp/zy-bootstrap.log"

      - name: ▶️ 启动训练 (tmux)
        if: github.event.inputs.action == 'start'
        run: |
          ssh -i ~/.ssh/zy_gpu_key -o StrictHostKeyChecking=no \
            "${{ secrets.ZY_GPU_USER }}@${{ secrets.ZY_GPU_HOST }}" \
            "bash /opt/guanghu/training/start-training.sh --tmux"

      - name: ⏹️ 停止训练
        if: github.event.inputs.action == 'stop'
        run: |
          ssh -i ~/.ssh/zy_gpu_key -o StrictHostKeyChecking=no \
            "${{ secrets.ZY_GPU_USER }}@${{ secrets.ZY_GPU_HOST }}" \
            "bash /opt/guanghu/training/start-training.sh --stop"

      - name: 📊 查询状态
        if: github.event.inputs.action == 'status'
        run: |
          ssh -i ~/.ssh/zy_gpu_key -o StrictHostKeyChecking=no \
            "${{ secrets.ZY_GPU_USER }}@${{ secrets.ZY_GPU_HOST }}" \
            "echo '=== 阶段 ===' && cat /opt/guanghu/training/.phase 2>/dev/null || echo idle; \
             echo '=== tmux ===' && (tmux ls 2>/dev/null || echo no-tmux); \
             echo '=== nvidia-smi ===' && nvidia-smi --query-gpu=index,name,utilization.gpu,memory.used,memory.total,temperature.gpu --format=csv 2>/dev/null || echo no-gpu; \
             echo '=== disk ===' && df -h /data/guanghu 2>/dev/null || df -h /; \
             echo '=== last log lines ===' && tail -n 40 /opt/guanghu/training/training.log 2>/dev/null || echo no-log"

      - name: 📝 摘要
        if: always()
        run: |
          {
            echo "## 训练机操作: \`${{ github.event.inputs.action }}\`"
            echo ""
            echo "- 服务器: \`${{ secrets.ZY_GPU_HOST }}\`"
            echo "- 时间: $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
            echo "- Run #: ${{ github.run_number }}"
          } >> "$GITHUB_STEP_SUMMARY"