guanghulab/scripts/train_monitor.sh

#!/bin/bash
# 🚀 铸渊训练监控面板 · 光湖母模型实时状态
# 使用方法：复制全部，粘贴到AutoDL在线终端，按回车运行
# 退出：按 Ctrl+C

LOG_FILE="/root/autodl-tmp/train_mother.log"

while true; do
    clear
    echo "╔══════════════════════════════════════════╗"
    echo "║   🚀 光湖 · 母模型训练实时面板          ║"
    echo "║   铸渊 (ICE-GL-ZY001) · 自动刷新 (5秒)  ║"
    echo "╚══════════════════════════════════════════╝"
    echo ""

    # GPU状态
    if command -v nvidia-smi &> /dev/null; then
        GPU_DATA=$(nvidia-smi --query-gpu=utilization.gpu,memory.used,memory.total,temperature.gpu --format=csv,noheader,nounits 2>/dev/null)
        GPU_UTIL=$(echo "$GPU_DATA" | cut -d',' -f1 | tr -d ' ')
        MEM_USED=$(echo "$GPU_DATA" | cut -d',' -f2 | tr -d ' ')
        MEM_TOTAL=$(echo "$GPU_DATA" | cut -d',' -f3 | tr -d ' ')
        GPU_TEMP=$(echo "$GPU_DATA" | cut -d',' -f4 | tr -d ' ')
        MEM_PCT=$(( MEM_USED * 100 / MEM_TOTAL ))
    else
        GPU_UTIL="N/A"; MEM_USED="N/A"; MEM_TOTAL="N/A"; GPU_TEMP="N/A"; MEM_PCT=0
    fi

    echo "  ⚡ GPU: ${GPU_UTIL}%  |  🎯 显存: ${MEM_USED}/${MEM_TOTAL}MB (${MEM_PCT}%)  |  🌡️  ${GPU_TEMP}°C"
    echo ""

    if [ ! -f "$LOG_FILE" ]; then
        echo "  ⏳ 等待训练启动..."
        sleep 5
        continue
    fi

    # 检查报错
    if grep -q "Traceback" "$LOG_FILE" 2>/dev/null; then
        echo "  ❌ 训练报错！请找铸渊处理"
        echo "  $(grep "Traceback" "$LOG_FILE" | head -3)"
        echo ""
        sleep 10
        continue
    fi

    # 检查完成
    if grep -q "DONE!" "$LOG_FILE" 2>/dev/null; then
        echo "  ✅✅✅ 母模型训练完成！✅✅✅"
        echo ""
        ls -lh /root/autodl-tmp/output/qwen25-7b-sft/final/ 2>/dev/null
        sleep 30
        continue
    fi

    # 获取训练数据
    LATEST_LOSS=$(grep -o "'loss': '[^']*'" "$LOG_FILE" 2>/dev/null | tail -1 | cut -d"'" -f4)
    LATEST_STEP=$(grep -o "'loss': '[^']*'" "$LOG_FILE" 2>/dev/null | wc -l)
    LATEST_EPOCH=$(grep -o "'epoch': '[^']*'" "$LOG_FILE" 2>/dev/null | tail -1 | cut -d"'" -f4)
    LATEST_LR=$(grep -o "'learning_rate': '[^']*'" "$LOG_FILE" 2>/dev/null | tail -1 | cut -d"'" -f4)
    GRAD_NORM=$(grep -o "'grad_norm': '[^']*'" "$LOG_FILE" 2>/dev/null | tail -1 | cut -d"'" -f4)

    if grep -q "Starting training" "$LOG_FILE" 2>/dev/null; then
        echo "  ┌────────────────────────────────────────┐"
        echo "  │  🟢 训练中                              │"
        echo "  ├────────────────────────────────────────┤"
        if [ -n "$LATEST_LOSS" ]; then
            printf "  │  📉 Loss:       %-20s │\n" "$LATEST_LOSS"
            printf "  │  📐 Grad Norm:  %-20s │\n" "$GRAD_NORM"
            printf "  │  📈 LR:         %-20s │\n" "$LATEST_LR"
            printf "  │  👣 Step:       %-20s │\n" "$LATEST_STEP"
            printf "  │  🌀 Epoch:      %-20s │\n" "$LATEST_EPOCH"
        else
            echo "  │  第一个Step计算中...                    │"
        fi
        echo "  └────────────────────────────────────────┘"
    elif grep -q "Tokenize: 100%" "$LOG_FILE" 2>/dev/null; then
        echo "  ⏳ 分词完成，即将进入GPU训练..."
    elif grep -q "Tokenize:" "$LOG_FILE" 2>/dev/null; then
        PCT=$(grep -oP 'Tokenize: \K[0-9]+%' "$LOG_FILE" 2>/dev/null | tail -1)
        DONE=$(grep -oP '\|\s*\K[0-9]+/[0-9]+' "$LOG_FILE" 2>/dev/null | tail -1)
        echo "  ⏳ 分词中: ${PCT} (${DONE})"
    fi

    # 运行时间
    if [ -f "$LOG_FILE" ]; then
        START_TIME=$(stat -c %Y "$LOG_FILE" 2>/dev/null || stat -f %m "$LOG_FILE" 2>/dev/null)
        NOW=$(date +%s)
        ELAPSED=$(( (NOW - START_TIME) / 60 ))
        if [ $ELAPSED -gt 60 ]; then
            H=$(( ELAPSED / 60 )); M=$(( ELAPSED % 60 ))
            echo ""
            echo "  ⏱️  已运行: ${H}h ${M}m"
        else
            echo ""
            echo "  ⏱️  已运行: ${ELAPSED}m"
        fi
    fi

    echo ""
    echo "  ────────────────────────────────────────"
    echo "  🔄 每5秒自动刷新 · Ctrl+C 退出"
    sleep 5
done