66 lines
4.0 KiB
Bash
66 lines
4.0 KiB
Bash
|
#!/bin/bash
|
|||
|
|
# 📊 光湖 · 母模型训练全历史追踪器
|
|||
|
|
# 先显示所有历史，再持续追加新步骤，不刷新不覆盖
|
|||
|
|
|
|||
|
|
LOG_FILE="/root/autodl-tmp/train_mother.log"
|
|||
|
|
|
|||
|
|
echo ""
|
|||
|
|
echo "╔══════════════════════════════════════════════════════════╗"
|
|||
|
|
echo "║ 📊 光湖 · 母模型训练全历史追踪 ║"
|
|||
|
|
echo "║ 铸渊 · 先显示已有历史 → 再持续追加新步 ║"
|
|||
|
|
echo "╚══════════════════════════════════════════════════════════╝"
|
|||
|
|
echo ""
|
|||
|
|
|
|||
|
|
# ─── 第一步：显示已有的全部历史步骤 ───
|
|||
|
|
if [ -f "$LOG_FILE" ] && grep -q "'loss':" "$LOG_FILE" 2>/dev/null; then
|
|||
|
|
echo " 📜 已有训练记录："
|
|||
|
|
echo " ────────────────────────────────────────────────────────"
|
|||
|
|
echo " 时间 Loss GradNorm LR Epoch"
|
|||
|
|
echo " ────────────────────────────────────────────────────────"
|
|||
|
|
grep "'loss':" "$LOG_FILE" | while IFS= read -r line; do
|
|||
|
|
LOSS=$(echo "$line" | grep -o "'loss': '[^']*'" | cut -d"'" -f4)
|
|||
|
|
GN=$(echo "$line" | grep -o "'grad_norm': '[^']*'" | cut -d"'" -f4)
|
|||
|
|
LR=$(echo "$line" | grep -o "'learning_rate': '[^']*'" | cut -d"'" -f4)
|
|||
|
|
EPOCH=$(echo "$line" | grep -o "'epoch': '[^']*'" | cut -d"'" -f4)
|
|||
|
|
echo " --:--:-- ${LOSS} ${GN} ${LR} ${EPOCH}"
|
|||
|
|
done
|
|||
|
|
echo " ────────────────────────────────────────────────────────"
|
|||
|
|
echo ""
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
if [ -f "$LOG_FILE" ] && grep -q "Traceback" "$LOG_FILE" 2>/dev/null; then
|
|||
|
|
echo " ❌ 检测到训练报错！最后错误："
|
|||
|
|
grep "Traceback" "$LOG_FILE" | tail -3
|
|||
|
|
echo ""; exit 1
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
if [ -f "$LOG_FILE" ] && grep -q "DONE!" "$LOG_FILE" 2>/dev/null; then
|
|||
|
|
echo " ✅✅✅ 训练已完成！模型保存位置："
|
|||
|
|
ls -lh /root/autodl-tmp/output/qwen25-7b-sft/final/ 2>/dev/null
|
|||
|
|
echo ""; exit 0
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
echo " 🔄 等待新训练步骤...（Ctrl+C 退出）"
|
|||
|
|
echo " ────────────────────────────────────────────────────────"
|
|||
|
|
|
|||
|
|
LAST_COUNT=$(grep -c "'loss':" "$LOG_FILE" 2>/dev/null || echo 0)
|
|||
|
|
|
|||
|
|
while true; do
|
|||
|
|
NEW_COUNT=$(grep -c "'loss':" "$LOG_FILE" 2>/dev/null || echo 0)
|
|||
|
|
if [ "$NEW_COUNT" -gt "$LAST_COUNT" ]; then
|
|||
|
|
tail -n +$(( $(wc -l < "$LOG_FILE") - 10 )) "$LOG_FILE" 2>/dev/null | grep "'loss':" | tail -1 | while IFS= read -r line; do
|
|||
|
|
LOSS=$(echo "$line" | grep -o "'loss': '[^']*'" | cut -d"'" -f4)
|
|||
|
|
GN=$(echo "$line" | grep -o "'grad_norm': '[^']*'" | cut -d"'" -f4)
|
|||
|
|
LR=$(echo "$line" | grep -o "'learning_rate': '[^']*'" | cut -d"'" -f4)
|
|||
|
|
EPOCH=$(echo "$line" | grep -o "'epoch': '[^']*'" | cut -d"'" -f4)
|
|||
|
|
GPU=$(nvidia-smi --query-gpu=utilization.gpu,memory.used,temperature.gpu --format=csv,noheader,nounits 2>/dev/null)
|
|||
|
|
GU=$(echo "$GPU"|cut -d',' -f1|tr -d ' '); MT=$(echo "$GPU"|cut -d',' -f2|tr -d ' '); GT=$(echo "$GPU"|cut -d',' -f3|tr -d ' ')
|
|||
|
|
printf " %s 📉%-8s 📐%-8s 📈%-12s 🌀%-8s ⚡%s%% 🌡️%s°C\n" "$(date '+%H:%M:%S')" "$LOSS" "$GN" "$LR" "$EPOCH" "$GU" "$GT"
|
|||
|
|
done
|
|||
|
|
LAST_COUNT=$NEW_COUNT
|
|||
|
|
fi
|
|||
|
|
if grep -q "Traceback" "$LOG_FILE" 2>/dev/null; then echo ""; echo " ❌❌❌ 训练报错！请找铸渊"; exit 1; fi
|
|||
|
|
if grep -q "DONE!" "$LOG_FILE" 2>/dev/null; then echo ""; echo " ✅✅✅ 训练完成！"; ls -lh /root/autodl-tmp/output/qwen25-7b-sft/final/ 2>/dev/null; exit 0; fi
|
|||
|
|
sleep 5
|
|||
|
|
done
|