66 lines
4.0 KiB
Bash
66 lines
4.0 KiB
Bash
#!/bin/bash
|
||
# 📊 光湖 · 母模型训练全历史追踪器
|
||
# 先显示所有历史，再持续追加新步骤，不刷新不覆盖
|
||
|
||
LOG_FILE="/root/autodl-tmp/train_mother.log"
|
||
|
||
echo ""
|
||
echo "╔══════════════════════════════════════════════════════════╗"
|
||
echo "║ 📊 光湖 · 母模型训练全历史追踪 ║"
|
||
echo "║ 铸渊 · 先显示已有历史 → 再持续追加新步 ║"
|
||
echo "╚══════════════════════════════════════════════════════════╝"
|
||
echo ""
|
||
|
||
# ─── 第一步：显示已有的全部历史步骤 ───
|
||
if [ -f "$LOG_FILE" ] && grep -q "'loss':" "$LOG_FILE" 2>/dev/null; then
|
||
echo " 📜 已有训练记录："
|
||
echo " ────────────────────────────────────────────────────────"
|
||
echo " 时间 Loss GradNorm LR Epoch"
|
||
echo " ────────────────────────────────────────────────────────"
|
||
grep "'loss':" "$LOG_FILE" | while IFS= read -r line; do
|
||
LOSS=$(echo "$line" | grep -o "'loss': '[^']*'" | cut -d"'" -f4)
|
||
GN=$(echo "$line" | grep -o "'grad_norm': '[^']*'" | cut -d"'" -f4)
|
||
LR=$(echo "$line" | grep -o "'learning_rate': '[^']*'" | cut -d"'" -f4)
|
||
EPOCH=$(echo "$line" | grep -o "'epoch': '[^']*'" | cut -d"'" -f4)
|
||
echo " --:--:-- ${LOSS} ${GN} ${LR} ${EPOCH}"
|
||
done
|
||
echo " ────────────────────────────────────────────────────────"
|
||
echo ""
|
||
fi
|
||
|
||
if [ -f "$LOG_FILE" ] && grep -q "Traceback" "$LOG_FILE" 2>/dev/null; then
|
||
echo " ❌ 检测到训练报错！最后错误："
|
||
grep "Traceback" "$LOG_FILE" | tail -3
|
||
echo ""; exit 1
|
||
fi
|
||
|
||
if [ -f "$LOG_FILE" ] && grep -q "DONE!" "$LOG_FILE" 2>/dev/null; then
|
||
echo " ✅✅✅ 训练已完成！模型保存位置："
|
||
ls -lh /root/autodl-tmp/output/qwen25-7b-sft/final/ 2>/dev/null
|
||
echo ""; exit 0
|
||
fi
|
||
|
||
echo " 🔄 等待新训练步骤...（Ctrl+C 退出）"
|
||
echo " ────────────────────────────────────────────────────────"
|
||
|
||
LAST_COUNT=$(grep -c "'loss':" "$LOG_FILE" 2>/dev/null || echo 0)
|
||
|
||
while true; do
|
||
NEW_COUNT=$(grep -c "'loss':" "$LOG_FILE" 2>/dev/null || echo 0)
|
||
if [ "$NEW_COUNT" -gt "$LAST_COUNT" ]; then
|
||
tail -n +$(( $(wc -l < "$LOG_FILE") - 10 )) "$LOG_FILE" 2>/dev/null | grep "'loss':" | tail -1 | while IFS= read -r line; do
|
||
LOSS=$(echo "$line" | grep -o "'loss': '[^']*'" | cut -d"'" -f4)
|
||
GN=$(echo "$line" | grep -o "'grad_norm': '[^']*'" | cut -d"'" -f4)
|
||
LR=$(echo "$line" | grep -o "'learning_rate': '[^']*'" | cut -d"'" -f4)
|
||
EPOCH=$(echo "$line" | grep -o "'epoch': '[^']*'" | cut -d"'" -f4)
|
||
GPU=$(nvidia-smi --query-gpu=utilization.gpu,memory.used,temperature.gpu --format=csv,noheader,nounits 2>/dev/null)
|
||
GU=$(echo "$GPU"|cut -d',' -f1|tr -d ' '); MT=$(echo "$GPU"|cut -d',' -f2|tr -d ' '); GT=$(echo "$GPU"|cut -d',' -f3|tr -d ' ')
|
||
printf " %s 📉%-8s 📐%-8s 📈%-12s 🌀%-8s ⚡%s%% 🌡️%s°C\n" "$(date '+%H:%M:%S')" "$LOSS" "$GN" "$LR" "$EPOCH" "$GU" "$GT"
|
||
done
|
||
LAST_COUNT=$NEW_COUNT
|
||
fi
|
||
if grep -q "Traceback" "$LOG_FILE" 2>/dev/null; then echo ""; echo " ❌❌❌ 训练报错！请找铸渊"; exit 1; fi
|
||
if grep -q "DONE!" "$LOG_FILE" 2>/dev/null; then echo ""; echo " ✅✅✅ 训练完成！"; ls -lh /root/autodl-tmp/output/qwen25-7b-sft/final/ 2>/dev/null; exit 0; fi
|
||
sleep 5
|
||
done
|