bingshuo/guanghulab
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity
guanghulab/data/training/state.json

387 lines
11 KiB
JSON
Raw Permalink Normal View History

chore: import sanitized domestic snapshot for REPO-002 Source snapshot: ca48d3ddf926d79aa138306164169baf764bb829
2026-07-17 15:54:41 +08:00
{
"schema_version": "1.0",
"frozen": true,
"frozen_at": "2026-05-03T08:56:40Z",
"frozen_reason": "D72 决策：母模型训练转交 Notion 侧霜砚，仓库不再接收训练心跳。详见 docs/zhuyuan-handover/05-stop-sync.md",
"_doc": "训练实时状态真相源 · 已冻结 · 不再被 merge-event.js 写入 · 解冻方法见 README 训练仪表盘段落",
"task": {
"id": "ZY-TRAIN-001",
"name": "Qwen2.5-7B 全参数 SFT · 光湖世界观 + 人格体语料",
"model_base": "Qwen/Qwen2.5-7B",
"started_at": "2026-05-02T19:34:41.648Z",
"updated_at": "2026-05-03T09:29:05.896Z",
"ended_at": "2026-05-03T09:29:05.896Z"
},
"server": {
"host": "119.45.160.137",
"instance_name": "zy-gpu-train",
"instance_type": "GN10Xp.10XLARGE160",
"region": "ap-nanjing-3",
"gpu": "V100 32G × 4 (合计 128G)",
"os": "Ubuntu 22.04 LTS",
"cuda": "12.8.1",
"cudnn": "9.10.2",
"driver": "570.158.01"
},
"cos": {
"bucket": "sy-finetune-corpus-1317346199",
"region": "ap-guangzhou",
"raw_files": null,
"processed_files": null,
"checkpoints": null
},
"phase": "done",
"_phase_doc": "idle | bootstrapping | downloading-corpus | preprocessing | training | evaluating | done | error",
"phase_label": "训练完成",
"progress": {
"step": 0,
"total_steps": 0,
"epoch": 0,
"total_epochs": 0,
"percent": 0,
"loss": null,
"learning_rate": null,
"throughput_samples_per_sec": null,
"eta_seconds": null,
"elapsed_seconds": 0
},
"gpu_metrics": {
"snapshot_at": "2026-05-03T09:28:54Z",
"devices": [
{
"index": 0,
"memory_total_mib": 32768,
"memory_used_mib": 0,
"name": "Tesla V100-SXM2-32GB",
"power_w": 70.95,
"temperature_c": 57,
"util_percent": 0
},
{
"index": 1,
"memory_total_mib": 32768,
"memory_used_mib": 0,
"name": "Tesla V100-SXM2-32GB",
"power_w": 65.08,
"temperature_c": 52,
"util_percent": 0
},
{
"index": 2,
"memory_total_mib": 32768,
"memory_used_mib": 0,
"name": "Tesla V100-SXM2-32GB",
"power_w": 74.39,
"temperature_c": 54,
"util_percent": 0
},
{
"index": 3,
"memory_total_mib": 32768,
"memory_used_mib": 0,
"name": "Tesla V100-SXM2-32GB",
"power_w": 76.24,
"temperature_c": 57,
"util_percent": 0
}
]
},
"health": {
"status": "ok",
"_status_doc": "ok | warning | error | idle",
"message": "auto-run workflow 启动",
"last_heartbeat_at": "2026-05-03T09:29:05.896Z"
},
"timeline": [
{
"at": "2026-05-02T16:42:00Z",
"phase": "idle",
"level": "info",
"message": "铸渊副将已接管。GPU 训练机 119.45.160.137 已开机，5 个 Secrets 已配置，COS 桶语料已就位。"
},
{
"at": "2026-05-02T17:50:00Z",
"phase": "idle",
"level": "info",
"message": "ZY_DISPATCH_TOKEN 已配置。真实训练栈落地：train.py (Qwen2.5-7B 全参 SFT) + DeepSpeed ZeRO-3 + 模型自动下载 + 语料预处理。合并 PR 后 training-auto-run 工作流自动 SCP/setup/启动训练，仓库首页将开始实时刷新。"
},
{
"at": "2026-05-02T19:02:00.614Z",
"phase": "bootstrapping",
"level": "info",
"message": "Bootstrap started on VM-0-11-ubuntu"
},
{
"at": "2026-05-02T19:02:10.707Z",
"phase": "bootstrapping",
"level": "info",
"message": "apt 包安装完成"
},
{
"at": "2026-05-02T19:11:44.714Z",
"phase": "bootstrapping",
"level": "info",
"message": "torch + transformers + accelerate + deepspeed + modelscope 安装完成"
},
{
"at": "2026-05-02T19:12:56.711Z",
"phase": "preprocessing",
"level": "info",
"message": "Bootstrap done on VM-0-11-ubuntu. raw=2 files. Ready to start training."
},
{
"at": "2026-05-02T19:34:41.648Z",
"phase": "training",
"level": "info",
"message": "start-training.sh launched on VM-0-11-ubuntu"
},
{
"at": "2026-05-02T19:35:48.820Z",
"phase": "done",
"level": "info",
"message": "train.py exited cleanly on VM-0-11-ubuntu"
},
{
"at": "2026-05-03T05:33:56.539Z",
"phase": "training",
"level": "info",
"message": "start-training.sh launched on VM-0-11-ubuntu"
},
{
"at": "2026-05-03T05:37:02.879Z",
"phase": "done",
"level": "info",
"message": "train.py exited cleanly on VM-0-11-ubuntu"
},
{
"at": "2026-05-03T05:41:36.877Z",
"phase": "training",
"level": "info",
"message": "start-training.sh launched on VM-0-11-ubuntu"
},
{
"at": "2026-05-03T05:42:46.428Z",
"phase": "done",
"level": "info",
"message": "train.py exited cleanly on VM-0-11-ubuntu"
},
{
"at": "2026-05-03T06:17:34.391Z",
"phase": "bootstrapping",
"level": "info",
"message": "commit 9884be1f5bdc7c847fdf24038c44aacc7a7d912f 合并到 main, 自动训练流程启动"
},
{
"at": "2026-05-03T06:18:14.923Z",
"phase": "bootstrapping",
"level": "info",
"message": "Bootstrap started on VM-0-11-ubuntu"
},
{
"at": "2026-05-03T06:18:27.782Z",
"phase": "bootstrapping",
"level": "info",
"message": "torch + transformers + accelerate + deepspeed + modelscope 安装完成"
},
{
"at": "2026-05-03T06:19:36.454Z",
"phase": "preprocessing",
"level": "info",
"message": "running preprocess-corpus.py"
},
{
"at": "2026-05-03T06:19:54.961Z",
"phase": "preprocessing",
"level": "info",
"message": "SFT 样本数=11456 写入 /data/guanghu/processed/sft.jsonl"
},
{
"at": "2026-05-03T06:20:05.467Z",
"phase": "training",
"level": "info",
"message": "deepspeed --num_gpus=4 train.py"
},
{
"at": "2026-05-03T06:24:24.164Z",
"phase": "done",
"level": "info",
"message": "train.py exited cleanly on VM-0-11-ubuntu"
},
{
"at": "2026-05-03T06:25:17.707Z",
"phase": "training",
"level": "info",
"message": "start-training.sh launched on VM-0-11-ubuntu"
},
{
"at": "2026-05-03T06:30:35.282Z",
"phase": "done",
"level": "info",
"message": "train.py exited cleanly on VM-0-11-ubuntu"
},
{
"at": "2026-05-03T06:48:05.610Z",
"phase": "bootstrapping",
"level": "info",
"message": "commit 7961cfb4610270537130ef3675c9bba44b0e7649 合并到 main, 自动训练流程启动"
},
{
"at": "2026-05-03T06:48:40.985Z",
"phase": "bootstrapping",
"level": "info",
"message": "Bootstrap started on VM-0-11-ubuntu"
},
{
"at": "2026-05-03T06:49:50.545Z",
"phase": "bootstrapping",
"level": "info",
"message": "ModelScope snapshot_download qwen/Qwen2.5-7B"
},
{
"at": "2026-05-03T06:51:00.558Z",
"phase": "training",
"level": "info",
"message": "deepspeed --num_gpus=4 train.py"
},
{
"at": "2026-05-03T06:52:15.860Z",
"phase": "bootstrapping",
"level": "info",
"message": "ModelScope snapshot_download qwen/Qwen2.5-7B"
},
{
"at": "2026-05-03T06:53:19.509Z",
"phase": "preprocessing",
"level": "info",
"message": "Bootstrap done on VM-0-11-ubuntu. raw=2 files. Ready to start training."
},
{
"at": "2026-05-03T06:55:29.729Z",
"phase": "training",
"level": "info",
"message": "start-training.sh launched on VM-0-11-ubuntu"
},
{
"at": "2026-05-03T06:56:36.190Z",
"phase": "done",
"level": "info",
"message": "train.py exited cleanly on VM-0-11-ubuntu"
},
{
"at": "2026-05-03T07:13:49.951Z",
"phase": "bootstrapping",
"level": "info",
"message": "commit 52f694cc468b9208e10c9fc44d48eb79a12f9a8c 合并到 main, 自动训练流程启动"
},
{
"at": "2026-05-03T07:14:39.031Z",
"phase": "bootstrapping",
"level": "info",
"message": "Bootstrap started on VM-0-11-ubuntu"
},
{
"at": "2026-05-03T07:14:49.274Z",
"phase": "bootstrapping",
"level": "info",
"message": "coscmd configured for sy-finetune-corpus-1317346199 (ap-guangzhou)"
},
{
"at": "2026-05-03T07:15:53.858Z",
"phase": "bootstrapping",
"level": "info",
"message": "Qwen2.5-7B 已就绪 14.20 GiB at /data/guanghu/models/Qwen2.5-7B"
},
{
"at": "2026-05-03T07:16:12.460Z",
"phase": "preprocessing",
"level": "info",
"message": "SFT 样本数=11456 写入 /data/guanghu/processed/sft.jsonl"
},
{
"at": "2026-05-03T07:17:17.765Z",
"phase": "done",
"level": "info",
"message": "train.py exited cleanly on VM-0-11-ubuntu"
},
{
"at": "2026-05-03T07:20:28.940Z",
"phase": "bootstrapping",
"level": "info",
"message": "Bootstrap started on VM-0-11-ubuntu"
},
{
"at": "2026-05-03T07:21:32.568Z",
"phase": "downloading-corpus",
"level": "info",
"message": "raw=2 files downloaded to /data/guanghu/raw"
},
{
"at": "2026-05-03T07:22:46.429Z",
"phase": "preprocessing",
"level": "info",
"message": "Bootstrap done on VM-0-11-ubuntu. raw=2 files. Ready to start training."
},
{
"at": "2026-05-03T07:24:00.244Z",
"phase": "training",
"level": "info",
"message": "start-training.sh launched on VM-0-11-ubuntu"
},
{
"at": "2026-05-03T07:25:11.486Z",
"phase": "done",
"level": "info",
"message": "train.py exited cleanly on VM-0-11-ubuntu"
},
{
"at": "2026-05-03T07:28:39.771Z",
"phase": "training",
"level": "info",
"message": "start-training.sh launched on VM-0-11-ubuntu"
},
{
"at": "2026-05-03T07:53:32.974Z",
"phase": "done",
"level": "info",
"message": "train.py exited cleanly on VM-0-11-ubuntu"
},
{
"at": "2026-05-03T08:11:20.490Z",
"phase": "training",
"level": "info",
"message": "start-training.sh launched on VM-0-11-ubuntu"
},
{
"at": "2026-05-03T08:36:12.139Z",
"phase": "done",
"level": "info",
"message": "train.py exited cleanly on VM-0-11-ubuntu"
},
{
"at": "2026-05-03T08:54:57.053Z",
"phase": "training",
"level": "info",
"message": "start-training.sh launched on VM-0-11-ubuntu"
},
{
"at": "2026-05-03T08:56:05.810Z",
"phase": "done",
"level": "info",
"message": "train.py exited cleanly on VM-0-11-ubuntu"
},
{
"at": "2026-05-03T08:58:42.082Z",
"phase": "training",
"level": "info",
"message": "start-training.sh launched on VM-0-11-ubuntu"
},
{
"at": "2026-05-03T09:29:05.896Z",
"phase": "done",
"level": "info",
"message": "train.py exited cleanly on VM-0-11-ubuntu"
}
],
"_timeline_doc": "最新事件在尾部 · 渲染时取最近 10 条倒序展示 · 由 progress-reporter 追加"
}
Reference in New Issue Copy Permalink