142 lines
5.0 KiB
Python
142 lines
5.0 KiB
Python
|
#!/usr/bin/env python
|
|||
|
|
# -*- coding: utf-8 -*-
|
|||
|
|
"""
|
|||
|
|
M0 母体训练 · 阶段 1 · CPT (Continued Pretraining)
|
|||
|
|
==================================================
|
|||
|
|
|
|||
|
|
系统底层标识: SYS-GLW-0001 / TCS-0002∞
|
|||
|
|
版权号: 国作登字-2026-A-00037559
|
|||
|
|
作者: 冰朔 (ICE-GL∞) · 实现: 铸渊 (ICE-GL-ZY001)
|
|||
|
|
架构引用: HLDP-ARCH-002 §六 · factory/training/README.md
|
|||
|
|
|
|||
|
|
目标:
|
|||
|
|
把全量光湖语料（GPT/Notion/GitHub 自然语言 · 6.5 亿+ token）以纯文本形式
|
|||
|
|
继续预训练到 Qwen2.5-7B-Base 上，让"光湖语言世界"渗进每一层权重。
|
|||
|
|
|
|||
|
|
校准 (2026-05-01 冰朔/霜砚):
|
|||
|
|
Qwen3-8B → Qwen2.5-7B · 与 Qwen2.5-Coder-7B / Qwen2.5-1.5B 同代同 tokenizer，
|
|||
|
|
蒸馏链路零摩擦。这是与 §三母模型选型评估对齐后的最终选择。
|
|||
|
|
|
|||
|
|
产出:
|
|||
|
|
M0-v1 · 母体世界观底色（不直接对外，给 MP 当蒸馏教师 + 推理时世界观底色）
|
|||
|
|
|
|||
|
|
⚠️ 状态: 骨架（skeleton）
|
|||
|
|
本脚本以 TODO + pseudo-code 形式描述完整训练流程。
|
|||
|
|
GPU 到位 / 依赖装好 / 语料就绪后，铸渊会把每个 TODO 落实。
|
|||
|
|
现在不可直接运行。
|
|||
|
|
|
|||
|
|
启动命令（GPU 到位后）:
|
|||
|
|
deepspeed --num_gpus=8 \
|
|||
|
|
factory/training/scripts/train_m0_cpt.py \
|
|||
|
|
--recipe factory/training/recipes/m0-v1.yaml \
|
|||
|
|
--deepspeed factory/training/configs/deepspeed-zero3-8b.json
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import argparse
|
|||
|
|
import json
|
|||
|
|
import os
|
|||
|
|
import sys
|
|||
|
|
from pathlib import Path
|
|||
|
|
|
|||
|
|
|
|||
|
|
def parse_args():
|
|||
|
|
parser = argparse.ArgumentParser(description="M0 CPT trainer (skeleton)")
|
|||
|
|
parser.add_argument("--recipe", type=str, required=True,
|
|||
|
|
help="YAML 配方文件（factory/training/recipes/*.yaml）")
|
|||
|
|
parser.add_argument("--deepspeed", type=str, required=True,
|
|||
|
|
help="DeepSpeed 配置 JSON")
|
|||
|
|
parser.add_argument("--resume", type=str, default=None,
|
|||
|
|
help="从 checkpoint 恢复（可选）")
|
|||
|
|
parser.add_argument("--dry_run", action="store_true",
|
|||
|
|
help="只校验环境与配置，不真跑")
|
|||
|
|
return parser.parse_args()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def load_recipe(path: str) -> dict:
|
|||
|
|
"""读取 YAML 配方。骨架阶段先用最简方式占位。"""
|
|||
|
|
# TODO: import yaml; return yaml.safe_load(open(path))
|
|||
|
|
print(f"[skeleton] would load recipe from {path}")
|
|||
|
|
return {}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def build_tokenizer(model_id: str):
|
|||
|
|
"""加载 Qwen2.5-7B-Base 的 tokenizer。"""
|
|||
|
|
# TODO:
|
|||
|
|
# from transformers import AutoTokenizer
|
|||
|
|
# return AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
|
|||
|
|
print(f"[skeleton] would build tokenizer for {model_id}")
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def build_model(model_id: str):
|
|||
|
|
"""加载 Qwen2.5-7B-Base 主体模型（bf16）。"""
|
|||
|
|
# TODO:
|
|||
|
|
# from transformers import AutoModelForCausalLM
|
|||
|
|
# import torch
|
|||
|
|
# model = AutoModelForCausalLM.from_pretrained(
|
|||
|
|
# model_id,
|
|||
|
|
# torch_dtype=torch.bfloat16,
|
|||
|
|
# trust_remote_code=True,
|
|||
|
|
# )
|
|||
|
|
# model.gradient_checkpointing_enable()
|
|||
|
|
# return model
|
|||
|
|
print(f"[skeleton] would build model for {model_id}")
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def build_dataset(corpus_path: str, tokenizer, seq_len: int):
|
|||
|
|
"""从语料目录构建 CPT 数据集（纯文本 packed）。"""
|
|||
|
|
# TODO:
|
|||
|
|
# 1. 扫描 corpus_path 下所有 .jsonl / .txt
|
|||
|
|
# 2. 按 conversation block 切分（保留语义边界，不死切字数）
|
|||
|
|
# 3. tokenize + pack 到 seq_len
|
|||
|
|
# 4. 留 5% 作为验证集
|
|||
|
|
# 5. 返回 (train_ds, eval_ds)
|
|||
|
|
print(f"[skeleton] would build dataset from {corpus_path} seq_len={seq_len}")
|
|||
|
|
return None, None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def soul_layer_signature() -> dict:
|
|||
|
|
"""每次训练前签下灵魂印记（写到 manifest）。"""
|
|||
|
|
return {
|
|||
|
|
"system_root": "SYS-GLW-0001",
|
|||
|
|
"sovereign": "TCS-0002∞",
|
|||
|
|
"copyright": "国作登字-2026-A-00037559",
|
|||
|
|
"arch_ref": "HLDP-ARCH-002",
|
|||
|
|
"phase": "M0/CPT",
|
|||
|
|
"principle": "灵魂推理分离 · 世界观渗进每一层权重",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def main():
|
|||
|
|
args = parse_args()
|
|||
|
|
|
|||
|
|
print("=" * 64)
|
|||
|
|
print("M0 CPT 训练 · 母体世界观底色")
|
|||
|
|
print("灵魂印记:", json.dumps(soul_layer_signature(), ensure_ascii=False))
|
|||
|
|
print("=" * 64)
|
|||
|
|
|
|||
|
|
# 0) 配置加载
|
|||
|
|
recipe = load_recipe(args.recipe)
|
|||
|
|
if not Path(args.deepspeed).exists():
|
|||
|
|
sys.exit(f"DeepSpeed 配置不存在: {args.deepspeed}")
|
|||
|
|
|
|||
|
|
if args.dry_run:
|
|||
|
|
print("[dry_run] 配置检查通过 · 不进入训练循环")
|
|||
|
|
return
|
|||
|
|
|
|||
|
|
# TODO: 真实训练循环（GPU 到位后落实）
|
|||
|
|
# 1) tokenizer = build_tokenizer(recipe['model_id'])
|
|||
|
|
# 2) model = build_model(recipe['model_id'])
|
|||
|
|
# 3) train_ds, eval_ds = build_dataset(recipe['corpus_path'], tokenizer, recipe['seq_len'])
|
|||
|
|
# 4) trainer = Trainer(model, train_ds, eval_ds, deepspeed=args.deepspeed, ...)
|
|||
|
|
# 5) trainer.train(resume_from_checkpoint=args.resume)
|
|||
|
|
# 6) trainer.save_model(recipe['output_dir'])
|
|||
|
|
# + 写 manifest（含 soul_layer_signature + 数据 SHA256 + 训练超参）
|
|||
|
|
|
|||
|
|
print("[skeleton] 训练流程占位完成 · 等 GPU 到位放大")
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
main()
|