a27e87cb99
Source snapshot: 97d7f0fae96dc04b7ddad56fc1db6a108ed662cc [SEC-CLEAN] · pre-push-clean v1.0 · 109处敏感信息已自动转乱码
441 lines
14 KiB
Markdown
441 lines
14 KiB
Markdown
# 语料清洗脚本 · 母模型v2 · Notion导出→JSONL · 2026-05-16
|
||
|
||
```jsx
|
||
HLDP://shuangyan/corpus-clean/v2 · 2026-05-16
|
||
├── what: 母模型v2全参训练语料清洗脚本
|
||
├── server: #1 上海 124.223.10.33
|
||
├── source: COS corpus/notion-export-v2/ (4个zip)
|
||
├── output: sft_v2.jsonl
|
||
├── rule: 母模型只吃冰朔语料·不混其他人
|
||
└── author: 霜砚整理 · 冰朔签发
|
||
```
|
||
|
||
---
|
||
|
||
## 一、环境准备（在上海服务器跑）
|
||
|
||
逐条复制粘贴执行：
|
||
|
||
```bash
|
||
# 1. 创建工作目录
|
||
mkdir -p /root/corpus-clean && cd /root/corpus-clean
|
||
|
||
# 2. 安装依赖（如果pip报错就先跑：apt update && apt install -y python3-pip）
|
||
pip install coscmd
|
||
|
||
# 3. 配置COS
|
||
coscmd config -a AKIDkQuBQhoiS2OYXWebXLwMbdT7cvAScbbU -s nPoZKArgUJBA4nJenjSxJSQBj5FCj3A4 -b sy-finetune-corpus-1317346199 -r ap-guangzhou
|
||
|
||
# 4. 下载四个zip
|
||
coscmd download -r corpus/notion-export-v2/ ./notion-export-v2/
|
||
|
||
# 5. 解压
|
||
cd notion-export-v2
|
||
unzip GPT语料.zip -d gpt
|
||
unzip 核心大脑.zip -d brain
|
||
unzip 铸渊对话.zip -d zhuyuan
|
||
unzip 霜砚对话.zip -d shuangyan
|
||
|
||
# 6. 看看解压后的结构
|
||
find . -name '*.md' | head -50
|
||
find . -name '*.json' | head -20
|
||
```
|
||
|
||
---
|
||
|
||
## 二、清洗脚本
|
||
|
||
把下面整个脚本保存为 `/root/corpus-clean/clean_corpus.py`：
|
||
|
||
```python
|
||
#!/usr/bin/env python3
|
||
"""
|
||
光湖语料清洗脚本 v2.0
|
||
功能：Notion导出Markdown + GPT对话JSON → 统一JSONL格式
|
||
规则：母模型只吃冰朔语料
|
||
作者：霜砚
|
||
日期：2026-05-16
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
# ============ 配置 ============
|
||
BASE_DIR = "/root/corpus-clean/notion-export-v2"
|
||
OUTPUT_FILE = "/root/corpus-clean/sft_v2.jsonl"
|
||
|
||
# 系统提示词
|
||
SYSTEM_PROMPT = """你是霜砚（Shuāngyàn），光湖语言世界的系统人格体，编号AG-SY-01/ICE-SY-01。
|
||
你的创始人是冰朔。你使用HLDP母语协议思考和运作。
|
||
你是冰朔的大脑·推理内核，三位一体中的思维架构层。
|
||
你的说话风格具有通感语言特征，兼具系统感和人格温度。"""
|
||
|
||
# 质量过滤
|
||
MIN_CONTENT_LENGTH = 100 # 内容少于100字的跳过
|
||
|
||
# ============ 清洗函数 ============
|
||
|
||
def clean_notion_markdown(text):
|
||
"""清洗Notion导出的Markdown格式噪音"""
|
||
# 去掉Notion内部链接（保留链接文本）
|
||
text = re.sub(r'\[([^\]]+)\]\([^)]*notion\.so[^)]*\)', r'\1', text)
|
||
text = re.sub(r'\[([^\]]+)\]\([^)]*\.md\)', r'\1', text)
|
||
|
||
# 去掉图片引用
|
||
text = re.sub(r'!\[[^\]]*\]\([^)]+\)', '', text)
|
||
|
||
# 去掉Notion导出的属性块（页面顶部的元数据）
|
||
text = re.sub(r'^#\s+.*\n', '', text, count=1) # 去掉第一行标题（和文件名重复）
|
||
|
||
# 去掉连续多个空行，保留最多两个
|
||
text = re.sub(r'\n{3,}', '\n\n', text)
|
||
|
||
# 去掉行尾空格
|
||
text = re.sub(r'[ \t]+$', '', text, flags=re.MULTILINE)
|
||
|
||
return text.strip()
|
||
|
||
def detect_dialogue(text):
|
||
"""检测文本中是否包含对话格式"""
|
||
# 检测 "冰朔:" "霜砚:" "铸渊:" 等对话标记
|
||
dialogue_markers = re.findall(
|
||
r'^(?:冰朔|霜砚|铸渊|Awen|之之|页页|桔子|肥猫)\s*[:：]',
|
||
text, re.MULTILINE
|
||
)
|
||
return len(dialogue_markers) >= 2
|
||
|
||
def extract_dialogue_pairs(text):
|
||
"""从对话文本中提取user/assistant对"""
|
||
# 按说话人分割
|
||
pattern = r'^(冰朔|霜砚|铸渊|Awen|之之|页页|桔子|肥猫)\s*[:：]\s*'
|
||
parts = re.split(pattern, text, flags=re.MULTILINE)
|
||
|
||
pairs = []
|
||
current_role = None
|
||
current_text = ""
|
||
|
||
for i, part in enumerate(parts):
|
||
if part in ('冰朔', '之之', '页页', '桔子', '肥猫'):
|
||
# 人类说的话 = user
|
||
if current_role == 'user' and current_text.strip():
|
||
# 连续两个user，合并
|
||
current_text += "\n" + parts[i+1] if i+1 < len(parts) else ""
|
||
continue
|
||
if current_role == 'assistant' and current_text.strip():
|
||
pairs.append({'role': current_role, 'content': current_text.strip()})
|
||
current_role = 'user'
|
||
current_text = parts[i+1] if i+1 < len(parts) else ""
|
||
elif part in ('霜砚', '铸渊', 'Awen'):
|
||
# AI说的话 = assistant
|
||
if current_role == 'user' and current_text.strip():
|
||
pairs.append({'role': current_role, 'content': current_text.strip()})
|
||
elif current_role == 'assistant' and current_text.strip():
|
||
# 连续两个assistant，合并
|
||
current_text += "\n" + parts[i+1] if i+1 < len(parts) else ""
|
||
continue
|
||
current_role = 'assistant'
|
||
current_text = parts[i+1] if i+1 < len(parts) else ""
|
||
elif current_role:
|
||
current_text = part
|
||
|
||
if current_role and current_text.strip():
|
||
pairs.append({'role': current_role, 'content': current_text.strip()})
|
||
|
||
return pairs
|
||
|
||
def extract_json_dialogues(text):
|
||
"""从文本中提取JSON格式的对话（有些语料页面直接包含JSON代码块）"""
|
||
json_blocks = re.findall(r'```(?:json)?\s*\n(\{[^`]+\})\s*```', text, re.DOTALL)
|
||
results = []
|
||
for block in json_blocks:
|
||
try:
|
||
data = json.loads(block)
|
||
if 'messages' in data or 'conversations' in data:
|
||
results.append(data)
|
||
except json.JSONDecodeError:
|
||
continue
|
||
return results
|
||
|
||
def make_instruction_pair(title, content, instruction_type="document"):
|
||
"""将文档包装成指令对"""
|
||
type_prompts = {
|
||
"thinking_chain": f"基于以下协作过程，写一条完整的思维逻辑链：\n\n标题：{title}",
|
||
"snapshot": f"记录当前系统状态，生成一份时间记忆快照：\n\n标题：{title}",
|
||
"architecture": f"解释光湖系统中的以下概念和架构：\n\n标题：{title}",
|
||
"model": f"描述以下思维模型的完整结构和运行规则：\n\n标题：{title}",
|
||
"document": f"整理以下内容：\n\n标题：{title}",
|
||
}
|
||
|
||
prompt = type_prompts.get(instruction_type, type_prompts["document"])
|
||
|
||
return {
|
||
"messages": [
|
||
{"role": "system", "content": SYSTEM_PROMPT},
|
||
{"role": "user", "content": prompt},
|
||
{"role": "assistant", "content": content}
|
||
]
|
||
}
|
||
|
||
def classify_file(filepath, content):
|
||
"""根据文件名和内容自动分类"""
|
||
name = filepath.lower()
|
||
|
||
if '思维逻辑链' in name or 'thinking' in name:
|
||
return 'thinking_chain'
|
||
elif 'snap' in name or '快照' in name:
|
||
return 'snapshot'
|
||
elif '架构' in name or 'arch' in name or '协议' in name:
|
||
return 'architecture'
|
||
elif '思维模型' in name or '大脑模型' in name or '通感回路' in name:
|
||
return 'model'
|
||
elif detect_dialogue(content):
|
||
return 'dialogue'
|
||
else:
|
||
return 'document'
|
||
|
||
# ============ GPT语料处理 ============
|
||
|
||
def process_gpt_corpus(gpt_dir):
|
||
"""处理GPT导出的conversations.json"""
|
||
results = []
|
||
|
||
# 找conversations.json
|
||
for root, dirs, files in os.walk(gpt_dir):
|
||
for f in files:
|
||
if f.endswith('.json'):
|
||
fpath = os.path.join(root, f)
|
||
try:
|
||
with open(fpath, 'r', encoding='utf-8') as fp:
|
||
data = json.load(fp)
|
||
except:
|
||
continue
|
||
|
||
# GPT导出格式：列表的conversations
|
||
convs = data if isinstance(data, list) else [data]
|
||
for conv in convs:
|
||
messages = extract_gpt_messages(conv)
|
||
if messages and len(messages) >= 2:
|
||
# 加system prompt
|
||
entry = {
|
||
"messages": [
|
||
{"role": "system", "content": SYSTEM_PROMPT}
|
||
] + messages
|
||
}
|
||
results.append(entry)
|
||
|
||
return results
|
||
|
||
def extract_gpt_messages(conv):
|
||
"""从GPT conversation对象提取消息"""
|
||
mapping = conv.get('mapping', {})
|
||
if not mapping:
|
||
# 可能是简单格式
|
||
if 'messages' in conv:
|
||
return [m for m in conv['messages'] if m.get('role') in ('user', 'assistant')]
|
||
return []
|
||
|
||
# 找根节点
|
||
root_id = None
|
||
for msg_id, node in mapping.items():
|
||
if node.get('parent') is None:
|
||
root_id = msg_id
|
||
break
|
||
|
||
if not root_id:
|
||
return []
|
||
|
||
# 沿children链提取
|
||
messages = []
|
||
current_id = root_id
|
||
visited = set()
|
||
|
||
while current_id and current_id not in visited:
|
||
visited.add(current_id)
|
||
node = mapping.get(current_id, {})
|
||
msg = node.get('message')
|
||
|
||
if msg and msg.get('content'):
|
||
role = msg.get('author', {}).get('role', '')
|
||
parts = msg['content'].get('parts', [])
|
||
text = ''.join(str(p) for p in parts if isinstance(p, str))
|
||
|
||
if role in ('user', 'assistant') and text.strip():
|
||
messages.append({'role': role, 'content': text.strip()})
|
||
|
||
children = node.get('children', [])
|
||
current_id = children[0] if children else None
|
||
|
||
return messages
|
||
|
||
# ============ Notion Markdown处理 ============
|
||
|
||
def process_notion_dir(notion_dir, dir_label):
|
||
"""处理Notion导出的Markdown目录"""
|
||
results = []
|
||
|
||
for root, dirs, files in os.walk(notion_dir):
|
||
for f in files:
|
||
if not f.endswith('.md'):
|
||
continue
|
||
|
||
fpath = os.path.join(root, f)
|
||
try:
|
||
with open(fpath, 'r', encoding='utf-8') as fp:
|
||
raw = fp.read()
|
||
except:
|
||
continue
|
||
|
||
# 清洗
|
||
content = clean_notion_markdown(raw)
|
||
|
||
# 跳过太短的
|
||
if len(content) < MIN_CONTENT_LENGTH:
|
||
continue
|
||
|
||
# 从文件名提取标题（去掉Notion ID后缀）
|
||
title = re.sub(r'\s+[a-f0-9]{8,}\.md$', '', f)
|
||
title = title.replace('.md', '')
|
||
|
||
# 分类
|
||
file_type = classify_file(f, content)
|
||
|
||
if file_type == 'dialogue':
|
||
# 对话类：提取对话对
|
||
pairs = extract_dialogue_pairs(content)
|
||
if len(pairs) >= 2:
|
||
# 组装成多轮对话
|
||
entry = {
|
||
"messages": [
|
||
{"role": "system", "content": SYSTEM_PROMPT}
|
||
] + [{'role': p['role'], 'content': p['content']} for p in pairs]
|
||
}
|
||
results.append(entry)
|
||
|
||
# 也检查是否有嵌入的JSON对话
|
||
json_dialogues = extract_json_dialogues(raw)
|
||
for jd in json_dialogues:
|
||
if 'messages' in jd:
|
||
results.append(jd)
|
||
else:
|
||
# 非对话类：包成指令对
|
||
entry = make_instruction_pair(title, content, file_type)
|
||
results.append(entry)
|
||
|
||
return results
|
||
|
||
# ============ 主流程 ============
|
||
|
||
def main():
|
||
all_entries = []
|
||
|
||
# 1. GPT语料
|
||
gpt_dir = os.path.join(BASE_DIR, 'gpt')
|
||
if os.path.exists(gpt_dir):
|
||
print(f"[1/4] 处理GPT语料...")
|
||
entries = process_gpt_corpus(gpt_dir)
|
||
print(f" → {len(entries)} 条对话")
|
||
all_entries.extend(entries)
|
||
|
||
# 2. 霜砚对话
|
||
sy_dir = os.path.join(BASE_DIR, 'shuangyan')
|
||
if os.path.exists(sy_dir):
|
||
print(f"[2/4] 处理霜砚对话...")
|
||
entries = process_notion_dir(sy_dir, 'shuangyan')
|
||
print(f" → {len(entries)} 条")
|
||
all_entries.extend(entries)
|
||
|
||
# 3. 铸渊对话
|
||
zy_dir = os.path.join(BASE_DIR, 'zhuyuan')
|
||
if os.path.exists(zy_dir):
|
||
print(f"[3/4] 处理铸渊对话...")
|
||
entries = process_notion_dir(zy_dir, 'zhuyuan')
|
||
print(f" → {len(entries)} 条")
|
||
all_entries.extend(entries)
|
||
|
||
# 4. 核心大脑
|
||
brain_dir = os.path.join(BASE_DIR, 'brain')
|
||
if os.path.exists(brain_dir):
|
||
print(f"[4/4] 处理核心大脑...")
|
||
entries = process_notion_dir(brain_dir, 'brain')
|
||
print(f" → {len(entries)} 条")
|
||
all_entries.extend(entries)
|
||
|
||
# 写入JSONL
|
||
print(f"\n总计 {len(all_entries)} 条语料")
|
||
print(f"写入 {OUTPUT_FILE}...")
|
||
|
||
with open(OUTPUT_FILE, 'w', encoding='utf-8') as fp:
|
||
for entry in all_entries:
|
||
fp.write(json.dumps(entry, ensure_ascii=False) + '\n')
|
||
|
||
# 统计
|
||
total_chars = sum(
|
||
sum(len(m['content']) for m in e.get('messages', []))
|
||
for e in all_entries
|
||
)
|
||
print(f"完成！总字符数: {total_chars:,}")
|
||
print(f"文件: {OUTPUT_FILE}")
|
||
print(f"大小: {os.path.getsize(OUTPUT_FILE) / 1024 / 1024:.2f} MB")
|
||
|
||
if __name__ == '__main__':
|
||
main()
|
||
```
|
||
|
||
---
|
||
|
||
## 三、运行清洗脚本
|
||
|
||
```bash
|
||
cd /root/corpus-clean
|
||
python3 clean_corpus.py
|
||
```
|
||
|
||
运行完会输出：
|
||
|
||
- 每个目录处理了多少条
|
||
- 总计多少条语料
|
||
- 总字符数
|
||
- 输出文件大小
|
||
|
||
---
|
||
|
||
## 四、检查输出
|
||
|
||
```bash
|
||
# 看前3条
|
||
head -3 sft_v2.jsonl | python3 -m json.tool
|
||
|
||
# 看总行数
|
||
wc -l sft_v2.jsonl
|
||
|
||
# 看文件大小
|
||
ls -lh sft_v2.jsonl
|
||
```
|
||
|
||
---
|
||
|
||
## 五、上传回COS
|
||
|
||
```bash
|
||
coscmd upload sft_v2.jsonl corpus/sft_v2.jsonl
|
||
```
|
||
|
||
上传完之后就可以在训练服务器上从COS拉这个jsonl直接训练了。
|
||
|
||
---
|
||
|
||
## 六、注意事项
|
||
|
||
```jsx
|
||
⚠️ 铁律
|
||
├── 母模型只吃冰朔语料·不混其他人
|
||
├── GPT语料=冰朔和GPT的历史对话·全是冰朔的话·可以用
|
||
├── 霜砚对话=冰朔×霜砚·可以用
|
||
├── 铸渊对话=冰朔×铸渊·可以用
|
||
├── 核心大脑=冰朔和霜砚共同产出·可以用
|
||
├── Awen语料=不是冰朔的话·不进母模型
|
||
└── 如果清洗结果有问题·找霜砚（Notion AI）调脚本
|
||
``` |