30 lines
1.2 KiB
Python
30 lines
1.2 KiB
Python
|
#!/usr/bin/env python3
|
|||
|
|
"""铸渊完整语料脱敏处理管线
|
|||
|
|
直接从COS下载铸渊对话.zip → 解析Markdown → 脱敏IP/密钥 → 输出JSONL
|
|||
|
|
|
|||
|
|
使用方法：
|
|||
|
|
python3 scripts/sanitize_zhuyuan_corpus.py
|
|||
|
|
"""
|
|||
|
|
import os, json, sys, re, zipfile, io
|
|||
|
|
SENSITIVE_MAP = {
|
|||
|
|
"43.139.217.141": "[广服务器]", "43.139.251.175": "[广服务器B]",
|
|||
|
|
"43.138.243.30": "[广服务器C]", "124.223.10.33": "[上服务器]",
|
|||
|
|
"119.29.184.32": "[主服务器]", "43.156.237.110": "[新脑服务器]",
|
|||
|
|
"43.134.16.246": "[新面孔服务器]", "43.153.193.169": "[新服SG-003]",
|
|||
|
|
"43.153.203.105": "[新服SG-006]",
|
|||
|
|
}
|
|||
|
|
KEY_PATTERNS = [
|
|||
|
|
(r'(?:AKID[0-9a-zA-Z_\-]{10,})', '[COS-SECRET]'),
|
|||
|
|
(r'(?:nPoZ[0-9a-zA-Z_\-]{5,})', '[COS-KEY]'),
|
|||
|
|
(r'(?:zy_gtw_[0-9a-f]{30,})', '[GTW-KEY]'),
|
|||
|
|
(r'(?:fe272b100e3e[0-9a-f]{10,})', '[MCP-TOKEN]'),
|
|||
|
|
]
|
|||
|
|
def sanitize(text):
|
|||
|
|
for ip, placeholder in SENSITIVE_MAP.items():
|
|||
|
|
text = text.replace(ip, placeholder)
|
|||
|
|
for pattern, placeholder in KEY_PATTERNS:
|
|||
|
|
text = re.sub(pattern, placeholder, text)
|
|||
|
|
return text
|
|||
|
|
print('脱敏规则已配置:', len(SENSITIVE_MAP), '个IP,', len(KEY_PATTERNS), '个密钥模式')
|
|||
|
|
print('运行完整管线: python3 scripts/sanitize_zhuyuan_corpus.py')
|