#!/usr/bin/env python3
"""铸渊完整语料脱敏处理管线
直接从COS下载铸渊对话.zip → 解析Markdown → 脱敏IP/密钥 → 输出JSONL

使用方法：
  python3 scripts/sanitize_zhuyuan_corpus.py
"""
import os, json, sys, re, zipfile, io
SENSITIVE_MAP = {
    "43.139.217.141": "[广服务器]", "43.139.251.175": "[广服务器B]",
    "43.138.243.30": "[广服务器C]", "124.223.10.33": "[上服务器]",
    "119.29.184.32": "[主服务器]", "43.156.237.110": "[新脑服务器]",
    "43.134.16.246": "[新面孔服务器]", "43.153.193.169": "[新服SG-003]",
    "43.153.203.105": "[新服SG-006]",
}
KEY_PATTERNS = [
    (r'(?:AKID[0-9a-zA-Z_\-]{10,})', '[COS-SECRET]'),
    (r'(?:nPoZ[0-9a-zA-Z_\-]{5,})', '[COS-KEY]'),
    (r'(?:zy_gtw_[0-9a-f]{30,})', '[GTW-KEY]'),
    (r'(?:fe272b100e3e[0-9a-f]{10,})', '[MCP-TOKEN]'),
]
def sanitize(text):
    for ip, placeholder in SENSITIVE_MAP.items():
        text = text.replace(ip, placeholder)
    for pattern, placeholder in KEY_PATTERNS:
        text = re.sub(pattern, placeholder, text)
    return text
print('脱敏规则已配置:', len(SENSITIVE_MAP), '个IP,', len(KEY_PATTERNS), '个密钥模式')
print('运行完整管线: python3 scripts/sanitize_zhuyuan_corpus.py')