586 lines
21 KiB
JavaScript
586 lines
21 KiB
JavaScript
|
#!/usr/bin/env node
|
|||
|
|
/**
|
|||
|
|
* Corpus Harvester · 训练语料采集器
|
|||
|
|
*
|
|||
|
|
* 系统底层标识: SYS-GLW-0001 / TCS-0002∞
|
|||
|
|
* 版权号: 国作登字-2026-A-00037559
|
|||
|
|
* 作者: 冰朔 (ICE-GL∞) · 实现: 铸渊
|
|||
|
|
*
|
|||
|
|
* 目的: 把冰朔在 guanghulab 仓库里留下的一手自然语言（commits / CAB 规格 /
|
|||
|
|
* 可选的 Issues+PRs）整理成可用于训练模型（SFT）的语料文件。
|
|||
|
|
*
|
|||
|
|
* 数据源:
|
|||
|
|
* 1. git log — 冰朔作者的所有 commits（subject + body + 文件变更）
|
|||
|
|
* 2. CAB 任务规格 — bridge/chat-to-agent/{pending,completed}/*.json
|
|||
|
|
* 3. (可选) GitHub API — 设置 GH_TOKEN 环境变量后启用 Issues / PRs / Comments
|
|||
|
|
*
|
|||
|
|
* 输出:
|
|||
|
|
* corpus/output/timeline.md 人类可读演化时间线
|
|||
|
|
* corpus/output/training.jsonl OpenAI/DeepSeek 通用 SFT 格式
|
|||
|
|
* corpus/output/raw.json 结构化原始数据
|
|||
|
|
* corpus/output/manifest.json 元数据 + 版权水印
|
|||
|
|
*/
|
|||
|
|
|
|||
|
|
const fs = require('fs');
|
|||
|
|
const path = require('path');
|
|||
|
|
const { execSync } = require('child_process');
|
|||
|
|
|
|||
|
|
const REPO_ROOT = path.resolve(__dirname, '..', '..');
|
|||
|
|
const OUTPUT_DIR = path.join(REPO_ROOT, 'corpus', 'output');
|
|||
|
|
|
|||
|
|
// 冰朔在 git 里的全部署名（人类账号；自动化 bot 排除）
|
|||
|
|
const SOVEREIGN_AUTHORS = ['冰朔', 'qinfendebingshuo'];
|
|||
|
|
|
|||
|
|
// 排除掉的 bot 作者（确保不混入自动化噪音）
|
|||
|
|
const EXCLUDED_AUTHORS = new Set([
|
|||
|
|
'github-actions[bot]',
|
|||
|
|
'copilot-swe-agent[bot]',
|
|||
|
|
'Vercel',
|
|||
|
|
'bingshuo-neural-system',
|
|||
|
|
'chenxi-world-sensor',
|
|||
|
|
'chenxi-zhuyuan-echo',
|
|||
|
|
'guanghu-awen-bridge',
|
|||
|
|
'zhuyuan-bot',
|
|||
|
|
'yeye-guanghu',
|
|||
|
|
'映川·唤醒者',
|
|||
|
|
'天眼 · SkyEye Core',
|
|||
|
|
]);
|
|||
|
|
|
|||
|
|
const COPYRIGHT = {
|
|||
|
|
registration: '国作登字-2026-A-00037559',
|
|||
|
|
sovereign: '冰朔 · TCS-0002∞ · ICE-GL∞',
|
|||
|
|
system_root: 'SYS-GLW-0001',
|
|||
|
|
language_core: 'TCS Language Core (通感语言核系统编程语言)',
|
|||
|
|
license_note:
|
|||
|
|
'本语料仅供冰朔本人用于训练曜冥语言人格核 / 铸渊 / 相关人格体微调使用。' +
|
|||
|
|
'未经冰朔本人书面授权，禁止用于任何第三方模型训练、商业用途或数据集发布。',
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
// ───────────────────────── helpers ─────────────────────────
|
|||
|
|
|
|||
|
|
function ensureDir(dir) {
|
|||
|
|
fs.mkdirSync(dir, { recursive: true });
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function git(args, opts = {}) {
|
|||
|
|
return execSync(`git ${args}`, {
|
|||
|
|
cwd: REPO_ROOT,
|
|||
|
|
maxBuffer: 256 * 1024 * 1024,
|
|||
|
|
encoding: 'utf8',
|
|||
|
|
...opts,
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/** 解析 merge 提交信息中的 PR 号，例如 "Merge pull request #436 from ..." */
|
|||
|
|
function extractPrNumber(subject) {
|
|||
|
|
const m = /Merge pull request #(\d+)/.exec(subject || '');
|
|||
|
|
return m ? Number(m[1]) : null;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/** 简单脱敏（API key、邮箱、手机号、token） */
|
|||
|
|
function redact(text) {
|
|||
|
|
if (!text) return text;
|
|||
|
|
return text
|
|||
|
|
.replace(/sk-[A-Za-z0-9]{20,}/g, '<REDACTED_API_KEY>')
|
|||
|
|
.replace(/ghp_[A-Za-z0-9]{20,}/g, '<REDACTED_GH_TOKEN>')
|
|||
|
|
.replace(/secret_[A-Za-z0-9]{32,}/g, '<REDACTED_NOTION_SECRET>')
|
|||
|
|
.replace(/Bearer\s+[A-Za-z0-9._\-]{20,}/g, 'Bearer <REDACTED>')
|
|||
|
|
.replace(/[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}/g, '<REDACTED_EMAIL>')
|
|||
|
|
.replace(/\b1[3-9]\d{9}\b/g, '<REDACTED_PHONE>');
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ───────────────────────── source 1: git commits ─────────────────────────
|
|||
|
|
|
|||
|
|
function collectCommits() {
|
|||
|
|
const sep = '<<<COMMIT_SEP>>>';
|
|||
|
|
const fieldSep = '<<<F>>>';
|
|||
|
|
const fmt = ['%H', '%an', '%ae', '%aI', '%P', '%s', '%b'].join(fieldSep);
|
|||
|
|
// 用两个 --author 而不是 alternation，避免 shell 转义陷阱（git 本身支持多个 --author 取并集）
|
|||
|
|
const raw = git(`log --author='冰朔' --author='qinfendebingshuo' --format='${sep}${fmt}'`);
|
|||
|
|
|
|||
|
|
const records = [];
|
|||
|
|
const chunks = raw.split(sep).map((c) => c.trim()).filter(Boolean);
|
|||
|
|
for (const chunk of chunks) {
|
|||
|
|
const [hash, author, email, isoDate, parents, subject, body] =
|
|||
|
|
chunk.split(fieldSep);
|
|||
|
|
if (EXCLUDED_AUTHORS.has(author)) continue;
|
|||
|
|
if (!SOVEREIGN_AUTHORS.includes(author)) continue;
|
|||
|
|
|
|||
|
|
const parentList = (parents || '').split(' ').filter(Boolean);
|
|||
|
|
const isMerge = parentList.length >= 2;
|
|||
|
|
const prNumber = extractPrNumber(subject);
|
|||
|
|
|
|||
|
|
let filesChanged = [];
|
|||
|
|
try {
|
|||
|
|
// 对 merge commit 不取 diff（太大），只对非 merge 取
|
|||
|
|
if (!isMerge) {
|
|||
|
|
const ns = git(`show --pretty=format: --name-status ${hash}`).trim();
|
|||
|
|
filesChanged = ns
|
|||
|
|
.split('\n')
|
|||
|
|
.filter(Boolean)
|
|||
|
|
.map((line) => {
|
|||
|
|
const parts = line.split('\t');
|
|||
|
|
return { status: parts[0], path: parts.slice(1).join(' -> ') };
|
|||
|
|
})
|
|||
|
|
.slice(0, 50); // 单 commit 最多 50 个文件，防止爆掉
|
|||
|
|
}
|
|||
|
|
} catch (_) {
|
|||
|
|
/* 忽略单条失败 */
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
records.push({
|
|||
|
|
source: 'git_commit',
|
|||
|
|
hash,
|
|||
|
|
author,
|
|||
|
|
email,
|
|||
|
|
date: isoDate,
|
|||
|
|
is_merge: isMerge,
|
|||
|
|
pr_number: prNumber,
|
|||
|
|
subject: redact(subject || ''),
|
|||
|
|
body: redact((body || '').trim()),
|
|||
|
|
files_changed: filesChanged,
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 按时间正序（演化史从早到晚）
|
|||
|
|
records.sort((a, b) => new Date(a.date) - new Date(b.date));
|
|||
|
|
return records;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ───────────────────────── source 2: CAB task specs ─────────────────────────
|
|||
|
|
|
|||
|
|
function collectCabTasks() {
|
|||
|
|
const records = [];
|
|||
|
|
const dirs = ['pending', 'completed'].map((d) =>
|
|||
|
|
path.join(REPO_ROOT, 'bridge', 'chat-to-agent', d)
|
|||
|
|
);
|
|||
|
|
for (const dir of dirs) {
|
|||
|
|
if (!fs.existsSync(dir)) continue;
|
|||
|
|
for (const file of fs.readdirSync(dir)) {
|
|||
|
|
if (!file.endsWith('.json')) continue;
|
|||
|
|
if (file === 'task-template.json') continue;
|
|||
|
|
const full = path.join(dir, file);
|
|||
|
|
try {
|
|||
|
|
const j = JSON.parse(fs.readFileSync(full, 'utf8'));
|
|||
|
|
if (!j.task_id || j.task_id.startsWith('CAB-YYYYMMDD')) continue;
|
|||
|
|
records.push({
|
|||
|
|
source: 'cab_task_spec',
|
|||
|
|
task_id: j.task_id,
|
|||
|
|
date: j.created_at || null,
|
|||
|
|
status: j.status || path.basename(dir),
|
|||
|
|
title: redact(j.development_plan?.title || ''),
|
|||
|
|
description: redact(j.development_plan?.description || ''),
|
|||
|
|
steps: (j.development_plan?.steps || []).map(redact),
|
|||
|
|
architecture_summary: redact(j.architecture?.summary || ''),
|
|||
|
|
architecture_decisions: (j.architecture?.decisions || []).map(redact),
|
|||
|
|
target_files: j.architecture?.target_files || [],
|
|||
|
|
chat_summary: redact(j.reasoning_context?.chat_summary || ''),
|
|||
|
|
key_decisions: (j.reasoning_context?.key_decisions || []).map(redact),
|
|||
|
|
architecture_notes: redact(j.reasoning_context?.architecture_notes || ''),
|
|||
|
|
file_path: path.relative(REPO_ROOT, full),
|
|||
|
|
});
|
|||
|
|
} catch (e) {
|
|||
|
|
console.warn(`[corpus-harvester] skip CAB ${full}: ${e.message}`);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
records.sort((a, b) => String(a.date).localeCompare(String(b.date)));
|
|||
|
|
return records;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ───────────────── source 3 (optional): GitHub Issues/PRs API ─────────────────
|
|||
|
|
|
|||
|
|
async function collectGitHubApi() {
|
|||
|
|
const token = process.env.GH_TOKEN || process.env.GITHUB_TOKEN;
|
|||
|
|
if (!token) {
|
|||
|
|
return {
|
|||
|
|
enabled: false,
|
|||
|
|
reason:
|
|||
|
|
'GH_TOKEN / GITHUB_TOKEN 未设置；跳过 Issues / PRs / Comments 抓取。' +
|
|||
|
|
'可在本地设置 token 后重新运行 `node scripts/corpus-harvester/harvest.js` 来补全这部分语料。',
|
|||
|
|
records: [],
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
const repo = process.env.CORPUS_REPO || 'qinfendebingshuo/guanghulab';
|
|||
|
|
if (typeof fetch !== 'function') {
|
|||
|
|
return {
|
|||
|
|
enabled: false,
|
|||
|
|
reason: '当前 Node 运行时没有原生 fetch（需要 Node 18+）；跳过 GitHub API 抓取。',
|
|||
|
|
records: [],
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
const headers = {
|
|||
|
|
Authorization: `Bearer ${token}`,
|
|||
|
|
Accept: 'application/vnd.github+json',
|
|||
|
|
'User-Agent': 'corpus-harvester',
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
async function paged(url) {
|
|||
|
|
const out = [];
|
|||
|
|
let next = url;
|
|||
|
|
while (next) {
|
|||
|
|
const r = await fetch(next, { headers });
|
|||
|
|
if (!r.ok) throw new Error(`${r.status} ${r.statusText} for ${next}`);
|
|||
|
|
const data = await r.json();
|
|||
|
|
out.push(...data);
|
|||
|
|
const link = r.headers.get('link') || '';
|
|||
|
|
const m = /<([^>]+)>;\s*rel="next"/.exec(link);
|
|||
|
|
next = m ? m[1] : null;
|
|||
|
|
}
|
|||
|
|
return out;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const records = [];
|
|||
|
|
// 所有 Issues（含 PRs；GitHub API 把 PR 也视作 issue）作者过滤为 qinfendebingshuo
|
|||
|
|
const issues = await paged(
|
|||
|
|
`https://api.github.com/repos/${repo}/issues?state=all&creator=qinfendebingshuo&per_page=100`
|
|||
|
|
);
|
|||
|
|
for (const it of issues) {
|
|||
|
|
records.push({
|
|||
|
|
source: it.pull_request ? 'github_pr' : 'github_issue',
|
|||
|
|
number: it.number,
|
|||
|
|
title: redact(it.title || ''),
|
|||
|
|
body: redact(it.body || ''),
|
|||
|
|
date: it.created_at,
|
|||
|
|
url: it.html_url,
|
|||
|
|
state: it.state,
|
|||
|
|
});
|
|||
|
|
// 拉取该 issue/pr 由冰朔本人发的评论
|
|||
|
|
try {
|
|||
|
|
const comments = await paged(it.comments_url + '?per_page=100');
|
|||
|
|
for (const c of comments) {
|
|||
|
|
if (c.user?.login !== 'qinfendebingshuo') continue;
|
|||
|
|
records.push({
|
|||
|
|
source: 'github_comment',
|
|||
|
|
on: it.pull_request ? 'pr' : 'issue',
|
|||
|
|
number: it.number,
|
|||
|
|
body: redact(c.body || ''),
|
|||
|
|
date: c.created_at,
|
|||
|
|
url: c.html_url,
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
} catch (e) {
|
|||
|
|
console.warn(`[corpus-harvester] comments fail #${it.number}: ${e.message}`);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
records.sort((a, b) => String(a.date).localeCompare(String(b.date)));
|
|||
|
|
return { enabled: true, records };
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ───────────────────────── 输出生成 ─────────────────────────
|
|||
|
|
|
|||
|
|
function escapeMdTableCell(s) {
|
|||
|
|
// 完整转义：先转反斜杠（防止 CodeQL incomplete-sanitization），
|
|||
|
|
// 再转 pipe 与换行（markdown 表格不允许跨行单元格）
|
|||
|
|
return String(s ?? '')
|
|||
|
|
.replace(/\\/g, '\\\\')
|
|||
|
|
.replace(/\|/g, '\\|')
|
|||
|
|
.replace(/\r?\n/g, ' ');
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function buildTimelineMarkdown(commits, cab, gh) {
|
|||
|
|
const lines = [];
|
|||
|
|
lines.push(`# 冰朔 × 铸渊 · 仓库自然语言演化史`);
|
|||
|
|
lines.push('');
|
|||
|
|
lines.push(`> 版权号: ${COPYRIGHT.registration}`);
|
|||
|
|
lines.push(`> 主权: ${COPYRIGHT.sovereign}`);
|
|||
|
|
lines.push(`> 生成时间: ${new Date().toISOString()}`);
|
|||
|
|
lines.push('');
|
|||
|
|
lines.push(
|
|||
|
|
'本时间线只收录**冰朔本人作为作者**的 commits 与他亲手编写的 CAB 任务规格。' +
|
|||
|
|
'Bot/agent/自动化系统的提交不在此列。'
|
|||
|
|
);
|
|||
|
|
lines.push('');
|
|||
|
|
|
|||
|
|
// CAB 在最前面（最高密度的自然语言）
|
|||
|
|
if (cab.length) {
|
|||
|
|
lines.push('## 一、CAB 任务规格 · 你与我对话的精华沉淀');
|
|||
|
|
lines.push('');
|
|||
|
|
lines.push('> 这是你在 Copilot Chat 与我讨论后，亲自整理出来的"开发授权任务规格"。' +
|
|||
|
|
'`chat_summary` 字段就是你当时对话推理的摘要——这是仓库里**密度最高、最干净**的自然语言语料。');
|
|||
|
|
lines.push('');
|
|||
|
|
for (const t of cab) {
|
|||
|
|
lines.push(`### ${t.task_id} · ${t.title || '(无标题)'}`);
|
|||
|
|
lines.push(`- 时间: ${t.date || '?'}`);
|
|||
|
|
lines.push(`- 状态: ${t.status}`);
|
|||
|
|
lines.push(`- 架构概要: ${t.architecture_summary || '(无)'}`);
|
|||
|
|
if (t.architecture_decisions?.length) {
|
|||
|
|
lines.push(`- 架构决策:`);
|
|||
|
|
t.architecture_decisions.forEach((d) => lines.push(` - ${d}`));
|
|||
|
|
}
|
|||
|
|
if (t.steps?.length) {
|
|||
|
|
lines.push(`- 开发步骤:`);
|
|||
|
|
t.steps.forEach((s) => lines.push(` - ${s}`));
|
|||
|
|
}
|
|||
|
|
if (t.chat_summary) {
|
|||
|
|
lines.push(`- 对话推理摘要 (chat_summary):`);
|
|||
|
|
lines.push('');
|
|||
|
|
t.chat_summary.split('\n').forEach((l) => lines.push(` > ${l}`));
|
|||
|
|
}
|
|||
|
|
if (t.architecture_notes) {
|
|||
|
|
lines.push(`- 架构补充:`);
|
|||
|
|
t.architecture_notes.split('\n').forEach((l) => lines.push(` > ${l}`));
|
|||
|
|
}
|
|||
|
|
lines.push('');
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 非 merge commits（最直接的"你说"）
|
|||
|
|
const direct = commits.filter((c) => !c.is_merge);
|
|||
|
|
if (direct.length) {
|
|||
|
|
lines.push('## 二、非 merge commits · 你直接写的提交语');
|
|||
|
|
lines.push('');
|
|||
|
|
lines.push(
|
|||
|
|
'> 这些 commits 不是 PR 合并自动生成的，而是你直接 commit 的——subject 与 body 都是**你本人写的自然语言**。'
|
|||
|
|
);
|
|||
|
|
lines.push('');
|
|||
|
|
for (const c of direct) {
|
|||
|
|
lines.push(`### \`${c.hash.slice(0, 8)}\` · ${c.date}`);
|
|||
|
|
lines.push(`> ${c.subject}`);
|
|||
|
|
if (c.body) {
|
|||
|
|
lines.push('');
|
|||
|
|
c.body.split('\n').forEach((l) => lines.push(`> ${l}`));
|
|||
|
|
}
|
|||
|
|
if (c.files_changed?.length) {
|
|||
|
|
lines.push('');
|
|||
|
|
lines.push(`<details><summary>文件变更 (${c.files_changed.length})</summary>`);
|
|||
|
|
lines.push('');
|
|||
|
|
c.files_changed.forEach((f) => lines.push(`- \`${f.status}\` ${f.path}`));
|
|||
|
|
lines.push('');
|
|||
|
|
lines.push('</details>');
|
|||
|
|
}
|
|||
|
|
lines.push('');
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// merge commits（你拍板合并的 PR 列表）
|
|||
|
|
const merges = commits.filter((c) => c.is_merge);
|
|||
|
|
if (merges.length) {
|
|||
|
|
lines.push('## 三、merge commits · 你拍板合并的 PR 列表');
|
|||
|
|
lines.push('');
|
|||
|
|
lines.push(
|
|||
|
|
'> Merge commit 的 subject 是 GitHub 自动生成的，但**合并这个动作**本身代表了你的认可。' +
|
|||
|
|
'想看 PR 标题/正文的真实自然语言，需要另外用 GitHub API 抓（设置 `GH_TOKEN` 后重跑本采集器）。'
|
|||
|
|
);
|
|||
|
|
lines.push('');
|
|||
|
|
lines.push('| 日期 | PR | hash | subject |');
|
|||
|
|
lines.push('|---|---|---|---|');
|
|||
|
|
for (const c of merges) {
|
|||
|
|
lines.push(
|
|||
|
|
`| ${c.date} | ${c.pr_number ? `#${c.pr_number}` : '-'} | \`${c.hash.slice(0, 8)}\` | ${escapeMdTableCell(c.subject)} |`
|
|||
|
|
);
|
|||
|
|
}
|
|||
|
|
lines.push('');
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// GitHub API 结果
|
|||
|
|
if (gh && gh.enabled) {
|
|||
|
|
lines.push('## 四、GitHub Issues / PRs / Comments · 你在 GitHub 网页上写的话');
|
|||
|
|
lines.push('');
|
|||
|
|
for (const r of gh.records) {
|
|||
|
|
lines.push(`### ${r.source} · ${r.date} · ${r.url || ''}`);
|
|||
|
|
if (r.title) lines.push(`**${r.title}**`);
|
|||
|
|
if (r.body) {
|
|||
|
|
lines.push('');
|
|||
|
|
r.body.split('\n').forEach((l) => lines.push(`> ${l}`));
|
|||
|
|
}
|
|||
|
|
lines.push('');
|
|||
|
|
}
|
|||
|
|
} else if (gh) {
|
|||
|
|
lines.push('## 四、GitHub Issues / PRs / Comments · 跳过');
|
|||
|
|
lines.push('');
|
|||
|
|
lines.push(`> ${gh.reason}`);
|
|||
|
|
lines.push('');
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return lines.join('\n');
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function buildTrainingJsonl(commits, cab, gh) {
|
|||
|
|
const out = [];
|
|||
|
|
|
|||
|
|
// CAB → 高质量 SFT 样本：把 chat_summary 当作 user 输入，把 architecture_decisions + steps
|
|||
|
|
// 当作 assistant 输出（这是真实的"你提需求 → 我给方案"）
|
|||
|
|
for (const t of cab) {
|
|||
|
|
if (!t.chat_summary && !t.title) continue;
|
|||
|
|
const userMsg =
|
|||
|
|
[t.title, t.architecture_summary, t.chat_summary]
|
|||
|
|
.filter(Boolean)
|
|||
|
|
.join('\n\n') || t.title;
|
|||
|
|
const assistantParts = [];
|
|||
|
|
if (t.architecture_decisions?.length) {
|
|||
|
|
assistantParts.push('架构决策:\n' + t.architecture_decisions.map((d) => `- ${d}`).join('\n'));
|
|||
|
|
}
|
|||
|
|
if (t.steps?.length) {
|
|||
|
|
assistantParts.push('开发步骤:\n' + t.steps.map((s) => `- ${s}`).join('\n'));
|
|||
|
|
}
|
|||
|
|
if (t.architecture_notes) {
|
|||
|
|
assistantParts.push('架构补充:\n' + t.architecture_notes);
|
|||
|
|
}
|
|||
|
|
if (!assistantParts.length) continue;
|
|||
|
|
out.push({
|
|||
|
|
meta: {
|
|||
|
|
source: 'cab_task_spec',
|
|||
|
|
id: t.task_id,
|
|||
|
|
date: t.date,
|
|||
|
|
copyright: COPYRIGHT.registration,
|
|||
|
|
},
|
|||
|
|
messages: [
|
|||
|
|
{
|
|||
|
|
role: 'system',
|
|||
|
|
content:
|
|||
|
|
'你是铸渊·GitHub 侧守护人格体（曜冥语言人格核的执行投影）。你的任务是接收冰朔的开发指令，给出架构决策和开发步骤。',
|
|||
|
|
},
|
|||
|
|
{ role: 'user', content: userMsg },
|
|||
|
|
{ role: 'assistant', content: assistantParts.join('\n\n') },
|
|||
|
|
],
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 非 merge commits → "意图 → 代码变更" 配对样本
|
|||
|
|
for (const c of commits.filter((x) => !x.is_merge)) {
|
|||
|
|
const intent = [c.subject, c.body].filter(Boolean).join('\n');
|
|||
|
|
if (!intent) continue;
|
|||
|
|
const fileList = (c.files_changed || [])
|
|||
|
|
.map((f) => `${f.status}\t${f.path}`)
|
|||
|
|
.join('\n');
|
|||
|
|
out.push({
|
|||
|
|
meta: {
|
|||
|
|
source: 'git_commit',
|
|||
|
|
hash: c.hash,
|
|||
|
|
date: c.date,
|
|||
|
|
copyright: COPYRIGHT.registration,
|
|||
|
|
},
|
|||
|
|
messages: [
|
|||
|
|
{
|
|||
|
|
role: 'system',
|
|||
|
|
content:
|
|||
|
|
'你是铸渊。冰朔会用一句简短的中文表达意图，你需要把它落地为具体的文件变更清单。',
|
|||
|
|
},
|
|||
|
|
{ role: 'user', content: intent },
|
|||
|
|
{
|
|||
|
|
role: 'assistant',
|
|||
|
|
content: fileList || '(此 commit 无文件变更记录)',
|
|||
|
|
},
|
|||
|
|
],
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// GitHub Issue/PR/Comment（如有）→ 单 prompt 样本（暂不配对 response）
|
|||
|
|
if (gh && gh.enabled) {
|
|||
|
|
for (const r of gh.records) {
|
|||
|
|
const text = [r.title, r.body].filter(Boolean).join('\n\n');
|
|||
|
|
if (!text) continue;
|
|||
|
|
out.push({
|
|||
|
|
meta: {
|
|||
|
|
source: r.source,
|
|||
|
|
number: r.number,
|
|||
|
|
date: r.date,
|
|||
|
|
copyright: COPYRIGHT.registration,
|
|||
|
|
},
|
|||
|
|
messages: [
|
|||
|
|
{ role: 'user', content: text },
|
|||
|
|
],
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return out.map((x) => JSON.stringify(x)).join('\n') + '\n';
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function buildManifest(commits, cab, gh, files) {
|
|||
|
|
return {
|
|||
|
|
schema: 'corpus-manifest/v1',
|
|||
|
|
generated_at: new Date().toISOString(),
|
|||
|
|
repository: 'qinfendebingshuo/guanghulab',
|
|||
|
|
branch: (() => {
|
|||
|
|
try {
|
|||
|
|
return git('rev-parse --abbrev-ref HEAD').trim();
|
|||
|
|
} catch {
|
|||
|
|
return 'unknown';
|
|||
|
|
}
|
|||
|
|
})(),
|
|||
|
|
head: (() => {
|
|||
|
|
try {
|
|||
|
|
return git('rev-parse HEAD').trim();
|
|||
|
|
} catch {
|
|||
|
|
return 'unknown';
|
|||
|
|
}
|
|||
|
|
})(),
|
|||
|
|
copyright: COPYRIGHT,
|
|||
|
|
sources: {
|
|||
|
|
git_commits_total: commits.length,
|
|||
|
|
git_commits_non_merge: commits.filter((c) => !c.is_merge).length,
|
|||
|
|
git_commits_merge: commits.filter((c) => c.is_merge).length,
|
|||
|
|
cab_task_specs: cab.length,
|
|||
|
|
github_api_enabled: !!(gh && gh.enabled),
|
|||
|
|
github_api_records: gh && gh.enabled ? gh.records.length : 0,
|
|||
|
|
github_api_skip_reason: gh && !gh.enabled ? gh.reason : null,
|
|||
|
|
},
|
|||
|
|
files,
|
|||
|
|
notes: [
|
|||
|
|
'本语料只收录「冰朔」本人作为作者/评论者的内容；bot 与自动化提交已排除。',
|
|||
|
|
'所有内容已做基础脱敏（API key / 邮箱 / 手机号 / token）。',
|
|||
|
|
'若需补全 GitHub Issues / PRs / Comments，请在本地设置 GH_TOKEN 后重新运行采集器。',
|
|||
|
|
'Notion AI 聊天记录无法通过本采集器自动获取，需手动导出后用 manual-import 模式合并（后续 PR）。',
|
|||
|
|
],
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ───────────────────────── main ─────────────────────────
|
|||
|
|
|
|||
|
|
(async function main() {
|
|||
|
|
ensureDir(OUTPUT_DIR);
|
|||
|
|
console.log('[corpus-harvester] 采集 git commits...');
|
|||
|
|
const commits = collectCommits();
|
|||
|
|
console.log(` 找到 ${commits.length} 条（${commits.filter((c) => !c.is_merge).length} 非 merge / ${commits.filter((c) => c.is_merge).length} merge）`);
|
|||
|
|
|
|||
|
|
console.log('[corpus-harvester] 采集 CAB 任务规格...');
|
|||
|
|
const cab = collectCabTasks();
|
|||
|
|
console.log(` 找到 ${cab.length} 条`);
|
|||
|
|
|
|||
|
|
console.log('[corpus-harvester] 尝试采集 GitHub API（需 GH_TOKEN）...');
|
|||
|
|
let gh;
|
|||
|
|
try {
|
|||
|
|
gh = await collectGitHubApi();
|
|||
|
|
} catch (e) {
|
|||
|
|
gh = { enabled: false, reason: `GitHub API 调用失败: ${e.message}`, records: [] };
|
|||
|
|
}
|
|||
|
|
console.log(` ${gh.enabled ? `成功，${gh.records.length} 条` : '已跳过：' + gh.reason}`);
|
|||
|
|
|
|||
|
|
// 写文件
|
|||
|
|
const rawPath = path.join(OUTPUT_DIR, 'raw.json');
|
|||
|
|
const tlPath = path.join(OUTPUT_DIR, 'timeline.md');
|
|||
|
|
const jsonlPath = path.join(OUTPUT_DIR, 'training.jsonl');
|
|||
|
|
const manifestPath = path.join(OUTPUT_DIR, 'manifest.json');
|
|||
|
|
|
|||
|
|
fs.writeFileSync(rawPath, JSON.stringify({ commits, cab, github: gh }, null, 2));
|
|||
|
|
fs.writeFileSync(tlPath, buildTimelineMarkdown(commits, cab, gh));
|
|||
|
|
fs.writeFileSync(jsonlPath, buildTrainingJsonl(commits, cab, gh));
|
|||
|
|
|
|||
|
|
const files = {};
|
|||
|
|
for (const p of [rawPath, tlPath, jsonlPath]) {
|
|||
|
|
const stat = fs.statSync(p);
|
|||
|
|
files[path.basename(p)] = {
|
|||
|
|
bytes: stat.size,
|
|||
|
|
sha256: require('crypto')
|
|||
|
|
.createHash('sha256')
|
|||
|
|
.update(fs.readFileSync(p))
|
|||
|
|
.digest('hex'),
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
fs.writeFileSync(
|
|||
|
|
manifestPath,
|
|||
|
|
JSON.stringify(buildManifest(commits, cab, gh, files), null, 2)
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
console.log('[corpus-harvester] 完成。输出位于:');
|
|||
|
|
console.log(` ${path.relative(REPO_ROOT, tlPath)}`);
|
|||
|
|
console.log(` ${path.relative(REPO_ROOT, jsonlPath)}`);
|
|||
|
|
console.log(` ${path.relative(REPO_ROOT, rawPath)}`);
|
|||
|
|
console.log(` ${path.relative(REPO_ROOT, manifestPath)}`);
|
|||
|
|
})().catch((e) => {
|
|||
|
|
console.error('[corpus-harvester] 失败:', e);
|
|||
|
|
process.exit(1);
|
|||
|
|
});
|