493 lines
19 KiB
JavaScript
493 lines
19 KiB
JavaScript
/**
|
||
* ═══════════════════════════════════════════════════════════
|
||
* 🧠 COS训练触发器 · 端到端训练管线
|
||
* ═══════════════════════════════════════════════════════════
|
||
*
|
||
* 签发: 铸渊 · ICE-GL-ZY001
|
||
* 版权: 国作登字-2026-A-00037559
|
||
*
|
||
* 扫描COS桶中的新语料 → 解压/转换TCS格式 → 启动训练会话
|
||
*
|
||
* 设计:
|
||
* 1. 扫描cold桶，列出所有文件（含非压缩文件和文件夹）
|
||
* 2. 对比tcs-structured/目录，找出未处理的语料
|
||
* 3. 自动提取/转换为TCS结构化格式
|
||
* 4. 启动训练会话，用LLM分析和分类
|
||
* 5. 输出处理结果，写入日志
|
||
*
|
||
* 运行方式:
|
||
* node scripts/cos-training-trigger.js [scan|extract|train|full]
|
||
*
|
||
* scan — 仅扫描，输出未处理语料列表
|
||
* extract — 扫描并解压/转换为TCS格式
|
||
* train — 对已有TCS语料启动训练
|
||
* full — 完整流程: 扫描 → 提取 → 训练
|
||
*/
|
||
|
||
'use strict';
|
||
|
||
const path = require('path');
|
||
const fs = require('fs');
|
||
|
||
// ─── 路径 ───
|
||
const ROOT = path.resolve(__dirname, '..');
|
||
const COS_MODULE = path.join(ROOT, 'server', 'age-os', 'mcp-server', 'cos');
|
||
const EXTRACTOR_MODULE = path.join(ROOT, 'server', 'age-os', 'mcp-server', 'tools', 'corpus-extractor-ops');
|
||
const TRAINING_MODULE = path.join(ROOT, 'server', 'age-os', 'mcp-server', 'tools', 'training-agent-ops');
|
||
|
||
// ─── 延迟加载模块（允许在CI中跳过数据库依赖） ───
|
||
let cos, extractor, trainer;
|
||
|
||
function loadModules() {
|
||
cos = require(COS_MODULE);
|
||
extractor = require(EXTRACTOR_MODULE);
|
||
trainer = require(TRAINING_MODULE);
|
||
}
|
||
|
||
// ─── 配置 ───
|
||
const DEFAULT_BUCKET = 'cold';
|
||
const DEFAULT_PERSONA = 'zhuyuan';
|
||
const PROCESSED_PREFIX = 'tcs-structured/';
|
||
const MAX_EXTRACT_PER_RUN = 20; // 每次最多处理文件数
|
||
const MAX_TRAIN_PER_RUN = 5; // 每次最多训练文件数
|
||
const MAX_EXTRACT_FILE_SIZE = 200 * 1024 * 1024; // 200MB — 超过此阈值的文件使用分块策略
|
||
|
||
// ─── 排除路径（不视为语料的目录/文件） ───
|
||
const EXCLUDED_PREFIXES = [
|
||
'tcs-structured/',
|
||
'training-sessions/',
|
||
'training-results/',
|
||
'training-memory/',
|
||
];
|
||
|
||
// ─── 支持的语料文件扩展名（含非压缩格式） ───
|
||
const CORPUS_EXTENSIONS = [
|
||
'.zip', '.gz', '.tar.gz', '.tgz', '.json.gz', // 压缩格式
|
||
'.json', '.jsonl', '.md', '.txt', '.csv', // 非压缩格式
|
||
];
|
||
|
||
/**
|
||
* 判断文件是否为语料文件
|
||
*/
|
||
function isCorpusFile(key) {
|
||
// 排除处理结果目录
|
||
for (const prefix of EXCLUDED_PREFIXES) {
|
||
if (key.startsWith(prefix)) return false;
|
||
}
|
||
// 匹配扩展名
|
||
const lower = key.toLowerCase();
|
||
return CORPUS_EXTENSIONS.some(ext => lower.endsWith(ext));
|
||
}
|
||
|
||
/**
|
||
* 判断是否为语料目录（如 repo-archive/）
|
||
*/
|
||
function isCorpusDirectory(key) {
|
||
for (const prefix of EXCLUDED_PREFIXES) {
|
||
if (key.startsWith(prefix)) return false;
|
||
}
|
||
return key.endsWith('/');
|
||
}
|
||
|
||
/**
|
||
* 从已处理列表中判断某文件是否已处理
|
||
*/
|
||
function isProcessed(rawKey, processedFiles) {
|
||
// 从rawKey提取基础文件名（处理多重扩展名如.tar.gz）
|
||
let baseName = rawKey.split('/').pop();
|
||
// 移除所有已知的语料文件扩展名
|
||
for (const ext of ['.tar.gz', '.json.gz', '.tgz', '.zip', '.gz', '.jsonl', '.json', '.md', '.txt', '.csv']) {
|
||
if (baseName.toLowerCase().endsWith(ext)) {
|
||
baseName = baseName.slice(0, -ext.length);
|
||
break;
|
||
}
|
||
}
|
||
return processedFiles.some(f => f.key.includes(baseName));
|
||
}
|
||
|
||
// ═══════════════════════════════════════════
|
||
// 命令: scan — 扫描未处理语料
|
||
// ═══════════════════════════════════════════
|
||
|
||
async function cmdScan(bucket) {
|
||
console.log('═══ COS训练触发器 · 语料扫描 ═══\n');
|
||
|
||
const bucketName = bucket || DEFAULT_BUCKET;
|
||
|
||
// 列出所有文件
|
||
const allFiles = await cos.list(bucketName, '', 500);
|
||
// 列出已处理文件
|
||
const processed = await cos.list(bucketName, PROCESSED_PREFIX, 500);
|
||
|
||
const processedFiles = processed.files.filter(f => f.key.endsWith('.tcs.json'));
|
||
|
||
// 分类
|
||
const corpusFiles = [];
|
||
const corpusDirs = [];
|
||
|
||
for (const file of allFiles.files) {
|
||
if (isCorpusFile(file.key)) {
|
||
const alreadyProcessed = isProcessed(file.key, processedFiles);
|
||
corpusFiles.push({
|
||
key: file.key,
|
||
size_bytes: file.size_bytes,
|
||
processed: alreadyProcessed
|
||
});
|
||
} else if (isCorpusDirectory(file.key)) {
|
||
corpusDirs.push({ key: file.key });
|
||
}
|
||
}
|
||
|
||
const pending = corpusFiles.filter(f => !f.processed);
|
||
|
||
console.log(`桶: ${bucketName}`);
|
||
console.log(`总文件数: ${allFiles.files.length}`);
|
||
console.log(`语料文件: ${corpusFiles.length}`);
|
||
console.log(`语料目录: ${corpusDirs.length}`);
|
||
console.log(`已处理: ${corpusFiles.length - pending.length}`);
|
||
console.log(`待处理: ${pending.length}`);
|
||
console.log(`已生成TCS: ${processedFiles.length}`);
|
||
|
||
if (pending.length > 0) {
|
||
console.log('\n📋 待处理语料:');
|
||
for (const f of pending) {
|
||
console.log(` 📄 ${f.key} (${formatBytes(f.size_bytes)})`);
|
||
}
|
||
}
|
||
|
||
if (corpusDirs.length > 0) {
|
||
console.log('\n📁 语料目录:');
|
||
for (const d of corpusDirs) {
|
||
console.log(` 📂 ${d.key}`);
|
||
}
|
||
|
||
// 扫描目录内的文件
|
||
for (const dir of corpusDirs) {
|
||
try {
|
||
const dirFiles = await cos.list(bucketName, dir.key, 100);
|
||
const dirCorpus = dirFiles.files.filter(f => isCorpusFile(f.key));
|
||
if (dirCorpus.length > 0) {
|
||
console.log(` └── ${dir.key} 内含 ${dirCorpus.length} 个语料文件`);
|
||
for (const f of dirCorpus) {
|
||
const alreadyProcessed = isProcessed(f.key, processedFiles);
|
||
if (!alreadyProcessed) {
|
||
pending.push({
|
||
key: f.key,
|
||
size_bytes: f.size_bytes,
|
||
processed: false
|
||
});
|
||
console.log(` 📄 ${f.key} (${formatBytes(f.size_bytes)}) [待处理]`);
|
||
}
|
||
}
|
||
}
|
||
} catch (err) {
|
||
console.log(` └── ${dir.key} 扫描失败: ${err.message}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
// 写入GitHub Actions输出
|
||
if (process.env.GITHUB_OUTPUT) {
|
||
const outputLines = [
|
||
`pending=${pending.length}`,
|
||
`total_corpus=${corpusFiles.length}`,
|
||
`processed=${processedFiles.length}`,
|
||
`has_new_corpus=${pending.length > 0 ? 'true' : 'false'}`,
|
||
`pending_files=${pending.map(f => f.key).join(',')}`
|
||
];
|
||
fs.appendFileSync(process.env.GITHUB_OUTPUT, outputLines.join('\n') + '\n');
|
||
}
|
||
|
||
return { pending, processedFiles, corpusDirs };
|
||
}
|
||
|
||
// ═══════════════════════════════════════════
|
||
// 命令: extract — 提取/转换语料为TCS格式
|
||
// ═══════════════════════════════════════════
|
||
|
||
async function cmdExtract(bucket) {
|
||
console.log('═══ COS训练触发器 · 语料提取 ═══\n');
|
||
|
||
const bucketName = bucket || DEFAULT_BUCKET;
|
||
const { pending } = await cmdScan(bucketName);
|
||
|
||
if (pending.length === 0) {
|
||
console.log('\n✅ 无待处理语料');
|
||
writeGitHubOutput('extracted=0', 'extract_status=skipped');
|
||
return { extracted: 0, errors: 0 };
|
||
}
|
||
|
||
const toProcess = pending.slice(0, MAX_EXTRACT_PER_RUN);
|
||
console.log(`\n🔄 开始提取 ${toProcess.length}/${pending.length} 个文件...\n`);
|
||
|
||
let extracted = 0;
|
||
let errors = 0;
|
||
let skipped = 0;
|
||
const results = [];
|
||
|
||
for (const file of toProcess) {
|
||
try {
|
||
// 大文件预警
|
||
const sizeMB = (file.size_bytes / 1024 / 1024).toFixed(1);
|
||
if (file.size_bytes > MAX_EXTRACT_FILE_SIZE) {
|
||
console.log(` 📦 处理: ${file.key} (${sizeMB}MB · 超大文件，使用分块策略)...`);
|
||
} else {
|
||
console.log(` 📦 处理: ${file.key} (${sizeMB}MB)...`);
|
||
}
|
||
|
||
const result = await extractor.cosExtractCorpus({
|
||
bucket: bucketName,
|
||
key: file.key,
|
||
output_bucket: bucketName,
|
||
output_prefix: PROCESSED_PREFIX
|
||
});
|
||
|
||
// 根据返回状态分类计数
|
||
if (result.status === 'zip_detected') {
|
||
skipped++;
|
||
results.push({ key: file.key, status: 'skipped', reason: 'zip_needs_special_tool' });
|
||
console.log(` ⏭️ 跳过: ${file.key} — ZIP文件需要专用工具处理`);
|
||
} else if (result.status === 'skipped_too_large') {
|
||
skipped++;
|
||
results.push({ key: file.key, status: 'skipped', reason: 'too_large', size_mb: result.size_mb });
|
||
console.log(` ⏭️ 跳过: ${file.key} — ${result.message}`);
|
||
} else if (result.status === 'partial_extract') {
|
||
extracted++;
|
||
results.push({ key: file.key, status: 'partial', output: result.output?.key, message: result.message });
|
||
console.log(` 🔶 部分提取: ${result.message}`);
|
||
} else {
|
||
extracted++;
|
||
results.push({ key: file.key, status: 'success', output: result.output?.key });
|
||
console.log(` ✅ 完成: ${result.output?.key || '已处理'} (${result.entries || 0} 条目)`);
|
||
}
|
||
} catch (err) {
|
||
errors++;
|
||
results.push({ key: file.key, status: 'error', error: err.message });
|
||
console.log(` ❌ 失败: ${file.key} — ${err.message}`);
|
||
}
|
||
}
|
||
|
||
console.log(`\n═══ 提取完毕 ═══`);
|
||
console.log(`✅ 成功: ${extracted}`);
|
||
console.log(`⏭️ 跳过: ${skipped}`);
|
||
console.log(`❌ 失败: ${errors}`);
|
||
console.log(`⏳ 剩余: ${pending.length - toProcess.length}`);
|
||
|
||
writeGitHubOutput(
|
||
`extracted=${extracted}`,
|
||
`extract_skipped=${skipped}`,
|
||
`extract_errors=${errors}`,
|
||
`extract_status=${errors > 0 ? 'partial' : 'success'}`
|
||
);
|
||
|
||
return { extracted, errors, skipped, results };
|
||
}
|
||
|
||
// ═══════════════════════════════════════════
|
||
// 命令: train — 对TCS语料启动训练
|
||
// ═══════════════════════════════════════════
|
||
|
||
async function cmdTrain(bucket, personaId) {
|
||
console.log('═══ COS训练触发器 · 训练处理 ═══\n');
|
||
|
||
const bucketName = bucket || DEFAULT_BUCKET;
|
||
const persona = personaId || DEFAULT_PERSONA;
|
||
|
||
// 列出可用的TCS语料
|
||
const processed = await cos.list(bucketName, PROCESSED_PREFIX, 500);
|
||
const tcsFiles = processed.files.filter(f => f.key.endsWith('.tcs.json'));
|
||
|
||
if (tcsFiles.length === 0) {
|
||
console.log('⚠️ 无TCS结构化语料可训练。请先运行 extract 命令。');
|
||
writeGitHubOutput('trained=0', 'train_status=no_corpus');
|
||
return { trained: 0, errors: 0 };
|
||
}
|
||
|
||
console.log(`📚 找到 ${tcsFiles.length} 个TCS语料文件`);
|
||
|
||
// 检查已有训练结果，避免重复处理
|
||
let existingResults = [];
|
||
try {
|
||
const existing = await cos.list(bucketName, `training-results/${persona}/`, 100);
|
||
existingResults = existing.files.filter(f => f.key.endsWith('.json'));
|
||
} catch (err) {
|
||
console.log(`⚠️ 无法读取已有训练结果: ${err.message}`);
|
||
}
|
||
|
||
// 启动训练会话
|
||
console.log(`\n🧠 启动训练会话 · 人格体: ${persona}`);
|
||
|
||
let session;
|
||
try {
|
||
session = await trainer.trainingStartSession({
|
||
persona_id: persona,
|
||
corpus_bucket: bucketName,
|
||
corpus_prefix: PROCESSED_PREFIX,
|
||
session_name: `自动训练-${new Date().toISOString().slice(0, 10)}`
|
||
});
|
||
console.log(`✅ 会话已启动: ${session.session_id}`);
|
||
console.log(` 可用模型: ${session.models.available.map(m => m.name).join(', ') || '无'}`);
|
||
} catch (err) {
|
||
console.log(`❌ 训练会话启动失败: ${err.message}`);
|
||
writeGitHubOutput('trained=0', `train_status=session_error`, `train_error=${err.message}`);
|
||
return { trained: 0, errors: 1, error: err.message };
|
||
}
|
||
|
||
// 检查是否有可用的LLM模型
|
||
if (!session.models.available || session.models.available.length === 0) {
|
||
console.log('⚠️ 无可用LLM模型（需要配置 ZY_DEEPSEEK_API_KEY 等密钥）');
|
||
console.log(' 训练会话已记录，等待LLM密钥配置后再次运行。');
|
||
writeGitHubOutput('trained=0', 'train_status=no_llm_keys');
|
||
return { trained: 0, errors: 0, note: '无LLM密钥' };
|
||
}
|
||
|
||
// 处理语料
|
||
const toTrain = tcsFiles.slice(0, MAX_TRAIN_PER_RUN);
|
||
let trained = 0;
|
||
let trainErrors = 0;
|
||
const trainResults = [];
|
||
|
||
for (const tcsFile of toTrain) {
|
||
try {
|
||
console.log(` 🔬 训练处理: ${tcsFile.key}...`);
|
||
const result = await trainer.trainingProcessCorpus({
|
||
corpus_bucket: bucketName,
|
||
corpus_key: tcsFile.key,
|
||
persona_id: persona,
|
||
max_entries: 10
|
||
});
|
||
trained++;
|
||
trainResults.push({ key: tcsFile.key, status: 'success', ...result });
|
||
console.log(` ✅ 完成: ${result.classified}/${result.total} 分类成功`);
|
||
} catch (err) {
|
||
trainErrors++;
|
||
trainResults.push({ key: tcsFile.key, status: 'error', error: err.message });
|
||
console.log(` ❌ 失败: ${tcsFile.key} — ${err.message}`);
|
||
}
|
||
}
|
||
|
||
console.log(`\n═══ 训练完毕 ═══`);
|
||
console.log(`✅ 成功: ${trained}`);
|
||
console.log(`❌ 失败: ${trainErrors}`);
|
||
|
||
writeGitHubOutput(
|
||
`trained=${trained}`,
|
||
`train_errors=${trainErrors}`,
|
||
`train_status=${trainErrors > 0 ? 'partial' : 'success'}`
|
||
);
|
||
|
||
return { trained, errors: trainErrors, results: trainResults, session_id: session.session_id };
|
||
}
|
||
|
||
// ═══════════════════════════════════════════
|
||
// 命令: full — 完整流程
|
||
// ═══════════════════════════════════════════
|
||
|
||
async function cmdFull(bucket, personaId) {
|
||
console.log('╔═══════════════════════════════════════════╗');
|
||
console.log('║ COS训练触发器 · 完整训练管线 ║');
|
||
console.log('║ 铸渊 · ICE-GL-ZY001 ║');
|
||
console.log('╚═══════════════════════════════════════════╝\n');
|
||
|
||
const bucketName = bucket || DEFAULT_BUCKET;
|
||
const persona = personaId || DEFAULT_PERSONA;
|
||
const startTime = Date.now();
|
||
|
||
// 第一步: 提取语料
|
||
console.log('📍 第一步: 提取语料\n');
|
||
const extractResult = await cmdExtract(bucketName);
|
||
|
||
// 第二步: 训练处理
|
||
console.log('\n📍 第二步: 训练处理\n');
|
||
const trainResult = await cmdTrain(bucketName, persona);
|
||
|
||
// 汇总
|
||
const duration = Date.now() - startTime;
|
||
console.log('\n╔═══════════════════════════════════════════╗');
|
||
console.log('║ 完整训练管线 · 运行完毕 ║');
|
||
console.log('╚═══════════════════════════════════════════╝');
|
||
console.log(` 提取: ${extractResult.extracted} 成功 / ${extractResult.skipped || 0} 跳过 / ${extractResult.errors} 失败`);
|
||
console.log(` 训练: ${trainResult.trained} 成功 / ${trainResult.errors} 失败`);
|
||
console.log(` 耗时: ${(duration / 1000).toFixed(1)}s`);
|
||
|
||
writeGitHubOutput(
|
||
`pipeline_status=${(extractResult.errors + trainResult.errors) > 0 ? 'partial' : 'success'}`,
|
||
`pipeline_duration_ms=${duration}`
|
||
);
|
||
|
||
return { extract: extractResult, train: trainResult, duration_ms: duration };
|
||
}
|
||
|
||
// ═══════════════════════════════════════════
|
||
// 辅助函数
|
||
// ═══════════════════════════════════════════
|
||
|
||
function formatBytes(bytes) {
|
||
if (bytes < 1024) return `${bytes}B`;
|
||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}KB`;
|
||
return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
|
||
}
|
||
|
||
function writeGitHubOutput(...lines) {
|
||
if (process.env.GITHUB_OUTPUT) {
|
||
fs.appendFileSync(process.env.GITHUB_OUTPUT, lines.join('\n') + '\n');
|
||
}
|
||
}
|
||
|
||
// ═══════════════════════════════════════════
|
||
// CLI 入口
|
||
// ═══════════════════════════════════════════
|
||
|
||
async function main() {
|
||
const args = process.argv.slice(2);
|
||
const command = args[0] || 'scan';
|
||
const bucket = args.find(a => a.startsWith('--bucket='))?.split('=')[1] || DEFAULT_BUCKET;
|
||
const persona = args.find(a => a.startsWith('--persona='))?.split('=')[1] || DEFAULT_PERSONA;
|
||
|
||
// 加载模块
|
||
try {
|
||
loadModules();
|
||
} catch (err) {
|
||
console.error(`❌ 模块加载失败: ${err.message}`);
|
||
console.error(' 请确保 server/age-os/mcp-server/ 依赖已安装');
|
||
process.exit(1);
|
||
}
|
||
|
||
switch (command) {
|
||
case 'scan':
|
||
await cmdScan(bucket);
|
||
break;
|
||
case 'extract':
|
||
await cmdExtract(bucket);
|
||
break;
|
||
case 'train':
|
||
await cmdTrain(bucket, persona);
|
||
break;
|
||
case 'full':
|
||
await cmdFull(bucket, persona);
|
||
break;
|
||
default:
|
||
console.log('COS训练触发器 · 铸渊 · ICE-GL-ZY001');
|
||
console.log('');
|
||
console.log('用法:');
|
||
console.log(' node scripts/cos-training-trigger.js scan — 扫描未处理语料');
|
||
console.log(' node scripts/cos-training-trigger.js extract — 提取/转换为TCS格式');
|
||
console.log(' node scripts/cos-training-trigger.js train — 启动训练处理');
|
||
console.log(' node scripts/cos-training-trigger.js full — 完整流程');
|
||
console.log('');
|
||
console.log('选项:');
|
||
console.log(' --bucket=cold|hot|team — 指定COS桶（默认cold）');
|
||
console.log(' --persona=zhuyuan — 指定人格体（默认zhuyuan）');
|
||
break;
|
||
}
|
||
}
|
||
|
||
main().catch(err => {
|
||
console.error('COS训练触发器异常:', err.message);
|
||
if (process.env.GITHUB_OUTPUT) {
|
||
fs.appendFileSync(process.env.GITHUB_OUTPUT, 'pipeline_status=error\n');
|
||
fs.appendFileSync(process.env.GITHUB_OUTPUT, `pipeline_error=${err.message}\n`);
|
||
}
|
||
process.exit(1);
|
||
});
|