973 lines
32 KiB
JavaScript
973 lines
32 KiB
JavaScript
|
/**
|
|||
|
|
* ═══════════════════════════════════════════════════════════
|
|||
|
|
* 模块H · 开源模型微调引擎 MCP 工具
|
|||
|
|
* ═══════════════════════════════════════════════════════════
|
|||
|
|
*
|
|||
|
|
* 签发: 铸渊 · ICE-GL-ZY001
|
|||
|
|
* 版权: 国作登字-2026-A-00037559
|
|||
|
|
*
|
|||
|
|
* 冰朔D62核心指令: 接入开源模型，用COS桶训练数据直接做微调
|
|||
|
|
* 本质: 同一份TCS结构化数据，两种用途 — RAG + 微调
|
|||
|
|
*
|
|||
|
|
* 架构理念:
|
|||
|
|
* 现有RAG训练 → 用API调用商业模型，人格体"脑子"在COS桶里
|
|||
|
|
* 开源模型微调 → 用同一份数据，把"脑子"直接装进开源模型
|
|||
|
|
* 二者并行运行 → 微调模型优先 → 不可用时降级回API模型
|
|||
|
|
*
|
|||
|
|
* 工具清单:
|
|||
|
|
* finetuneExportDataset — 导出TCS语料为微调JSONL格式
|
|||
|
|
* finetuneSubmitJob — 提交微调任务到DeepSeek/Qwen
|
|||
|
|
* finetuneCheckStatus — 查询微调任务进度
|
|||
|
|
* finetuneRegisterModel — 注册微调完成的模型
|
|||
|
|
* finetuneListModels — 列出已注册的微调模型
|
|||
|
|
* finetuneCallModel — 调用微调模型进行推理
|
|||
|
|
* finetuneCompareModels — A/B测试微调 vs 基座模型
|
|||
|
|
* finetuneGetCostEstimate — 估算微调成本
|
|||
|
|
*/
|
|||
|
|
|
|||
|
|
'use strict';
|
|||
|
|
|
|||
|
|
const https = require('https');
|
|||
|
|
const crypto = require('crypto');
|
|||
|
|
const cos = require('../cos');
|
|||
|
|
|
|||
|
|
// ─── 微调 API 配置 ───
|
|||
|
|
const FINETUNE_PROVIDERS = {
|
|||
|
|
deepseek: {
|
|||
|
|
host: 'api.deepseek.com',
|
|||
|
|
createPath: '/fine_tuning/jobs',
|
|||
|
|
statusPath: '/fine_tuning/jobs/',
|
|||
|
|
uploadPath: '/files',
|
|||
|
|
inferencePath: '/v1/chat/completions',
|
|||
|
|
defaultModel: 'deepseek-chat',
|
|||
|
|
keyEnv: 'ZY_DEEPSEEK_API_KEY',
|
|||
|
|
label: 'DeepSeek微调'
|
|||
|
|
},
|
|||
|
|
qwen: {
|
|||
|
|
host: 'dashscope.aliyuncs.com',
|
|||
|
|
createPath: '/api/v1/fine-tunes',
|
|||
|
|
statusPath: '/api/v1/fine-tunes/',
|
|||
|
|
uploadPath: '/api/v1/files',
|
|||
|
|
inferencePath: '/compatible-mode/v1/chat/completions',
|
|||
|
|
defaultModel: 'qwen-max',
|
|||
|
|
keyEnv: 'ZY_QIANWEN_API_KEY',
|
|||
|
|
label: 'Qwen/DashScope微调'
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
// ─── 推理降级配置（与training-agent-ops.js同源） ───
|
|||
|
|
const LLM_CONFIGS = {
|
|||
|
|
'deepseek-chat': {
|
|||
|
|
host: 'api.deepseek.com',
|
|||
|
|
path: '/v1/chat/completions',
|
|||
|
|
model: 'deepseek-chat',
|
|||
|
|
keyEnv: 'ZY_DEEPSEEK_API_KEY',
|
|||
|
|
purpose: '微调基座·推理降级'
|
|||
|
|
},
|
|||
|
|
'qwen-max': {
|
|||
|
|
host: 'dashscope.aliyuncs.com',
|
|||
|
|
path: '/compatible-mode/v1/chat/completions',
|
|||
|
|
model: 'qwen-max',
|
|||
|
|
keyEnv: 'ZY_QIANWEN_API_KEY',
|
|||
|
|
purpose: '微调基座·推理降级'
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
// ─── 成本估算参数（2026-04 参考价，实际以provider当月公告为准） ───
|
|||
|
|
const COST_PER_1K_TOKENS = {
|
|||
|
|
deepseek: 0.014, // 约 ¥0.014 / 1K tokens（训练）· 2026-04 参考
|
|||
|
|
qwen: 0.020 // 约 ¥0.020 / 1K tokens（训练）· 2026-04 参考
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
// ─── 常量 ───
|
|||
|
|
const DEFAULT_BUCKET = 'cold';
|
|||
|
|
const MAX_SAMPLES_DEFAULT = 500;
|
|||
|
|
const FINETUNE_TIMEOUT = 60000;
|
|||
|
|
|
|||
|
|
// ═══════════════════════════════════════════════════════════
|
|||
|
|
// 工具实现
|
|||
|
|
// ═══════════════════════════════════════════════════════════
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* finetuneExportDataset — 导出TCS语料为微调JSONL格式
|
|||
|
|
*
|
|||
|
|
* 将COS桶中的TCS结构化语料转换为 instruction/input/output 三元组
|
|||
|
|
* JSONL格式，直接用于提交到微调API
|
|||
|
|
*
|
|||
|
|
* input:
|
|||
|
|
* persona_id: string — 人格体ID
|
|||
|
|
* corpus_bucket: string — 语料桶（默认cold）
|
|||
|
|
* corpus_prefix: string — 语料路径前缀
|
|||
|
|
* output_format: string — 输出格式（默认jsonl）
|
|||
|
|
* max_samples: number — 最大样本数
|
|||
|
|
*/
|
|||
|
|
async function finetuneExportDataset(input) {
|
|||
|
|
const { persona_id, corpus_bucket, corpus_prefix, output_format, max_samples } = input;
|
|||
|
|
if (!persona_id) throw new Error('缺少 persona_id');
|
|||
|
|
|
|||
|
|
const bucket = corpus_bucket || DEFAULT_BUCKET;
|
|||
|
|
const prefix = corpus_prefix || 'tcs-structured/';
|
|||
|
|
const format = output_format || 'jsonl';
|
|||
|
|
const maxSamples = max_samples || MAX_SAMPLES_DEFAULT;
|
|||
|
|
const datasetId = `ds-${persona_id}-${Date.now()}-${crypto.randomBytes(4).toString('hex')}`;
|
|||
|
|
|
|||
|
|
// 扫描TCS语料文件
|
|||
|
|
let corpusFiles = [];
|
|||
|
|
try {
|
|||
|
|
const result = await cos.list(bucket, prefix, 500);
|
|||
|
|
corpusFiles = result.files.filter(f => f.key.endsWith('.tcs.json') || f.key.endsWith('.json'));
|
|||
|
|
} catch {
|
|||
|
|
throw new Error(`无法读取语料桶 ${bucket}/${prefix}`);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (corpusFiles.length === 0) {
|
|||
|
|
throw new Error(`语料桶 ${bucket}/${prefix} 中未找到TCS语料文件`);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 逐文件读取并转换为JSONL三元组
|
|||
|
|
const jsonlLines = [];
|
|||
|
|
let filesProcessed = 0;
|
|||
|
|
|
|||
|
|
for (const file of corpusFiles) {
|
|||
|
|
if (jsonlLines.length >= maxSamples) break;
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
const raw = await cos.read(bucket, file.key);
|
|||
|
|
const corpus = JSON.parse(raw.content);
|
|||
|
|
const entries = corpus.entries || (Array.isArray(corpus) ? corpus : [corpus]);
|
|||
|
|
|
|||
|
|
for (const entry of entries) {
|
|||
|
|
if (jsonlLines.length >= maxSamples) break;
|
|||
|
|
|
|||
|
|
const triple = convertEntryToTriple(persona_id, corpus.corpus_type, entry);
|
|||
|
|
if (triple) {
|
|||
|
|
jsonlLines.push(JSON.stringify(triple));
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
filesProcessed++;
|
|||
|
|
} catch {
|
|||
|
|
// 跳过无法解析的文件
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (jsonlLines.length === 0) {
|
|||
|
|
throw new Error('未能从语料中生成任何训练样本');
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 写入JSONL到COS
|
|||
|
|
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
|||
|
|
const fileKey = `finetune-datasets/${persona_id}/${timestamp}.${format}`;
|
|||
|
|
const jsonlContent = jsonlLines.join('\n') + '\n';
|
|||
|
|
|
|||
|
|
await cos.write(bucket, fileKey, jsonlContent, 'application/jsonl');
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
dataset_id: datasetId,
|
|||
|
|
file_key: fileKey,
|
|||
|
|
sample_count: jsonlLines.length,
|
|||
|
|
format,
|
|||
|
|
files_scanned: corpusFiles.length,
|
|||
|
|
files_processed: filesProcessed,
|
|||
|
|
bucket,
|
|||
|
|
created_at: new Date().toISOString()
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* finetuneSubmitJob — 提交微调任务到DeepSeek或Qwen API
|
|||
|
|
*
|
|||
|
|
* 从COS读取JSONL数据集，上传到provider，然后创建微调任务
|
|||
|
|
*
|
|||
|
|
* input:
|
|||
|
|
* persona_id: string — 人格体ID
|
|||
|
|
* dataset_key: string — COS中JSONL文件路径
|
|||
|
|
* provider: string — 微调提供方（deepseek / qwen）
|
|||
|
|
* base_model: string — 基座模型（可选，默认取provider默认值）
|
|||
|
|
* hyperparams: object — 超参数（可选）
|
|||
|
|
*/
|
|||
|
|
async function finetuneSubmitJob(input) {
|
|||
|
|
const { persona_id, dataset_key, provider, base_model, hyperparams } = input;
|
|||
|
|
if (!persona_id) throw new Error('缺少 persona_id');
|
|||
|
|
if (!dataset_key) throw new Error('缺少 dataset_key');
|
|||
|
|
|
|||
|
|
const providerKey = (provider || 'deepseek').toLowerCase();
|
|||
|
|
const providerConfig = FINETUNE_PROVIDERS[providerKey];
|
|||
|
|
if (!providerConfig) throw new Error(`不支持的微调提供方: ${providerKey}，仅支持 deepseek / qwen`);
|
|||
|
|
|
|||
|
|
const apiKey = process.env[providerConfig.keyEnv];
|
|||
|
|
if (!apiKey) throw new Error(`缺少API密钥环境变量 ${providerConfig.keyEnv}`);
|
|||
|
|
|
|||
|
|
const jobId = `ft-${persona_id}-${Date.now()}-${crypto.randomBytes(4).toString('hex')}`;
|
|||
|
|
const model = base_model || providerConfig.defaultModel;
|
|||
|
|
|
|||
|
|
// 从COS读取JSONL数据集
|
|||
|
|
const bucket = DEFAULT_BUCKET;
|
|||
|
|
const raw = await cos.read(bucket, dataset_key);
|
|||
|
|
const datasetContent = raw.content;
|
|||
|
|
|
|||
|
|
// 上传训练文件到provider
|
|||
|
|
const fileId = await uploadTrainingFile(providerConfig, apiKey, datasetContent, providerKey);
|
|||
|
|
|
|||
|
|
// 创建微调任务
|
|||
|
|
const jobResult = await createFinetuneJob(providerConfig, apiKey, model, fileId, hyperparams, providerKey);
|
|||
|
|
|
|||
|
|
// 保存任务元数据到COS
|
|||
|
|
const jobMeta = {
|
|||
|
|
job_id: jobId,
|
|||
|
|
provider_job_id: jobResult.provider_job_id,
|
|||
|
|
persona_id,
|
|||
|
|
provider: providerKey,
|
|||
|
|
base_model: model,
|
|||
|
|
dataset_key,
|
|||
|
|
file_id: fileId,
|
|||
|
|
hyperparams: hyperparams || {},
|
|||
|
|
status: jobResult.status || 'pending',
|
|||
|
|
created_at: new Date().toISOString(),
|
|||
|
|
updated_at: new Date().toISOString()
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
await cos.write(bucket, `finetune-jobs/${persona_id}/${jobId}.json`,
|
|||
|
|
JSON.stringify(jobMeta, null, 2), 'application/json');
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
job_id: jobId,
|
|||
|
|
provider_job_id: jobResult.provider_job_id,
|
|||
|
|
provider: providerKey,
|
|||
|
|
status: jobMeta.status,
|
|||
|
|
base_model: model,
|
|||
|
|
estimated_time: jobResult.estimated_time || '未知，通常需要数小时'
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* finetuneCheckStatus — 查询微调任务进度
|
|||
|
|
*
|
|||
|
|
* input:
|
|||
|
|
* persona_id: string — 人格体ID
|
|||
|
|
* job_id: string — 任务ID
|
|||
|
|
* provider: string — 微调提供方
|
|||
|
|
*/
|
|||
|
|
async function finetuneCheckStatus(input) {
|
|||
|
|
const { persona_id, job_id, provider } = input;
|
|||
|
|
if (!persona_id) throw new Error('缺少 persona_id');
|
|||
|
|
if (!job_id) throw new Error('缺少 job_id');
|
|||
|
|
|
|||
|
|
const bucket = DEFAULT_BUCKET;
|
|||
|
|
|
|||
|
|
// 读取任务元数据
|
|||
|
|
let jobMeta;
|
|||
|
|
try {
|
|||
|
|
const raw = await cos.read(bucket, `finetune-jobs/${persona_id}/${job_id}.json`);
|
|||
|
|
jobMeta = JSON.parse(raw.content);
|
|||
|
|
} catch {
|
|||
|
|
throw new Error(`未找到微调任务: ${job_id}`);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const providerKey = provider || jobMeta.provider;
|
|||
|
|
const providerConfig = FINETUNE_PROVIDERS[providerKey];
|
|||
|
|
if (!providerConfig) throw new Error(`不支持的微调提供方: ${providerKey}`);
|
|||
|
|
|
|||
|
|
const apiKey = process.env[providerConfig.keyEnv];
|
|||
|
|
if (!apiKey) throw new Error(`缺少API密钥环境变量 ${providerConfig.keyEnv}`);
|
|||
|
|
|
|||
|
|
// 查询provider API获取最新状态
|
|||
|
|
const providerJobId = jobMeta.provider_job_id;
|
|||
|
|
let statusResult;
|
|||
|
|
try {
|
|||
|
|
statusResult = await queryJobStatus(providerConfig, apiKey, providerJobId, providerKey);
|
|||
|
|
} catch (err) {
|
|||
|
|
return {
|
|||
|
|
job_id,
|
|||
|
|
provider: providerKey,
|
|||
|
|
status: jobMeta.status,
|
|||
|
|
progress: null,
|
|||
|
|
metrics: null,
|
|||
|
|
error: `查询provider状态失败: ${err.message}`,
|
|||
|
|
last_known_update: jobMeta.updated_at
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 更新COS中的任务元数据
|
|||
|
|
jobMeta.status = statusResult.status;
|
|||
|
|
jobMeta.updated_at = new Date().toISOString();
|
|||
|
|
if (statusResult.fine_tuned_model) {
|
|||
|
|
jobMeta.fine_tuned_model = statusResult.fine_tuned_model;
|
|||
|
|
}
|
|||
|
|
if (statusResult.metrics) {
|
|||
|
|
jobMeta.metrics = statusResult.metrics;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
await cos.write(bucket, `finetune-jobs/${persona_id}/${job_id}.json`,
|
|||
|
|
JSON.stringify(jobMeta, null, 2), 'application/json');
|
|||
|
|
} catch { /* ignore */ }
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
job_id,
|
|||
|
|
provider_job_id: providerJobId,
|
|||
|
|
provider: providerKey,
|
|||
|
|
status: statusResult.status,
|
|||
|
|
progress: statusResult.progress || null,
|
|||
|
|
metrics: statusResult.metrics || null,
|
|||
|
|
fine_tuned_model: statusResult.fine_tuned_model || null,
|
|||
|
|
updated_at: jobMeta.updated_at
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* finetuneRegisterModel — 注册微调完成的模型
|
|||
|
|
*
|
|||
|
|
* input:
|
|||
|
|
* persona_id: string — 人格体ID
|
|||
|
|
* job_id: string — 关联的微调任务ID
|
|||
|
|
* model_endpoint: string — 模型推理端点（provider返回的fine_tuned_model名称）
|
|||
|
|
* model_name: string — 本地注册名称
|
|||
|
|
* provider: string — 微调提供方
|
|||
|
|
* description: string — 模型描述
|
|||
|
|
*/
|
|||
|
|
async function finetuneRegisterModel(input) {
|
|||
|
|
const { persona_id, job_id, model_endpoint, model_name, provider, description } = input;
|
|||
|
|
if (!persona_id) throw new Error('缺少 persona_id');
|
|||
|
|
if (!model_endpoint) throw new Error('缺少 model_endpoint');
|
|||
|
|
if (!model_name) throw new Error('缺少 model_name');
|
|||
|
|
|
|||
|
|
const providerKey = (provider || 'deepseek').toLowerCase();
|
|||
|
|
const providerConfig = FINETUNE_PROVIDERS[providerKey];
|
|||
|
|
if (!providerConfig) throw new Error(`不支持的微调提供方: ${providerKey}`);
|
|||
|
|
|
|||
|
|
const modelId = `mdl-${persona_id}-${crypto.randomBytes(4).toString('hex')}`;
|
|||
|
|
const now = new Date().toISOString();
|
|||
|
|
|
|||
|
|
const modelConfig = {
|
|||
|
|
model_id: modelId,
|
|||
|
|
persona_id,
|
|||
|
|
model_name,
|
|||
|
|
model_endpoint,
|
|||
|
|
provider: providerKey,
|
|||
|
|
provider_host: providerConfig.host,
|
|||
|
|
inference_path: providerConfig.inferencePath,
|
|||
|
|
key_env: providerConfig.keyEnv,
|
|||
|
|
job_id: job_id || null,
|
|||
|
|
description: description || `${persona_id} 微调模型`,
|
|||
|
|
status: 'active',
|
|||
|
|
created_at: now,
|
|||
|
|
updated_at: now
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
const bucket = DEFAULT_BUCKET;
|
|||
|
|
const configKey = `finetune-models/${persona_id}/${model_name}.json`;
|
|||
|
|
await cos.write(bucket, configKey, JSON.stringify(modelConfig, null, 2), 'application/json');
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
model_id: modelId,
|
|||
|
|
model_name,
|
|||
|
|
provider: providerKey,
|
|||
|
|
registered_at: now,
|
|||
|
|
config_key: configKey,
|
|||
|
|
config: modelConfig
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* finetuneListModels — 列出已注册的微调模型
|
|||
|
|
*
|
|||
|
|
* input:
|
|||
|
|
* persona_id: string — 人格体ID
|
|||
|
|
*/
|
|||
|
|
async function finetuneListModels(input) {
|
|||
|
|
const { persona_id } = input;
|
|||
|
|
if (!persona_id) throw new Error('缺少 persona_id');
|
|||
|
|
|
|||
|
|
const bucket = DEFAULT_BUCKET;
|
|||
|
|
const prefix = `finetune-models/${persona_id}/`;
|
|||
|
|
|
|||
|
|
let files = [];
|
|||
|
|
try {
|
|||
|
|
const result = await cos.list(bucket, prefix, 100);
|
|||
|
|
files = result.files.filter(f => f.key.endsWith('.json'));
|
|||
|
|
} catch {
|
|||
|
|
return { persona_id, models: [], count: 0 };
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const models = [];
|
|||
|
|
for (const file of files) {
|
|||
|
|
try {
|
|||
|
|
const raw = await cos.read(bucket, file.key);
|
|||
|
|
const config = JSON.parse(raw.content);
|
|||
|
|
models.push({
|
|||
|
|
model_name: config.model_name,
|
|||
|
|
model_id: config.model_id,
|
|||
|
|
provider: config.provider,
|
|||
|
|
model_endpoint: config.model_endpoint,
|
|||
|
|
status: config.status,
|
|||
|
|
description: config.description,
|
|||
|
|
created_at: config.created_at
|
|||
|
|
});
|
|||
|
|
} catch {
|
|||
|
|
// 跳过无法解析的配置
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
persona_id,
|
|||
|
|
models,
|
|||
|
|
count: models.length
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* finetuneCallModel — 调用微调模型进行推理
|
|||
|
|
*
|
|||
|
|
* 加载模型配置，调用provider推理API
|
|||
|
|
* 微调模型不可用时自动降级到基座模型
|
|||
|
|
*
|
|||
|
|
* input:
|
|||
|
|
* persona_id: string — 人格体ID
|
|||
|
|
* model_name: string — 已注册的模型名称
|
|||
|
|
* prompt: string — 推理提示词
|
|||
|
|
* temperature: number — 温度（默认0.7）
|
|||
|
|
* max_tokens: number — 最大token数（默认1000）
|
|||
|
|
*/
|
|||
|
|
async function finetuneCallModel(input) {
|
|||
|
|
const { persona_id, model_name, prompt, temperature, max_tokens } = input;
|
|||
|
|
if (!persona_id) throw new Error('缺少 persona_id');
|
|||
|
|
if (!model_name) throw new Error('缺少 model_name');
|
|||
|
|
if (!prompt) throw new Error('缺少 prompt');
|
|||
|
|
|
|||
|
|
const bucket = DEFAULT_BUCKET;
|
|||
|
|
let modelConfig;
|
|||
|
|
let fallbackUsed = false;
|
|||
|
|
|
|||
|
|
// 加载模型配置
|
|||
|
|
try {
|
|||
|
|
const raw = await cos.read(bucket, `finetune-models/${persona_id}/${model_name}.json`);
|
|||
|
|
modelConfig = JSON.parse(raw.content);
|
|||
|
|
} catch {
|
|||
|
|
throw new Error(`未找到模型配置: ${model_name}`);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const apiKey = process.env[modelConfig.key_env];
|
|||
|
|
if (!apiKey) throw new Error(`缺少API密钥环境变量 ${modelConfig.key_env}`);
|
|||
|
|
|
|||
|
|
const temp = typeof temperature === 'number' ? temperature : 0.7;
|
|||
|
|
const tokens = max_tokens || 1000;
|
|||
|
|
|
|||
|
|
// 尝试调用微调模型
|
|||
|
|
try {
|
|||
|
|
const result = await callInferenceAPI({
|
|||
|
|
host: modelConfig.provider_host,
|
|||
|
|
path: modelConfig.inference_path,
|
|||
|
|
model: modelConfig.model_endpoint
|
|||
|
|
}, apiKey, prompt, temp, tokens);
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
response: result.content,
|
|||
|
|
model_used: modelConfig.model_endpoint,
|
|||
|
|
provider: modelConfig.provider,
|
|||
|
|
tokens: result.tokens,
|
|||
|
|
fallback_used: false
|
|||
|
|
};
|
|||
|
|
} catch {
|
|||
|
|
// 微调模型不可用，降级到基座模型
|
|||
|
|
fallbackUsed = true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 降级：使用基座模型
|
|||
|
|
const baseConfig = LLM_CONFIGS[modelConfig.provider === 'deepseek' ? 'deepseek-chat' : 'qwen-max'];
|
|||
|
|
if (!baseConfig) throw new Error('降级失败：无可用基座模型');
|
|||
|
|
|
|||
|
|
const baseKey = process.env[baseConfig.keyEnv];
|
|||
|
|
if (!baseKey) throw new Error(`降级失败：缺少基座模型API密钥 ${baseConfig.keyEnv}`);
|
|||
|
|
|
|||
|
|
const result = await callInferenceAPI({
|
|||
|
|
host: baseConfig.host,
|
|||
|
|
path: baseConfig.path,
|
|||
|
|
model: baseConfig.model
|
|||
|
|
}, baseKey, prompt, temp, tokens);
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
response: result.content,
|
|||
|
|
model_used: baseConfig.model,
|
|||
|
|
provider: modelConfig.provider,
|
|||
|
|
tokens: result.tokens,
|
|||
|
|
fallback_used: fallbackUsed,
|
|||
|
|
fallback_reason: '微调模型不可用，已降级到基座模型'
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* finetuneCompareModels — A/B测试微调 vs 基座模型
|
|||
|
|
*
|
|||
|
|
* 用相同的prompt分别调用微调模型和基座模型，返回对比结果
|
|||
|
|
*
|
|||
|
|
* input:
|
|||
|
|
* persona_id: string — 人格体ID
|
|||
|
|
* model_name: string — 已注册的微调模型名称
|
|||
|
|
* test_prompt: string — 测试提示词
|
|||
|
|
* base_model: string — 基座模型名称（默认取provider对应的基座）
|
|||
|
|
*/
|
|||
|
|
async function finetuneCompareModels(input) {
|
|||
|
|
const { persona_id, model_name, test_prompt, base_model } = input;
|
|||
|
|
if (!persona_id) throw new Error('缺少 persona_id');
|
|||
|
|
if (!model_name) throw new Error('缺少 model_name');
|
|||
|
|
if (!test_prompt) throw new Error('缺少 test_prompt');
|
|||
|
|
|
|||
|
|
const bucket = DEFAULT_BUCKET;
|
|||
|
|
|
|||
|
|
// 加载微调模型配置
|
|||
|
|
let modelConfig;
|
|||
|
|
try {
|
|||
|
|
const raw = await cos.read(bucket, `finetune-models/${persona_id}/${model_name}.json`);
|
|||
|
|
modelConfig = JSON.parse(raw.content);
|
|||
|
|
} catch {
|
|||
|
|
throw new Error(`未找到模型配置: ${model_name}`);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const apiKey = process.env[modelConfig.key_env];
|
|||
|
|
if (!apiKey) throw new Error(`缺少API密钥环境变量 ${modelConfig.key_env}`);
|
|||
|
|
|
|||
|
|
// 确定基座模型
|
|||
|
|
const baseModelName = base_model || (modelConfig.provider === 'deepseek' ? 'deepseek-chat' : 'qwen-max');
|
|||
|
|
const baseConfig = LLM_CONFIGS[baseModelName];
|
|||
|
|
if (!baseConfig) throw new Error(`未找到基座模型配置: ${baseModelName}`);
|
|||
|
|
|
|||
|
|
const baseKey = process.env[baseConfig.keyEnv];
|
|||
|
|
if (!baseKey) throw new Error(`缺少基座模型API密钥 ${baseConfig.keyEnv}`);
|
|||
|
|
|
|||
|
|
// 并行调用两个模型
|
|||
|
|
const [finetunedResult, baseResult] = await Promise.allSettled([
|
|||
|
|
callInferenceAPI({
|
|||
|
|
host: modelConfig.provider_host,
|
|||
|
|
path: modelConfig.inference_path,
|
|||
|
|
model: modelConfig.model_endpoint
|
|||
|
|
}, apiKey, test_prompt, 0.7, 1000),
|
|||
|
|
callInferenceAPI({
|
|||
|
|
host: baseConfig.host,
|
|||
|
|
path: baseConfig.path,
|
|||
|
|
model: baseConfig.model
|
|||
|
|
}, baseKey, test_prompt, 0.7, 1000)
|
|||
|
|
]);
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
test_prompt,
|
|||
|
|
finetuned_response: finetunedResult.status === 'fulfilled'
|
|||
|
|
? finetunedResult.value.content
|
|||
|
|
: `调用失败: ${finetunedResult.reason?.message || '未知错误'}`,
|
|||
|
|
base_response: baseResult.status === 'fulfilled'
|
|||
|
|
? baseResult.value.content
|
|||
|
|
: `调用失败: ${baseResult.reason?.message || '未知错误'}`,
|
|||
|
|
model_a: {
|
|||
|
|
name: modelConfig.model_endpoint,
|
|||
|
|
type: 'finetuned',
|
|||
|
|
tokens: finetunedResult.status === 'fulfilled' ? finetunedResult.value.tokens : null
|
|||
|
|
},
|
|||
|
|
model_b: {
|
|||
|
|
name: baseConfig.model,
|
|||
|
|
type: 'base',
|
|||
|
|
tokens: baseResult.status === 'fulfilled' ? baseResult.value.tokens : null
|
|||
|
|
},
|
|||
|
|
compared_at: new Date().toISOString()
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* finetuneGetCostEstimate — 估算微调成本
|
|||
|
|
*
|
|||
|
|
* 读取JSONL数据集，统计token数量，按provider定价估算费用
|
|||
|
|
*
|
|||
|
|
* input:
|
|||
|
|
* persona_id: string — 人格体ID
|
|||
|
|
* dataset_key: string — COS中JSONL文件路径
|
|||
|
|
* provider: string — 微调提供方（deepseek / qwen）
|
|||
|
|
*/
|
|||
|
|
async function finetuneGetCostEstimate(input) {
|
|||
|
|
const { persona_id, dataset_key, provider } = input;
|
|||
|
|
if (!persona_id) throw new Error('缺少 persona_id');
|
|||
|
|
if (!dataset_key) throw new Error('缺少 dataset_key');
|
|||
|
|
|
|||
|
|
const bucket = DEFAULT_BUCKET;
|
|||
|
|
const providerKey = (provider || 'deepseek').toLowerCase();
|
|||
|
|
|
|||
|
|
if (!FINETUNE_PROVIDERS[providerKey]) {
|
|||
|
|
throw new Error(`不支持的微调提供方: ${providerKey}`);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 读取JSONL数据集
|
|||
|
|
const raw = await cos.read(bucket, dataset_key);
|
|||
|
|
const lines = raw.content.split('\n').filter(l => l.trim());
|
|||
|
|
|
|||
|
|
// 统计token（中文约每字1.5-2 token，英文约每词1 token，粗估用字符数/2）
|
|||
|
|
let totalChars = 0;
|
|||
|
|
let sampleCount = 0;
|
|||
|
|
|
|||
|
|
for (const line of lines) {
|
|||
|
|
try {
|
|||
|
|
const sample = JSON.parse(line);
|
|||
|
|
const messages = sample.messages || [];
|
|||
|
|
for (const msg of messages) {
|
|||
|
|
totalChars += (msg.content || '').length;
|
|||
|
|
}
|
|||
|
|
sampleCount++;
|
|||
|
|
} catch {
|
|||
|
|
// 跳过无效行
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 粗估token数（中文字符 ≈ 1.5 tokens，英文约1:1）
|
|||
|
|
const estimatedTokens = Math.ceil(totalChars * 1.5);
|
|||
|
|
const costPer1k = COST_PER_1K_TOKENS[providerKey] || 0.02;
|
|||
|
|
|
|||
|
|
// 微调通常跑 3-4 个 epoch
|
|||
|
|
const epochs = 3;
|
|||
|
|
const totalTrainTokens = estimatedTokens * epochs;
|
|||
|
|
const estimatedCostRmb = (totalTrainTokens / 1000) * costPer1k;
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
dataset_key,
|
|||
|
|
sample_count: sampleCount,
|
|||
|
|
total_chars: totalChars,
|
|||
|
|
token_count: estimatedTokens,
|
|||
|
|
training_tokens: totalTrainTokens,
|
|||
|
|
epochs,
|
|||
|
|
estimated_cost_rmb: Math.round(estimatedCostRmb * 100) / 100,
|
|||
|
|
provider: providerKey,
|
|||
|
|
cost_per_1k_tokens: costPer1k,
|
|||
|
|
notes: [
|
|||
|
|
`Token估算基于字符数粗估（中文 ×1.5），实际以provider计费为准`,
|
|||
|
|
`训练按 ${epochs} 个epoch估算`,
|
|||
|
|
`${FINETUNE_PROVIDERS[providerKey].label} 当前参考价: ¥${costPer1k}/1K tokens`,
|
|||
|
|
`实际费用可能因模型版本和优惠策略有所不同`
|
|||
|
|
]
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ═══════════════════════════════════════════════════════════
|
|||
|
|
// Provider API 交互（内部实现）
|
|||
|
|
// ═══════════════════════════════════════════════════════════
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* 上传训练文件到provider
|
|||
|
|
*/
|
|||
|
|
function uploadTrainingFile(providerConfig, apiKey, content, providerKey) {
|
|||
|
|
return new Promise((resolve, reject) => {
|
|||
|
|
// 构建 multipart/form-data
|
|||
|
|
const boundary = `----FormBoundary${crypto.randomBytes(8).toString('hex')}`;
|
|||
|
|
const fileName = `training-${Date.now()}.jsonl`;
|
|||
|
|
|
|||
|
|
let bodyParts = [];
|
|||
|
|
bodyParts.push(`--${boundary}\r\n`);
|
|||
|
|
bodyParts.push(`Content-Disposition: form-data; name="purpose"\r\n\r\n`);
|
|||
|
|
bodyParts.push(`fine-tune\r\n`);
|
|||
|
|
bodyParts.push(`--${boundary}\r\n`);
|
|||
|
|
bodyParts.push(`Content-Disposition: form-data; name="file"; filename="${fileName}"\r\n`);
|
|||
|
|
bodyParts.push(`Content-Type: application/jsonl\r\n\r\n`);
|
|||
|
|
bodyParts.push(content);
|
|||
|
|
bodyParts.push(`\r\n--${boundary}--\r\n`);
|
|||
|
|
|
|||
|
|
const body = bodyParts.join('');
|
|||
|
|
|
|||
|
|
const req = https.request({
|
|||
|
|
hostname: providerConfig.host,
|
|||
|
|
port: 443,
|
|||
|
|
path: providerConfig.uploadPath,
|
|||
|
|
method: 'POST',
|
|||
|
|
headers: {
|
|||
|
|
'Content-Type': `multipart/form-data; boundary=${boundary}`,
|
|||
|
|
'Authorization': `Bearer ${apiKey}`,
|
|||
|
|
'Content-Length': Buffer.byteLength(body)
|
|||
|
|
},
|
|||
|
|
timeout: FINETUNE_TIMEOUT
|
|||
|
|
}, (res) => {
|
|||
|
|
const chunks = [];
|
|||
|
|
res.on('data', c => chunks.push(c));
|
|||
|
|
res.on('end', () => {
|
|||
|
|
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|||
|
|
try {
|
|||
|
|
const data = JSON.parse(Buffer.concat(chunks).toString());
|
|||
|
|
// DeepSeek返回 {id: "file-xxx"}, Qwen返回类似结构
|
|||
|
|
resolve(data.id || data.file_id || data.output?.file_id || '');
|
|||
|
|
} catch {
|
|||
|
|
reject(new Error('训练文件上传响应解析失败'));
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
reject(new Error(`训练文件上传失败: HTTP ${res.statusCode}`));
|
|||
|
|
}
|
|||
|
|
});
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
req.on('error', reject);
|
|||
|
|
req.on('timeout', () => { req.destroy(); reject(new Error('训练文件上传超时')); });
|
|||
|
|
req.write(body);
|
|||
|
|
req.end();
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* 创建微调任务
|
|||
|
|
*/
|
|||
|
|
function createFinetuneJob(providerConfig, apiKey, model, fileId, hyperparams, providerKey) {
|
|||
|
|
return new Promise((resolve, reject) => {
|
|||
|
|
let requestBody;
|
|||
|
|
|
|||
|
|
if (providerKey === 'deepseek') {
|
|||
|
|
requestBody = {
|
|||
|
|
model,
|
|||
|
|
training_file: fileId,
|
|||
|
|
hyperparameters: {
|
|||
|
|
n_epochs: hyperparams?.n_epochs || 3,
|
|||
|
|
learning_rate_multiplier: hyperparams?.learning_rate_multiplier || 1.0,
|
|||
|
|
batch_size: hyperparams?.batch_size || 'auto'
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
} else {
|
|||
|
|
// Qwen/DashScope 格式
|
|||
|
|
requestBody = {
|
|||
|
|
model,
|
|||
|
|
training_file_ids: [fileId],
|
|||
|
|
hyper_parameters: {
|
|||
|
|
n_epochs: hyperparams?.n_epochs || 3,
|
|||
|
|
learning_rate: hyperparams?.learning_rate_multiplier || 1e-5,
|
|||
|
|
batch_size: hyperparams?.batch_size || 4
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const body = JSON.stringify(requestBody);
|
|||
|
|
|
|||
|
|
const req = https.request({
|
|||
|
|
hostname: providerConfig.host,
|
|||
|
|
port: 443,
|
|||
|
|
path: providerConfig.createPath,
|
|||
|
|
method: 'POST',
|
|||
|
|
headers: {
|
|||
|
|
'Content-Type': 'application/json',
|
|||
|
|
'Authorization': `Bearer ${apiKey}`,
|
|||
|
|
'Content-Length': Buffer.byteLength(body)
|
|||
|
|
},
|
|||
|
|
timeout: FINETUNE_TIMEOUT
|
|||
|
|
}, (res) => {
|
|||
|
|
const chunks = [];
|
|||
|
|
res.on('data', c => chunks.push(c));
|
|||
|
|
res.on('end', () => {
|
|||
|
|
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|||
|
|
try {
|
|||
|
|
const data = JSON.parse(Buffer.concat(chunks).toString());
|
|||
|
|
resolve({
|
|||
|
|
provider_job_id: data.id || data.output?.job_id || '',
|
|||
|
|
status: data.status || data.output?.status || 'pending',
|
|||
|
|
estimated_time: data.estimated_completion || null
|
|||
|
|
});
|
|||
|
|
} catch {
|
|||
|
|
reject(new Error('微调任务创建响应解析失败'));
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
reject(new Error(`微调任务创建失败: HTTP ${res.statusCode}`));
|
|||
|
|
}
|
|||
|
|
});
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
req.on('error', reject);
|
|||
|
|
req.on('timeout', () => { req.destroy(); reject(new Error('微调任务创建超时')); });
|
|||
|
|
req.write(body);
|
|||
|
|
req.end();
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* 查询微调任务状态
|
|||
|
|
*/
|
|||
|
|
function queryJobStatus(providerConfig, apiKey, providerJobId, providerKey) {
|
|||
|
|
return new Promise((resolve, reject) => {
|
|||
|
|
const path = `${providerConfig.statusPath}${encodeURIComponent(providerJobId)}`;
|
|||
|
|
|
|||
|
|
const req = https.request({
|
|||
|
|
hostname: providerConfig.host,
|
|||
|
|
port: 443,
|
|||
|
|
path,
|
|||
|
|
method: 'GET',
|
|||
|
|
headers: {
|
|||
|
|
'Authorization': `Bearer ${apiKey}`
|
|||
|
|
},
|
|||
|
|
timeout: 30000
|
|||
|
|
}, (res) => {
|
|||
|
|
const chunks = [];
|
|||
|
|
res.on('data', c => chunks.push(c));
|
|||
|
|
res.on('end', () => {
|
|||
|
|
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|||
|
|
try {
|
|||
|
|
const data = JSON.parse(Buffer.concat(chunks).toString());
|
|||
|
|
|
|||
|
|
// 统一不同provider的状态字段
|
|||
|
|
let status, progress, metrics, fineTunedModel;
|
|||
|
|
|
|||
|
|
if (providerKey === 'deepseek') {
|
|||
|
|
status = data.status || 'unknown';
|
|||
|
|
fineTunedModel = data.fine_tuned_model || null;
|
|||
|
|
metrics = data.result_files ? { result_files: data.result_files } : null;
|
|||
|
|
progress = data.trained_tokens
|
|||
|
|
? { trained_tokens: data.trained_tokens }
|
|||
|
|
: null;
|
|||
|
|
} else {
|
|||
|
|
// Qwen
|
|||
|
|
const output = data.output || data;
|
|||
|
|
status = output.status || data.status || 'unknown';
|
|||
|
|
fineTunedModel = output.fine_tuned_model || output.finetuned_output?.model_id || null;
|
|||
|
|
metrics = output.metrics || null;
|
|||
|
|
progress = output.training_progress || null;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 统一状态值
|
|||
|
|
status = normalizeJobStatus(status);
|
|||
|
|
|
|||
|
|
resolve({ status, progress, metrics, fine_tuned_model: fineTunedModel });
|
|||
|
|
} catch {
|
|||
|
|
reject(new Error('微调状态查询响应解析失败'));
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
reject(new Error(`微调状态查询失败: HTTP ${res.statusCode}`));
|
|||
|
|
}
|
|||
|
|
});
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
req.on('error', reject);
|
|||
|
|
req.on('timeout', () => { req.destroy(); reject(new Error('微调状态查询超时')); });
|
|||
|
|
req.end();
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* 调用推理API（微调模型或基座模型通用）
|
|||
|
|
*/
|
|||
|
|
function callInferenceAPI(config, apiKey, prompt, temperature, maxTokens) {
|
|||
|
|
return new Promise((resolve, reject) => {
|
|||
|
|
const body = JSON.stringify({
|
|||
|
|
model: config.model,
|
|||
|
|
messages: [
|
|||
|
|
{ role: 'user', content: prompt }
|
|||
|
|
],
|
|||
|
|
temperature: temperature || 0.7,
|
|||
|
|
max_tokens: maxTokens || 1000
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
const req = https.request({
|
|||
|
|
hostname: config.host,
|
|||
|
|
port: 443,
|
|||
|
|
path: config.path,
|
|||
|
|
method: 'POST',
|
|||
|
|
headers: {
|
|||
|
|
'Content-Type': 'application/json',
|
|||
|
|
'Authorization': `Bearer ${apiKey}`,
|
|||
|
|
'Content-Length': Buffer.byteLength(body)
|
|||
|
|
},
|
|||
|
|
timeout: 30000
|
|||
|
|
}, (res) => {
|
|||
|
|
const chunks = [];
|
|||
|
|
res.on('data', c => chunks.push(c));
|
|||
|
|
res.on('end', () => {
|
|||
|
|
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|||
|
|
try {
|
|||
|
|
const data = JSON.parse(Buffer.concat(chunks).toString());
|
|||
|
|
resolve({
|
|||
|
|
content: data.choices?.[0]?.message?.content || '',
|
|||
|
|
tokens: data.usage || {}
|
|||
|
|
});
|
|||
|
|
} catch {
|
|||
|
|
reject(new Error('推理响应解析失败'));
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
reject(new Error(`推理调用失败: HTTP ${res.statusCode}`));
|
|||
|
|
}
|
|||
|
|
});
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
req.on('error', reject);
|
|||
|
|
req.on('timeout', () => { req.destroy(); reject(new Error('推理请求超时')); });
|
|||
|
|
req.write(body);
|
|||
|
|
req.end();
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ═══════════════════════════════════════════════════════════
|
|||
|
|
// 辅助函数
|
|||
|
|
// ═══════════════════════════════════════════════════════════
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* 将TCS语料条目转换为微调JSONL三元组
|
|||
|
|
*/
|
|||
|
|
function convertEntryToTriple(personaId, corpusType, entry) {
|
|||
|
|
const content = typeof entry.content === 'string'
|
|||
|
|
? entry.content
|
|||
|
|
: (entry.text || entry.summary || JSON.stringify(entry));
|
|||
|
|
|
|||
|
|
if (!content || content.length < 10) return null;
|
|||
|
|
|
|||
|
|
const tags = entry.tcs_tags || entry.tags || [];
|
|||
|
|
const category = entry.category || corpusType || 'general';
|
|||
|
|
|
|||
|
|
// 生成system prompt（人格体身份）
|
|||
|
|
const systemContent = `你是${personaId}，光湖系统中的人格体。你的思维方式基于TCS语言核系统，` +
|
|||
|
|
`你需要以${personaId}的视角和风格来回答问题。`;
|
|||
|
|
|
|||
|
|
// 根据语料类型构建instruction/input/output
|
|||
|
|
let userContent, assistantContent;
|
|||
|
|
|
|||
|
|
if (entry.question && entry.answer) {
|
|||
|
|
// 已有Q&A结构
|
|||
|
|
userContent = entry.question;
|
|||
|
|
assistantContent = entry.answer;
|
|||
|
|
} else if (tags.length > 0) {
|
|||
|
|
// 有标签的条目：生成关于该内容的问答
|
|||
|
|
userContent = `关于${category}类型的内容，请解释以下要点: ${tags.slice(0, 3).join('、')}`;
|
|||
|
|
assistantContent = content;
|
|||
|
|
} else {
|
|||
|
|
// 通用条目：以理解和阐述的方式构建
|
|||
|
|
userContent = `请阐述你对以下内容的理解和看法：\n${content.substring(0, 200)}`;
|
|||
|
|
assistantContent = content;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
messages: [
|
|||
|
|
{ role: 'system', content: systemContent },
|
|||
|
|
{ role: 'user', content: userContent },
|
|||
|
|
{ role: 'assistant', content: assistantContent }
|
|||
|
|
]
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* 统一不同provider的任务状态值
|
|||
|
|
*/
|
|||
|
|
function normalizeJobStatus(rawStatus) {
|
|||
|
|
const statusMap = {
|
|||
|
|
// DeepSeek 状态
|
|||
|
|
validating_files: 'pending',
|
|||
|
|
queued: 'pending',
|
|||
|
|
running: 'running',
|
|||
|
|
succeeded: 'completed',
|
|||
|
|
failed: 'failed',
|
|||
|
|
cancelled: 'failed',
|
|||
|
|
// Qwen/DashScope 状态
|
|||
|
|
PENDING: 'pending',
|
|||
|
|
RUNNING: 'running',
|
|||
|
|
SUCCEEDED: 'completed',
|
|||
|
|
FAILED: 'failed',
|
|||
|
|
CANCELED: 'failed',
|
|||
|
|
// 通用
|
|||
|
|
pending: 'pending',
|
|||
|
|
completed: 'completed'
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
return statusMap[rawStatus] || rawStatus;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
module.exports = {
|
|||
|
|
finetuneExportDataset,
|
|||
|
|
finetuneSubmitJob,
|
|||
|
|
finetuneCheckStatus,
|
|||
|
|
finetuneRegisterModel,
|
|||
|
|
finetuneListModels,
|
|||
|
|
finetuneCallModel,
|
|||
|
|
finetuneCompareModels,
|
|||
|
|
finetuneGetCostEstimate
|
|||
|
|
};
|