313 lines
9.7 KiB
JavaScript
313 lines
9.7 KiB
JavaScript
|
/*
|
||
|
|
* 推理客户端
|
||
|
|
* - 读 inference-endpoint.json (PR-3 的 refresh-autodl-endpoint.yml 写入)
|
||
|
|
* - 提供 health() / fetch() / pipeChat()
|
||
|
|
*
|
||
|
|
* cc-002: 入口前再 strip 一次 system role (虽然前端不发, 但作为后端不补这层防线).
|
||
|
|
* cc-003: 不假设 host:port 是固定的, 文件变了下次请求自动用新值.
|
||
|
|
* cc-004: 错误信息走中文.
|
||
|
|
*/
|
||
|
|
"use strict";
|
||
|
|
|
||
|
|
const fs = require("fs");
|
||
|
|
const http = require("http");
|
||
|
|
const https = require("https");
|
||
|
|
const { URL } = require("url");
|
||
|
|
|
||
|
|
let endpointPath = null;
|
||
|
|
let activeModelPath = null;
|
||
|
|
let lastEndpoint = null;
|
||
|
|
let lastEndpointMtime = 0;
|
||
|
|
let activeModel = "mother";
|
||
|
|
|
||
|
|
function init({ endpointPath: epp, activeModelPath: amp }) {
|
||
|
|
endpointPath = epp;
|
||
|
|
activeModelPath = amp;
|
||
|
|
// 加载已保存的 active model
|
||
|
|
try {
|
||
|
|
if (fs.existsSync(activeModelPath)) {
|
||
|
|
const j = JSON.parse(fs.readFileSync(activeModelPath, "utf8"));
|
||
|
|
if (j && (j.name === "mother" || j.name === "coder")) activeModel = j.name;
|
||
|
|
}
|
||
|
|
} catch (_) {
|
||
|
|
/* ignore */
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
function readEndpoint() {
|
||
|
|
if (!endpointPath || !fs.existsSync(endpointPath)) {
|
||
|
|
lastEndpoint = null;
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
try {
|
||
|
|
const stat = fs.statSync(endpointPath);
|
||
|
|
if (stat.mtimeMs !== lastEndpointMtime) {
|
||
|
|
const j = JSON.parse(fs.readFileSync(endpointPath, "utf8"));
|
||
|
|
lastEndpoint = j;
|
||
|
|
lastEndpointMtime = stat.mtimeMs;
|
||
|
|
}
|
||
|
|
return lastEndpoint;
|
||
|
|
} catch (e) {
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
function getActiveModel() {
|
||
|
|
return activeModel;
|
||
|
|
}
|
||
|
|
|
||
|
|
function setActiveModel(name) {
|
||
|
|
if (name !== "mother" && name !== "coder") {
|
||
|
|
throw Object.assign(new Error("模型名称只允许 mother 或 coder"), { status: 400 });
|
||
|
|
}
|
||
|
|
activeModel = name;
|
||
|
|
try {
|
||
|
|
fs.writeFileSync(
|
||
|
|
activeModelPath,
|
||
|
|
JSON.stringify({ name, updated_at: new Date().toISOString() }, null, 2)
|
||
|
|
);
|
||
|
|
} catch (_) {
|
||
|
|
/* ignore — 即使写不下去也不挡前端 */
|
||
|
|
}
|
||
|
|
return name;
|
||
|
|
}
|
||
|
|
|
||
|
|
function buildUrl(ep, urlPath) {
|
||
|
|
if (!ep || !ep.host) return null;
|
||
|
|
const scheme = ep.scheme === "http" ? "http" : "https";
|
||
|
|
const port = ep.port ? `:${ep.port}` : "";
|
||
|
|
const base = `${scheme}://${ep.host}${port}`;
|
||
|
|
return new URL(urlPath, base);
|
||
|
|
}
|
||
|
|
|
||
|
|
function pickClient(u) {
|
||
|
|
return u.protocol === "https:" ? https : http;
|
||
|
|
}
|
||
|
|
|
||
|
|
function buildAuthHeaders(ep) {
|
||
|
|
const h = {};
|
||
|
|
if (ep && ep.bearer) h["Authorization"] = "Bearer " + ep.bearer;
|
||
|
|
return h;
|
||
|
|
}
|
||
|
|
|
||
|
|
function _strip_system_messages(messages) {
|
||
|
|
if (!Array.isArray(messages)) return [];
|
||
|
|
return messages.filter((m) => m && m.role && m.role !== "system");
|
||
|
|
}
|
||
|
|
|
||
|
|
async function health() {
|
||
|
|
const ep = readEndpoint();
|
||
|
|
if (!ep) {
|
||
|
|
return { ready: false, message: "推理端点未配置 · 等 refresh-autodl-endpoint.yml 写 inference-endpoint.json" };
|
||
|
|
}
|
||
|
|
const url = buildUrl(ep, "/v1/health");
|
||
|
|
if (!url) return { ready: false, message: "推理端点配置无效" };
|
||
|
|
return new Promise((resolve) => {
|
||
|
|
const req = pickClient(url).request(
|
||
|
|
url,
|
||
|
|
{ method: "GET", timeout: 5000, headers: buildAuthHeaders(ep) },
|
||
|
|
(res) => {
|
||
|
|
let body = "";
|
||
|
|
res.on("data", (c) => (body += c));
|
||
|
|
res.on("end", () => {
|
||
|
|
if (res.statusCode !== 200) {
|
||
|
|
return resolve({ ready: false, message: `推理端返回 ${res.statusCode}` });
|
||
|
|
}
|
||
|
|
try {
|
||
|
|
const j = JSON.parse(body);
|
||
|
|
resolve({
|
||
|
|
ready: !!j.ready,
|
||
|
|
model: j.model || activeModel,
|
||
|
|
gpu: j.gpu || null,
|
||
|
|
endpoint: { host: ep.host, port: ep.port }
|
||
|
|
});
|
||
|
|
} catch (_) {
|
||
|
|
resolve({ ready: false, message: "推理端响应不是合法 JSON" });
|
||
|
|
}
|
||
|
|
});
|
||
|
|
}
|
||
|
|
);
|
||
|
|
req.on("error", (e) => resolve({ ready: false, message: "推理端连接失败: " + e.message }));
|
||
|
|
req.on("timeout", () => {
|
||
|
|
req.destroy();
|
||
|
|
resolve({ ready: false, message: "推理端连接超时" });
|
||
|
|
});
|
||
|
|
req.end();
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
async function switchUpstreamModel(name) {
|
||
|
|
const ep = readEndpoint();
|
||
|
|
if (!ep) throw Object.assign(new Error("推理端点未配置, 无法切换"), { status: 503 });
|
||
|
|
const url = buildUrl(ep, "/v1/switch-model");
|
||
|
|
return new Promise((resolve, reject) => {
|
||
|
|
const body = JSON.stringify({ name });
|
||
|
|
const req = pickClient(url).request(
|
||
|
|
url,
|
||
|
|
{
|
||
|
|
method: "POST",
|
||
|
|
timeout: 30000,
|
||
|
|
headers: {
|
||
|
|
"Content-Type": "application/json",
|
||
|
|
"Content-Length": Buffer.byteLength(body),
|
||
|
|
...buildAuthHeaders(ep)
|
||
|
|
}
|
||
|
|
},
|
||
|
|
(res) => {
|
||
|
|
let buf = "";
|
||
|
|
res.on("data", (c) => (buf += c));
|
||
|
|
res.on("end", () => {
|
||
|
|
if (res.statusCode >= 200 && res.statusCode < 300) {
|
||
|
|
try { resolve(JSON.parse(buf)); } catch (_) { resolve({ ok: true }); }
|
||
|
|
} else {
|
||
|
|
reject(Object.assign(new Error(`推理端切模型失败 (${res.statusCode})`), { status: 502 }));
|
||
|
|
}
|
||
|
|
});
|
||
|
|
}
|
||
|
|
);
|
||
|
|
req.on("error", (e) => reject(Object.assign(new Error("切模型连接失败: " + e.message), { status: 502 })));
|
||
|
|
req.on("timeout", () => {
|
||
|
|
req.destroy();
|
||
|
|
reject(Object.assign(new Error("切模型超时"), { status: 504 }));
|
||
|
|
});
|
||
|
|
req.write(body);
|
||
|
|
req.end();
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* pipeChat — 字节级 SSE 转发.
|
||
|
|
* 入口前 strip system, 之后服务端只做 auth+rate-limit, 不重组 SSE.
|
||
|
|
*
|
||
|
|
* 注意: 这里是 byte-pipe — 上游写什么字节, 浏览器就收什么字节
|
||
|
|
* (cc-002 的"不重组"原则).
|
||
|
|
*
|
||
|
|
* @param {object} payload OpenAI 兼容 chat.completions 请求体
|
||
|
|
* @param {object} res Express response (将被 pipe)
|
||
|
|
* @returns {object} { fullText } 在流结束后, 用于持久化 assistant 消息
|
||
|
|
*/
|
||
|
|
function pipeChat(payload, res) {
|
||
|
|
return new Promise((resolve, reject) => {
|
||
|
|
const ep = readEndpoint();
|
||
|
|
if (!ep) {
|
||
|
|
const err = Object.assign(new Error("推理端点未配置"), { status: 503 });
|
||
|
|
return reject(err);
|
||
|
|
}
|
||
|
|
const url = buildUrl(ep, "/v1/chat/completions");
|
||
|
|
if (!url) return reject(Object.assign(new Error("推理端点配置无效"), { status: 503 }));
|
||
|
|
|
||
|
|
// cc-002 后端最后一道防线: 剥 system
|
||
|
|
const safe = {
|
||
|
|
...payload,
|
||
|
|
stream: true,
|
||
|
|
model: payload.model || activeModel,
|
||
|
|
messages: _strip_system_messages(payload.messages)
|
||
|
|
};
|
||
|
|
const body = JSON.stringify(safe);
|
||
|
|
|
||
|
|
// 写 SSE 头
|
||
|
|
res.status(200);
|
||
|
|
res.setHeader("Content-Type", "text/event-stream; charset=utf-8");
|
||
|
|
res.setHeader("Cache-Control", "no-cache, no-transform");
|
||
|
|
res.setHeader("Connection", "keep-alive");
|
||
|
|
res.setHeader("X-Accel-Buffering", "no");
|
||
|
|
if (typeof res.flushHeaders === "function") res.flushHeaders();
|
||
|
|
|
||
|
|
let fullText = "";
|
||
|
|
|
||
|
|
const upstream = pickClient(url).request(
|
||
|
|
url,
|
||
|
|
{
|
||
|
|
method: "POST",
|
||
|
|
timeout: 120000,
|
||
|
|
headers: {
|
||
|
|
"Content-Type": "application/json",
|
||
|
|
Accept: "text/event-stream",
|
||
|
|
"Content-Length": Buffer.byteLength(body),
|
||
|
|
...buildAuthHeaders(ep)
|
||
|
|
}
|
||
|
|
},
|
||
|
|
(upRes) => {
|
||
|
|
if (upRes.statusCode !== 200) {
|
||
|
|
let buf = "";
|
||
|
|
upRes.on("data", (c) => (buf += c));
|
||
|
|
upRes.on("end", () => {
|
||
|
|
// 没开始 stream 之前, 推理端返回非 200 — 报中文错给浏览器
|
||
|
|
const msg = `推理端拒绝请求 (${upRes.statusCode}): ${buf.slice(0, 200)}`;
|
||
|
|
try {
|
||
|
|
res.write(`data: ${JSON.stringify({ error: true, message: msg })}\n\n`);
|
||
|
|
res.write("data: [DONE]\n\n");
|
||
|
|
} finally {
|
||
|
|
res.end();
|
||
|
|
reject(Object.assign(new Error(msg), { status: 502 }));
|
||
|
|
}
|
||
|
|
});
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
// byte-pipe: 上游写什么, 我们写什么. 同时旁路解析以便落库.
|
||
|
|
upRes.on("data", (chunk) => {
|
||
|
|
res.write(chunk);
|
||
|
|
// 旁路解析 OpenAI 兼容 SSE — 失败也不影响 pipe
|
||
|
|
try {
|
||
|
|
const text = chunk.toString("utf8");
|
||
|
|
for (const block of text.split("\n\n")) {
|
||
|
|
for (const line of block.split("\n")) {
|
||
|
|
const t = line.trim();
|
||
|
|
if (!t.startsWith("data:")) continue;
|
||
|
|
const data = t.slice(5).trim();
|
||
|
|
if (!data || data === "[DONE]") continue;
|
||
|
|
const j = JSON.parse(data);
|
||
|
|
const piece = j && j.choices && j.choices[0] && j.choices[0].delta && j.choices[0].delta.content;
|
||
|
|
if (piece) fullText += piece;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
} catch (_) {
|
||
|
|
/* ignore parse error — pipe 不能因为解析失败断 */
|
||
|
|
}
|
||
|
|
});
|
||
|
|
upRes.on("end", () => {
|
||
|
|
res.end();
|
||
|
|
resolve({ fullText });
|
||
|
|
});
|
||
|
|
upRes.on("error", (e) => {
|
||
|
|
try { res.end(); } catch (_) {}
|
||
|
|
reject(Object.assign(new Error("推理端流断: " + e.message), { status: 502 }));
|
||
|
|
});
|
||
|
|
}
|
||
|
|
);
|
||
|
|
|
||
|
|
upstream.on("error", (e) => {
|
||
|
|
// 还没开始写 SSE body 时直接 reject; 已经写了就关流 (检查 writableEnded 防双关)
|
||
|
|
if (!res.headersSent) {
|
||
|
|
return reject(Object.assign(new Error("推理端连接失败: " + e.message), { status: 502 }));
|
||
|
|
}
|
||
|
|
if (!res.writableEnded) {
|
||
|
|
try {
|
||
|
|
res.write(`data: ${JSON.stringify({ error: true, message: "推理端连接断: " + e.message })}\n\n`);
|
||
|
|
res.write("data: [DONE]\n\n");
|
||
|
|
res.end();
|
||
|
|
} catch (_) {
|
||
|
|
/* socket 已断, 写不进去就算了 */
|
||
|
|
}
|
||
|
|
}
|
||
|
|
reject(Object.assign(new Error("推理端连接失败: " + e.message), { status: 502 }));
|
||
|
|
});
|
||
|
|
upstream.on("timeout", () => {
|
||
|
|
upstream.destroy();
|
||
|
|
});
|
||
|
|
|
||
|
|
upstream.write(body);
|
||
|
|
upstream.end();
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
module.exports = {
|
||
|
|
init,
|
||
|
|
health,
|
||
|
|
pipeChat,
|
||
|
|
switchUpstreamModel,
|
||
|
|
getActiveModel,
|
||
|
|
setActiveModel,
|
||
|
|
_strip_system_messages
|
||
|
|
};
|