feat(authz): add atomic mobile knowledge writes
This commit is contained in:
parent
a3282f7fe1
commit
5e2ba5b006
6 changed files with 527 additions and 19 deletions
|
|
@ -3,6 +3,10 @@
|
|||
const childProcess = require("node:child_process");
|
||||
const crypto = require("node:crypto");
|
||||
const fs = require("node:fs");
|
||||
const os = require("node:os");
|
||||
const path = require("node:path");
|
||||
|
||||
const MAX_KNOWLEDGE_PAGE_BYTES = 1024 * 1024;
|
||||
|
||||
class HoloLakeKnowledgeProvider {
|
||||
constructor(options = {}) {
|
||||
|
|
@ -63,14 +67,104 @@ class HoloLakeKnowledgeProvider {
|
|||
};
|
||||
}
|
||||
|
||||
git(args, maxBuffer = 1024 * 1024) {
|
||||
writePage(request = {}) {
|
||||
const pagePath = validateKnowledgePagePath(request.path);
|
||||
if (typeof request.content !== "string") {
|
||||
throw new Error("knowledge_page_content_invalid");
|
||||
}
|
||||
const content = Buffer.from(request.content, "utf8");
|
||||
if (content.length > MAX_KNOWLEDGE_PAGE_BYTES) {
|
||||
throw new Error("knowledge_page_too_large");
|
||||
}
|
||||
if (content.includes(0)) {
|
||||
throw new Error("knowledge_page_content_invalid");
|
||||
}
|
||||
const current = this.manifest().commit;
|
||||
const baseCommit = String(request.baseCommit || "");
|
||||
if (!/^[0-9a-f]{40}$/.test(baseCommit) || !safeEqual(baseCommit, current)) {
|
||||
throw new Error("knowledge_commit_conflict");
|
||||
}
|
||||
|
||||
const temporary = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), "hololake-knowledge-index-"),
|
||||
);
|
||||
const indexFile = path.join(temporary, "index");
|
||||
const gitEnv = {
|
||||
...process.env,
|
||||
GIT_INDEX_FILE: indexFile,
|
||||
GIT_AUTHOR_NAME: "HoloLake Knowledge",
|
||||
GIT_AUTHOR_EMAIL: "knowledge@guanghulake.invalid",
|
||||
GIT_COMMITTER_NAME: "HoloLake Knowledge",
|
||||
GIT_COMMITTER_EMAIL: "knowledge@guanghulake.invalid",
|
||||
};
|
||||
try {
|
||||
this.git(["read-tree", current], 1024 * 1024, { env: gitEnv });
|
||||
const blob = this.git(
|
||||
["hash-object", "-w", "--stdin"],
|
||||
MAX_KNOWLEDGE_PAGE_BYTES + 1024,
|
||||
{ env: gitEnv, input: content },
|
||||
).toString("utf8").trim();
|
||||
if (!/^[0-9a-f]{40}$/.test(blob)) {
|
||||
throw new Error("knowledge_blob_invalid");
|
||||
}
|
||||
this.git(
|
||||
["update-index", "--add", "--cacheinfo", "100644", blob, pagePath],
|
||||
1024 * 1024,
|
||||
{ env: gitEnv },
|
||||
);
|
||||
const tree = this.git(["write-tree"], 1024 * 1024, { env: gitEnv })
|
||||
.toString("utf8")
|
||||
.trim();
|
||||
if (!/^[0-9a-f]{40}$/.test(tree)) {
|
||||
throw new Error("knowledge_tree_invalid");
|
||||
}
|
||||
const nextCommit = this.git(
|
||||
[
|
||||
"commit-tree",
|
||||
tree,
|
||||
"-p",
|
||||
current,
|
||||
"-m",
|
||||
`HoloLake knowledge: ${pagePath}`,
|
||||
],
|
||||
1024 * 1024,
|
||||
{ env: gitEnv },
|
||||
).toString("utf8").trim();
|
||||
if (!/^[0-9a-f]{40}$/.test(nextCommit)) {
|
||||
throw new Error("knowledge_commit_invalid");
|
||||
}
|
||||
try {
|
||||
this.git(
|
||||
["update-ref", this.ref, nextCommit, current],
|
||||
1024 * 1024,
|
||||
{ env: gitEnv },
|
||||
);
|
||||
} catch {
|
||||
throw new Error("knowledge_commit_conflict");
|
||||
}
|
||||
return {
|
||||
schema: "guanghu.hololake-knowledge-write-receipt/v1",
|
||||
repository: this.repositoryId,
|
||||
path: pagePath,
|
||||
base_commit: current,
|
||||
commit: nextCommit,
|
||||
content_sha256: crypto.createHash("sha256").update(content).digest("hex"),
|
||||
};
|
||||
} finally {
|
||||
fs.rmSync(temporary, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
git(args, maxBuffer = 1024 * 1024, options = {}) {
|
||||
return childProcess.execFileSync(
|
||||
"git",
|
||||
[`--git-dir=${this.repositoryPath}`, ...args],
|
||||
{
|
||||
encoding: "buffer",
|
||||
maxBuffer,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
stdio: [options.input === undefined ? "ignore" : "pipe", "pipe", "pipe"],
|
||||
env: options.env || process.env,
|
||||
input: options.input,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
|
@ -119,6 +213,7 @@ class HoloLakeAiGateway {
|
|||
this.maxMessages,
|
||||
this.maxInputCharacters,
|
||||
);
|
||||
const tools = validateTools(request.tools);
|
||||
const maxTokens = Number(request.max_tokens || 1024);
|
||||
if (
|
||||
!Number.isInteger(maxTokens)
|
||||
|
|
@ -128,18 +223,23 @@ class HoloLakeAiGateway {
|
|||
throw new Error("ai_output_limit_exceeded");
|
||||
}
|
||||
|
||||
const upstreamBody = {
|
||||
model,
|
||||
messages,
|
||||
max_tokens: maxTokens,
|
||||
stream: false,
|
||||
};
|
||||
if (tools.length > 0) {
|
||||
upstreamBody.tools = tools;
|
||||
upstreamBody.tool_choice = request.tool_choice === "none" ? "none" : "auto";
|
||||
}
|
||||
const response = await this.fetchImpl(`${provider.baseUrl}/chat/completions`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
authorization: `Bearer ${provider.apiKey}`,
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
messages,
|
||||
max_tokens: maxTokens,
|
||||
stream: false,
|
||||
}),
|
||||
body: JSON.stringify(upstreamBody),
|
||||
signal: AbortSignal.timeout(provider.timeoutMs),
|
||||
});
|
||||
const text = await response.text();
|
||||
|
|
@ -215,26 +315,144 @@ function validateMessages(messages, maxMessages, maxInputCharacters) {
|
|||
if (messages.length > maxMessages) throw new Error("ai_input_too_large");
|
||||
let characters = 0;
|
||||
const normalized = messages.map(message => {
|
||||
if (
|
||||
!message
|
||||
|| !["system", "user", "assistant"].includes(message.role)
|
||||
|| typeof message.content !== "string"
|
||||
) {
|
||||
if (!message || !["system", "user", "assistant", "tool"].includes(message.role)) {
|
||||
throw new Error("ai_message_invalid");
|
||||
}
|
||||
characters += message.content.length;
|
||||
return { role: message.role, content: message.content };
|
||||
if (message.role === "tool") {
|
||||
if (
|
||||
typeof message.content !== "string"
|
||||
|| !/^[A-Za-z0-9._:-]{1,160}$/.test(String(message.tool_call_id || ""))
|
||||
) {
|
||||
throw new Error("ai_message_invalid");
|
||||
}
|
||||
characters += message.content.length;
|
||||
return {
|
||||
role: "tool",
|
||||
tool_call_id: message.tool_call_id,
|
||||
content: message.content,
|
||||
};
|
||||
}
|
||||
const content = message.content === null && message.role === "assistant"
|
||||
? null
|
||||
: message.content;
|
||||
if (typeof content !== "string" && content !== null) {
|
||||
throw new Error("ai_message_invalid");
|
||||
}
|
||||
characters += typeof content === "string" ? content.length : 0;
|
||||
const normalizedMessage = { role: message.role, content };
|
||||
if (message.role === "assistant" && message.tool_calls !== undefined) {
|
||||
normalizedMessage.tool_calls = validateToolCalls(message.tool_calls);
|
||||
characters += normalizedMessage.tool_calls.reduce(
|
||||
(total, call) => total + call.function.arguments.length,
|
||||
0,
|
||||
);
|
||||
}
|
||||
return normalizedMessage;
|
||||
});
|
||||
if (characters > maxInputCharacters) throw new Error("ai_input_too_large");
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function validateToolCalls(value) {
|
||||
if (!Array.isArray(value) || value.length < 1 || value.length > 32) {
|
||||
throw new Error("ai_message_invalid");
|
||||
}
|
||||
return value.map(call => {
|
||||
if (
|
||||
!call
|
||||
|| call.type !== "function"
|
||||
|| !/^[A-Za-z0-9._:-]{1,160}$/.test(String(call.id || ""))
|
||||
|| !call.function
|
||||
|| !/^[A-Za-z0-9_-]{1,80}$/.test(String(call.function.name || ""))
|
||||
|| typeof call.function.arguments !== "string"
|
||||
|| call.function.arguments.length > 32 * 1024
|
||||
) {
|
||||
throw new Error("ai_message_invalid");
|
||||
}
|
||||
return {
|
||||
id: call.id,
|
||||
type: "function",
|
||||
function: {
|
||||
name: call.function.name,
|
||||
arguments: call.function.arguments,
|
||||
},
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function validateTools(value) {
|
||||
if (value === undefined || value === null) return [];
|
||||
if (!Array.isArray(value) || value.length > 32) {
|
||||
throw new Error("ai_tools_invalid");
|
||||
}
|
||||
const encoded = JSON.stringify(value);
|
||||
if (Buffer.byteLength(encoded) > 128 * 1024) {
|
||||
throw new Error("ai_tools_invalid");
|
||||
}
|
||||
return value.map(tool => {
|
||||
if (
|
||||
!tool
|
||||
|| tool.type !== "function"
|
||||
|| !tool.function
|
||||
|| !/^[A-Za-z0-9_-]{1,80}$/.test(String(tool.function.name || ""))
|
||||
|| typeof tool.function.description !== "string"
|
||||
|| tool.function.description.length > 4_000
|
||||
|| !tool.function.parameters
|
||||
|| typeof tool.function.parameters !== "object"
|
||||
|| Array.isArray(tool.function.parameters)
|
||||
) {
|
||||
throw new Error("ai_tools_invalid");
|
||||
}
|
||||
return {
|
||||
type: "function",
|
||||
function: {
|
||||
name: tool.function.name,
|
||||
description: tool.function.description,
|
||||
parameters: JSON.parse(JSON.stringify(tool.function.parameters)),
|
||||
},
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function safeEqual(left, right) {
|
||||
const a = Buffer.from(String(left));
|
||||
const b = Buffer.from(String(right));
|
||||
return a.length === b.length && crypto.timingSafeEqual(a, b);
|
||||
}
|
||||
|
||||
function validateKnowledgePagePath(value) {
|
||||
const pagePath = String(value || "").normalize("NFC");
|
||||
if (
|
||||
pagePath.length < 1
|
||||
|| pagePath.length > 512
|
||||
|| pagePath.startsWith("/")
|
||||
|| pagePath.endsWith("/")
|
||||
|| pagePath.includes("\\")
|
||||
|| /[\0-\x1f\x7f]/u.test(pagePath)
|
||||
|| !pagePath.endsWith(".md")
|
||||
) {
|
||||
throw new Error("knowledge_page_path_invalid");
|
||||
}
|
||||
const components = pagePath.split("/");
|
||||
if (
|
||||
components.length < 1
|
||||
|| components.length > 16
|
||||
|| components.some(component => (
|
||||
!component
|
||||
|| component === "."
|
||||
|| component === ".."
|
||||
|| component.length > 120
|
||||
|| component.toLocaleLowerCase("en-US") === ".git"
|
||||
))
|
||||
) {
|
||||
throw new Error("knowledge_page_path_invalid");
|
||||
}
|
||||
if (components.some(component => component.normalize("NFKC") === "本地密钥")) {
|
||||
throw new Error("knowledge_secret_page_forbidden");
|
||||
}
|
||||
return pagePath;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
HoloLakeAiGateway,
|
||||
HoloLakeKnowledgeProvider,
|
||||
|
|
|
|||
Loading…
Reference in a new issue