184 lines
8.1 KiB
JavaScript
184 lines
8.1 KiB
JavaScript
#!/usr/bin/env node
|
|
import crypto from "node:crypto";
|
|
import fs from "node:fs";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import { spawn } from "node:child_process";
|
|
import { pipeline } from "node:stream/promises";
|
|
|
|
function parseArgs(argv) {
|
|
const args = {};
|
|
for (let index = 0; index < argv.length; index += 2) {
|
|
const key = argv[index];
|
|
if (!key?.startsWith("--") || !argv[index + 1]) throw new Error("invalid_arguments");
|
|
args[key.slice(2)] = argv[index + 1];
|
|
}
|
|
return args;
|
|
}
|
|
|
|
function atomicWrite(file, value) {
|
|
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
const temporary = `${file}.${process.pid}.tmp`;
|
|
fs.writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 });
|
|
fs.renameSync(temporary, file);
|
|
}
|
|
|
|
async function digestFile(file) {
|
|
const hash = crypto.createHash("sha256");
|
|
for await (const chunk of fs.createReadStream(file)) hash.update(chunk);
|
|
return hash.digest("hex");
|
|
}
|
|
|
|
async function countTopLevelObjects(file) {
|
|
let inString = false;
|
|
let escaped = false;
|
|
let arrayDepth = 0;
|
|
let objectDepth = 0;
|
|
let count = 0;
|
|
let sawArray = false;
|
|
for await (const chunk of fs.createReadStream(file, { encoding: "utf8" })) {
|
|
for (const character of chunk) {
|
|
if (inString) {
|
|
if (escaped) escaped = false;
|
|
else if (character === "\\") escaped = true;
|
|
else if (character === '"') inString = false;
|
|
continue;
|
|
}
|
|
if (character === '"') inString = true;
|
|
else if (character === "[") { arrayDepth += 1; sawArray = true; }
|
|
else if (character === "]") arrayDepth -= 1;
|
|
else if (character === "{") {
|
|
objectDepth += 1;
|
|
if (arrayDepth === 1 && objectDepth === 1) count += 1;
|
|
} else if (character === "}") objectDepth -= 1;
|
|
if (arrayDepth < 0 || objectDepth < 0) throw new Error("invalid_json_structure");
|
|
}
|
|
}
|
|
if (inString || arrayDepth !== 0 || objectDepth !== 0 || !sawArray) throw new Error("invalid_or_incomplete_conversations_json");
|
|
return count;
|
|
}
|
|
|
|
function collectLines(command, args, input = null) {
|
|
return new Promise((resolve, reject) => {
|
|
const child = spawn(command, args, { stdio: [input ? "pipe" : "ignore", "pipe", "pipe"] });
|
|
const lines = [];
|
|
let pending = "";
|
|
let errorText = "";
|
|
child.stdout.setEncoding("utf8");
|
|
child.stdout.on("data", (chunk) => {
|
|
pending += chunk;
|
|
const parts = pending.split("\n");
|
|
pending = parts.pop() || "";
|
|
lines.push(...parts);
|
|
});
|
|
child.stderr.setEncoding("utf8");
|
|
child.stderr.on("data", (chunk) => { errorText += chunk; });
|
|
child.on("error", reject);
|
|
child.on("close", (code) => {
|
|
if (pending) lines.push(pending);
|
|
if (code !== 0) reject(new Error(`${command}_failed:${errorText.slice(0, 160)}`));
|
|
else resolve(lines);
|
|
});
|
|
if (input) input.pipe(child.stdin);
|
|
});
|
|
}
|
|
|
|
async function innerZipEntries(outer, member) {
|
|
const temporaryRoot = fs.mkdtempSync(path.join(os.tmpdir(), "guanghu-notion-part-"));
|
|
const temporaryPart = path.join(temporaryRoot, "part.zip");
|
|
const unzip = spawn("unzip", ["-p", outer, member], { stdio: ["ignore", "pipe", "pipe"] });
|
|
let unzipError = "";
|
|
unzip.stderr.setEncoding("utf8");
|
|
unzip.stderr.on("data", (chunk) => { unzipError += chunk; });
|
|
const unzipExit = new Promise((resolve, reject) => {
|
|
unzip.on("error", reject);
|
|
unzip.on("close", resolve);
|
|
});
|
|
try {
|
|
await pipeline(unzip.stdout, fs.createWriteStream(temporaryPart, { mode: 0o600 }));
|
|
const unzipCode = await unzipExit;
|
|
if (unzipCode !== 0) throw new Error(`unzip_failed:${unzipError.slice(0, 160)}`);
|
|
return await collectLines("bsdtar", ["-tf", temporaryPart]);
|
|
} finally {
|
|
fs.rmSync(temporaryRoot, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
async function notionCounts(file) {
|
|
const outerMembers = (await collectLines("unzip", ["-Z1", file])).filter((name) => name.toLowerCase().endsWith(".zip"));
|
|
if (outerMembers.length === 0) throw new Error("notion_export_parts_missing");
|
|
const counts = { part_count: outerMembers.length, entry_count: 0, markdown_count: 0, csv_count: 0, html_count: 0, media_count: 0 };
|
|
for (const member of outerMembers) {
|
|
for (const name of await innerZipEntries(file, member)) {
|
|
counts.entry_count += 1;
|
|
const lower = name.toLowerCase();
|
|
if (lower.endsWith(".md")) counts.markdown_count += 1;
|
|
else if (lower.endsWith(".csv")) counts.csv_count += 1;
|
|
else if (/\.(html?|xhtml)$/u.test(lower)) counts.html_count += 1;
|
|
else if (/\.(png|jpe?g|gif|webp|pdf|mp4|mov|mp3|wav)$/u.test(lower)) counts.media_count += 1;
|
|
}
|
|
}
|
|
return counts;
|
|
}
|
|
|
|
function notionDirectoryCounts(directory) {
|
|
const counts = { part_count: 0, entry_count: 0, markdown_count: 0, csv_count: 0, html_count: 0, media_count: 0 };
|
|
const roots = fs.readdirSync(directory, { withFileTypes: true }).filter((entry) => entry.isDirectory() && entry.name.startsWith("Export-")).map((entry) => path.join(directory, entry.name));
|
|
if (roots.length === 0) throw new Error("notion_export_directories_missing");
|
|
counts.part_count = roots.length;
|
|
for (const root of roots) {
|
|
const pending = [root];
|
|
while (pending.length) {
|
|
const current = pending.pop();
|
|
for (const entry of fs.readdirSync(current, { withFileTypes: true })) {
|
|
const target = path.join(current, entry.name);
|
|
if (entry.isDirectory()) pending.push(target);
|
|
else if (entry.isFile()) {
|
|
counts.entry_count += 1;
|
|
const lower = entry.name.toLowerCase();
|
|
if (lower.endsWith(".md")) counts.markdown_count += 1;
|
|
else if (lower.endsWith(".csv")) counts.csv_count += 1;
|
|
else if (/\.(html?|xhtml)$/u.test(lower)) counts.html_count += 1;
|
|
else if (/\.(png|jpe?g|gif|webp|pdf|mp4|mov|mp3|wav)$/u.test(lower)) counts.media_count += 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return counts;
|
|
}
|
|
|
|
export async function buildHistorySourceManifest({ gpt, notion, notionDirectory }) {
|
|
for (const file of [gpt, notion]) {
|
|
if (!file || !fs.statSync(file).isFile()) throw new Error("history_source_file_missing");
|
|
}
|
|
if (!notionDirectory || !fs.statSync(notionDirectory).isDirectory()) throw new Error("notion_working_directory_missing");
|
|
const [gptSha, gptCount, notionSha, notionArchiveInventory] = await Promise.all([
|
|
digestFile(gpt),
|
|
countTopLevelObjects(gpt),
|
|
digestFile(notion),
|
|
notionCounts(notion),
|
|
]);
|
|
const notionInventory = notionDirectoryCounts(notionDirectory);
|
|
if (notionInventory.markdown_count !== notionArchiveInventory.markdown_count) throw new Error("notion_directory_archive_count_mismatch");
|
|
return {
|
|
schema: "guanghu.persona-private-history-source-manifest/v1",
|
|
generated_at: new Date().toISOString(),
|
|
privacy_class: "PERSONA_PRIVATE",
|
|
contains_original_language: false,
|
|
contains_page_titles: false,
|
|
contains_absolute_paths: false,
|
|
duplicate_ingestion_forbidden: true,
|
|
sources: [
|
|
{ source_id: "GPT-PRIMARY-001", source_type: "GPT_CONVERSATIONS_JSON", source_sha256: gptSha, item_count: gptCount, archive_bytes: fs.statSync(gpt).size, original_source_immutable: true },
|
|
{ source_id: "NOTION-PRIMARY-001", source_type: "NOTION_EXPORT", source_sha256: notionSha, item_count: notionInventory.markdown_count, archive_bytes: fs.statSync(notion).size, inventory: notionInventory, archive_inventory_verified: true, working_source: "EXTRACTED_DIRECTORY", original_source_immutable: true },
|
|
],
|
|
};
|
|
}
|
|
|
|
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
const args = parseArgs(process.argv.slice(2));
|
|
if (!args.gpt || !args.notion || !args["notion-directory"] || !args.output) throw new Error("required: --gpt --notion --notion-directory --output");
|
|
const manifest = await buildHistorySourceManifest({ gpt: path.resolve(args.gpt), notion: path.resolve(args.notion), notionDirectory: path.resolve(args["notion-directory"]) });
|
|
atomicWrite(path.resolve(args.output), manifest);
|
|
process.stdout.write(`${JSON.stringify({ outcome: "PASS", output: path.resolve(args.output), sources: manifest.sources.map(({ source_id, source_type, source_sha256, item_count }) => ({ source_id, source_type, source_sha256, item_count })) }, null, 2)}\n`);
|
|
}
|