346 lines
11 KiB
JavaScript
346 lines
11 KiB
JavaScript
|
#!/usr/bin/env node
|
||
|
|
|
||
|
|
import fs from "node:fs";
|
||
|
|
import path from "node:path";
|
||
|
|
import crypto from "node:crypto";
|
||
|
|
import { DatabaseSync } from "node:sqlite";
|
||
|
|
|
||
|
|
function arg(name, fallback) {
|
||
|
|
const index = process.argv.indexOf(name);
|
||
|
|
return index >= 0 && process.argv[index + 1] ? process.argv[index + 1] : fallback;
|
||
|
|
}
|
||
|
|
|
||
|
|
const configPath = path.resolve(arg("--config", "../index-config.json"));
|
||
|
|
const dbPath = path.resolve(arg("--db", "../index/notion-world-index.sqlite"));
|
||
|
|
const reportPath = path.resolve(arg("--report", "../index/corpus-inventory.json"));
|
||
|
|
const config = JSON.parse(fs.readFileSync(configPath, "utf8"));
|
||
|
|
const sourceRoot = path.resolve(config.source_roots[0]);
|
||
|
|
const previewBytes = Number(config.preview_bytes ?? 4096);
|
||
|
|
const storedPreviewBytes = 512;
|
||
|
|
const threshold = Number(config.candidate_score_threshold ?? 6);
|
||
|
|
const tempDbPath = `${dbPath}.building`;
|
||
|
|
|
||
|
|
fs.mkdirSync(path.dirname(dbPath), { recursive: true });
|
||
|
|
fs.mkdirSync(path.dirname(reportPath), { recursive: true });
|
||
|
|
if (fs.existsSync(tempDbPath)) fs.rmSync(tempDbPath);
|
||
|
|
|
||
|
|
function normalizePageId(hex) {
|
||
|
|
const value = hex.toLowerCase();
|
||
|
|
return `${value.slice(0, 8)}-${value.slice(8, 12)}-${value.slice(12, 16)}-${value.slice(16, 20)}-${value.slice(20)}`;
|
||
|
|
}
|
||
|
|
|
||
|
|
function pageIdentity(filename) {
|
||
|
|
const basename = filename.replace(/\.md$/i, "");
|
||
|
|
const match = basename.match(/(?:^| )([0-9a-f]{32})$/i);
|
||
|
|
if (!match) return { title: basename, pageId: null };
|
||
|
|
return {
|
||
|
|
title: basename.slice(0, match.index).trim(),
|
||
|
|
pageId: normalizePageId(match[1])
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
function logicalPath(relativePath, title) {
|
||
|
|
return path.posix.join(path.posix.dirname(relativePath), title);
|
||
|
|
}
|
||
|
|
|
||
|
|
function readPreview(file) {
|
||
|
|
const fd = fs.openSync(file, "r");
|
||
|
|
try {
|
||
|
|
const buffer = Buffer.allocUnsafe(previewBytes);
|
||
|
|
const length = fs.readSync(fd, buffer, 0, previewBytes, 0);
|
||
|
|
return buffer.subarray(0, length).toString("utf8").replaceAll("\u0000", "");
|
||
|
|
} finally {
|
||
|
|
fs.closeSync(fd);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
function classify(relativePath, title, preview) {
|
||
|
|
const haystack = `${relativePath}\n${title}\n${preview}`.toLowerCase();
|
||
|
|
const titlePath = `${relativePath}\n${title}`.toLowerCase();
|
||
|
|
let score = 0;
|
||
|
|
const positive = [];
|
||
|
|
const negative = [];
|
||
|
|
|
||
|
|
for (const term of config.positive_terms) {
|
||
|
|
const normalized = term.toLowerCase();
|
||
|
|
if (titlePath.includes(normalized)) {
|
||
|
|
score += 4;
|
||
|
|
positive.push(term);
|
||
|
|
} else if (haystack.includes(normalized)) {
|
||
|
|
score += 1;
|
||
|
|
positive.push(term);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
for (const term of config.negative_terms) {
|
||
|
|
const normalized = term.toLowerCase();
|
||
|
|
if (titlePath.includes(normalized)) {
|
||
|
|
score -= 5;
|
||
|
|
negative.push(term);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
const alwaysInclude = config.always_include_path_fragments.some(fragment =>
|
||
|
|
relativePath.includes(fragment)
|
||
|
|
);
|
||
|
|
if (alwaysInclude) score += 100;
|
||
|
|
|
||
|
|
const domainHints = [];
|
||
|
|
const hints = [
|
||
|
|
["MAIN", ["光湖主域", "domain-main", "dom-main"]],
|
||
|
|
["SUB", ["光湖分域", "domain-sub", "dom-sub"]],
|
||
|
|
["ZERO", ["光湖零域", "domain-zero", "dom-zero"]],
|
||
|
|
["ZEROSENSE", ["光湖零感域", "零感域", "zero-sense", "zerosense"]],
|
||
|
|
["FIFTH", ["第五域", "fifth domain", "sys-5th", "domain-fifth"]]
|
||
|
|
];
|
||
|
|
for (const [domain, terms] of hints) {
|
||
|
|
if (terms.some(term => haystack.includes(term))) domainHints.push(domain);
|
||
|
|
}
|
||
|
|
|
||
|
|
return {
|
||
|
|
score,
|
||
|
|
candidate: score >= threshold,
|
||
|
|
alwaysInclude,
|
||
|
|
positive,
|
||
|
|
negative,
|
||
|
|
domainHints
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
function* walk(root) {
|
||
|
|
const stack = [root];
|
||
|
|
while (stack.length) {
|
||
|
|
const directory = stack.pop();
|
||
|
|
const entries = fs.readdirSync(directory, { withFileTypes: true });
|
||
|
|
for (const entry of entries) {
|
||
|
|
if (entry.name === ".DS_Store") continue;
|
||
|
|
const fullPath = path.join(directory, entry.name);
|
||
|
|
if (entry.isDirectory()) stack.push(fullPath);
|
||
|
|
else if (entry.isFile() && entry.name.toLowerCase().endsWith(".md")) yield fullPath;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
const db = new DatabaseSync(tempDbPath);
|
||
|
|
db.exec(`
|
||
|
|
PRAGMA journal_mode = DELETE;
|
||
|
|
PRAGMA synchronous = NORMAL;
|
||
|
|
PRAGMA temp_store = MEMORY;
|
||
|
|
CREATE TABLE pages (
|
||
|
|
file_path TEXT PRIMARY KEY,
|
||
|
|
relative_path TEXT NOT NULL,
|
||
|
|
export_batch TEXT NOT NULL,
|
||
|
|
title TEXT NOT NULL,
|
||
|
|
page_id TEXT,
|
||
|
|
logical_path TEXT NOT NULL,
|
||
|
|
parent_logical_path TEXT,
|
||
|
|
parent_file_path TEXT,
|
||
|
|
size_bytes INTEGER NOT NULL,
|
||
|
|
mtime_ms INTEGER NOT NULL,
|
||
|
|
first_heading TEXT,
|
||
|
|
preview TEXT,
|
||
|
|
relevance_score INTEGER NOT NULL,
|
||
|
|
is_candidate INTEGER NOT NULL,
|
||
|
|
always_include INTEGER NOT NULL,
|
||
|
|
positive_terms TEXT NOT NULL,
|
||
|
|
negative_terms TEXT NOT NULL,
|
||
|
|
domain_hints TEXT NOT NULL
|
||
|
|
);
|
||
|
|
CREATE INDEX pages_page_id_idx ON pages(page_id);
|
||
|
|
CREATE INDEX pages_logical_path_idx ON pages(logical_path);
|
||
|
|
CREATE INDEX pages_candidate_idx ON pages(is_candidate, relevance_score DESC);
|
||
|
|
CREATE INDEX pages_title_idx ON pages(title);
|
||
|
|
CREATE TABLE links (
|
||
|
|
source_file_path TEXT NOT NULL,
|
||
|
|
target_raw TEXT NOT NULL,
|
||
|
|
target_file_path TEXT,
|
||
|
|
target_page_id TEXT,
|
||
|
|
link_kind TEXT NOT NULL,
|
||
|
|
resolved INTEGER NOT NULL,
|
||
|
|
PRIMARY KEY(source_file_path, target_raw)
|
||
|
|
);
|
||
|
|
CREATE INDEX links_source_idx ON links(source_file_path);
|
||
|
|
CREATE INDEX links_target_path_idx ON links(target_file_path);
|
||
|
|
CREATE INDEX links_target_page_idx ON links(target_page_id);
|
||
|
|
CREATE TABLE identifiers (
|
||
|
|
source_file_path TEXT NOT NULL,
|
||
|
|
identifier TEXT NOT NULL,
|
||
|
|
PRIMARY KEY(source_file_path, identifier)
|
||
|
|
);
|
||
|
|
CREATE INDEX identifiers_value_idx ON identifiers(identifier);
|
||
|
|
CREATE VIRTUAL TABLE page_search USING fts5(
|
||
|
|
file_path UNINDEXED,
|
||
|
|
title,
|
||
|
|
relative_path,
|
||
|
|
tokenize='unicode61'
|
||
|
|
);
|
||
|
|
`);
|
||
|
|
|
||
|
|
const insertPage = db.prepare(`
|
||
|
|
INSERT INTO pages (
|
||
|
|
file_path, relative_path, export_batch, title, page_id, logical_path,
|
||
|
|
parent_logical_path, size_bytes, mtime_ms, first_heading, preview,
|
||
|
|
relevance_score, is_candidate, always_include, positive_terms,
|
||
|
|
negative_terms, domain_hints
|
||
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||
|
|
`);
|
||
|
|
const insertSearch = db.prepare(
|
||
|
|
"INSERT INTO page_search(file_path, title, relative_path) VALUES (?, ?, ?)"
|
||
|
|
);
|
||
|
|
|
||
|
|
const pageFiles = [...walk(sourceRoot)];
|
||
|
|
console.log(`DISCOVERED ${pageFiles.length} markdown files`);
|
||
|
|
|
||
|
|
db.exec("BEGIN");
|
||
|
|
let indexed = 0;
|
||
|
|
for (const file of pageFiles) {
|
||
|
|
const relativePath = path.relative(sourceRoot, file).split(path.sep).join("/");
|
||
|
|
const exportBatch = relativePath.split("/")[0];
|
||
|
|
const { title, pageId } = pageIdentity(path.basename(file));
|
||
|
|
const preview = readPreview(file);
|
||
|
|
const headingMatch = preview.match(/^#\s+(.+)$/m);
|
||
|
|
const classification = classify(relativePath, title, preview);
|
||
|
|
const stat = fs.statSync(file);
|
||
|
|
const logical = logicalPath(relativePath, title);
|
||
|
|
const parentLogical = path.posix.dirname(relativePath) === "."
|
||
|
|
? null
|
||
|
|
: path.posix.dirname(relativePath);
|
||
|
|
|
||
|
|
insertPage.run(
|
||
|
|
file,
|
||
|
|
relativePath,
|
||
|
|
exportBatch,
|
||
|
|
title,
|
||
|
|
pageId,
|
||
|
|
logical,
|
||
|
|
parentLogical,
|
||
|
|
stat.size,
|
||
|
|
stat.mtimeMs,
|
||
|
|
headingMatch?.[1]?.trim() ?? null,
|
||
|
|
preview.slice(0, storedPreviewBytes),
|
||
|
|
classification.score,
|
||
|
|
classification.candidate ? 1 : 0,
|
||
|
|
classification.alwaysInclude ? 1 : 0,
|
||
|
|
JSON.stringify(classification.positive),
|
||
|
|
JSON.stringify(classification.negative),
|
||
|
|
JSON.stringify(classification.domainHints)
|
||
|
|
);
|
||
|
|
insertSearch.run(file, title, relativePath);
|
||
|
|
indexed++;
|
||
|
|
if (indexed % 10000 === 0) console.log(`INDEXED ${indexed}/${pageFiles.length}`);
|
||
|
|
}
|
||
|
|
db.exec("COMMIT");
|
||
|
|
|
||
|
|
db.exec(`
|
||
|
|
UPDATE pages
|
||
|
|
SET parent_file_path = (
|
||
|
|
SELECT parent.file_path
|
||
|
|
FROM pages AS parent
|
||
|
|
WHERE parent.logical_path = pages.parent_logical_path
|
||
|
|
AND parent.export_batch = pages.export_batch
|
||
|
|
ORDER BY parent.size_bytes DESC
|
||
|
|
LIMIT 1
|
||
|
|
)
|
||
|
|
WHERE parent_logical_path IS NOT NULL;
|
||
|
|
`);
|
||
|
|
|
||
|
|
const candidateRows = db.prepare(
|
||
|
|
"SELECT file_path, relative_path FROM pages WHERE is_candidate = 1"
|
||
|
|
).all();
|
||
|
|
console.log(`CANDIDATES ${candidateRows.length}`);
|
||
|
|
|
||
|
|
const insertLink = db.prepare(`
|
||
|
|
INSERT OR IGNORE INTO links(
|
||
|
|
source_file_path, target_raw, target_file_path, target_page_id, link_kind, resolved
|
||
|
|
) VALUES (?, ?, ?, ?, ?, ?)
|
||
|
|
`);
|
||
|
|
const insertIdentifier = db.prepare(
|
||
|
|
"INSERT OR IGNORE INTO identifiers(source_file_path, identifier) VALUES (?, ?)"
|
||
|
|
);
|
||
|
|
const markdownLink = /\[[^\]]*\]\((?:<([^>]+)>|([^) \t\r\n]+))(?:\s+["'][^"']*["'])?\)/g;
|
||
|
|
const notionId = /(?:notion\.(?:so|com)\/(?:p\/)?(?:[^/?#]*-)?|\/p\/)([0-9a-f]{32})(?:[?#/]|$)/i;
|
||
|
|
const identifierPattern = /\b(?:SYS|HLDP-DOMAIN|DOM|GLS|TCS|ICE|GH|BC|CH|AGE|PER|5TH|DEV|AG|NAV|FD-MIG)-[A-Z0-9∞]+(?:-[A-Z0-9∞]+)*\b/gi;
|
||
|
|
|
||
|
|
db.exec("BEGIN");
|
||
|
|
let relationCount = 0;
|
||
|
|
for (const row of candidateRows) {
|
||
|
|
const content = fs.readFileSync(row.file_path, "utf8");
|
||
|
|
const sourceDirectory = path.dirname(row.file_path);
|
||
|
|
|
||
|
|
for (const match of content.matchAll(markdownLink)) {
|
||
|
|
const targetRaw = match[1] ?? match[2];
|
||
|
|
if (!targetRaw || targetRaw.startsWith("#") || targetRaw.startsWith("mailto:")) continue;
|
||
|
|
const notionMatch = targetRaw.match(notionId);
|
||
|
|
if (/^https?:/i.test(targetRaw)) {
|
||
|
|
insertLink.run(
|
||
|
|
row.file_path,
|
||
|
|
targetRaw,
|
||
|
|
null,
|
||
|
|
notionMatch ? normalizePageId(notionMatch[1]) : null,
|
||
|
|
notionMatch ? "notion" : "external",
|
||
|
|
notionMatch ? 1 : 0
|
||
|
|
);
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
|
||
|
|
const cleanTarget = targetRaw.split("#")[0];
|
||
|
|
let decoded = cleanTarget;
|
||
|
|
try { decoded = decodeURIComponent(cleanTarget); } catch {}
|
||
|
|
const resolvedPath = path.resolve(sourceDirectory, decoded);
|
||
|
|
const exists = fs.existsSync(resolvedPath);
|
||
|
|
const identity = pageIdentity(path.basename(resolvedPath));
|
||
|
|
insertLink.run(
|
||
|
|
row.file_path,
|
||
|
|
targetRaw,
|
||
|
|
exists ? resolvedPath : null,
|
||
|
|
identity.pageId,
|
||
|
|
"local",
|
||
|
|
exists ? 1 : 0
|
||
|
|
);
|
||
|
|
relationCount++;
|
||
|
|
}
|
||
|
|
|
||
|
|
const identifiers = new Set(content.match(identifierPattern) ?? []);
|
||
|
|
for (const identifier of identifiers) {
|
||
|
|
insertIdentifier.run(row.file_path, identifier.toUpperCase());
|
||
|
|
}
|
||
|
|
}
|
||
|
|
db.exec("COMMIT");
|
||
|
|
|
||
|
|
db.exec("PRAGMA optimize");
|
||
|
|
const stats = {
|
||
|
|
schema: "guanghu.notion-reconstruction-inventory/v0.1",
|
||
|
|
generated_at: new Date().toISOString(),
|
||
|
|
source_root: sourceRoot,
|
||
|
|
source_size_bytes: Number(
|
||
|
|
db.prepare("SELECT COALESCE(SUM(size_bytes), 0) AS n FROM pages").get().n
|
||
|
|
),
|
||
|
|
markdown_pages: Number(db.prepare("SELECT COUNT(*) AS n FROM pages").get().n),
|
||
|
|
unique_notion_page_ids: Number(
|
||
|
|
db.prepare("SELECT COUNT(DISTINCT page_id) AS n FROM pages WHERE page_id IS NOT NULL").get().n
|
||
|
|
),
|
||
|
|
duplicate_page_id_groups: Number(
|
||
|
|
db.prepare(`
|
||
|
|
SELECT COUNT(*) AS n FROM (
|
||
|
|
SELECT page_id FROM pages
|
||
|
|
WHERE page_id IS NOT NULL
|
||
|
|
GROUP BY page_id HAVING COUNT(*) > 1
|
||
|
|
)
|
||
|
|
`).get().n
|
||
|
|
),
|
||
|
|
candidate_pages: candidateRows.length,
|
||
|
|
candidate_links: Number(db.prepare("SELECT COUNT(*) AS n FROM links").get().n),
|
||
|
|
unresolved_candidate_links: Number(
|
||
|
|
db.prepare("SELECT COUNT(*) AS n FROM links WHERE link_kind = 'local' AND resolved = 0").get().n
|
||
|
|
),
|
||
|
|
identifiers: Number(db.prepare("SELECT COUNT(*) AS n FROM identifiers").get().n),
|
||
|
|
database_sha256: null
|
||
|
|
};
|
||
|
|
|
||
|
|
db.close();
|
||
|
|
if (fs.existsSync(dbPath)) fs.rmSync(dbPath);
|
||
|
|
fs.renameSync(tempDbPath, dbPath);
|
||
|
|
const databaseHash = crypto.createHash("sha256");
|
||
|
|
for await (const chunk of fs.createReadStream(dbPath)) databaseHash.update(chunk);
|
||
|
|
stats.database_sha256 = databaseHash.digest("hex");
|
||
|
|
fs.writeFileSync(reportPath, `${JSON.stringify(stats, null, 2)}\n`);
|
||
|
|
console.log(JSON.stringify(stats, null, 2));
|