docs: publish HoloLake system architecture baseline

language-world/reconstruction/tools/build-notion-index.mjs

@@ -0,0 +1,346 @@
+#!/usr/bin/env node
+
+import fs from "node:fs";
+import path from "node:path";
+import crypto from "node:crypto";
+import { DatabaseSync } from "node:sqlite";
+
+function arg(name, fallback) {
+  const index = process.argv.indexOf(name);
+  return index >= 0 && process.argv[index + 1] ? process.argv[index + 1] : fallback;
+}
+
+const configPath = path.resolve(arg("--config", "../index-config.json"));
+const dbPath = path.resolve(arg("--db", "../index/notion-world-index.sqlite"));
+const reportPath = path.resolve(arg("--report", "../index/corpus-inventory.json"));
+const config = JSON.parse(fs.readFileSync(configPath, "utf8"));
+const sourceRoot = path.resolve(config.source_roots[0]);
+const previewBytes = Number(config.preview_bytes ?? 4096);
+const storedPreviewBytes = 512;
+const threshold = Number(config.candidate_score_threshold ?? 6);
+const tempDbPath = `${dbPath}.building`;
+
+fs.mkdirSync(path.dirname(dbPath), { recursive: true });
+fs.mkdirSync(path.dirname(reportPath), { recursive: true });
+if (fs.existsSync(tempDbPath)) fs.rmSync(tempDbPath);
+
+function normalizePageId(hex) {
+  const value = hex.toLowerCase();
+  return `${value.slice(0, 8)}-${value.slice(8, 12)}-${value.slice(12, 16)}-${value.slice(16, 20)}-${value.slice(20)}`;
+}
+
+function pageIdentity(filename) {
+  const basename = filename.replace(/\.md$/i, "");
+  const match = basename.match(/(?:^| )([0-9a-f]{32})$/i);
+  if (!match) return { title: basename, pageId: null };
+  return {
+    title: basename.slice(0, match.index).trim(),
+    pageId: normalizePageId(match[1])
+  };
+}
+
+function logicalPath(relativePath, title) {
+  return path.posix.join(path.posix.dirname(relativePath), title);
+}
+
+function readPreview(file) {
+  const fd = fs.openSync(file, "r");
+  try {
+    const buffer = Buffer.allocUnsafe(previewBytes);
+    const length = fs.readSync(fd, buffer, 0, previewBytes, 0);
+    return buffer.subarray(0, length).toString("utf8").replaceAll("\u0000", "");
+  } finally {
+    fs.closeSync(fd);
+  }
+}
+
+function classify(relativePath, title, preview) {
+  const haystack = `${relativePath}\n${title}\n${preview}`.toLowerCase();
+  const titlePath = `${relativePath}\n${title}`.toLowerCase();
+  let score = 0;
+  const positive = [];
+  const negative = [];
+
+  for (const term of config.positive_terms) {
+    const normalized = term.toLowerCase();
+    if (titlePath.includes(normalized)) {
+      score += 4;
+      positive.push(term);
+    } else if (haystack.includes(normalized)) {
+      score += 1;
+      positive.push(term);
+    }
+  }
+  for (const term of config.negative_terms) {
+    const normalized = term.toLowerCase();
+    if (titlePath.includes(normalized)) {
+      score -= 5;
+      negative.push(term);
+    }
+  }
+
+  const alwaysInclude = config.always_include_path_fragments.some(fragment =>
+    relativePath.includes(fragment)
+  );
+  if (alwaysInclude) score += 100;
+
+  const domainHints = [];
+  const hints = [
+    ["MAIN", ["光湖主域", "domain-main", "dom-main"]],
+    ["SUB", ["光湖分域", "domain-sub", "dom-sub"]],
+    ["ZERO", ["光湖零域", "domain-zero", "dom-zero"]],
+    ["ZEROSENSE", ["光湖零感域", "零感域", "zero-sense", "zerosense"]],
+    ["FIFTH", ["第五域", "fifth domain", "sys-5th", "domain-fifth"]]
+  ];
+  for (const [domain, terms] of hints) {
+    if (terms.some(term => haystack.includes(term))) domainHints.push(domain);
+  }
+
+  return {
+    score,
+    candidate: score >= threshold,
+    alwaysInclude,
+    positive,
+    negative,
+    domainHints
+  };
+}
+
+function* walk(root) {
+  const stack = [root];
+  while (stack.length) {
+    const directory = stack.pop();
+    const entries = fs.readdirSync(directory, { withFileTypes: true });
+    for (const entry of entries) {
+      if (entry.name === ".DS_Store") continue;
+      const fullPath = path.join(directory, entry.name);
+      if (entry.isDirectory()) stack.push(fullPath);
+      else if (entry.isFile() && entry.name.toLowerCase().endsWith(".md")) yield fullPath;
+    }
+  }
+}
+
+const db = new DatabaseSync(tempDbPath);
+db.exec(`
+  PRAGMA journal_mode = DELETE;
+  PRAGMA synchronous = NORMAL;
+  PRAGMA temp_store = MEMORY;
+  CREATE TABLE pages (
+    file_path TEXT PRIMARY KEY,
+    relative_path TEXT NOT NULL,
+    export_batch TEXT NOT NULL,
+    title TEXT NOT NULL,
+    page_id TEXT,
+    logical_path TEXT NOT NULL,
+    parent_logical_path TEXT,
+    parent_file_path TEXT,
+    size_bytes INTEGER NOT NULL,
+    mtime_ms INTEGER NOT NULL,
+    first_heading TEXT,
+    preview TEXT,
+    relevance_score INTEGER NOT NULL,
+    is_candidate INTEGER NOT NULL,
+    always_include INTEGER NOT NULL,
+    positive_terms TEXT NOT NULL,
+    negative_terms TEXT NOT NULL,
+    domain_hints TEXT NOT NULL
+  );
+  CREATE INDEX pages_page_id_idx ON pages(page_id);
+  CREATE INDEX pages_logical_path_idx ON pages(logical_path);
+  CREATE INDEX pages_candidate_idx ON pages(is_candidate, relevance_score DESC);
+  CREATE INDEX pages_title_idx ON pages(title);
+  CREATE TABLE links (
+    source_file_path TEXT NOT NULL,
+    target_raw TEXT NOT NULL,
+    target_file_path TEXT,
+    target_page_id TEXT,
+    link_kind TEXT NOT NULL,
+    resolved INTEGER NOT NULL,
+    PRIMARY KEY(source_file_path, target_raw)
+  );
+  CREATE INDEX links_source_idx ON links(source_file_path);
+  CREATE INDEX links_target_path_idx ON links(target_file_path);
+  CREATE INDEX links_target_page_idx ON links(target_page_id);
+  CREATE TABLE identifiers (
+    source_file_path TEXT NOT NULL,
+    identifier TEXT NOT NULL,
+    PRIMARY KEY(source_file_path, identifier)
+  );
+  CREATE INDEX identifiers_value_idx ON identifiers(identifier);
+  CREATE VIRTUAL TABLE page_search USING fts5(
+    file_path UNINDEXED,
+    title,
+    relative_path,
+    tokenize='unicode61'
+  );
+`);
+
+const insertPage = db.prepare(`
+  INSERT INTO pages (
+    file_path, relative_path, export_batch, title, page_id, logical_path,
+    parent_logical_path, size_bytes, mtime_ms, first_heading, preview,
+    relevance_score, is_candidate, always_include, positive_terms,
+    negative_terms, domain_hints
+  ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+`);
+const insertSearch = db.prepare(
+  "INSERT INTO page_search(file_path, title, relative_path) VALUES (?, ?, ?)"
+);
+
+const pageFiles = [...walk(sourceRoot)];
+console.log(`DISCOVERED ${pageFiles.length} markdown files`);
+
+db.exec("BEGIN");
+let indexed = 0;
+for (const file of pageFiles) {
+  const relativePath = path.relative(sourceRoot, file).split(path.sep).join("/");
+  const exportBatch = relativePath.split("/")[0];
+  const { title, pageId } = pageIdentity(path.basename(file));
+  const preview = readPreview(file);
+  const headingMatch = preview.match(/^#\s+(.+)$/m);
+  const classification = classify(relativePath, title, preview);
+  const stat = fs.statSync(file);
+  const logical = logicalPath(relativePath, title);
+  const parentLogical = path.posix.dirname(relativePath) === "."
+    ? null
+    : path.posix.dirname(relativePath);
+
+  insertPage.run(
+    file,
+    relativePath,
+    exportBatch,
+    title,
+    pageId,
+    logical,
+    parentLogical,
+    stat.size,
+    stat.mtimeMs,
+    headingMatch?.[1]?.trim() ?? null,
+    preview.slice(0, storedPreviewBytes),
+    classification.score,
+    classification.candidate ? 1 : 0,
+    classification.alwaysInclude ? 1 : 0,
+    JSON.stringify(classification.positive),
+    JSON.stringify(classification.negative),
+    JSON.stringify(classification.domainHints)
+  );
+  insertSearch.run(file, title, relativePath);
+  indexed++;
+  if (indexed % 10000 === 0) console.log(`INDEXED ${indexed}/${pageFiles.length}`);
+}
+db.exec("COMMIT");
+
+db.exec(`
+  UPDATE pages
+  SET parent_file_path = (
+    SELECT parent.file_path
+    FROM pages AS parent
+    WHERE parent.logical_path = pages.parent_logical_path
+      AND parent.export_batch = pages.export_batch
+    ORDER BY parent.size_bytes DESC
+    LIMIT 1
+  )
+  WHERE parent_logical_path IS NOT NULL;
+`);
+
+const candidateRows = db.prepare(
+  "SELECT file_path, relative_path FROM pages WHERE is_candidate = 1"
+).all();
+console.log(`CANDIDATES ${candidateRows.length}`);
+
+const insertLink = db.prepare(`
+  INSERT OR IGNORE INTO links(
+    source_file_path, target_raw, target_file_path, target_page_id, link_kind, resolved
+  ) VALUES (?, ?, ?, ?, ?, ?)
+`);
+const insertIdentifier = db.prepare(
+  "INSERT OR IGNORE INTO identifiers(source_file_path, identifier) VALUES (?, ?)"
+);
+const markdownLink = /\[[^\]]*\]\((?:<([^>]+)>|([^) \t\r\n]+))(?:\s+["'][^"']*["'])?\)/g;
+const notionId = /(?:notion\.(?:so|com)\/(?:p\/)?(?:[^/?#]*-)?|\/p\/)([0-9a-f]{32})(?:[?#/]|$)/i;
+const identifierPattern = /\b(?:SYS|HLDP-DOMAIN|DOM|GLS|TCS|ICE|GH|BC|CH|AGE|PER|5TH|DEV|AG|NAV|FD-MIG)-[A-Z0-9∞]+(?:-[A-Z0-9∞]+)*\b/gi;
+
+db.exec("BEGIN");
+let relationCount = 0;
+for (const row of candidateRows) {
+  const content = fs.readFileSync(row.file_path, "utf8");
+  const sourceDirectory = path.dirname(row.file_path);
+
+  for (const match of content.matchAll(markdownLink)) {
+    const targetRaw = match[1] ?? match[2];
+    if (!targetRaw || targetRaw.startsWith("#") || targetRaw.startsWith("mailto:")) continue;
+    const notionMatch = targetRaw.match(notionId);
+    if (/^https?:/i.test(targetRaw)) {
+      insertLink.run(
+        row.file_path,
+        targetRaw,
+        null,
+        notionMatch ? normalizePageId(notionMatch[1]) : null,
+        notionMatch ? "notion" : "external",
+        notionMatch ? 1 : 0
+      );
+      continue;
+    }
+
+    const cleanTarget = targetRaw.split("#")[0];
+    let decoded = cleanTarget;
+    try { decoded = decodeURIComponent(cleanTarget); } catch {}
+    const resolvedPath = path.resolve(sourceDirectory, decoded);
+    const exists = fs.existsSync(resolvedPath);
+    const identity = pageIdentity(path.basename(resolvedPath));
+    insertLink.run(
+      row.file_path,
+      targetRaw,
+      exists ? resolvedPath : null,
+      identity.pageId,
+      "local",
+      exists ? 1 : 0
+    );
+    relationCount++;
+  }
+
+  const identifiers = new Set(content.match(identifierPattern) ?? []);
+  for (const identifier of identifiers) {
+    insertIdentifier.run(row.file_path, identifier.toUpperCase());
+  }
+}
+db.exec("COMMIT");
+
+db.exec("PRAGMA optimize");
+const stats = {
+  schema: "guanghu.notion-reconstruction-inventory/v0.1",
+  generated_at: new Date().toISOString(),
+  source_root: sourceRoot,
+  source_size_bytes: Number(
+    db.prepare("SELECT COALESCE(SUM(size_bytes), 0) AS n FROM pages").get().n
+  ),
+  markdown_pages: Number(db.prepare("SELECT COUNT(*) AS n FROM pages").get().n),
+  unique_notion_page_ids: Number(
+    db.prepare("SELECT COUNT(DISTINCT page_id) AS n FROM pages WHERE page_id IS NOT NULL").get().n
+  ),
+  duplicate_page_id_groups: Number(
+    db.prepare(`
+      SELECT COUNT(*) AS n FROM (
+        SELECT page_id FROM pages
+        WHERE page_id IS NOT NULL
+        GROUP BY page_id HAVING COUNT(*) > 1
+      )
+    `).get().n
+  ),
+  candidate_pages: candidateRows.length,
+  candidate_links: Number(db.prepare("SELECT COUNT(*) AS n FROM links").get().n),
+  unresolved_candidate_links: Number(
+    db.prepare("SELECT COUNT(*) AS n FROM links WHERE link_kind = 'local' AND resolved = 0").get().n
+  ),
+  identifiers: Number(db.prepare("SELECT COUNT(*) AS n FROM identifiers").get().n),
+  database_sha256: null
+};
+
+db.close();
+if (fs.existsSync(dbPath)) fs.rmSync(dbPath);
+fs.renameSync(tempDbPath, dbPath);
+const databaseHash = crypto.createHash("sha256");
+for await (const chunk of fs.createReadStream(dbPath)) databaseHash.update(chunk);
+stats.database_sha256 = databaseHash.digest("hex");
+fs.writeFileSync(reportPath, `${JSON.stringify(stats, null, 2)}\n`);
+console.log(JSON.stringify(stats, null, 2));

language-world/reconstruction/tools/import-selected-sources.mjs

@@ -0,0 +1,55 @@
+#!/usr/bin/env node
+
+import { createHash } from "node:crypto";
+import { mkdir, readFile, writeFile, copyFile } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+const reconstructionRoot = path.resolve(here, "..");
+const configPath = path.join(reconstructionRoot, "selected-sources.json");
+const outputRoot = path.join(reconstructionRoot, "sources");
+const manifestPath = path.join(reconstructionRoot, "selected-sources-manifest.json");
+
+const config = JSON.parse(await readFile(configPath, "utf8"));
+const manifest = [];
+
+for (const source of config.sources) {
+  const bytes = await readFile(source.path);
+  const text = bytes.toString("utf8");
+  const title = text.match(/^#\s+(.+)$/m)?.[1]?.trim() ?? path.basename(source.path);
+  const targetDir = path.join(outputRoot, source.group);
+  const targetPath = path.join(targetDir, `${source.id}.source.txt`);
+  await mkdir(targetDir, { recursive: true });
+  await copyFile(source.path, targetPath);
+  manifest.push({
+    id: source.id,
+    title,
+    group: source.group,
+    status: source.status,
+    privacy: source.privacy,
+    role: source.role,
+    notion_url: `https://app.notion.com/p/${source.id}`,
+    source_path: source.path,
+    imported_path: path.relative(reconstructionRoot, targetPath),
+    bytes: bytes.length,
+    sha256: createHash("sha256").update(bytes).digest("hex")
+  });
+}
+
+await writeFile(
+  manifestPath,
+  `${JSON.stringify({
+    schema: "guanghu.selected-notion-source-manifest/v0.1",
+    generated_at: new Date().toISOString(),
+    source_count: manifest.length,
+    copy_mode: "VERBATIM_BYTES_AS_SOURCE_TEXT",
+    sources: manifest
+  }, null, 2)}\n`
+);
+
+console.log(JSON.stringify({
+  source_count: manifest.length,
+  manifest: manifestPath,
+  output_root: outputRoot
+}, null, 2));

language-world/reconstruction/tools/validate-reconstruction.mjs

@@ -0,0 +1,111 @@
+#!/usr/bin/env node
+
+import { createHash } from "node:crypto";
+import { readdir, readFile, stat } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+const root = path.resolve(here, "..");
+const knowledgeRoot = path.resolve(root, "../..");
+const worldRoot = path.join(root, "world");
+const manifest = JSON.parse(await readFile(path.join(root, "selected-sources-manifest.json"), "utf8"));
+const errors = [];
+let linksChecked = 0;
+let jsonFilesChecked = 0;
+let sourceHashesChecked = 0;
+
+async function filesUnder(directory) {
+  const found = [];
+  for (const entry of await readdir(directory, { withFileTypes: true })) {
+    if (entry.name === ".git") continue;
+    const full = path.join(directory, entry.name);
+    if (entry.isDirectory()) found.push(...await filesUnder(full));
+    else found.push(full);
+  }
+  return found;
+}
+
+for (const file of (await filesUnder(root)).filter((item) => item.endsWith(".json"))) {
+  try {
+    JSON.parse(await readFile(file, "utf8"));
+    jsonFilesChecked += 1;
+  } catch (error) {
+    errors.push({ type: "INVALID_JSON", file, message: error.message });
+  }
+}
+
+for (const source of manifest.sources) {
+  const imported = path.join(root, source.imported_path);
+  for (const [kind, file] of [["source", source.source_path], ["imported", imported]]) {
+    try {
+      const bytes = await readFile(file);
+      const sha256 = createHash("sha256").update(bytes).digest("hex");
+      if (sha256 !== source.sha256) {
+        errors.push({ type: "HASH_MISMATCH", id: source.id, kind, file });
+      }
+    } catch (error) {
+      errors.push({ type: "SOURCE_READ_FAILED", id: source.id, kind, file, message: error.message });
+    }
+  }
+  sourceHashesChecked += 1;
+}
+
+const markdownFiles = (await filesUnder(knowledgeRoot)).filter((item) => item.endsWith(".md"));
+for (const file of markdownFiles) {
+  const text = await readFile(file, "utf8");
+  const linkPattern = /\[[^\]]*]\(([^)]+)\)/g;
+  for (const match of text.matchAll(linkPattern)) {
+    let target = match[1].trim();
+    if (!target || target.startsWith("#") || /^(https?:|mailto:)/.test(target)) continue;
+    if (target.startsWith("<") && target.endsWith(">")) target = target.slice(1, -1);
+    target = decodeURIComponent(target.split("#")[0]);
+    const resolved = path.resolve(path.dirname(file), target);
+    linksChecked += 1;
+    try {
+      await stat(resolved);
+    } catch {
+      errors.push({ type: "BROKEN_RELATIVE_LINK", file, target });
+    }
+  }
+}
+
+for (const source of manifest.sources) {
+  if (/^domains\/(main|sub|zero|zero-sense)/.test(source.group) && source.privacy === "PRIVATE_FIFTH") {
+    errors.push({ type: "PRIVATE_SOURCE_IN_PUBLIC_DOMAIN", id: source.id, group: source.group });
+  }
+}
+
+for (const domain of ["main", "sub", "zero", "zero-sense"]) {
+  const file = path.join(worldRoot, "domains", domain, "INDEX.md");
+  const text = await readFile(file, "utf8");
+  if (text.includes("sources/domains/fifth") || text.includes("sources/supplemental/fd-mig-001")) {
+    errors.push({ type: "PUBLIC_DOMAIN_LINKS_PRIVATE_FIFTH", file });
+  }
+}
+
+const routes = JSON.parse(await readFile(path.join(worldRoot, "routing/login-route-map.json"), "utf8"));
+const routeById = new Map(routes.routes.map((route) => [route.route_id, route]));
+const expected = {
+  "GLW-LOGIN-BINGSHUO-FIFTH": ["光湖语言世界", "第五域", "永恒湖心系统", "心跳核心频道"],
+  "GLW-LOGIN-ZHIZHI-FIFTH": ["光湖语言世界", "第五域", "永恒湖心系统", "爱之核心子系统", "明天见频道"]
+};
+for (const [routeId, steps] of Object.entries(expected)) {
+  if (JSON.stringify(routeById.get(routeId)?.steps) !== JSON.stringify(steps)) {
+    errors.push({ type: "CURRENT_ROUTE_MISMATCH", route_id: routeId });
+  }
+}
+
+console.log(JSON.stringify({
+  schema: "guanghu.reconstruction-validation/v0.1",
+  status: errors.length === 0 ? "PASS" : "FAIL",
+  markdown_pages_checked: markdownFiles.length,
+  relative_links_checked: linksChecked,
+  json_files_checked: jsonFilesChecked,
+  source_hash_pairs_checked: sourceHashesChecked,
+  privacy_boundary_checked: true,
+  current_routes_checked: Object.keys(expected).length,
+  errors
+}, null, 2));
+
+process.exitCode = errors.length === 0 ? 0 : 1;