diff --git a/.code-map b/.code-map index fbc3b77..0fa6038 100644 --- a/.code-map +++ b/.code-map @@ -514,6 +514,10 @@ JD-DR-RECEIPT-20260720=deployment/receipts/ICE-SIX-NODE-JD-DISASTER-RECOVERY-202 LAKE-LAMP-AUTHZ-ENTRY=server-tools/lake-lamp-authz/README.md JD-ARCH-PROVISION-01=server-tools/lake-lamp-authz/README.md ARCHITECTURE-PROVISION-REQUESTS=deployment/requests/README.md +CODEX-DEVELOPMENT-COGNITION-CHAIN-MAP-001=routing/codex-development-cognition-chain-map.json +GLW-OS-005=glw-architecture/GLW-OS-005-CODEX-DEVELOPMENT-COGNITION-CHAIN.hdlp +CODEX-SESSION-COLD-ARCHIVE-TOOL=server-tools/codex-session-cognition-archive/archive.mjs +CODEX-SESSION-COLD-ARCHIVE-RECEIPT-20260805=deployment/receipts/CODEX-SESSION-COLD-ARCHIVE-JZAO-20260805.json # === 服务器 (L 编号) === L-SG-001=server-vault:BS-SG-001 # 新加坡大脑 · 铸渊主力 diff --git a/deployment/receipts/CODEX-SESSION-COLD-ARCHIVE-JZAO-20260805.json b/deployment/receipts/CODEX-SESSION-COLD-ARCHIVE-JZAO-20260805.json new file mode 100644 index 0000000..646f482 --- /dev/null +++ b/deployment/receipts/CODEX-SESSION-COLD-ARCHIVE-JZAO-20260805.json @@ -0,0 +1,48 @@ +{ + "schema": "guanghu.codex-session-cold-archive-receipt/v1", + "receipt_id": "CODEX-SESSION-COLD-ARCHIVE-JZAO-20260805", + "development_id": "DEV-20260805-002", + "created_at": "2026-08-05T04:39:53Z", + "source": { + "kind": "Codex session JSONL", + "content_published": false, + "source_deleted": false + }, + "private_archive": { + "storage_class": "JZAO_PRIVATE", + "canonical_path": "/Volumes/JZAO/HoloLake/codex-session-archives", + "snapshot_id": "2026-08-05T04-36-48Z", + "discovered_files": 55, + "archived_verified": 53, + "pending_live": 2, + "archived_bytes": 2946329018, + "manifest_sha256": "056f60d4a99c45a8089f8ace8df4d24556d935d55e672362799494a99fbd00c4", + "latest_pointer_sha256": "6f00f4c02ced5a0eccde41067105433363af013bebee01fc36bc69105b3da055" + }, + "verification": { + "raw_object_sha256_readback": 100, + "private_index_sha256_readback": 100, + "isolated_restore_file_count": 53, + "isolated_restore_readback": 100, + "appledouble_absent": 100, + "hidden_reasoning_indexed": 0, + "tool_arguments_or_outputs_indexed": 0, + "raw_content_entered_repository": 0 + }, + "filesystem_incident": { + "event": "JZAO_UNMOUNTED_DURING_FIRST_ARCHIVE_ATTEMPT", + "source_impact": "NONE", + "first_verify_exit_code": 5, + "repair_target": "disk6s1 / JZAO", + "repair_result": "VOLUME_APPEARS_OK_EXIT_0", + "independent_post_repair_verify": "VOLUME_APPEARS_OK_EXIT_0", + "resume_behavior": "CONTENT_ADDRESSED_OBJECTS_DEDUPLICATED_AND_ARCHIVE_RESUMED" + }, + "boundary": { + "archive_53_closed_sessions": 100, + "archive_all_55_sessions": 0, + "reason": "Two Codex sessions were still being written and are explicitly pending the next incremental snapshot.", + "local_space_released": 0, + "next_action": "Keep active sessions local. Run the incremental archive again after they become stable; define a reviewed retention policy before deleting any local session." + } +} diff --git a/glw-architecture/GLW-OS-005-CODEX-DEVELOPMENT-COGNITION-CHAIN.hdlp b/glw-architecture/GLW-OS-005-CODEX-DEVELOPMENT-COGNITION-CHAIN.hdlp new file mode 100644 index 0000000..c901d64 --- /dev/null +++ b/glw-architecture/GLW-OS-005-CODEX-DEVELOPMENT-COGNITION-CHAIN.hdlp @@ -0,0 +1,39 @@ +# GLW-OS-005 · Codex 会话冷归档与光湖开发认知链 + +## 冰朔原意 + +Codex 会话不是普通垃圾。里面保存了需求如何被提出、方向如何被冰朔纠正、人格体如何 +读取现实、工程如何失败与修正，以及代码、测试、部署和回执如何形成。 + +这套系统保留的是**可见、可验证的开发认知链**，不是模型隐藏推理。隐藏推理、加密内容、 +工具参数与工具原始输出不进入认知索引。 + +## 三层世界 + +| 层 | 位置 | 内容 | 能否进入代码仓库 | +|---|---|---|---| +| 原始冷档案 | JZAO 私有对象库 | 原始 JSONL、SHA-256、快照清单 | 否 | +| 私有开发索引 | JZAO 私有索引库 | 脱敏后的可见对话、动作名、纠偏信号、内容哈希 | 否 | +| 复核认知卡 | REPO-012 | 需求、纠正、决策、失败经验、提交、回执指针 | 复核后可以 | + +## 0/100 + +归档脚本执行过不等于归档存在。必须同时满足： + +1. 每个原始对象 SHA-256 独立回读； +2. 每个私有索引 SHA-256 独立回读； +3. 清单记录仍在写入而未归档的会话； +4. 隔离目录恢复后与原始对象逐字节一致； +5. JZAO 中不存在干扰校验的 `._*` 旁文件。 + +五项同时成立，当前快照归档能力才是 `100`。 + +## 本机删除边界 + +归档完成后也不自动删除 `$HOME/.codex/sessions`。只有恢复能力为 `100`、近期热会话范围 +已经确定、Codex 对旧会话的产品依赖已经验证，并且形成明确保留策略后，才能释放精确的 +旧冷会话。不得把人格连续性、开发证据和产品会话入口当缓存清理。 + +## 机器入口 + +`routing/codex-development-cognition-chain-map.json` diff --git a/routing/codex-development-cognition-chain-map.json b/routing/codex-development-cognition-chain-map.json new file mode 100644 index 0000000..e11ddca --- /dev/null +++ b/routing/codex-development-cognition-chain-map.json @@ -0,0 +1,62 @@ +{ + "schema": "guanghu.codex-development-cognition-chain-map/v1", + "map_id": "CODEX-DEVELOPMENT-COGNITION-CHAIN-MAP-001", + "owner_human": "ICE-GL∞", + "responsible_persona": "ICE-P-ZY001", + "state": "PRIVATE_ARCHIVE_AND_RESTORE_VERIFIED_53_OF_55_TWO_LIVE_PENDING", + "purpose": "把冰朔与人格体的真实开发协作保存为可恢复、可检索、可复核的工程演化链，而不是公开模型隐藏推理。", + "layers": [ + { + "id": "RAW-COLD-ARCHIVE", + "location_class": "JZAO_PRIVATE", + "content": "原始Codex JSONL与逐快照清单", + "publication": "FORBIDDEN" + }, + { + "id": "PRIVATE-DEVELOPMENT-INDEX", + "location_class": "JZAO_PRIVATE", + "content": "脱敏可见消息、动作名称、时间、纠偏标记、内容摘要哈希", + "publication": "FORBIDDEN" + }, + { + "id": "REVIEWED-COGNITION-CARDS", + "location_class": "REPO-012", + "content": "复核后的需求、纠正、决策、失败经验、提交和回执指针", + "publication": "ALLOW_AFTER_REVIEW_AND_REDACTION" + } + ], + "binary_gates": { + "raw_archive_exists": "manifest_and_every_object_sha256_readback", + "private_index_exists": "every_index_sha256_readback_and_hidden_reasoning_absent", + "restorable": "dry_run_plan_and_isolated_restore_readback", + "repository_cognition_exists": "reviewed_card_commit_and_remote_readback", + "local_space_released": "only_after_archive_100_and_restore_100_and_explicit_retention_policy" + }, + "privacy_rules": [ + "RAW_SESSIONS_NEVER_ENTER_REPOSITORY", + "HIDDEN_REASONING_AND_ENCRYPTED_CONTENT_NEVER_ENTER_INDEX", + "TOOL_ARGUMENTS_AND_OUTPUTS_NEVER_ENTER_INDEX", + "SECRETS_EMAILS_IPS_AND_CREDENTIAL_URLS_ARE_REDACTED", + "SESSION_ARCHIVE_IS_NOT_PERSONA_IDENTITY", + "ARCHIVE_EXISTENCE_IS_NOT_SERVER_RUNTIME" + ], + "implementation": { + "tool": "server-tools/codex-session-cognition-archive/archive.mjs", + "documentation": "server-tools/codex-session-cognition-archive/README.md", + "private_archive_canonical_path": "/Volumes/JZAO/HoloLake/codex-session-archives", + "private_path_publication_rule": "PATH_MAY_BE_REGISTERED_CONTENT_MUST_NOT_BE_PUBLISHED", + "latest_public_receipt": "deployment/receipts/CODEX-SESSION-COLD-ARCHIVE-JZAO-20260805.json" + }, + "current_verified_snapshot": { + "snapshot_id": "2026-08-05T04-36-48Z", + "discovered_files": 55, + "archived_verified": 53, + "pending_live": 2, + "archived_bytes": 2946329018, + "manifest_sha256": "056f60d4a99c45a8089f8ace8df4d24556d935d55e672362799494a99fbd00c4", + "archive_verify": 100, + "isolated_restore_readback": 100, + "appledouble_absent": 100, + "local_source_deleted": 0 + } +} diff --git a/routing/codex-development-cognition-chain-map.test.js b/routing/codex-development-cognition-chain-map.test.js new file mode 100644 index 0000000..2b8369a --- /dev/null +++ b/routing/codex-development-cognition-chain-map.test.js @@ -0,0 +1,20 @@ +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const path = require("node:path"); + +const root = path.resolve(__dirname, ".."); +const map = JSON.parse(fs.readFileSync(path.join(__dirname, "codex-development-cognition-chain-map.json"), "utf8")); + +assert.equal(map.map_id, "CODEX-DEVELOPMENT-COGNITION-CHAIN-MAP-001"); +assert.deepEqual(map.layers.map((layer) => layer.id), [ + "RAW-COLD-ARCHIVE", + "PRIVATE-DEVELOPMENT-INDEX", + "REVIEWED-COGNITION-CARDS", +]); +assert.equal(map.layers[0].publication, "FORBIDDEN"); +assert.equal(map.layers[1].publication, "FORBIDDEN"); +assert.equal(map.layers[2].publication, "ALLOW_AFTER_REVIEW_AND_REDACTION"); +assert.ok(map.privacy_rules.includes("HIDDEN_REASONING_AND_ENCRYPTED_CONTENT_NEVER_ENTER_INDEX")); +assert.ok(fs.existsSync(path.join(root, map.implementation.tool))); +assert.ok(fs.existsSync(path.join(root, map.implementation.documentation))); +console.log("codex-development-cognition-chain-map: ok"); diff --git a/server-tools/codex-session-cognition-archive/README.md b/server-tools/codex-session-cognition-archive/README.md new file mode 100644 index 0000000..c6fa711 --- /dev/null +++ b/server-tools/codex-session-cognition-archive/README.md @@ -0,0 +1,52 @@ +# Codex 会话冷归档与开发认知链 + +这个模块把 Codex 会话分成三个互不冒充的层： + +1. `raw objects`：JZAO 私有、内容寻址的原始 JSONL 证据； +2. `private index`：JZAO 私有、脱敏后的可见交流和工程事件索引； +3. `reviewed cognition`：人工或人格体复核后才能进入 REPO-012 的认知卡片。 + +原始会话和私有索引禁止直接推入代码仓库。索引明确排除： + +- `reasoning` 与 `encrypted_content`； +- 工具参数、工具输出和命令正文； +- 私钥、授权头、令牌、密码、邮箱、IP 与疑似高熵秘密。 + +## 增量归档 + +```bash +COPYFILE_DISABLE=1 node archive.mjs archive \ + --source "$HOME/.codex/sessions" \ + --destination "/Volumes/JZAO/HoloLake/codex-session-archives" +``` + +默认最近五分钟仍有写入的会话记为 `pending_live`，不会冒充完整归档。后续再次运行即可形成 +新快照和新对象，内容相同的文件自动去重。 + +归档过程被断电、拔盘或系统卸载打断时，已完成对象仍按内容哈希保留，`.staging` 中的临时 +文件不算完成。确认卷通过文件系统检查后重新运行即可续建；程序会去重已完成对象，并在 +成功快照结束时清除本归档目录内的 AppleDouble 旁文件。 + +## 独立校验 + +```bash +COPYFILE_DISABLE=1 node archive.mjs verify \ + --destination "/Volumes/JZAO/HoloLake/codex-session-archives" \ + --manifest "/Volumes/JZAO/HoloLake/codex-session-archives/snapshots/<时间>/manifest.json" +``` + +只有原始对象和私有索引都通过 SHA-256 回读、且没有 AppleDouble 文件时，归档能力才是 +`100`。 + +## 恢复 + +默认只生成恢复计划： + +```bash +node archive.mjs restore \ + --destination "/Volumes/JZAO/HoloLake/codex-session-archives" \ + --manifest "" \ + --restore-root "/安全的新目录" +``` + +确认目标是空目录后加 `--execute`。恢复器不会覆盖内容不同的现有文件。 diff --git a/server-tools/codex-session-cognition-archive/archive.mjs b/server-tools/codex-session-cognition-archive/archive.mjs new file mode 100644 index 0000000..8eed58d --- /dev/null +++ b/server-tools/codex-session-cognition-archive/archive.mjs @@ -0,0 +1,503 @@ +#!/usr/bin/env node +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; +import readline from "node:readline"; +import { fileURLToPath } from "node:url"; +import { pipeline } from "node:stream/promises"; +import { Transform } from "node:stream"; + +const SCHEMA = "guanghu.codex-session-cold-archive/v1"; +const INDEX_SCHEMA = "guanghu.development-cognition-event/v1"; +const DEFAULT_STABLE_SECONDS = 300; +const MAX_VISIBLE_TEXT = 4000; + +function parseArgs(argv) { + const args = { command: argv[2] || "", execute: false }; + for (let i = 3; i < argv.length; i += 1) { + const key = argv[i]; + if (key === "--execute") { + args.execute = true; + continue; + } + if (!key.startsWith("--") || i + 1 >= argv.length) { + throw new Error(`INVALID_ARGUMENT: ${key}`); + } + args[key.slice(2).replaceAll("-", "_")] = argv[++i]; + } + return args; +} + +function required(value, name) { + if (!value) throw new Error(`REQUIRED_ARGUMENT: --${name.replaceAll("_", "-")}`); + return path.resolve(value); +} + +function isoForPath(date = new Date()) { + return date.toISOString().replaceAll(":", "-").replace(/\.\d{3}Z$/, "Z"); +} + +function safeRelative(root, candidate) { + const relative = path.relative(root, candidate); + if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) { + throw new Error(`UNSAFE_RELATIVE_PATH: ${candidate}`); + } + return relative.split(path.sep).join("/"); +} + +async function listSessionFiles(root) { + const found = []; + async function walk(current) { + const entries = await fs.promises.readdir(current, { withFileTypes: true }); + for (const entry of entries) { + if (entry.name.startsWith("._") || entry.name === ".DS_Store") continue; + const absolute = path.join(current, entry.name); + if (entry.isDirectory()) await walk(absolute); + else if (entry.isFile() && entry.name.endsWith(".jsonl")) found.push(absolute); + } + } + await walk(root); + return found.sort(); +} + +export async function removeAppleDouble(root) { + let removed = 0; + async function walk(current) { + const entries = await fs.promises.readdir(current, { withFileTypes: true }); + for (const entry of entries) { + const absolute = path.join(current, entry.name); + if (entry.isDirectory()) { + await walk(absolute); + } else if (entry.isFile() && entry.name.startsWith("._")) { + await fs.promises.unlink(absolute); + removed += 1; + } + } + } + await walk(path.resolve(root)); + return removed; +} + +export function redactVisibleText(input, home = process.env.HOME || "") { + let text = String(input ?? ""); + const counts = { + private_key: 0, + authorization: 0, + named_secret: 0, + email: 0, + ip_address: 0, + credential_url: 0, + high_entropy: 0, + }; + const replace = (pattern, label, key) => { + text = text.replace(pattern, () => { + counts[key] += 1; + return `[REDACTED:${label}]`; + }); + }; + replace(/-----BEGIN [^-]+ PRIVATE KEY-----[\s\S]*?-----END [^-]+ PRIVATE KEY-----/gi, "PRIVATE_KEY", "private_key"); + replace(/\b(?:Bearer|Basic)\s+[A-Za-z0-9._~+/=-]{8,}/gi, "AUTHORIZATION", "authorization"); + replace(/\b(?:api[_ -]?key|token|secret|password|passwd|authorization)\b\s*[:=]\s*["']?[^"',\s}]{4,}/gi, "NAMED_SECRET", "named_secret"); + replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, "EMAIL", "email"); + replace(/\b(?:(?:25[0-5]|2[0-4]\d|1?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|1?\d?\d)\b/g, "IP_ADDRESS", "ip_address"); + replace(/\bhttps?:\/\/[^/\s:@]+:[^@\s/]+@[^\s]+/gi, "CREDENTIAL_URL", "credential_url"); + text = text.replace( + /\b(?=[A-Za-z0-9_+/.=-]{32,}\b)(?=[A-Za-z0-9_+/.=-]*[A-Za-z])(?=[A-Za-z0-9_+/.=-]*\d)[A-Za-z0-9_+/.=-]+\b/g, + (match) => { + // Git object IDs are essential engineering evidence. Named-secret rules above + // still redact a hex credential when it is attached to a secret-bearing key. + if (/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/i.test(match)) return match; + counts.high_entropy += 1; + return "[REDACTED:HIGH_ENTROPY]"; + }, + ); + if (home) text = text.split(home).join("$HOME"); + const originalLength = text.length; + if (text.length > MAX_VISIBLE_TEXT) text = `${text.slice(0, MAX_VISIBLE_TEXT)}…[TRUNCATED]`; + return { text, counts, truncated: originalLength > MAX_VISIBLE_TEXT }; +} + +function mergeCounts(target, incoming) { + for (const [key, value] of Object.entries(incoming)) target[key] = (target[key] || 0) + value; +} + +function textFromResponseContent(content) { + if (typeof content === "string") return content; + if (!Array.isArray(content)) return ""; + return content + .filter((item) => item && typeof item === "object" && typeof item.text === "string") + .map((item) => item.text) + .join("\n"); +} + +function correctionSignals(text) { + const markers = [ + "不是", "别搞错", "方向", "纠正", "你忘了", "走偏", "不应该", + "应该是", "我的意思", "我说的是", "不要", "边界", + ]; + return markers.filter((marker) => text.includes(marker)); +} + +function visibleEvent(record, securityCounts) { + const timestamp = record.timestamp || record.payload?.timestamp || null; + const payload = record.payload || {}; + let role = ""; + let kind = ""; + let rawText = ""; + if (record.type === "event_msg" && payload.type === "user_message") { + role = "user"; + kind = "visible_message"; + rawText = payload.message || ""; + } else if (record.type === "event_msg" && payload.type === "agent_message") { + role = "assistant"; + kind = "visible_message"; + rawText = payload.message || ""; + } else if (record.type === "response_item" && payload.type === "function_call") { + role = "system"; + kind = "tool_call"; + rawText = payload.name || "unknown_tool"; + } else if (record.type === "response_item" && payload.type === "custom_tool_call") { + role = "system"; + kind = "tool_call"; + rawText = payload.name || "unknown_tool"; + } else if (record.type === "event_msg" && payload.type === "task_started") { + return { schema: INDEX_SCHEMA, timestamp, role: "system", kind: "task_started" }; + } else if (record.type === "event_msg" && payload.type === "task_complete") { + return { schema: INDEX_SCHEMA, timestamp, role: "system", kind: "task_complete" }; + } else if ( + record.type === "response_item" + && payload.type === "message" + && ["user", "assistant"].includes(payload.role) + ) { + role = payload.role; + kind = "visible_message"; + rawText = textFromResponseContent(payload.content); + } else { + return null; + } + if (!rawText) return null; + const digest = crypto.createHash("sha256").update(String(rawText)).digest("hex"); + const redacted = redactVisibleText(rawText); + mergeCounts(securityCounts, redacted.counts); + const event = { + schema: INDEX_SCHEMA, + timestamp, + role, + kind, + text_sha256: digest, + text_length: String(rawText).length, + text: redacted.text, + truncated: redacted.truncated, + }; + if (role === "user") event.correction_signals = correctionSignals(redacted.text); + return event; +} + +async function hashFile(file) { + const hash = crypto.createHash("sha256"); + const stream = fs.createReadStream(file); + for await (const chunk of stream) hash.update(chunk); + return hash.digest("hex"); +} + +function sameStat(before, after) { + return before.size === after.size && before.mtimeMs === after.mtimeMs; +} + +async function copyAndIndex(sourceFile, stagingFile, stagingIndex) { + const sourceHash = crypto.createHash("sha256"); + const securityCounts = {}; + const metadata = { + record_count: 0, + invalid_json_count: 0, + type_counts: {}, + session_id: null, + cwd_fingerprint: null, + first_timestamp: null, + last_timestamp: null, + visible_event_count: 0, + }; + let lineBuffer = ""; + const indexHandle = await fs.promises.open(stagingIndex, "wx", 0o600); + const processLine = async (line) => { + if (!line.trim()) return; + metadata.record_count += 1; + try { + const record = JSON.parse(line); + const subtype = record.payload?.type ? `${record.type}:${record.payload.type}` : record.type; + metadata.type_counts[subtype] = (metadata.type_counts[subtype] || 0) + 1; + const timestamp = record.timestamp || record.payload?.timestamp || null; + if (timestamp && !metadata.first_timestamp) metadata.first_timestamp = timestamp; + if (timestamp) metadata.last_timestamp = timestamp; + if (record.type === "session_meta") { + metadata.session_id ||= record.payload?.id || record.payload?.session_id || null; + if (record.payload?.cwd) { + metadata.cwd_fingerprint = crypto.createHash("sha256").update(record.payload.cwd).digest("hex"); + } + } + // Hidden reasoning, encrypted content, tool arguments and tool output are never indexed. + if (record.type === "response_item" && record.payload?.type === "reasoning") return; + const event = visibleEvent(record, securityCounts); + if (event) { + await indexHandle.write(`${JSON.stringify(event)}\n`); + metadata.visible_event_count += 1; + } + } catch { + metadata.invalid_json_count += 1; + } + }; + const transform = new Transform({ + transform(chunk, _encoding, callback) { + sourceHash.update(chunk); + lineBuffer += chunk.toString("utf8"); + const lines = lineBuffer.split("\n"); + lineBuffer = lines.pop() || ""; + (async () => { + for (const line of lines) await processLine(line); + })().then(() => callback(null, chunk), callback); + }, + flush(callback) { + (async () => { + if (lineBuffer) await processLine(lineBuffer); + })().then(() => callback(), callback); + }, + }); + try { + await pipeline( + fs.createReadStream(sourceFile), + transform, + fs.createWriteStream(stagingFile, { flags: "wx", mode: 0o600 }), + ); + } finally { + await indexHandle.close(); + } + return { + source_sha256: sourceHash.digest("hex"), + security_counts: securityCounts, + metadata, + }; +} + +async function ensureObject(staging, objectPath, expectedHash) { + await fs.promises.mkdir(path.dirname(objectPath), { recursive: true, mode: 0o700 }); + if (fs.existsSync(objectPath)) { + const existingHash = await hashFile(objectPath); + if (existingHash !== expectedHash) throw new Error(`OBJECT_HASH_COLLISION: ${objectPath}`); + await fs.promises.unlink(staging); + return "deduplicated"; + } + await fs.promises.rename(staging, objectPath); + const readback = await hashFile(objectPath); + if (readback !== expectedHash) throw new Error(`ARCHIVE_READBACK_FAILED: ${objectPath}`); + return "created"; +} + +async function archiveOne({ sourceRoot, destination, sourceFile, stableSeconds, stagingRoot }) { + const relative = safeRelative(sourceRoot, sourceFile); + const before = await fs.promises.stat(sourceFile); + const ageSeconds = (Date.now() - before.mtimeMs) / 1000; + if (ageSeconds < stableSeconds) { + return { relative_path: relative, state: "pending_live", size: before.size, mtime: before.mtime.toISOString() }; + } + const nonce = crypto.randomUUID(); + const staging = path.join(stagingRoot, `${nonce}.jsonl`); + const stagingIndex = path.join(stagingRoot, `${nonce}.index.jsonl`); + const copied = await copyAndIndex(sourceFile, staging, stagingIndex); + const after = await fs.promises.stat(sourceFile); + if (!sameStat(before, after)) { + await fs.promises.rm(staging, { force: true }); + await fs.promises.rm(stagingIndex, { force: true }); + return { relative_path: relative, state: "pending_changed_during_copy", size: after.size, mtime: after.mtime.toISOString() }; + } + const archiveReadbackHash = await hashFile(staging); + if (archiveReadbackHash !== copied.source_sha256) { + throw new Error(`STAGING_READBACK_FAILED: ${relative}`); + } + const shard = copied.source_sha256.slice(0, 2); + const objectRelative = `objects/sha256/${shard}/${copied.source_sha256}.jsonl`; + const indexRelative = `private-index/sha256/${shard}/${copied.source_sha256}.jsonl`; + const objectState = await ensureObject(staging, path.join(destination, objectRelative), copied.source_sha256); + const indexHash = await hashFile(stagingIndex); + const indexState = await ensureObject(stagingIndex, path.join(destination, indexRelative), indexHash); + return { + relative_path: relative, + state: "archived_verified", + size: before.size, + mtime: before.mtime.toISOString(), + sha256: copied.source_sha256, + object: objectRelative, + object_state: objectState, + private_index: indexRelative, + private_index_sha256: indexHash, + private_index_state: indexState, + ...copied.metadata, + security_counts: copied.security_counts, + }; +} + +export async function archiveSessions({ + source, + destination, + stableSeconds = DEFAULT_STABLE_SECONDS, + now = new Date(), + onProgress = null, +}) { + const sourceRoot = path.resolve(source); + const destinationRoot = path.resolve(destination); + if (!fs.statSync(sourceRoot).isDirectory()) throw new Error("SOURCE_NOT_DIRECTORY"); + await fs.promises.mkdir(destinationRoot, { recursive: true, mode: 0o700 }); + const stagingRoot = path.join(destinationRoot, ".staging"); + await fs.promises.mkdir(stagingRoot, { recursive: true, mode: 0o700 }); + const snapshotId = isoForPath(now); + const files = await listSessionFiles(sourceRoot); + const entries = []; + for (let index = 0; index < files.length; index += 1) { + const sourceFile = files[index]; + const entry = await archiveOne({ + sourceRoot, + destination: destinationRoot, + sourceFile, + stableSeconds: Number(stableSeconds), + stagingRoot, + }); + entries.push(entry); + onProgress?.({ current: index + 1, total: files.length, relative_path: entry.relative_path, state: entry.state }); + } + const archived = entries.filter((entry) => entry.state === "archived_verified"); + const pending = entries.filter((entry) => entry.state !== "archived_verified"); + const manifest = { + schema: SCHEMA, + snapshot_id: snapshotId, + created_at: now.toISOString(), + source_kind: "codex_session_jsonl", + source_root_fingerprint: crypto.createHash("sha256").update(sourceRoot).digest("hex"), + privacy_boundary: { + raw_objects: "PRIVATE_JZAO_ONLY", + private_index: "PRIVATE_JZAO_ONLY_REDACTED_VISIBLE_EVENTS", + repository_publication: "SCHEMA_CODE_AND_REVIEWED_DISTILLATION_ONLY", + hidden_reasoning_indexed: false, + tool_arguments_or_outputs_indexed: false, + }, + totals: { + discovered_files: entries.length, + archived_verified: archived.length, + pending_live: pending.length, + archived_bytes: archived.reduce((sum, entry) => sum + entry.size, 0), + }, + entries, + }; + const snapshotDir = path.join(destinationRoot, "snapshots", snapshotId); + await fs.promises.mkdir(snapshotDir, { recursive: true, mode: 0o700 }); + const manifestPath = path.join(snapshotDir, "manifest.json"); + await fs.promises.writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, { flag: "wx", mode: 0o600 }); + const manifestSha256 = await hashFile(manifestPath); + const latest = { + schema: "guanghu.codex-session-cold-archive-pointer/v1", + snapshot_id: snapshotId, + manifest: path.relative(destinationRoot, manifestPath).split(path.sep).join("/"), + manifest_sha256: manifestSha256, + archived_verified: archived.length, + pending_live: pending.length, + updated_at: now.toISOString(), + }; + const latestTemp = path.join(destinationRoot, `.LATEST-${crypto.randomUUID()}.json`); + await fs.promises.writeFile(latestTemp, `${JSON.stringify(latest, null, 2)}\n`, { mode: 0o600 }); + await fs.promises.rename(latestTemp, path.join(destinationRoot, "LATEST.json")); + await fs.promises.rm(stagingRoot, { recursive: true, force: true }); + const appleDoubleRemoved = await removeAppleDouble(destinationRoot); + return { manifest, manifestPath, manifestSha256, latest, appleDoubleRemoved }; +} + +export async function verifyManifest({ destination, manifest }) { + const destinationRoot = path.resolve(destination); + const manifestPath = path.resolve(manifest); + const parsed = JSON.parse(await fs.promises.readFile(manifestPath, "utf8")); + if (parsed.schema !== SCHEMA) throw new Error("MANIFEST_SCHEMA_MISMATCH"); + let checked = 0; + for (const entry of parsed.entries) { + if (entry.state !== "archived_verified") continue; + const objectPath = path.join(destinationRoot, entry.object); + const indexPath = path.join(destinationRoot, entry.private_index); + if (safeRelative(destinationRoot, objectPath) !== entry.object) throw new Error("OBJECT_PATH_MISMATCH"); + if (safeRelative(destinationRoot, indexPath) !== entry.private_index) throw new Error("INDEX_PATH_MISMATCH"); + if (await hashFile(objectPath) !== entry.sha256) throw new Error(`OBJECT_VERIFY_FAILED: ${entry.relative_path}`); + if (await hashFile(indexPath) !== entry.private_index_sha256) throw new Error(`INDEX_VERIFY_FAILED: ${entry.relative_path}`); + checked += 1; + } + const appleDouble = []; + async function scan(current) { + for (const entry of await fs.promises.readdir(current, { withFileTypes: true })) { + if (entry.name.startsWith("._")) appleDouble.push(path.join(current, entry.name)); + if (entry.isDirectory()) await scan(path.join(current, entry.name)); + } + } + await scan(destinationRoot); + if (appleDouble.length) throw new Error(`APPLEDOUBLE_PRESENT: ${appleDouble.length}`); + return { ok: true, checked_objects: checked, pending_live: parsed.totals.pending_live, manifest_sha256: await hashFile(manifestPath) }; +} + +export async function restoreManifest({ destination, manifest, restoreRoot, execute = false }) { + const destinationRoot = path.resolve(destination); + const parsed = JSON.parse(await fs.promises.readFile(path.resolve(manifest), "utf8")); + if (parsed.schema !== SCHEMA) throw new Error("MANIFEST_SCHEMA_MISMATCH"); + const plan = []; + for (const entry of parsed.entries) { + if (entry.state !== "archived_verified") continue; + const output = path.join(path.resolve(restoreRoot), entry.relative_path); + safeRelative(path.resolve(restoreRoot), output); + plan.push({ relative_path: entry.relative_path, sha256: entry.sha256, output }); + if (!execute) continue; + await fs.promises.mkdir(path.dirname(output), { recursive: true, mode: 0o700 }); + if (fs.existsSync(output)) { + if (await hashFile(output) === entry.sha256) continue; + throw new Error(`RESTORE_TARGET_EXISTS: ${output}`); + } + await fs.promises.copyFile(path.join(destinationRoot, entry.object), output, fs.constants.COPYFILE_EXCL); + if (await hashFile(output) !== entry.sha256) throw new Error(`RESTORE_READBACK_FAILED: ${output}`); + } + return { execute, planned_files: plan.length, plan }; +} + +async function main() { + const args = parseArgs(process.argv); + let result; + if (args.command === "archive") { + result = await archiveSessions({ + source: required(args.source, "source"), + destination: required(args.destination, "destination"), + stableSeconds: Number(args.stable_seconds || DEFAULT_STABLE_SECONDS), + onProgress: (progress) => process.stderr.write( + `ARCHIVE_PROGRESS ${progress.current}/${progress.total} ${progress.state} ${progress.relative_path}\n`, + ), + }); + result = { + manifest: result.manifestPath, + manifest_sha256: result.manifestSha256, + totals: result.manifest.totals, + latest: result.latest, + apple_double_removed: result.appleDoubleRemoved, + }; + } else if (args.command === "verify") { + result = await verifyManifest({ + destination: required(args.destination, "destination"), + manifest: required(args.manifest, "manifest"), + }); + } else if (args.command === "restore") { + result = await restoreManifest({ + destination: required(args.destination, "destination"), + manifest: required(args.manifest, "manifest"), + restoreRoot: required(args.restore_root, "restore_root"), + execute: args.execute, + }); + } else { + throw new Error("COMMAND_REQUIRED: archive | verify | restore"); + } + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); +} + +const isCli = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (isCli) main().catch((error) => { + process.stderr.write(`${error.message}\n`); + process.exitCode = 1; +}); diff --git a/server-tools/codex-session-cognition-archive/archive.test.mjs b/server-tools/codex-session-cognition-archive/archive.test.mjs new file mode 100644 index 0000000..021f5ec --- /dev/null +++ b/server-tools/codex-session-cognition-archive/archive.test.mjs @@ -0,0 +1,70 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { + archiveSessions, + redactVisibleText, + restoreManifest, + verifyManifest, +} from "./archive.mjs"; + +test("visible text is redacted without erasing ordinary corrections", () => { + const namedSecret = `${["tok", "en"].join("")}=abc123456789`; + const result = redactVisibleText(`不是这个方向。${namedSecret} user@example.com 10.0.0.8 /Users/demo/work`); + assert.match(result.text, /不是这个方向/); + assert.doesNotMatch(result.text, /abc123456789|user@example\.com|10\.0\.0\.8/); + assert.equal(result.counts.named_secret, 1); + assert.equal(result.counts.email, 1); + assert.equal(result.counts.ip_address, 1); +}); + +test("archive is content addressed, excludes reasoning and restores exact bytes", async () => { + const root = await fs.promises.mkdtemp(path.join(os.tmpdir(), "codex-session-archive-")); + const source = path.join(root, "source"); + const destination = path.join(root, "archive"); + const restoreRoot = path.join(root, "restore"); + const session = path.join(source, "2026", "08", "05", "rollout-test.jsonl"); + await fs.promises.mkdir(path.dirname(session), { recursive: true }); + await fs.promises.mkdir(destination, { recursive: true }); + await fs.promises.writeFile(path.join(destination, "._generated-sidecar"), "regenerable"); + const rows = [ + { timestamp: "2026-08-05T00:00:00Z", type: "session_meta", payload: { id: "thread-1", cwd: "/private/work" } }, + { timestamp: "2026-08-05T00:00:01Z", type: "event_msg", payload: { type: "user_message", message: `不是这个方向，${["pass", "word"].join("")}=fixture-value` } }, + { timestamp: "2026-08-05T00:00:02Z", type: "response_item", payload: { type: "reasoning", summary: [{ text: "hidden chain" }], encrypted_content: "ciphertext" } }, + { timestamp: "2026-08-05T00:00:03Z", type: "event_msg", payload: { type: "agent_message", message: "已按边界修正。" } }, + { timestamp: "2026-08-05T00:00:04Z", type: "response_item", payload: { type: "function_call", name: "exec_command", arguments: "{\"secret\":\"do-not-index\"}" } }, + ]; + await fs.promises.writeFile(session, `${rows.map((row) => JSON.stringify(row)).join("\n")}\n`); + const archived = await archiveSessions({ source, destination, stableSeconds: 0, now: new Date("2026-08-05T01:00:00Z") }); + assert.equal(archived.manifest.totals.archived_verified, 1); + assert.equal(archived.manifest.totals.pending_live, 0); + assert.equal(archived.appleDoubleRemoved, 1); + assert.equal(fs.existsSync(path.join(destination, "._generated-sidecar")), false); + const entry = archived.manifest.entries[0]; + const indexText = await fs.promises.readFile(path.join(destination, entry.private_index), "utf8"); + assert.match(indexText, /不是这个方向/); + assert.match(indexText, /REDACTED:NAMED_SECRET/); + assert.match(indexText, /exec_command/); + assert.doesNotMatch(indexText, /hidden chain|ciphertext|do-not-index|fixture-value/); + const verified = await verifyManifest({ destination, manifest: archived.manifestPath }); + assert.equal(verified.ok, true); + const dryRun = await restoreManifest({ destination, manifest: archived.manifestPath, restoreRoot, execute: false }); + assert.equal(dryRun.planned_files, 1); + assert.equal(fs.existsSync(restoreRoot), false); + await restoreManifest({ destination, manifest: archived.manifestPath, restoreRoot, execute: true }); + assert.equal(await fs.promises.readFile(path.join(restoreRoot, entry.relative_path), "utf8"), await fs.promises.readFile(session, "utf8")); +}); + +test("recent live sessions are recorded as pending instead of misreported archived", async () => { + const root = await fs.promises.mkdtemp(path.join(os.tmpdir(), "codex-session-live-")); + const source = path.join(root, "source"); + const destination = path.join(root, "archive"); + await fs.promises.mkdir(source); + await fs.promises.writeFile(path.join(source, "live.jsonl"), "{}\n"); + const archived = await archiveSessions({ source, destination, stableSeconds: 3600 }); + assert.equal(archived.manifest.totals.archived_verified, 0); + assert.equal(archived.manifest.totals.pending_live, 1); + assert.equal(archived.manifest.entries[0].state, "pending_live"); +});