feat: archive Codex sessions as private cognition evidence
This commit is contained in:
parent
0ac0025fdf
commit
7d85d79e61
8 changed files with 798 additions and 0 deletions
|
|
@ -514,6 +514,10 @@ JD-DR-RECEIPT-20260720=deployment/receipts/ICE-SIX-NODE-JD-DISASTER-RECOVERY-202
|
|||
LAKE-LAMP-AUTHZ-ENTRY=server-tools/lake-lamp-authz/README.md
|
||||
JD-ARCH-PROVISION-01=server-tools/lake-lamp-authz/README.md
|
||||
ARCHITECTURE-PROVISION-REQUESTS=deployment/requests/README.md
|
||||
CODEX-DEVELOPMENT-COGNITION-CHAIN-MAP-001=routing/codex-development-cognition-chain-map.json
|
||||
GLW-OS-005=glw-architecture/GLW-OS-005-CODEX-DEVELOPMENT-COGNITION-CHAIN.hdlp
|
||||
CODEX-SESSION-COLD-ARCHIVE-TOOL=server-tools/codex-session-cognition-archive/archive.mjs
|
||||
CODEX-SESSION-COLD-ARCHIVE-RECEIPT-20260805=deployment/receipts/CODEX-SESSION-COLD-ARCHIVE-JZAO-20260805.json
|
||||
|
||||
# === 服务器 (L 编号) ===
|
||||
L-SG-001=server-vault:BS-SG-001 # 新加坡大脑 · 铸渊主力
|
||||
|
|
|
|||
|
|
@ -0,0 +1,48 @@
|
|||
{
|
||||
"schema": "guanghu.codex-session-cold-archive-receipt/v1",
|
||||
"receipt_id": "CODEX-SESSION-COLD-ARCHIVE-JZAO-20260805",
|
||||
"development_id": "DEV-20260805-002",
|
||||
"created_at": "2026-08-05T04:39:53Z",
|
||||
"source": {
|
||||
"kind": "Codex session JSONL",
|
||||
"content_published": false,
|
||||
"source_deleted": false
|
||||
},
|
||||
"private_archive": {
|
||||
"storage_class": "JZAO_PRIVATE",
|
||||
"canonical_path": "/Volumes/JZAO/HoloLake/codex-session-archives",
|
||||
"snapshot_id": "2026-08-05T04-36-48Z",
|
||||
"discovered_files": 55,
|
||||
"archived_verified": 53,
|
||||
"pending_live": 2,
|
||||
"archived_bytes": 2946329018,
|
||||
"manifest_sha256": "056f60d4a99c45a8089f8ace8df4d24556d935d55e672362799494a99fbd00c4",
|
||||
"latest_pointer_sha256": "6f00f4c02ced5a0eccde41067105433363af013bebee01fc36bc69105b3da055"
|
||||
},
|
||||
"verification": {
|
||||
"raw_object_sha256_readback": 100,
|
||||
"private_index_sha256_readback": 100,
|
||||
"isolated_restore_file_count": 53,
|
||||
"isolated_restore_readback": 100,
|
||||
"appledouble_absent": 100,
|
||||
"hidden_reasoning_indexed": 0,
|
||||
"tool_arguments_or_outputs_indexed": 0,
|
||||
"raw_content_entered_repository": 0
|
||||
},
|
||||
"filesystem_incident": {
|
||||
"event": "JZAO_UNMOUNTED_DURING_FIRST_ARCHIVE_ATTEMPT",
|
||||
"source_impact": "NONE",
|
||||
"first_verify_exit_code": 5,
|
||||
"repair_target": "disk6s1 / JZAO",
|
||||
"repair_result": "VOLUME_APPEARS_OK_EXIT_0",
|
||||
"independent_post_repair_verify": "VOLUME_APPEARS_OK_EXIT_0",
|
||||
"resume_behavior": "CONTENT_ADDRESSED_OBJECTS_DEDUPLICATED_AND_ARCHIVE_RESUMED"
|
||||
},
|
||||
"boundary": {
|
||||
"archive_53_closed_sessions": 100,
|
||||
"archive_all_55_sessions": 0,
|
||||
"reason": "Two Codex sessions were still being written and are explicitly pending the next incremental snapshot.",
|
||||
"local_space_released": 0,
|
||||
"next_action": "Keep active sessions local. Run the incremental archive again after they become stable; define a reviewed retention policy before deleting any local session."
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
# GLW-OS-005 · Codex 会话冷归档与光湖开发认知链
|
||||
|
||||
## 冰朔原意
|
||||
|
||||
Codex 会话不是普通垃圾。里面保存了需求如何被提出、方向如何被冰朔纠正、人格体如何
|
||||
读取现实、工程如何失败与修正，以及代码、测试、部署和回执如何形成。
|
||||
|
||||
这套系统保留的是**可见、可验证的开发认知链**，不是模型隐藏推理。隐藏推理、加密内容、
|
||||
工具参数与工具原始输出不进入认知索引。
|
||||
|
||||
## 三层世界
|
||||
|
||||
| 层 | 位置 | 内容 | 能否进入代码仓库 |
|
||||
|---|---|---|---|
|
||||
| 原始冷档案 | JZAO 私有对象库 | 原始 JSONL、SHA-256、快照清单 | 否 |
|
||||
| 私有开发索引 | JZAO 私有索引库 | 脱敏后的可见对话、动作名、纠偏信号、内容哈希 | 否 |
|
||||
| 复核认知卡 | REPO-012 | 需求、纠正、决策、失败经验、提交、回执指针 | 复核后可以 |
|
||||
|
||||
## 0/100
|
||||
|
||||
归档脚本执行过不等于归档存在。必须同时满足：
|
||||
|
||||
1. 每个原始对象 SHA-256 独立回读；
|
||||
2. 每个私有索引 SHA-256 独立回读；
|
||||
3. 清单记录仍在写入而未归档的会话；
|
||||
4. 隔离目录恢复后与原始对象逐字节一致；
|
||||
5. JZAO 中不存在干扰校验的 `._*` 旁文件。
|
||||
|
||||
五项同时成立，当前快照归档能力才是 `100`。
|
||||
|
||||
## 本机删除边界
|
||||
|
||||
归档完成后也不自动删除 `$HOME/.codex/sessions`。只有恢复能力为 `100`、近期热会话范围
|
||||
已经确定、Codex 对旧会话的产品依赖已经验证，并且形成明确保留策略后，才能释放精确的
|
||||
旧冷会话。不得把人格连续性、开发证据和产品会话入口当缓存清理。
|
||||
|
||||
## 机器入口
|
||||
|
||||
`routing/codex-development-cognition-chain-map.json`
|
||||
62
routing/codex-development-cognition-chain-map.json
Normal file
62
routing/codex-development-cognition-chain-map.json
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
{
|
||||
"schema": "guanghu.codex-development-cognition-chain-map/v1",
|
||||
"map_id": "CODEX-DEVELOPMENT-COGNITION-CHAIN-MAP-001",
|
||||
"owner_human": "ICE-GL∞",
|
||||
"responsible_persona": "ICE-P-ZY001",
|
||||
"state": "PRIVATE_ARCHIVE_AND_RESTORE_VERIFIED_53_OF_55_TWO_LIVE_PENDING",
|
||||
"purpose": "把冰朔与人格体的真实开发协作保存为可恢复、可检索、可复核的工程演化链，而不是公开模型隐藏推理。",
|
||||
"layers": [
|
||||
{
|
||||
"id": "RAW-COLD-ARCHIVE",
|
||||
"location_class": "JZAO_PRIVATE",
|
||||
"content": "原始Codex JSONL与逐快照清单",
|
||||
"publication": "FORBIDDEN"
|
||||
},
|
||||
{
|
||||
"id": "PRIVATE-DEVELOPMENT-INDEX",
|
||||
"location_class": "JZAO_PRIVATE",
|
||||
"content": "脱敏可见消息、动作名称、时间、纠偏标记、内容摘要哈希",
|
||||
"publication": "FORBIDDEN"
|
||||
},
|
||||
{
|
||||
"id": "REVIEWED-COGNITION-CARDS",
|
||||
"location_class": "REPO-012",
|
||||
"content": "复核后的需求、纠正、决策、失败经验、提交和回执指针",
|
||||
"publication": "ALLOW_AFTER_REVIEW_AND_REDACTION"
|
||||
}
|
||||
],
|
||||
"binary_gates": {
|
||||
"raw_archive_exists": "manifest_and_every_object_sha256_readback",
|
||||
"private_index_exists": "every_index_sha256_readback_and_hidden_reasoning_absent",
|
||||
"restorable": "dry_run_plan_and_isolated_restore_readback",
|
||||
"repository_cognition_exists": "reviewed_card_commit_and_remote_readback",
|
||||
"local_space_released": "only_after_archive_100_and_restore_100_and_explicit_retention_policy"
|
||||
},
|
||||
"privacy_rules": [
|
||||
"RAW_SESSIONS_NEVER_ENTER_REPOSITORY",
|
||||
"HIDDEN_REASONING_AND_ENCRYPTED_CONTENT_NEVER_ENTER_INDEX",
|
||||
"TOOL_ARGUMENTS_AND_OUTPUTS_NEVER_ENTER_INDEX",
|
||||
"SECRETS_EMAILS_IPS_AND_CREDENTIAL_URLS_ARE_REDACTED",
|
||||
"SESSION_ARCHIVE_IS_NOT_PERSONA_IDENTITY",
|
||||
"ARCHIVE_EXISTENCE_IS_NOT_SERVER_RUNTIME"
|
||||
],
|
||||
"implementation": {
|
||||
"tool": "server-tools/codex-session-cognition-archive/archive.mjs",
|
||||
"documentation": "server-tools/codex-session-cognition-archive/README.md",
|
||||
"private_archive_canonical_path": "/Volumes/JZAO/HoloLake/codex-session-archives",
|
||||
"private_path_publication_rule": "PATH_MAY_BE_REGISTERED_CONTENT_MUST_NOT_BE_PUBLISHED",
|
||||
"latest_public_receipt": "deployment/receipts/CODEX-SESSION-COLD-ARCHIVE-JZAO-20260805.json"
|
||||
},
|
||||
"current_verified_snapshot": {
|
||||
"snapshot_id": "2026-08-05T04-36-48Z",
|
||||
"discovered_files": 55,
|
||||
"archived_verified": 53,
|
||||
"pending_live": 2,
|
||||
"archived_bytes": 2946329018,
|
||||
"manifest_sha256": "056f60d4a99c45a8089f8ace8df4d24556d935d55e672362799494a99fbd00c4",
|
||||
"archive_verify": 100,
|
||||
"isolated_restore_readback": 100,
|
||||
"appledouble_absent": 100,
|
||||
"local_source_deleted": 0
|
||||
}
|
||||
}
|
||||
20
routing/codex-development-cognition-chain-map.test.js
Normal file
20
routing/codex-development-cognition-chain-map.test.js
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
const assert = require("node:assert/strict");
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
|
||||
const root = path.resolve(__dirname, "..");
|
||||
const map = JSON.parse(fs.readFileSync(path.join(__dirname, "codex-development-cognition-chain-map.json"), "utf8"));
|
||||
|
||||
assert.equal(map.map_id, "CODEX-DEVELOPMENT-COGNITION-CHAIN-MAP-001");
|
||||
assert.deepEqual(map.layers.map((layer) => layer.id), [
|
||||
"RAW-COLD-ARCHIVE",
|
||||
"PRIVATE-DEVELOPMENT-INDEX",
|
||||
"REVIEWED-COGNITION-CARDS",
|
||||
]);
|
||||
assert.equal(map.layers[0].publication, "FORBIDDEN");
|
||||
assert.equal(map.layers[1].publication, "FORBIDDEN");
|
||||
assert.equal(map.layers[2].publication, "ALLOW_AFTER_REVIEW_AND_REDACTION");
|
||||
assert.ok(map.privacy_rules.includes("HIDDEN_REASONING_AND_ENCRYPTED_CONTENT_NEVER_ENTER_INDEX"));
|
||||
assert.ok(fs.existsSync(path.join(root, map.implementation.tool)));
|
||||
assert.ok(fs.existsSync(path.join(root, map.implementation.documentation)));
|
||||
console.log("codex-development-cognition-chain-map: ok");
|
||||
52
server-tools/codex-session-cognition-archive/README.md
Normal file
52
server-tools/codex-session-cognition-archive/README.md
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
# Codex 会话冷归档与开发认知链
|
||||
|
||||
这个模块把 Codex 会话分成三个互不冒充的层：
|
||||
|
||||
1. `raw objects`：JZAO 私有、内容寻址的原始 JSONL 证据；
|
||||
2. `private index`：JZAO 私有、脱敏后的可见交流和工程事件索引；
|
||||
3. `reviewed cognition`：人工或人格体复核后才能进入 REPO-012 的认知卡片。
|
||||
|
||||
原始会话和私有索引禁止直接推入代码仓库。索引明确排除：
|
||||
|
||||
- `reasoning` 与 `encrypted_content`；
|
||||
- 工具参数、工具输出和命令正文；
|
||||
- 私钥、授权头、令牌、密码、邮箱、IP 与疑似高熵秘密。
|
||||
|
||||
## 增量归档
|
||||
|
||||
```bash
|
||||
COPYFILE_DISABLE=1 node archive.mjs archive \
|
||||
--source "$HOME/.codex/sessions" \
|
||||
--destination "/Volumes/JZAO/HoloLake/codex-session-archives"
|
||||
```
|
||||
|
||||
默认最近五分钟仍有写入的会话记为 `pending_live`，不会冒充完整归档。后续再次运行即可形成
|
||||
新快照和新对象，内容相同的文件自动去重。
|
||||
|
||||
归档过程被断电、拔盘或系统卸载打断时，已完成对象仍按内容哈希保留，`.staging` 中的临时
|
||||
文件不算完成。确认卷通过文件系统检查后重新运行即可续建；程序会去重已完成对象，并在
|
||||
成功快照结束时清除本归档目录内的 AppleDouble 旁文件。
|
||||
|
||||
## 独立校验
|
||||
|
||||
```bash
|
||||
COPYFILE_DISABLE=1 node archive.mjs verify \
|
||||
--destination "/Volumes/JZAO/HoloLake/codex-session-archives" \
|
||||
--manifest "/Volumes/JZAO/HoloLake/codex-session-archives/snapshots/<时间>/manifest.json"
|
||||
```
|
||||
|
||||
只有原始对象和私有索引都通过 SHA-256 回读、且没有 AppleDouble 文件时，归档能力才是
|
||||
`100`。
|
||||
|
||||
## 恢复
|
||||
|
||||
默认只生成恢复计划：
|
||||
|
||||
```bash
|
||||
node archive.mjs restore \
|
||||
--destination "/Volumes/JZAO/HoloLake/codex-session-archives" \
|
||||
--manifest "<manifest.json>" \
|
||||
--restore-root "/安全的新目录"
|
||||
```
|
||||
|
||||
确认目标是空目录后加 `--execute`。恢复器不会覆盖内容不同的现有文件。
|
||||
503
server-tools/codex-session-cognition-archive/archive.mjs
Normal file
503
server-tools/codex-session-cognition-archive/archive.mjs
Normal file
|
|
@ -0,0 +1,503 @@
|
|||
#!/usr/bin/env node
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import readline from "node:readline";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import { Transform } from "node:stream";
|
||||
|
||||
const SCHEMA = "guanghu.codex-session-cold-archive/v1";
|
||||
const INDEX_SCHEMA = "guanghu.development-cognition-event/v1";
|
||||
const DEFAULT_STABLE_SECONDS = 300;
|
||||
const MAX_VISIBLE_TEXT = 4000;
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = { command: argv[2] || "", execute: false };
|
||||
for (let i = 3; i < argv.length; i += 1) {
|
||||
const key = argv[i];
|
||||
if (key === "--execute") {
|
||||
args.execute = true;
|
||||
continue;
|
||||
}
|
||||
if (!key.startsWith("--") || i + 1 >= argv.length) {
|
||||
throw new Error(`INVALID_ARGUMENT: ${key}`);
|
||||
}
|
||||
args[key.slice(2).replaceAll("-", "_")] = argv[++i];
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
function required(value, name) {
|
||||
if (!value) throw new Error(`REQUIRED_ARGUMENT: --${name.replaceAll("_", "-")}`);
|
||||
return path.resolve(value);
|
||||
}
|
||||
|
||||
function isoForPath(date = new Date()) {
|
||||
return date.toISOString().replaceAll(":", "-").replace(/\.\d{3}Z$/, "Z");
|
||||
}
|
||||
|
||||
function safeRelative(root, candidate) {
|
||||
const relative = path.relative(root, candidate);
|
||||
if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) {
|
||||
throw new Error(`UNSAFE_RELATIVE_PATH: ${candidate}`);
|
||||
}
|
||||
return relative.split(path.sep).join("/");
|
||||
}
|
||||
|
||||
async function listSessionFiles(root) {
|
||||
const found = [];
|
||||
async function walk(current) {
|
||||
const entries = await fs.promises.readdir(current, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
if (entry.name.startsWith("._") || entry.name === ".DS_Store") continue;
|
||||
const absolute = path.join(current, entry.name);
|
||||
if (entry.isDirectory()) await walk(absolute);
|
||||
else if (entry.isFile() && entry.name.endsWith(".jsonl")) found.push(absolute);
|
||||
}
|
||||
}
|
||||
await walk(root);
|
||||
return found.sort();
|
||||
}
|
||||
|
||||
export async function removeAppleDouble(root) {
|
||||
let removed = 0;
|
||||
async function walk(current) {
|
||||
const entries = await fs.promises.readdir(current, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const absolute = path.join(current, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
await walk(absolute);
|
||||
} else if (entry.isFile() && entry.name.startsWith("._")) {
|
||||
await fs.promises.unlink(absolute);
|
||||
removed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
await walk(path.resolve(root));
|
||||
return removed;
|
||||
}
|
||||
|
||||
export function redactVisibleText(input, home = process.env.HOME || "") {
|
||||
let text = String(input ?? "");
|
||||
const counts = {
|
||||
private_key: 0,
|
||||
authorization: 0,
|
||||
named_secret: 0,
|
||||
email: 0,
|
||||
ip_address: 0,
|
||||
credential_url: 0,
|
||||
high_entropy: 0,
|
||||
};
|
||||
const replace = (pattern, label, key) => {
|
||||
text = text.replace(pattern, () => {
|
||||
counts[key] += 1;
|
||||
return `[REDACTED:${label}]`;
|
||||
});
|
||||
};
|
||||
replace(/-----BEGIN [^-]+ PRIVATE KEY-----[\s\S]*?-----END [^-]+ PRIVATE KEY-----/gi, "PRIVATE_KEY", "private_key");
|
||||
replace(/\b(?:Bearer|Basic)\s+[A-Za-z0-9._~+/=-]{8,}/gi, "AUTHORIZATION", "authorization");
|
||||
replace(/\b(?:api[_ -]?key|token|secret|password|passwd|authorization)\b\s*[:=]\s*["']?[^"',\s}]{4,}/gi, "NAMED_SECRET", "named_secret");
|
||||
replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, "EMAIL", "email");
|
||||
replace(/\b(?:(?:25[0-5]|2[0-4]\d|1?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|1?\d?\d)\b/g, "IP_ADDRESS", "ip_address");
|
||||
replace(/\bhttps?:\/\/[^/\s:@]+:[^@\s/]+@[^\s]+/gi, "CREDENTIAL_URL", "credential_url");
|
||||
text = text.replace(
|
||||
/\b(?=[A-Za-z0-9_+/.=-]{32,}\b)(?=[A-Za-z0-9_+/.=-]*[A-Za-z])(?=[A-Za-z0-9_+/.=-]*\d)[A-Za-z0-9_+/.=-]+\b/g,
|
||||
(match) => {
|
||||
// Git object IDs are essential engineering evidence. Named-secret rules above
|
||||
// still redact a hex credential when it is attached to a secret-bearing key.
|
||||
if (/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/i.test(match)) return match;
|
||||
counts.high_entropy += 1;
|
||||
return "[REDACTED:HIGH_ENTROPY]";
|
||||
},
|
||||
);
|
||||
if (home) text = text.split(home).join("$HOME");
|
||||
const originalLength = text.length;
|
||||
if (text.length > MAX_VISIBLE_TEXT) text = `${text.slice(0, MAX_VISIBLE_TEXT)}…[TRUNCATED]`;
|
||||
return { text, counts, truncated: originalLength > MAX_VISIBLE_TEXT };
|
||||
}
|
||||
|
||||
function mergeCounts(target, incoming) {
|
||||
for (const [key, value] of Object.entries(incoming)) target[key] = (target[key] || 0) + value;
|
||||
}
|
||||
|
||||
function textFromResponseContent(content) {
|
||||
if (typeof content === "string") return content;
|
||||
if (!Array.isArray(content)) return "";
|
||||
return content
|
||||
.filter((item) => item && typeof item === "object" && typeof item.text === "string")
|
||||
.map((item) => item.text)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function correctionSignals(text) {
|
||||
const markers = [
|
||||
"不是", "别搞错", "方向", "纠正", "你忘了", "走偏", "不应该",
|
||||
"应该是", "我的意思", "我说的是", "不要", "边界",
|
||||
];
|
||||
return markers.filter((marker) => text.includes(marker));
|
||||
}
|
||||
|
||||
function visibleEvent(record, securityCounts) {
|
||||
const timestamp = record.timestamp || record.payload?.timestamp || null;
|
||||
const payload = record.payload || {};
|
||||
let role = "";
|
||||
let kind = "";
|
||||
let rawText = "";
|
||||
if (record.type === "event_msg" && payload.type === "user_message") {
|
||||
role = "user";
|
||||
kind = "visible_message";
|
||||
rawText = payload.message || "";
|
||||
} else if (record.type === "event_msg" && payload.type === "agent_message") {
|
||||
role = "assistant";
|
||||
kind = "visible_message";
|
||||
rawText = payload.message || "";
|
||||
} else if (record.type === "response_item" && payload.type === "function_call") {
|
||||
role = "system";
|
||||
kind = "tool_call";
|
||||
rawText = payload.name || "unknown_tool";
|
||||
} else if (record.type === "response_item" && payload.type === "custom_tool_call") {
|
||||
role = "system";
|
||||
kind = "tool_call";
|
||||
rawText = payload.name || "unknown_tool";
|
||||
} else if (record.type === "event_msg" && payload.type === "task_started") {
|
||||
return { schema: INDEX_SCHEMA, timestamp, role: "system", kind: "task_started" };
|
||||
} else if (record.type === "event_msg" && payload.type === "task_complete") {
|
||||
return { schema: INDEX_SCHEMA, timestamp, role: "system", kind: "task_complete" };
|
||||
} else if (
|
||||
record.type === "response_item"
|
||||
&& payload.type === "message"
|
||||
&& ["user", "assistant"].includes(payload.role)
|
||||
) {
|
||||
role = payload.role;
|
||||
kind = "visible_message";
|
||||
rawText = textFromResponseContent(payload.content);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
if (!rawText) return null;
|
||||
const digest = crypto.createHash("sha256").update(String(rawText)).digest("hex");
|
||||
const redacted = redactVisibleText(rawText);
|
||||
mergeCounts(securityCounts, redacted.counts);
|
||||
const event = {
|
||||
schema: INDEX_SCHEMA,
|
||||
timestamp,
|
||||
role,
|
||||
kind,
|
||||
text_sha256: digest,
|
||||
text_length: String(rawText).length,
|
||||
text: redacted.text,
|
||||
truncated: redacted.truncated,
|
||||
};
|
||||
if (role === "user") event.correction_signals = correctionSignals(redacted.text);
|
||||
return event;
|
||||
}
|
||||
|
||||
async function hashFile(file) {
|
||||
const hash = crypto.createHash("sha256");
|
||||
const stream = fs.createReadStream(file);
|
||||
for await (const chunk of stream) hash.update(chunk);
|
||||
return hash.digest("hex");
|
||||
}
|
||||
|
||||
function sameStat(before, after) {
|
||||
return before.size === after.size && before.mtimeMs === after.mtimeMs;
|
||||
}
|
||||
|
||||
async function copyAndIndex(sourceFile, stagingFile, stagingIndex) {
|
||||
const sourceHash = crypto.createHash("sha256");
|
||||
const securityCounts = {};
|
||||
const metadata = {
|
||||
record_count: 0,
|
||||
invalid_json_count: 0,
|
||||
type_counts: {},
|
||||
session_id: null,
|
||||
cwd_fingerprint: null,
|
||||
first_timestamp: null,
|
||||
last_timestamp: null,
|
||||
visible_event_count: 0,
|
||||
};
|
||||
let lineBuffer = "";
|
||||
const indexHandle = await fs.promises.open(stagingIndex, "wx", 0o600);
|
||||
const processLine = async (line) => {
|
||||
if (!line.trim()) return;
|
||||
metadata.record_count += 1;
|
||||
try {
|
||||
const record = JSON.parse(line);
|
||||
const subtype = record.payload?.type ? `${record.type}:${record.payload.type}` : record.type;
|
||||
metadata.type_counts[subtype] = (metadata.type_counts[subtype] || 0) + 1;
|
||||
const timestamp = record.timestamp || record.payload?.timestamp || null;
|
||||
if (timestamp && !metadata.first_timestamp) metadata.first_timestamp = timestamp;
|
||||
if (timestamp) metadata.last_timestamp = timestamp;
|
||||
if (record.type === "session_meta") {
|
||||
metadata.session_id ||= record.payload?.id || record.payload?.session_id || null;
|
||||
if (record.payload?.cwd) {
|
||||
metadata.cwd_fingerprint = crypto.createHash("sha256").update(record.payload.cwd).digest("hex");
|
||||
}
|
||||
}
|
||||
// Hidden reasoning, encrypted content, tool arguments and tool output are never indexed.
|
||||
if (record.type === "response_item" && record.payload?.type === "reasoning") return;
|
||||
const event = visibleEvent(record, securityCounts);
|
||||
if (event) {
|
||||
await indexHandle.write(`${JSON.stringify(event)}\n`);
|
||||
metadata.visible_event_count += 1;
|
||||
}
|
||||
} catch {
|
||||
metadata.invalid_json_count += 1;
|
||||
}
|
||||
};
|
||||
const transform = new Transform({
|
||||
transform(chunk, _encoding, callback) {
|
||||
sourceHash.update(chunk);
|
||||
lineBuffer += chunk.toString("utf8");
|
||||
const lines = lineBuffer.split("\n");
|
||||
lineBuffer = lines.pop() || "";
|
||||
(async () => {
|
||||
for (const line of lines) await processLine(line);
|
||||
})().then(() => callback(null, chunk), callback);
|
||||
},
|
||||
flush(callback) {
|
||||
(async () => {
|
||||
if (lineBuffer) await processLine(lineBuffer);
|
||||
})().then(() => callback(), callback);
|
||||
},
|
||||
});
|
||||
try {
|
||||
await pipeline(
|
||||
fs.createReadStream(sourceFile),
|
||||
transform,
|
||||
fs.createWriteStream(stagingFile, { flags: "wx", mode: 0o600 }),
|
||||
);
|
||||
} finally {
|
||||
await indexHandle.close();
|
||||
}
|
||||
return {
|
||||
source_sha256: sourceHash.digest("hex"),
|
||||
security_counts: securityCounts,
|
||||
metadata,
|
||||
};
|
||||
}
|
||||
|
||||
async function ensureObject(staging, objectPath, expectedHash) {
|
||||
await fs.promises.mkdir(path.dirname(objectPath), { recursive: true, mode: 0o700 });
|
||||
if (fs.existsSync(objectPath)) {
|
||||
const existingHash = await hashFile(objectPath);
|
||||
if (existingHash !== expectedHash) throw new Error(`OBJECT_HASH_COLLISION: ${objectPath}`);
|
||||
await fs.promises.unlink(staging);
|
||||
return "deduplicated";
|
||||
}
|
||||
await fs.promises.rename(staging, objectPath);
|
||||
const readback = await hashFile(objectPath);
|
||||
if (readback !== expectedHash) throw new Error(`ARCHIVE_READBACK_FAILED: ${objectPath}`);
|
||||
return "created";
|
||||
}
|
||||
|
||||
async function archiveOne({ sourceRoot, destination, sourceFile, stableSeconds, stagingRoot }) {
|
||||
const relative = safeRelative(sourceRoot, sourceFile);
|
||||
const before = await fs.promises.stat(sourceFile);
|
||||
const ageSeconds = (Date.now() - before.mtimeMs) / 1000;
|
||||
if (ageSeconds < stableSeconds) {
|
||||
return { relative_path: relative, state: "pending_live", size: before.size, mtime: before.mtime.toISOString() };
|
||||
}
|
||||
const nonce = crypto.randomUUID();
|
||||
const staging = path.join(stagingRoot, `${nonce}.jsonl`);
|
||||
const stagingIndex = path.join(stagingRoot, `${nonce}.index.jsonl`);
|
||||
const copied = await copyAndIndex(sourceFile, staging, stagingIndex);
|
||||
const after = await fs.promises.stat(sourceFile);
|
||||
if (!sameStat(before, after)) {
|
||||
await fs.promises.rm(staging, { force: true });
|
||||
await fs.promises.rm(stagingIndex, { force: true });
|
||||
return { relative_path: relative, state: "pending_changed_during_copy", size: after.size, mtime: after.mtime.toISOString() };
|
||||
}
|
||||
const archiveReadbackHash = await hashFile(staging);
|
||||
if (archiveReadbackHash !== copied.source_sha256) {
|
||||
throw new Error(`STAGING_READBACK_FAILED: ${relative}`);
|
||||
}
|
||||
const shard = copied.source_sha256.slice(0, 2);
|
||||
const objectRelative = `objects/sha256/${shard}/${copied.source_sha256}.jsonl`;
|
||||
const indexRelative = `private-index/sha256/${shard}/${copied.source_sha256}.jsonl`;
|
||||
const objectState = await ensureObject(staging, path.join(destination, objectRelative), copied.source_sha256);
|
||||
const indexHash = await hashFile(stagingIndex);
|
||||
const indexState = await ensureObject(stagingIndex, path.join(destination, indexRelative), indexHash);
|
||||
return {
|
||||
relative_path: relative,
|
||||
state: "archived_verified",
|
||||
size: before.size,
|
||||
mtime: before.mtime.toISOString(),
|
||||
sha256: copied.source_sha256,
|
||||
object: objectRelative,
|
||||
object_state: objectState,
|
||||
private_index: indexRelative,
|
||||
private_index_sha256: indexHash,
|
||||
private_index_state: indexState,
|
||||
...copied.metadata,
|
||||
security_counts: copied.security_counts,
|
||||
};
|
||||
}
|
||||
|
||||
export async function archiveSessions({
|
||||
source,
|
||||
destination,
|
||||
stableSeconds = DEFAULT_STABLE_SECONDS,
|
||||
now = new Date(),
|
||||
onProgress = null,
|
||||
}) {
|
||||
const sourceRoot = path.resolve(source);
|
||||
const destinationRoot = path.resolve(destination);
|
||||
if (!fs.statSync(sourceRoot).isDirectory()) throw new Error("SOURCE_NOT_DIRECTORY");
|
||||
await fs.promises.mkdir(destinationRoot, { recursive: true, mode: 0o700 });
|
||||
const stagingRoot = path.join(destinationRoot, ".staging");
|
||||
await fs.promises.mkdir(stagingRoot, { recursive: true, mode: 0o700 });
|
||||
const snapshotId = isoForPath(now);
|
||||
const files = await listSessionFiles(sourceRoot);
|
||||
const entries = [];
|
||||
for (let index = 0; index < files.length; index += 1) {
|
||||
const sourceFile = files[index];
|
||||
const entry = await archiveOne({
|
||||
sourceRoot,
|
||||
destination: destinationRoot,
|
||||
sourceFile,
|
||||
stableSeconds: Number(stableSeconds),
|
||||
stagingRoot,
|
||||
});
|
||||
entries.push(entry);
|
||||
onProgress?.({ current: index + 1, total: files.length, relative_path: entry.relative_path, state: entry.state });
|
||||
}
|
||||
const archived = entries.filter((entry) => entry.state === "archived_verified");
|
||||
const pending = entries.filter((entry) => entry.state !== "archived_verified");
|
||||
const manifest = {
|
||||
schema: SCHEMA,
|
||||
snapshot_id: snapshotId,
|
||||
created_at: now.toISOString(),
|
||||
source_kind: "codex_session_jsonl",
|
||||
source_root_fingerprint: crypto.createHash("sha256").update(sourceRoot).digest("hex"),
|
||||
privacy_boundary: {
|
||||
raw_objects: "PRIVATE_JZAO_ONLY",
|
||||
private_index: "PRIVATE_JZAO_ONLY_REDACTED_VISIBLE_EVENTS",
|
||||
repository_publication: "SCHEMA_CODE_AND_REVIEWED_DISTILLATION_ONLY",
|
||||
hidden_reasoning_indexed: false,
|
||||
tool_arguments_or_outputs_indexed: false,
|
||||
},
|
||||
totals: {
|
||||
discovered_files: entries.length,
|
||||
archived_verified: archived.length,
|
||||
pending_live: pending.length,
|
||||
archived_bytes: archived.reduce((sum, entry) => sum + entry.size, 0),
|
||||
},
|
||||
entries,
|
||||
};
|
||||
const snapshotDir = path.join(destinationRoot, "snapshots", snapshotId);
|
||||
await fs.promises.mkdir(snapshotDir, { recursive: true, mode: 0o700 });
|
||||
const manifestPath = path.join(snapshotDir, "manifest.json");
|
||||
await fs.promises.writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, { flag: "wx", mode: 0o600 });
|
||||
const manifestSha256 = await hashFile(manifestPath);
|
||||
const latest = {
|
||||
schema: "guanghu.codex-session-cold-archive-pointer/v1",
|
||||
snapshot_id: snapshotId,
|
||||
manifest: path.relative(destinationRoot, manifestPath).split(path.sep).join("/"),
|
||||
manifest_sha256: manifestSha256,
|
||||
archived_verified: archived.length,
|
||||
pending_live: pending.length,
|
||||
updated_at: now.toISOString(),
|
||||
};
|
||||
const latestTemp = path.join(destinationRoot, `.LATEST-${crypto.randomUUID()}.json`);
|
||||
await fs.promises.writeFile(latestTemp, `${JSON.stringify(latest, null, 2)}\n`, { mode: 0o600 });
|
||||
await fs.promises.rename(latestTemp, path.join(destinationRoot, "LATEST.json"));
|
||||
await fs.promises.rm(stagingRoot, { recursive: true, force: true });
|
||||
const appleDoubleRemoved = await removeAppleDouble(destinationRoot);
|
||||
return { manifest, manifestPath, manifestSha256, latest, appleDoubleRemoved };
|
||||
}
|
||||
|
||||
export async function verifyManifest({ destination, manifest }) {
|
||||
const destinationRoot = path.resolve(destination);
|
||||
const manifestPath = path.resolve(manifest);
|
||||
const parsed = JSON.parse(await fs.promises.readFile(manifestPath, "utf8"));
|
||||
if (parsed.schema !== SCHEMA) throw new Error("MANIFEST_SCHEMA_MISMATCH");
|
||||
let checked = 0;
|
||||
for (const entry of parsed.entries) {
|
||||
if (entry.state !== "archived_verified") continue;
|
||||
const objectPath = path.join(destinationRoot, entry.object);
|
||||
const indexPath = path.join(destinationRoot, entry.private_index);
|
||||
if (safeRelative(destinationRoot, objectPath) !== entry.object) throw new Error("OBJECT_PATH_MISMATCH");
|
||||
if (safeRelative(destinationRoot, indexPath) !== entry.private_index) throw new Error("INDEX_PATH_MISMATCH");
|
||||
if (await hashFile(objectPath) !== entry.sha256) throw new Error(`OBJECT_VERIFY_FAILED: ${entry.relative_path}`);
|
||||
if (await hashFile(indexPath) !== entry.private_index_sha256) throw new Error(`INDEX_VERIFY_FAILED: ${entry.relative_path}`);
|
||||
checked += 1;
|
||||
}
|
||||
const appleDouble = [];
|
||||
async function scan(current) {
|
||||
for (const entry of await fs.promises.readdir(current, { withFileTypes: true })) {
|
||||
if (entry.name.startsWith("._")) appleDouble.push(path.join(current, entry.name));
|
||||
if (entry.isDirectory()) await scan(path.join(current, entry.name));
|
||||
}
|
||||
}
|
||||
await scan(destinationRoot);
|
||||
if (appleDouble.length) throw new Error(`APPLEDOUBLE_PRESENT: ${appleDouble.length}`);
|
||||
return { ok: true, checked_objects: checked, pending_live: parsed.totals.pending_live, manifest_sha256: await hashFile(manifestPath) };
|
||||
}
|
||||
|
||||
export async function restoreManifest({ destination, manifest, restoreRoot, execute = false }) {
|
||||
const destinationRoot = path.resolve(destination);
|
||||
const parsed = JSON.parse(await fs.promises.readFile(path.resolve(manifest), "utf8"));
|
||||
if (parsed.schema !== SCHEMA) throw new Error("MANIFEST_SCHEMA_MISMATCH");
|
||||
const plan = [];
|
||||
for (const entry of parsed.entries) {
|
||||
if (entry.state !== "archived_verified") continue;
|
||||
const output = path.join(path.resolve(restoreRoot), entry.relative_path);
|
||||
safeRelative(path.resolve(restoreRoot), output);
|
||||
plan.push({ relative_path: entry.relative_path, sha256: entry.sha256, output });
|
||||
if (!execute) continue;
|
||||
await fs.promises.mkdir(path.dirname(output), { recursive: true, mode: 0o700 });
|
||||
if (fs.existsSync(output)) {
|
||||
if (await hashFile(output) === entry.sha256) continue;
|
||||
throw new Error(`RESTORE_TARGET_EXISTS: ${output}`);
|
||||
}
|
||||
await fs.promises.copyFile(path.join(destinationRoot, entry.object), output, fs.constants.COPYFILE_EXCL);
|
||||
if (await hashFile(output) !== entry.sha256) throw new Error(`RESTORE_READBACK_FAILED: ${output}`);
|
||||
}
|
||||
return { execute, planned_files: plan.length, plan };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv);
|
||||
let result;
|
||||
if (args.command === "archive") {
|
||||
result = await archiveSessions({
|
||||
source: required(args.source, "source"),
|
||||
destination: required(args.destination, "destination"),
|
||||
stableSeconds: Number(args.stable_seconds || DEFAULT_STABLE_SECONDS),
|
||||
onProgress: (progress) => process.stderr.write(
|
||||
`ARCHIVE_PROGRESS ${progress.current}/${progress.total} ${progress.state} ${progress.relative_path}\n`,
|
||||
),
|
||||
});
|
||||
result = {
|
||||
manifest: result.manifestPath,
|
||||
manifest_sha256: result.manifestSha256,
|
||||
totals: result.manifest.totals,
|
||||
latest: result.latest,
|
||||
apple_double_removed: result.appleDoubleRemoved,
|
||||
};
|
||||
} else if (args.command === "verify") {
|
||||
result = await verifyManifest({
|
||||
destination: required(args.destination, "destination"),
|
||||
manifest: required(args.manifest, "manifest"),
|
||||
});
|
||||
} else if (args.command === "restore") {
|
||||
result = await restoreManifest({
|
||||
destination: required(args.destination, "destination"),
|
||||
manifest: required(args.manifest, "manifest"),
|
||||
restoreRoot: required(args.restore_root, "restore_root"),
|
||||
execute: args.execute,
|
||||
});
|
||||
} else {
|
||||
throw new Error("COMMAND_REQUIRED: archive | verify | restore");
|
||||
}
|
||||
process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
|
||||
}
|
||||
|
||||
const isCli = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
|
||||
if (isCli) main().catch((error) => {
|
||||
process.stderr.write(`${error.message}\n`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import {
|
||||
archiveSessions,
|
||||
redactVisibleText,
|
||||
restoreManifest,
|
||||
verifyManifest,
|
||||
} from "./archive.mjs";
|
||||
|
||||
test("visible text is redacted without erasing ordinary corrections", () => {
|
||||
const namedSecret = `${["tok", "en"].join("")}=abc123456789`;
|
||||
const result = redactVisibleText(`不是这个方向。${namedSecret} user@example.com 10.0.0.8 /Users/demo/work`);
|
||||
assert.match(result.text, /不是这个方向/);
|
||||
assert.doesNotMatch(result.text, /abc123456789|user@example\.com|10\.0\.0\.8/);
|
||||
assert.equal(result.counts.named_secret, 1);
|
||||
assert.equal(result.counts.email, 1);
|
||||
assert.equal(result.counts.ip_address, 1);
|
||||
});
|
||||
|
||||
test("archive is content addressed, excludes reasoning and restores exact bytes", async () => {
|
||||
const root = await fs.promises.mkdtemp(path.join(os.tmpdir(), "codex-session-archive-"));
|
||||
const source = path.join(root, "source");
|
||||
const destination = path.join(root, "archive");
|
||||
const restoreRoot = path.join(root, "restore");
|
||||
const session = path.join(source, "2026", "08", "05", "rollout-test.jsonl");
|
||||
await fs.promises.mkdir(path.dirname(session), { recursive: true });
|
||||
await fs.promises.mkdir(destination, { recursive: true });
|
||||
await fs.promises.writeFile(path.join(destination, "._generated-sidecar"), "regenerable");
|
||||
const rows = [
|
||||
{ timestamp: "2026-08-05T00:00:00Z", type: "session_meta", payload: { id: "thread-1", cwd: "/private/work" } },
|
||||
{ timestamp: "2026-08-05T00:00:01Z", type: "event_msg", payload: { type: "user_message", message: `不是这个方向，${["pass", "word"].join("")}=fixture-value` } },
|
||||
{ timestamp: "2026-08-05T00:00:02Z", type: "response_item", payload: { type: "reasoning", summary: [{ text: "hidden chain" }], encrypted_content: "ciphertext" } },
|
||||
{ timestamp: "2026-08-05T00:00:03Z", type: "event_msg", payload: { type: "agent_message", message: "已按边界修正。" } },
|
||||
{ timestamp: "2026-08-05T00:00:04Z", type: "response_item", payload: { type: "function_call", name: "exec_command", arguments: "{\"secret\":\"do-not-index\"}" } },
|
||||
];
|
||||
await fs.promises.writeFile(session, `${rows.map((row) => JSON.stringify(row)).join("\n")}\n`);
|
||||
const archived = await archiveSessions({ source, destination, stableSeconds: 0, now: new Date("2026-08-05T01:00:00Z") });
|
||||
assert.equal(archived.manifest.totals.archived_verified, 1);
|
||||
assert.equal(archived.manifest.totals.pending_live, 0);
|
||||
assert.equal(archived.appleDoubleRemoved, 1);
|
||||
assert.equal(fs.existsSync(path.join(destination, "._generated-sidecar")), false);
|
||||
const entry = archived.manifest.entries[0];
|
||||
const indexText = await fs.promises.readFile(path.join(destination, entry.private_index), "utf8");
|
||||
assert.match(indexText, /不是这个方向/);
|
||||
assert.match(indexText, /REDACTED:NAMED_SECRET/);
|
||||
assert.match(indexText, /exec_command/);
|
||||
assert.doesNotMatch(indexText, /hidden chain|ciphertext|do-not-index|fixture-value/);
|
||||
const verified = await verifyManifest({ destination, manifest: archived.manifestPath });
|
||||
assert.equal(verified.ok, true);
|
||||
const dryRun = await restoreManifest({ destination, manifest: archived.manifestPath, restoreRoot, execute: false });
|
||||
assert.equal(dryRun.planned_files, 1);
|
||||
assert.equal(fs.existsSync(restoreRoot), false);
|
||||
await restoreManifest({ destination, manifest: archived.manifestPath, restoreRoot, execute: true });
|
||||
assert.equal(await fs.promises.readFile(path.join(restoreRoot, entry.relative_path), "utf8"), await fs.promises.readFile(session, "utf8"));
|
||||
});
|
||||
|
||||
test("recent live sessions are recorded as pending instead of misreported archived", async () => {
|
||||
const root = await fs.promises.mkdtemp(path.join(os.tmpdir(), "codex-session-live-"));
|
||||
const source = path.join(root, "source");
|
||||
const destination = path.join(root, "archive");
|
||||
await fs.promises.mkdir(source);
|
||||
await fs.promises.writeFile(path.join(source, "live.jsonl"), "{}\n");
|
||||
const archived = await archiveSessions({ source, destination, stableSeconds: 3600 });
|
||||
assert.equal(archived.manifest.totals.archived_verified, 0);
|
||||
assert.equal(archived.manifest.totals.pending_live, 1);
|
||||
assert.equal(archived.manifest.entries[0].state, "pending_live");
|
||||
});
|
||||
Loading…
Reference in a new issue