118 lines
4.8 KiB
Python
118 lines
4.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Index direct user messages from one Codex rollout as immutable source segments."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
|
|
SOURCE = Path(
|
|
"/Users/bingshuo/.codex/sessions/2026/08/23/"
|
|
"rollout-2026-08-23T00-26-28-01a02a4b-8ab1-7f53-8df8-787cbc2909ad.jsonl"
|
|
)
|
|
OUT = Path(
|
|
"/Volumes/JZAO/铸渊-ICE-GL-ZY001/CODEX-DEV-20260823/"
|
|
"mother-persona-neuron-extraction-20260823/corpus-manifests/"
|
|
"CORPUS-BS-CODEX-CURRENT-0001.direct-user-segments.jsonl"
|
|
)
|
|
AUDIT = OUT.with_name("CORPUS-BS-CODEX-CURRENT-0001.audit.json")
|
|
CORPUS_ID = "CORPUS-BS-CODEX-CURRENT-0001"
|
|
|
|
|
|
def digest(value: bytes) -> str:
|
|
return hashlib.sha256(value).hexdigest()
|
|
|
|
|
|
def stable_json(value: object) -> str:
|
|
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
|
|
|
|
|
def main() -> None:
|
|
if OUT.exists() or AUDIT.exists():
|
|
raise FileExistsError("current-language index already exists")
|
|
|
|
source_hash = hashlib.sha256()
|
|
source_size = 0
|
|
records: list[dict[str, object]] = []
|
|
seen_message_ids: set[str] = set()
|
|
|
|
with SOURCE.open("rb") as stream:
|
|
for raw_line in stream:
|
|
source_hash.update(raw_line)
|
|
source_size += len(raw_line)
|
|
envelope = json.loads(raw_line)
|
|
if envelope.get("type") != "response_item":
|
|
continue
|
|
payload = envelope.get("payload") or {}
|
|
if payload.get("type") != "message" or payload.get("role") != "user":
|
|
continue
|
|
message_id = payload.get("id")
|
|
if not isinstance(message_id, str) or message_id in seen_message_ids:
|
|
continue
|
|
seen_message_ids.add(message_id)
|
|
input_parts = [
|
|
item.get("text", "")
|
|
for item in payload.get("content", [])
|
|
if isinstance(item, dict) and item.get("type") == "input_text"
|
|
]
|
|
text_projection = "\n".join(input_parts)
|
|
content_canonical = stable_json(payload.get("content", []))
|
|
seed = f"{CORPUS_ID}:{message_id}:{digest(content_canonical.encode('utf-8'))}"
|
|
metadata = payload.get("internal_chat_message_metadata_passthrough") or {}
|
|
records.append(
|
|
{
|
|
"schema": "guanghu.codex-direct-language-segment/v1",
|
|
"segment_id": f"BSCX-SEG-{digest(seed.encode('utf-8'))[:24]}",
|
|
"corpus_id": CORPUS_ID,
|
|
"source_file": str(SOURCE),
|
|
"source_file_sha256_at_index_time": None,
|
|
"source_file_size_at_index_time": None,
|
|
"rollout_ordinal": envelope.get("ordinal"),
|
|
"message_id": message_id,
|
|
"turn_id": metadata.get("turn_id"),
|
|
"occurred_at_utc": envelope.get("timestamp"),
|
|
"speaker_identity": "BINGSHUO_DIRECT_CURRENT_CHANNEL",
|
|
"source_class": "S01_DIRECT_HUMAN_NATURAL_LANGUAGE",
|
|
"content_parts_canonical_sha256": digest(content_canonical.encode("utf-8")),
|
|
"text_projection_sha256": digest(text_projection.encode("utf-8")),
|
|
"text_projection_chars": len(text_projection),
|
|
"text_projection": text_projection,
|
|
"exact_locator": f"jsonl ordinal={envelope.get('ordinal')} payload.id={message_id}",
|
|
"promotion_state": "OBSERVED_DIRECT_LANGUAGE_ONLY",
|
|
}
|
|
)
|
|
|
|
complete_hash = source_hash.hexdigest()
|
|
for record in records:
|
|
record["source_file_sha256_at_index_time"] = complete_hash
|
|
record["source_file_size_at_index_time"] = source_size
|
|
|
|
OUT.parent.mkdir(parents=True, exist_ok=True)
|
|
with OUT.open("x", encoding="utf-8") as output:
|
|
for record in records:
|
|
output.write(stable_json(record) + "\n")
|
|
|
|
audit = {
|
|
"schema": "guanghu.codex-current-language-audit/v1",
|
|
"corpus_id": CORPUS_ID,
|
|
"indexed_at_utc": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
|
"source_file": str(SOURCE),
|
|
"source_file_sha256_at_index_time": complete_hash,
|
|
"source_file_size_at_index_time": source_size,
|
|
"direct_user_segment_count": len(records),
|
|
"selection_rule": "response_item/message/user/input_text only; event duplicates and compaction history excluded",
|
|
"output": str(OUT),
|
|
"output_sha256": digest(OUT.read_bytes()),
|
|
"neuron_promotion_state": "NONE",
|
|
}
|
|
with AUDIT.open("x", encoding="utf-8") as output:
|
|
json.dump(audit, output, ensure_ascii=False, indent=2)
|
|
output.write("\n")
|
|
print(json.dumps(audit, ensure_ascii=False, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|