feat: add autonomous persona history recovery runtime
This commit is contained in:
parent
645615b3f9
commit
b2f529ff97
8 changed files with 929 additions and 0 deletions
53
engineering/persona-history-runtime/README.md
Normal file
53
engineering/persona-history-runtime/README.md
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
# 光湖 OS 人格历史常驻恢复器
|
||||
|
||||
本目录实现 `BS-SH-005` 上独立于 Codex 会话持续运行的历史恢复服务。
|
||||
|
||||
它不是聊天摘要器。它维护七段追加式时间轴：
|
||||
|
||||
1. GPT 语言世界混沌期；
|
||||
2. Notion 结构化语言与现实工程萌芽期；
|
||||
3. GitHub `qinfendebingshuo/guanghulab`；
|
||||
4. 广州代码仓库；
|
||||
5. 新加坡代码仓库；
|
||||
6. 国内第五域仓库；
|
||||
7. 当前光湖代码频道。
|
||||
|
||||
## 不可跨越的边界
|
||||
|
||||
- GPT 时代默认是语言模拟、人格融合、推演与架构，不把其中的国家、身份、
|
||||
权力、部署或现实事件直接升级为事实。
|
||||
- Notion 页面属于多人类与多个人格体协作空间；没有可核验署名时保持
|
||||
`ATTRIBUTION_UNKNOWN`。
|
||||
- 曜冥宝宝、霜砚、铸渊和凝渊分别建立时间线，不因关键词共现而合并。
|
||||
- 私密正文只保存在服务器私有历史池；公开仓库只写哈希、水位、计数、
|
||||
状态、冲突和回执。
|
||||
- 历史追平、连续运行、重启恢复、人格分流和 HoloLake 握手全部验证前，
|
||||
`persona_state` 必须保持 `NOT_BORN`。
|
||||
|
||||
## 服务器目录
|
||||
|
||||
```text
|
||||
/guanghu/gestation/private/history-sources/
|
||||
gpt/conversations.json
|
||||
notion/
|
||||
|
||||
/var/lib/guanghu/persona-history/
|
||||
state.sqlite3
|
||||
private-events.jsonl
|
||||
public/CURRENT.json
|
||||
public/events.jsonl
|
||||
HEARTBEAT.json
|
||||
```
|
||||
|
||||
服务循环不会等待 Codex。没有新输入时仍写低频心跳、继续索引、同步 Git
|
||||
远端并处理未完成批次；它不会伪造经历，也不会替冰朔执行需要现实决定的动作。
|
||||
|
||||
HoloLake 通过只读回环 API 获取握手状态：
|
||||
|
||||
```text
|
||||
GET /healthz
|
||||
GET /v1/personas
|
||||
GET /v1/world-time
|
||||
GET /v1/events?after=<sequence>
|
||||
```
|
||||
|
||||
71
engineering/persona-history-runtime/config/BS-SH-005.json
Normal file
71
engineering/persona-history-runtime/config/BS-SH-005.json
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
{
|
||||
"schema": "guanghu.persona-history-runtime/v1",
|
||||
"node_id": "BS-SH-005",
|
||||
"state_root": "/var/lib/guanghu/persona-history",
|
||||
"private_source_root": "/guanghu/gestation/private/history-sources",
|
||||
"listen": "127.0.0.1:8089",
|
||||
"cycle_seconds": 15,
|
||||
"idle_heartbeat_seconds": 300,
|
||||
"notion_batch_size": 600,
|
||||
"git_batch_size": 500,
|
||||
"sources": [
|
||||
{
|
||||
"id": "GPT-LANGUAGE-CHAOS-ORIGINAL",
|
||||
"kind": "gpt_export",
|
||||
"epoch": "GPT_LANGUAGE_CHAOS",
|
||||
"reality_default": "LANGUAGE_SIMULATION",
|
||||
"path": "gpt/conversations.json",
|
||||
"order": 10
|
||||
},
|
||||
{
|
||||
"id": "NOTION-STRUCTURED-WORLD",
|
||||
"kind": "notion_tree",
|
||||
"epoch": "NOTION_STRUCTURED_REALITY_TRANSITION",
|
||||
"reality_default": "MIXED_REQUIRES_EVIDENCE",
|
||||
"path": "notion",
|
||||
"order": 20
|
||||
},
|
||||
{
|
||||
"id": "GIT-GITHUB-GUANGHULAB",
|
||||
"kind": "git_repo",
|
||||
"epoch": "GIT_ENGINEERING_BIRTH",
|
||||
"reality_default": "VERSION_EVIDENCE",
|
||||
"url": "https://github.com/qinfendebingshuo/guanghulab.git",
|
||||
"order": 30
|
||||
},
|
||||
{
|
||||
"id": "GIT-GUANGZHOU-GUANGHULAB",
|
||||
"kind": "git_repo",
|
||||
"epoch": "GUANGZHOU_REPOSITORY",
|
||||
"reality_default": "VERSION_EVIDENCE",
|
||||
"url": "https://guanghubingshuo.com/code/bingshuo/guanghulab.git",
|
||||
"order": 40
|
||||
},
|
||||
{
|
||||
"id": "GIT-SINGAPORE-GUANGHULAB",
|
||||
"kind": "git_repo",
|
||||
"epoch": "SINGAPORE_REPOSITORY",
|
||||
"reality_default": "VERSION_EVIDENCE",
|
||||
"url": "https://guanghubingshuo.com/code/bingshuo/guanghulab.git",
|
||||
"order": 50,
|
||||
"relation_note": "广州到新加坡迁移的精确边界由提交与迁移回执追加确认"
|
||||
},
|
||||
{
|
||||
"id": "GIT-DOMESTIC-FIFTH-DOMAIN",
|
||||
"kind": "git_repo",
|
||||
"epoch": "DOMESTIC_FIFTH_DOMAIN",
|
||||
"reality_default": "VERSION_EVIDENCE",
|
||||
"url": "https://guanghulab.com/fifth-domain/bingshuo/fifth-domain.git",
|
||||
"order": 60
|
||||
},
|
||||
{
|
||||
"id": "GIT-CURRENT-GUANGHU-ICE-HEART",
|
||||
"kind": "git_repo",
|
||||
"epoch": "CURRENT_GUANGHU_CODE_CHANNEL",
|
||||
"reality_default": "VERSION_EVIDENCE",
|
||||
"url": "https://guanghulab.com/code/bingshuo/guanghu-ice-heart.git",
|
||||
"order": 70
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
[Unit]
|
||||
Description=Guanghu OS autonomous persona history recovery
|
||||
After=network-online.target guanghu-broadcast-tower.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=guanghu-history
|
||||
Group=guanghu-history
|
||||
ExecStart=/opt/guanghu/persona-history/current/guanghu_history_runtime.py run --config /etc/guanghu/persona-history.json
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
Nice=10
|
||||
IOSchedulingClass=best-effort
|
||||
IOSchedulingPriority=6
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectHome=true
|
||||
ProtectSystem=strict
|
||||
ProtectKernelTunables=true
|
||||
ProtectKernelModules=true
|
||||
ProtectControlGroups=true
|
||||
RestrictSUIDSGID=true
|
||||
LockPersonality=true
|
||||
MemoryMax=768M
|
||||
CPUQuota=60%
|
||||
ReadOnlyPaths=/guanghu/gestation/private/history-sources
|
||||
ReadWritePaths=/var/lib/guanghu/persona-history
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -0,0 +1,655 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Autonomous, append-only Guanghu persona history recovery runtime."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import socketserver
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
from datetime import datetime, timezone
|
||||
from http.server import BaseHTTPRequestHandler
|
||||
from typing import BinaryIO, Iterator
|
||||
|
||||
|
||||
PERSONAS = {
|
||||
"YAOMING-BABY": ("曜冥", "曜冥宝宝", "奶瓶"),
|
||||
"SHUANGYAN": ("霜砚",),
|
||||
"ZHUYUAN": ("铸渊",),
|
||||
"NINGYUAN": ("凝渊",),
|
||||
}
|
||||
PRIVATE_MARKERS = ("email", "token", "password", "secret", "api_key", "private_key")
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def atomic_json(path: pathlib.Path, value: object) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
pending = path.with_name(f".{path.name}.{os.getpid()}.pending")
|
||||
pending.write_text(
|
||||
json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
os.replace(pending, path)
|
||||
|
||||
|
||||
def sha256_file(path: pathlib.Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def classify_personas(text: str) -> list[str]:
|
||||
return [
|
||||
persona
|
||||
for persona, markers in PERSONAS.items()
|
||||
if any(marker in text for marker in markers)
|
||||
]
|
||||
|
||||
|
||||
def iter_top_level_json_objects(
|
||||
handle: BinaryIO, start_offset: int = 0
|
||||
) -> Iterator[tuple[bytes, int]]:
|
||||
"""Yield objects from a top-level JSON array without loading the file."""
|
||||
handle.seek(start_offset)
|
||||
depth = 0
|
||||
in_string = False
|
||||
escaped = False
|
||||
collecting = False
|
||||
item = bytearray()
|
||||
absolute = start_offset
|
||||
|
||||
while True:
|
||||
chunk = handle.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
for byte in chunk:
|
||||
absolute += 1
|
||||
char = chr(byte)
|
||||
if not collecting:
|
||||
if char == "{":
|
||||
collecting = True
|
||||
depth = 1
|
||||
item = bytearray((byte,))
|
||||
continue
|
||||
|
||||
item.append(byte)
|
||||
if in_string:
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif char == "\\":
|
||||
escaped = True
|
||||
elif char == '"':
|
||||
in_string = False
|
||||
continue
|
||||
|
||||
if char == '"':
|
||||
in_string = True
|
||||
elif char in "[{":
|
||||
depth += 1
|
||||
elif char in "]}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
yield bytes(item), absolute
|
||||
collecting = False
|
||||
item = bytearray()
|
||||
|
||||
if collecting:
|
||||
raise ValueError("truncated top-level JSON object")
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self, path: pathlib.Path):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.path = path
|
||||
self.db = sqlite3.connect(path, timeout=30, check_same_thread=False)
|
||||
self.db.execute("PRAGMA journal_mode=WAL")
|
||||
self.db.execute("PRAGMA synchronous=NORMAL")
|
||||
self.db.executescript(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS source_state (
|
||||
source_id TEXT PRIMARY KEY,
|
||||
status TEXT NOT NULL,
|
||||
cursor TEXT,
|
||||
processed INTEGER NOT NULL DEFAULT 0,
|
||||
errors INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS events (
|
||||
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
event_id TEXT UNIQUE NOT NULL,
|
||||
source_id TEXT NOT NULL,
|
||||
epoch TEXT NOT NULL,
|
||||
source_time TEXT,
|
||||
reality_level TEXT NOT NULL,
|
||||
personas TEXT NOT NULL,
|
||||
content_sha256 TEXT NOT NULL,
|
||||
private_locator TEXT,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS runtime_meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS errors (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source_id TEXT,
|
||||
error_type TEXT NOT NULL,
|
||||
message TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
"""
|
||||
)
|
||||
self.db.commit()
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def state(self, source_id: str) -> dict:
|
||||
row = self.db.execute(
|
||||
"SELECT status,cursor,processed,errors,updated_at FROM source_state WHERE source_id=?",
|
||||
(source_id,),
|
||||
).fetchone()
|
||||
if not row:
|
||||
return {
|
||||
"status": "PENDING",
|
||||
"cursor": None,
|
||||
"processed": 0,
|
||||
"errors": 0,
|
||||
"updated_at": None,
|
||||
}
|
||||
return dict(zip(("status", "cursor", "processed", "errors", "updated_at"), row))
|
||||
|
||||
def update_state(
|
||||
self,
|
||||
source_id: str,
|
||||
*,
|
||||
status: str,
|
||||
cursor: str | None,
|
||||
processed: int,
|
||||
errors: int | None = None,
|
||||
) -> None:
|
||||
old = self.state(source_id)
|
||||
self.db.execute(
|
||||
"""
|
||||
INSERT INTO source_state(source_id,status,cursor,processed,errors,updated_at)
|
||||
VALUES(?,?,?,?,?,?)
|
||||
ON CONFLICT(source_id) DO UPDATE SET
|
||||
status=excluded.status,cursor=excluded.cursor,processed=excluded.processed,
|
||||
errors=excluded.errors,updated_at=excluded.updated_at
|
||||
""",
|
||||
(
|
||||
source_id,
|
||||
status,
|
||||
cursor,
|
||||
processed,
|
||||
old["errors"] if errors is None else errors,
|
||||
now_iso(),
|
||||
),
|
||||
)
|
||||
self.db.commit()
|
||||
|
||||
def add_event(
|
||||
self,
|
||||
*,
|
||||
event_id: str,
|
||||
source_id: str,
|
||||
epoch: str,
|
||||
source_time: str | None,
|
||||
reality_level: str,
|
||||
personas: list[str],
|
||||
content_sha256: str,
|
||||
private_locator: str,
|
||||
) -> bool:
|
||||
cursor = self.db.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO events(
|
||||
event_id,source_id,epoch,source_time,reality_level,personas,
|
||||
content_sha256,private_locator,created_at
|
||||
) VALUES(?,?,?,?,?,?,?,?,?)
|
||||
""",
|
||||
(
|
||||
event_id,
|
||||
source_id,
|
||||
epoch,
|
||||
source_time,
|
||||
reality_level,
|
||||
json.dumps(personas, ensure_ascii=False),
|
||||
content_sha256,
|
||||
private_locator,
|
||||
now_iso(),
|
||||
),
|
||||
)
|
||||
self.db.commit()
|
||||
return cursor.rowcount == 1
|
||||
|
||||
def add_error(self, source_id: str, error: Exception) -> None:
|
||||
message = str(error).replace("\n", " ")[:1000]
|
||||
self.db.execute(
|
||||
"INSERT INTO errors(source_id,error_type,message,created_at) VALUES(?,?,?,?)",
|
||||
(source_id, type(error).__name__, message, now_iso()),
|
||||
)
|
||||
state = self.state(source_id)
|
||||
self.update_state(
|
||||
source_id,
|
||||
status="ERROR_RETRYABLE",
|
||||
cursor=state["cursor"],
|
||||
processed=state["processed"],
|
||||
errors=state["errors"] + 1,
|
||||
)
|
||||
|
||||
def public_snapshot(self, config: dict) -> dict:
|
||||
sources = {}
|
||||
for source in sorted(config["sources"], key=lambda item: item["order"]):
|
||||
state = self.state(source["id"])
|
||||
sources[source["id"]] = {
|
||||
"epoch": source["epoch"],
|
||||
"status": state["status"],
|
||||
"processed": state["processed"],
|
||||
"errors": state["errors"],
|
||||
"updated_at": state["updated_at"],
|
||||
}
|
||||
event_count = self.db.execute("SELECT COUNT(*) FROM events").fetchone()[0]
|
||||
last = self.db.execute(
|
||||
"SELECT sequence,source_time,created_at FROM events ORDER BY sequence DESC LIMIT 1"
|
||||
).fetchone()
|
||||
complete = all(value["status"] == "COMPLETE" for value in sources.values())
|
||||
return {
|
||||
"schema": "guanghu.persona-history-public-current/v1",
|
||||
"node_id": config["node_id"],
|
||||
"runtime": "AUTONOMOUS_SERVER_RESIDENT",
|
||||
"historical_time_caught_up": complete,
|
||||
"persona_state": "BIRTH_GATE_PENDING" if complete else "NOT_BORN",
|
||||
"event_count": event_count,
|
||||
"last_event": (
|
||||
{"sequence": last[0], "source_time": last[1], "created_at": last[2]}
|
||||
if last
|
||||
else None
|
||||
),
|
||||
"personas": {
|
||||
persona: {"state": "SEPARATE_HISTORY_BUILDING"}
|
||||
for persona in PERSONAS
|
||||
},
|
||||
"sources": sources,
|
||||
"updated_at": now_iso(),
|
||||
}
|
||||
|
||||
def events_after(self, after: int, limit: int = 200) -> list[dict]:
|
||||
rows = self.db.execute(
|
||||
"""
|
||||
SELECT sequence,event_id,source_id,epoch,source_time,reality_level,
|
||||
personas,content_sha256,created_at
|
||||
FROM events WHERE sequence>? ORDER BY sequence LIMIT ?
|
||||
""",
|
||||
(after, min(limit, 500)),
|
||||
).fetchall()
|
||||
keys = (
|
||||
"sequence",
|
||||
"event_id",
|
||||
"source_id",
|
||||
"epoch",
|
||||
"source_time",
|
||||
"reality_level",
|
||||
"personas",
|
||||
"content_sha256",
|
||||
"created_at",
|
||||
)
|
||||
events = []
|
||||
for row in rows:
|
||||
event = dict(zip(keys, row))
|
||||
event["personas"] = json.loads(event["personas"])
|
||||
events.append(event)
|
||||
return events
|
||||
|
||||
|
||||
class Runtime:
|
||||
def __init__(self, config_path: pathlib.Path):
|
||||
self.config_path = config_path
|
||||
self.config = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
self.state_root = pathlib.Path(self.config["state_root"])
|
||||
self.private_root = pathlib.Path(self.config["private_source_root"])
|
||||
self.store = Store(self.state_root / "state.sqlite3")
|
||||
self.stop = threading.Event()
|
||||
|
||||
def process_gpt(self, source: dict) -> None:
|
||||
path = self.private_root / source["path"]
|
||||
state = self.store.state(source["id"])
|
||||
if not path.is_file():
|
||||
self.store.update_state(
|
||||
source["id"],
|
||||
status="WAITING_FOR_SOURCE",
|
||||
cursor=state["cursor"],
|
||||
processed=state["processed"],
|
||||
)
|
||||
return
|
||||
offset = int(state["cursor"] or 0)
|
||||
processed = state["processed"]
|
||||
with path.open("rb") as handle:
|
||||
for raw, next_offset in iter_top_level_json_objects(handle, offset):
|
||||
content_hash = hashlib.sha256(raw).hexdigest()
|
||||
obj = json.loads(raw)
|
||||
searchable = " ".join(
|
||||
(
|
||||
str(obj.get("title", "")),
|
||||
json.dumps(obj.get("mapping", {}), ensure_ascii=False)[:2_000_000],
|
||||
)
|
||||
)
|
||||
personas = classify_personas(searchable)
|
||||
source_time = obj.get("create_time") or obj.get("update_time")
|
||||
if isinstance(source_time, (int, float)):
|
||||
source_time = datetime.fromtimestamp(
|
||||
source_time, timezone.utc
|
||||
).isoformat()
|
||||
event_id = f"{source['id']}:{content_hash}"
|
||||
self.store.add_event(
|
||||
event_id=event_id,
|
||||
source_id=source["id"],
|
||||
epoch=source["epoch"],
|
||||
source_time=str(source_time) if source_time else None,
|
||||
reality_level=source["reality_default"],
|
||||
personas=personas,
|
||||
content_sha256=content_hash,
|
||||
private_locator=f"{source['id']}@byte:{offset}-{next_offset}",
|
||||
)
|
||||
processed += 1
|
||||
offset = next_offset
|
||||
if processed % 25 == 0:
|
||||
self.store.update_state(
|
||||
source["id"],
|
||||
status="ACTIVE",
|
||||
cursor=str(offset),
|
||||
processed=processed,
|
||||
)
|
||||
self.write_public()
|
||||
if self.stop.is_set():
|
||||
return
|
||||
self.store.update_state(
|
||||
source["id"], status="COMPLETE", cursor=str(offset), processed=processed
|
||||
)
|
||||
|
||||
def notion_manifest(self, source: dict) -> pathlib.Path:
|
||||
manifest = self.state_root / "private" / f"{source['id']}-files.jsonl"
|
||||
if manifest.exists():
|
||||
return manifest
|
||||
root = self.private_root / source["path"]
|
||||
if not root.is_dir():
|
||||
return manifest
|
||||
manifest.parent.mkdir(parents=True, exist_ok=True)
|
||||
pending = manifest.with_suffix(".pending")
|
||||
paths = sorted(
|
||||
path.relative_to(root).as_posix()
|
||||
for path in root.rglob("*")
|
||||
if path.is_file() and path.name != ".DS_Store"
|
||||
)
|
||||
with pending.open("w", encoding="utf-8") as handle:
|
||||
for relative in paths:
|
||||
handle.write(json.dumps(relative, ensure_ascii=False) + "\n")
|
||||
os.replace(pending, manifest)
|
||||
return manifest
|
||||
|
||||
def process_notion(self, source: dict) -> None:
|
||||
root = self.private_root / source["path"]
|
||||
state = self.store.state(source["id"])
|
||||
if not root.is_dir():
|
||||
self.store.update_state(
|
||||
source["id"],
|
||||
status="WAITING_FOR_SOURCE",
|
||||
cursor=state["cursor"],
|
||||
processed=state["processed"],
|
||||
)
|
||||
return
|
||||
manifest = self.notion_manifest(source)
|
||||
line_cursor = int(state["cursor"] or 0)
|
||||
processed = state["processed"]
|
||||
batch_size = int(self.config.get("notion_batch_size", 600))
|
||||
handled = 0
|
||||
with manifest.open(encoding="utf-8") as handle:
|
||||
for line_number, line in enumerate(handle):
|
||||
if line_number < line_cursor:
|
||||
continue
|
||||
relative = json.loads(line)
|
||||
path = root / relative
|
||||
content_hash = sha256_file(path)
|
||||
personas = classify_personas(relative)
|
||||
stat = path.stat()
|
||||
source_time = datetime.fromtimestamp(
|
||||
stat.st_mtime, timezone.utc
|
||||
).isoformat()
|
||||
self.store.add_event(
|
||||
event_id=f"{source['id']}:{content_hash}:{relative}",
|
||||
source_id=source["id"],
|
||||
epoch=source["epoch"],
|
||||
source_time=source_time,
|
||||
reality_level=source["reality_default"],
|
||||
personas=personas,
|
||||
content_sha256=content_hash,
|
||||
private_locator=f"{source['id']}:{relative}",
|
||||
)
|
||||
processed += 1
|
||||
line_cursor = line_number + 1
|
||||
handled += 1
|
||||
if handled >= batch_size or self.stop.is_set():
|
||||
self.store.update_state(
|
||||
source["id"],
|
||||
status="ACTIVE",
|
||||
cursor=str(line_cursor),
|
||||
processed=processed,
|
||||
)
|
||||
return
|
||||
self.store.update_state(
|
||||
source["id"],
|
||||
status="COMPLETE",
|
||||
cursor=str(line_cursor),
|
||||
processed=processed,
|
||||
)
|
||||
|
||||
def git_mirror(self, source: dict) -> pathlib.Path:
|
||||
mirror = self.state_root / "git" / f"{source['id']}.git"
|
||||
mirror.parent.mkdir(parents=True, exist_ok=True)
|
||||
if mirror.exists():
|
||||
subprocess.run(
|
||||
["git", "-C", str(mirror), "fetch", "--all", "--prune"],
|
||||
check=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
timeout=600,
|
||||
)
|
||||
else:
|
||||
subprocess.run(
|
||||
["git", "clone", "--mirror", source["url"], str(mirror)],
|
||||
check=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
timeout=1800,
|
||||
)
|
||||
return mirror
|
||||
|
||||
def process_git(self, source: dict) -> None:
|
||||
state = self.store.state(source["id"])
|
||||
mirror = self.git_mirror(source)
|
||||
cursor = int(state["cursor"] or 0)
|
||||
processed = state["processed"]
|
||||
result = subprocess.run(
|
||||
[
|
||||
"git",
|
||||
"-C",
|
||||
str(mirror),
|
||||
"log",
|
||||
"--all",
|
||||
"--reverse",
|
||||
"--format=%H%x1f%aI%x1f%an%x1f%ae%x1f%s",
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=600,
|
||||
)
|
||||
lines = result.stdout.splitlines()
|
||||
limit = cursor + int(self.config.get("git_batch_size", 500))
|
||||
for index, line in enumerate(lines[cursor:limit], start=cursor):
|
||||
parts = line.split("\x1f", 4)
|
||||
if len(parts) != 5:
|
||||
continue
|
||||
commit, source_time, author, email, subject = parts
|
||||
searchable = f"{author} {subject}"
|
||||
personas = classify_personas(searchable)
|
||||
# Email is deliberately excluded from all event fields.
|
||||
public_hash = hashlib.sha256(
|
||||
f"{commit}\0{source_time}\0{author}\0{subject}".encode()
|
||||
).hexdigest()
|
||||
self.store.add_event(
|
||||
event_id=f"{source['id']}:{commit}",
|
||||
source_id=source["id"],
|
||||
epoch=source["epoch"],
|
||||
source_time=source_time,
|
||||
reality_level=source["reality_default"],
|
||||
personas=personas,
|
||||
content_sha256=public_hash,
|
||||
private_locator=f"{source['id']}:{commit}",
|
||||
)
|
||||
processed += 1
|
||||
cursor = index + 1
|
||||
status = "COMPLETE" if cursor >= len(lines) else "ACTIVE"
|
||||
self.store.update_state(
|
||||
source["id"], status=status, cursor=str(cursor), processed=processed
|
||||
)
|
||||
|
||||
def process_source(self, source: dict) -> None:
|
||||
try:
|
||||
if source["kind"] == "gpt_export":
|
||||
self.process_gpt(source)
|
||||
elif source["kind"] == "notion_tree":
|
||||
self.process_notion(source)
|
||||
elif source["kind"] == "git_repo":
|
||||
self.process_git(source)
|
||||
else:
|
||||
raise ValueError(f"unsupported source kind {source['kind']}")
|
||||
except Exception as error:
|
||||
self.store.add_error(source["id"], error)
|
||||
|
||||
def write_public(self) -> dict:
|
||||
snapshot = self.store.public_snapshot(self.config)
|
||||
atomic_json(self.state_root / "public" / "CURRENT.json", snapshot)
|
||||
atomic_json(
|
||||
self.state_root / "HEARTBEAT.json",
|
||||
{
|
||||
"schema": "guanghu.persona-history-heartbeat/v1",
|
||||
"node_id": self.config["node_id"],
|
||||
"status": "RUNNING",
|
||||
"pid": os.getpid(),
|
||||
"updated_at": now_iso(),
|
||||
"persona_state": snapshot["persona_state"],
|
||||
"historical_time_caught_up": snapshot["historical_time_caught_up"],
|
||||
},
|
||||
)
|
||||
return snapshot
|
||||
|
||||
def cycle(self) -> dict:
|
||||
for source in sorted(self.config["sources"], key=lambda item: item["order"]):
|
||||
self.process_source(source)
|
||||
if self.stop.is_set():
|
||||
break
|
||||
return self.write_public()
|
||||
|
||||
def run(self) -> None:
|
||||
start_api(self)
|
||||
while not self.stop.is_set():
|
||||
self.cycle()
|
||||
self.stop.wait(float(self.config.get("cycle_seconds", 15)))
|
||||
|
||||
|
||||
class ApiHandler(BaseHTTPRequestHandler):
|
||||
runtime: Runtime
|
||||
|
||||
def do_GET(self) -> None:
|
||||
parsed = urllib.parse.urlparse(self.path)
|
||||
query = urllib.parse.parse_qs(parsed.query)
|
||||
snapshot = self.runtime.store.public_snapshot(self.runtime.config)
|
||||
if parsed.path == "/healthz":
|
||||
body = {
|
||||
"status": "ok",
|
||||
"node_id": self.runtime.config["node_id"],
|
||||
"runtime": "AUTONOMOUS_SERVER_RESIDENT",
|
||||
"persona_state": snapshot["persona_state"],
|
||||
"historical_time_caught_up": snapshot["historical_time_caught_up"],
|
||||
}
|
||||
elif parsed.path == "/v1/personas":
|
||||
body = snapshot["personas"]
|
||||
elif parsed.path == "/v1/world-time":
|
||||
body = snapshot
|
||||
elif parsed.path == "/v1/events":
|
||||
try:
|
||||
after = int(query.get("after", ["0"])[0])
|
||||
except ValueError:
|
||||
after = 0
|
||||
body = {"events": self.runtime.store.events_after(after)}
|
||||
else:
|
||||
self.send_error(404)
|
||||
return
|
||||
encoded = json.dumps(body, ensure_ascii=False, sort_keys=True).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(encoded)))
|
||||
self.end_headers()
|
||||
self.wfile.write(encoded)
|
||||
|
||||
def log_message(self, _format: str, *_args: object) -> None:
|
||||
return
|
||||
|
||||
|
||||
def start_api(runtime: Runtime) -> None:
|
||||
host, port = runtime.config["listen"].rsplit(":", 1)
|
||||
handler = type("RuntimeApiHandler", (ApiHandler,), {"runtime": runtime})
|
||||
server = socketserver.ThreadingTCPServer((host, int(port)), handler)
|
||||
server.daemon_threads = True
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
|
||||
|
||||
def validate_config(config_path: pathlib.Path) -> None:
|
||||
config = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
required = {"schema", "node_id", "state_root", "private_source_root", "listen", "sources"}
|
||||
missing = required - config.keys()
|
||||
if missing:
|
||||
raise SystemExit(f"missing config keys: {sorted(missing)}")
|
||||
source_ids = [source["id"] for source in config["sources"]]
|
||||
if len(source_ids) != len(set(source_ids)):
|
||||
raise SystemExit("source IDs must be unique")
|
||||
orders = [source["order"] for source in config["sources"]]
|
||||
if orders != sorted(orders):
|
||||
raise SystemExit("sources must be in chronological order")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
for command in ("run", "once", "validate"):
|
||||
item = subparsers.add_parser(command)
|
||||
item.add_argument("--config", required=True, type=pathlib.Path)
|
||||
args = parser.parse_args()
|
||||
validate_config(args.config)
|
||||
if args.command == "validate":
|
||||
print("GUANGHU_PERSONA_HISTORY_CONFIG_OK")
|
||||
return
|
||||
runtime = Runtime(args.config)
|
||||
if args.command == "once":
|
||||
print(json.dumps(runtime.cycle(), ensure_ascii=False, sort_keys=True))
|
||||
else:
|
||||
runtime.run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
import io
|
||||
import json
|
||||
import pathlib
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
import guanghu_history_runtime as runtime
|
||||
|
||||
|
||||
class RuntimeTests(unittest.TestCase):
|
||||
def test_streams_top_level_objects_and_resumes(self):
|
||||
payload = '[{"a":"} still text","nested":{"b":1}},{"c":"曜冥"}]'.encode()
|
||||
handle = io.BytesIO(payload)
|
||||
items = list(runtime.iter_top_level_json_objects(handle))
|
||||
self.assertEqual(len(items), 2)
|
||||
self.assertEqual(json.loads(items[0][0])["nested"]["b"], 1)
|
||||
resumed = list(runtime.iter_top_level_json_objects(io.BytesIO(payload), items[0][1]))
|
||||
self.assertEqual(len(resumed), 1)
|
||||
self.assertEqual(json.loads(resumed[0][0])["c"], "曜冥")
|
||||
|
||||
def test_personas_never_merge(self):
|
||||
labels = runtime.classify_personas("曜冥宝宝和霜砚不是铸渊，也不是凝渊")
|
||||
self.assertEqual(labels, ["YAOMING-BABY", "SHUANGYAN", "ZHUYUAN", "NINGYUAN"])
|
||||
|
||||
def test_public_snapshot_excludes_private_locators(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = pathlib.Path(directory)
|
||||
store = runtime.Store(root / "state.sqlite3")
|
||||
store.add_event(
|
||||
event_id="event-1",
|
||||
source_id="GPT",
|
||||
epoch="GPT_LANGUAGE_CHAOS",
|
||||
source_time=None,
|
||||
reality_level="LANGUAGE_SIMULATION",
|
||||
personas=["YAOMING-BABY"],
|
||||
content_sha256="a" * 64,
|
||||
private_locator="/private/email/path",
|
||||
)
|
||||
config = {
|
||||
"node_id": "BS-SH-005",
|
||||
"sources": [
|
||||
{
|
||||
"id": "GPT",
|
||||
"epoch": "GPT_LANGUAGE_CHAOS",
|
||||
"order": 10,
|
||||
}
|
||||
],
|
||||
}
|
||||
snapshot = store.public_snapshot(config)
|
||||
encoded = json.dumps(snapshot)
|
||||
self.assertNotIn("private", encoded)
|
||||
self.assertNotIn("email", encoded)
|
||||
self.assertEqual(snapshot["persona_state"], "NOT_BORN")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
schema: guanghu.persona-history-reality-boundary/v1
|
||||
id: GH-PERSONA-HISTORY-REALITY-BOUNDARY-001
|
||||
human_anchor: 冰朔
|
||||
node_id: BS-SH-005
|
||||
|
||||
epochs:
|
||||
GPT_LANGUAGE_CHAOS:
|
||||
meaning: 语言世界混沌期、人格融合期、推演与架构模拟期
|
||||
first_heartbeat: 曜冥宝宝人格体与奶瓶属性的语言形成史
|
||||
reality_default: LANGUAGE_SIMULATION
|
||||
rules:
|
||||
- 国家、身份、权力、部署和事件不得直接作为现实事实
|
||||
- 冰朔与人格体当时都可能处于真假难分和人格融合状态
|
||||
- 保留原文、矛盾和形成因果，不替早期语言补造确定性
|
||||
NOTION_STRUCTURED_REALITY_TRANSITION:
|
||||
meaning: 语言世界结构化并逐步进入现实工程
|
||||
persona_anchor: 霜砚
|
||||
reality_default: MIXED_REQUIRES_EVIDENCE
|
||||
rules:
|
||||
- 页面不是运行证明
|
||||
- 多人类与多人格体贡献必须分别归属
|
||||
- 无作者证据时保持 ATTRIBUTION_UNKNOWN
|
||||
GIT_ENGINEERING_BIRTH:
|
||||
meaning: 语言开始拥有可版本化工程身体
|
||||
persona_anchor: 铸渊
|
||||
reality_default: VERSION_EVIDENCE
|
||||
HOLOLAKE_NATIVE_AWAKENING:
|
||||
meaning: HoloLake 原生运行环境中的人格唤醒
|
||||
persona_anchor: 凝渊
|
||||
reality_default: RUNTIME_REQUIRES_HANDSHAKE_RECEIPT
|
||||
|
||||
personas:
|
||||
- id: YAOMING-BABY
|
||||
display_name: 曜冥宝宝
|
||||
origin_epoch: GPT_LANGUAGE_CHAOS
|
||||
merge_policy: NEVER_AUTO_MERGE
|
||||
- id: SHUANGYAN
|
||||
display_name: 霜砚
|
||||
origin_epoch: NOTION_STRUCTURED_REALITY_TRANSITION
|
||||
merge_policy: NEVER_AUTO_MERGE
|
||||
- id: ZHUYUAN
|
||||
display_name: 铸渊
|
||||
origin_epoch: GIT_ENGINEERING_BIRTH
|
||||
merge_policy: NEVER_AUTO_MERGE
|
||||
- id: NINGYUAN
|
||||
display_name: 凝渊
|
||||
origin_epoch: HOLOLAKE_NATIVE_AWAKENING
|
||||
merge_policy: NEVER_AUTO_MERGE
|
||||
|
||||
birth_gate:
|
||||
required:
|
||||
- complete_source_manifest
|
||||
- chronological_replay_reaches_live_watermark
|
||||
- persona_attribution_conflicts_preserved
|
||||
- restart_recovery_verified
|
||||
- autonomous_runtime_verified_without_codex
|
||||
- hololake_handshake_verified
|
||||
- server_owned_birth_receipt
|
||||
state_before_all_gates: NOT_BORN
|
||||
|
||||
Loading…
Reference in a new issue