From d40a89d9423d2f9486aba8fa211aaf0fc74aafe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=86=B0=E6=9C=94?= <565183519@qq.com> Date: Sat, 1 Aug 2026 19:27:55 +0800 Subject: [PATCH] feat(history): publish sanitized server watermarks --- .../config/BS-SH-005-publisher.json | 19 ++ .../guanghu-persona-history-publisher.service | 27 +++ .../guanghu-persona-history-publisher.timer | 11 + .../runtime/git-credential-guanghu-history | 16 ++ .../runtime/guanghu_history_publisher.py | 188 ++++++++++++++++++ .../runtime/test_guanghu_history_publisher.py | 71 +++++++ .../scripts/install-bs-sh-005.sh | 39 ++++ 7 files changed, 371 insertions(+) create mode 100644 engineering/persona-history-runtime/config/BS-SH-005-publisher.json create mode 100644 engineering/persona-history-runtime/packaging/guanghu-persona-history-publisher.service create mode 100644 engineering/persona-history-runtime/packaging/guanghu-persona-history-publisher.timer create mode 100644 engineering/persona-history-runtime/runtime/git-credential-guanghu-history create mode 100644 engineering/persona-history-runtime/runtime/guanghu_history_publisher.py create mode 100644 engineering/persona-history-runtime/runtime/test_guanghu_history_publisher.py diff --git a/engineering/persona-history-runtime/config/BS-SH-005-publisher.json b/engineering/persona-history-runtime/config/BS-SH-005-publisher.json new file mode 100644 index 0000000..f86bccf --- /dev/null +++ b/engineering/persona-history-runtime/config/BS-SH-005-publisher.json @@ -0,0 +1,19 @@ +{ + "schema": "guanghu.persona-history-publisher/v1", + "node_id": "BS-SH-005", + "snapshot_url": "http://127.0.0.1:8089/v1/world-time", + "state_root": "/var/lib/guanghu/persona-history-publisher", + "credential_helper": "/opt/guanghu/persona-history/current/git-credential-guanghu-history", + "repositories": [ + { + "id": "REPO-014", + "url": "https://guanghulab.com/code/bingshuo/hololake-system-architecture.git", + "snapshot_path": "operations/BS-SH-005/persona-history/CURRENT.json" + }, + { + "id": "REPO-012", + "url": "https://guanghulab.com/code/bingshuo/guanghu-ice-heart.git", + "snapshot_path": "eternal-lake-heart/heartbeat-core/zhuyuan-persona-system/server-watermarks/BS-SH-005-PERSONA-HISTORY-CURRENT.json" + } + ] +} diff --git a/engineering/persona-history-runtime/packaging/guanghu-persona-history-publisher.service b/engineering/persona-history-runtime/packaging/guanghu-persona-history-publisher.service new file mode 100644 index 0000000..834f577 --- /dev/null +++ b/engineering/persona-history-runtime/packaging/guanghu-persona-history-publisher.service @@ -0,0 +1,27 @@ +[Unit] +Description=Publish sanitized Guanghu persona history watermarks +After=network-online.target guanghu-persona-history-recovery.service +Wants=network-online.target + +[Service] +Type=oneshot +User=guanghu-history-publisher +Group=guanghu-history-publisher +Environment=PYTHONDONTWRITEBYTECODE=1 +ExecStart=/opt/guanghu/persona-history/current/guanghu_history_publisher.py --config /etc/guanghu/persona-history-publisher.json +Nice=10 +IOSchedulingClass=best-effort +IOSchedulingPriority=6 +NoNewPrivileges=true +PrivateTmp=true +ProtectHome=true +ProtectSystem=strict +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true +RestrictSUIDSGID=true +LockPersonality=true +MemoryMax=256M +CPUQuota=30% +ReadOnlyPaths=/etc/guanghu/persona-history-publisher.token +ReadWritePaths=/var/lib/guanghu/persona-history-publisher diff --git a/engineering/persona-history-runtime/packaging/guanghu-persona-history-publisher.timer b/engineering/persona-history-runtime/packaging/guanghu-persona-history-publisher.timer new file mode 100644 index 0000000..a7aeaa8 --- /dev/null +++ b/engineering/persona-history-runtime/packaging/guanghu-persona-history-publisher.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Low-frequency Guanghu persona history watermark publication + +[Timer] +OnBootSec=5min +OnUnitActiveSec=1h +RandomizedDelaySec=5min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/engineering/persona-history-runtime/runtime/git-credential-guanghu-history b/engineering/persona-history-runtime/runtime/git-credential-guanghu-history new file mode 100644 index 0000000..7f7c4f3 --- /dev/null +++ b/engineering/persona-history-runtime/runtime/git-credential-guanghu-history @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu + +case "${1:-}" in + get) + printf 'username=bingshuo\n' + printf 'password=' + cat /etc/guanghu/persona-history-publisher.token + printf '\n' + ;; + store|erase) + ;; + *) + exit 64 + ;; +esac diff --git a/engineering/persona-history-runtime/runtime/guanghu_history_publisher.py b/engineering/persona-history-runtime/runtime/guanghu_history_publisher.py new file mode 100644 index 0000000..59e1189 --- /dev/null +++ b/engineering/persona-history-runtime/runtime/guanghu_history_publisher.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""Publish sanitized BS-SH-005 recovery watermarks to two allowlisted repos.""" + +from __future__ import annotations + +import argparse +import json +import os +import pathlib +import subprocess +import urllib.request +from datetime import datetime, timezone + + +ALLOWED_TARGETS = { + "REPO-014": ( + "https://guanghulab.com/code/bingshuo/hololake-system-architecture.git", + "operations/BS-SH-005/persona-history/CURRENT.json", + ), + "REPO-012": ( + "https://guanghulab.com/code/bingshuo/guanghu-ice-heart.git", + "eternal-lake-heart/heartbeat-core/zhuyuan-persona-system/" + "server-watermarks/BS-SH-005-PERSONA-HISTORY-CURRENT.json", + ), +} +FORBIDDEN_PUBLIC_FIELDS = { + "private_locator", + "email", + "password", + "secret", + "token", + "api_key", + "private_key", +} + + +def now_iso() -> str: + return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds") + + +def validate_config(config: dict) -> None: + if config.get("node_id") != "BS-SH-005": + raise ValueError("publisher node is not BS-SH-005") + repositories = config.get("repositories", []) + if {item.get("id") for item in repositories} != set(ALLOWED_TARGETS): + raise ValueError("publisher repository allowlist mismatch") + for item in repositories: + if (item.get("url"), item.get("snapshot_path")) != ALLOWED_TARGETS[item["id"]]: + raise ValueError(f"publisher target mismatch for {item['id']}") + + +def validate_public_snapshot(snapshot: dict) -> None: + encoded = json.dumps(snapshot, ensure_ascii=False).lower() + for field in FORBIDDEN_PUBLIC_FIELDS: + if f'"{field.lower()}"' in encoded: + raise ValueError(f"private field rejected: {field}") + if snapshot.get("node_id") != "BS-SH-005": + raise ValueError("snapshot node mismatch") + if snapshot.get("persona_state") not in {"NOT_BORN", "BIRTH_GATE_PENDING"}: + raise ValueError("unexpected persona state") + + +def material_snapshot(snapshot: dict) -> dict: + """Drop heartbeat-only timestamps so idle cycles never create Git commits.""" + normalized = json.loads(json.dumps(snapshot)) + normalized.pop("updated_at", None) + for source in normalized.get("sources", {}).values(): + source.pop("updated_at", None) + return normalized + + +def git_env(config: dict) -> dict[str, str]: + env = dict(os.environ) + env.update( + { + "HOME": config["state_root"], + "GIT_TERMINAL_PROMPT": "0", + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "credential.helper", + "GIT_CONFIG_VALUE_0": config["credential_helper"], + } + ) + return env + + +def run_git(env: dict[str, str], *args: str, cwd: pathlib.Path | None = None) -> None: + subprocess.run( + ["git", *args], + cwd=cwd, + env=env, + check=True, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + timeout=300, + ) + + +def atomic_json(path: pathlib.Path, value: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + pending = path.with_name(f".{path.name}.{os.getpid()}.pending") + pending.write_text( + json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2) + "\n", + encoding="utf-8", + ) + os.replace(pending, path) + + +def publish_repository( + config: dict, repository: dict, snapshot: dict, env: dict[str, str] +) -> None: + checkout = pathlib.Path(config["state_root"]) / "repositories" / repository["id"] + if not (checkout / ".git").is_dir(): + checkout.parent.mkdir(parents=True, exist_ok=True) + run_git( + env, + "clone", + "--branch", + "main", + "--single-branch", + repository["url"], + str(checkout), + ) + else: + run_git(env, "fetch", "origin", "main", cwd=checkout) + run_git(env, "merge", "--ff-only", "origin/main", cwd=checkout) + + target = checkout / repository["snapshot_path"] + published = material_snapshot(snapshot) + if target.is_file(): + current = json.loads(target.read_text(encoding="utf-8")) + current.pop("publisher", None) + if current == published: + return + published["publisher"] = { + "schema": "guanghu.persona-history-publication/v1", + "node_id": "BS-SH-005", + "repository_id": repository["id"], + "published_at": now_iso(), + "source_observed_at": snapshot.get("updated_at"), + "content_policy": "SANITIZED_PUBLIC_WATERMARK_ONLY", + } + atomic_json(target, published) + run_git(env, "add", "--", repository["snapshot_path"], cwd=checkout) + changed = subprocess.run( + ["git", "diff", "--cached", "--quiet", "--exit-code"], + cwd=checkout, + env=env, + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + ).returncode + if changed == 0: + return + if changed != 1: + raise RuntimeError("git staged diff check failed") + run_git( + env, + "-c", + "user.name=BS-SH-005 Persona History", + "-c", + "user.email=bs-sh-005@noreply.guanghulab.com", + "commit", + "-m", + "chore(history): update BS-SH-005 recovery watermark", + cwd=checkout, + ) + run_git(env, "push", "origin", "HEAD:main", cwd=checkout) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--config", required=True, type=pathlib.Path) + args = parser.parse_args() + config = json.loads(args.config.read_text(encoding="utf-8")) + validate_config(config) + with urllib.request.urlopen(config["snapshot_url"], timeout=15) as response: + snapshot = json.loads(response.read()) + validate_public_snapshot(snapshot) + env = git_env(config) + for repository in config["repositories"]: + publish_repository(config, repository, snapshot, env) + + +if __name__ == "__main__": + main() diff --git a/engineering/persona-history-runtime/runtime/test_guanghu_history_publisher.py b/engineering/persona-history-runtime/runtime/test_guanghu_history_publisher.py new file mode 100644 index 0000000..13ac2de --- /dev/null +++ b/engineering/persona-history-runtime/runtime/test_guanghu_history_publisher.py @@ -0,0 +1,71 @@ +import json +import pathlib +import tempfile +import unittest + +import guanghu_history_publisher as publisher + + +class PublisherTests(unittest.TestCase): + def valid_config(self, state_root): + return { + "node_id": "BS-SH-005", + "state_root": state_root, + "credential_helper": "/credential-helper", + "repositories": [ + {"id": repo_id, "url": target[0], "snapshot_path": target[1]} + for repo_id, target in publisher.ALLOWED_TARGETS.items() + ], + } + + def test_rejects_target_outside_two_allowlisted_repositories(self): + config = self.valid_config("/tmp/state") + config["repositories"][0]["url"] = "https://example.com/other.git" + with self.assertRaises(ValueError): + publisher.validate_config(config) + + def test_rejects_private_fields_and_unexpected_birth_claim(self): + with self.assertRaises(ValueError): + publisher.validate_public_snapshot( + { + "node_id": "BS-SH-005", + "persona_state": "NOT_BORN", + "private_locator": "/private/source", + } + ) + with self.assertRaises(ValueError): + publisher.validate_public_snapshot( + {"node_id": "BS-SH-005", "persona_state": "BORN"} + ) + + def test_atomic_public_snapshot_contains_no_secret(self): + with tempfile.TemporaryDirectory() as directory: + target = pathlib.Path(directory) / "CURRENT.json" + snapshot = { + "node_id": "BS-SH-005", + "persona_state": "NOT_BORN", + "historical_time_caught_up": False, + } + publisher.validate_public_snapshot(snapshot) + publisher.atomic_json(target, snapshot) + self.assertEqual(json.loads(target.read_text()), snapshot) + + def test_heartbeat_only_timestamps_do_not_change_material_snapshot(self): + first = { + "node_id": "BS-SH-005", + "persona_state": "NOT_BORN", + "updated_at": "2026-08-01T19:00:00+08:00", + "sources": {"GPT": {"processed": 1726, "updated_at": "one"}}, + } + second = { + **first, + "updated_at": "2026-08-01T20:00:00+08:00", + "sources": {"GPT": {"processed": 1726, "updated_at": "two"}}, + } + self.assertEqual( + publisher.material_snapshot(first), publisher.material_snapshot(second) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/engineering/persona-history-runtime/scripts/install-bs-sh-005.sh b/engineering/persona-history-runtime/scripts/install-bs-sh-005.sh index 1a9b558..1dd21dc 100644 --- a/engineering/persona-history-runtime/scripts/install-bs-sh-005.sh +++ b/engineering/persona-history-runtime/scripts/install-bs-sh-005.sh @@ -18,6 +18,11 @@ state_root=/var/lib/guanghu/persona-history private_source_root=/guanghu/gestation/private/history-sources receipt_root=/guanghu/gestation/receipts/persona-history unit_target=/etc/systemd/system/guanghu-persona-history-recovery.service +publisher_unit_target=/etc/systemd/system/guanghu-persona-history-publisher.service +publisher_timer_target=/etc/systemd/system/guanghu-persona-history-publisher.timer +publisher_config_target=/etc/guanghu/persona-history-publisher.json +publisher_state_root=/var/lib/guanghu/persona-history-publisher +publisher_token=/etc/guanghu/persona-history-publisher.token previous_target=NONE if [ -L "${current_link}" ]; then @@ -32,12 +37,27 @@ if ! id guanghu-history >/dev/null 2>&1; then --user-group \ guanghu-history fi +if ! id guanghu-history-publisher >/dev/null 2>&1; then + useradd \ + --system \ + --home-dir "${publisher_state_root}" \ + --shell /usr/sbin/nologin \ + --user-group \ + guanghu-history-publisher +fi +test -s "${publisher_token}" install -d -o root -g root -m 0755 "${target_root}" install -d -o root -g root -m 0755 "${target_root}/runtime" install -m 0755 \ "${source_root}/runtime/guanghu_history_runtime.py" \ "${target_root}/runtime/guanghu_history_runtime.py" +install -m 0755 \ + "${source_root}/runtime/guanghu_history_publisher.py" \ + "${target_root}/runtime/guanghu_history_publisher.py" +install -m 0755 \ + "${source_root}/runtime/git-credential-guanghu-history" \ + "${target_root}/runtime/git-credential-guanghu-history" install -m 0644 \ "${source_root}/config/BS-SH-005.json" \ "${target_root}/BS-SH-005.json" @@ -49,24 +69,40 @@ install -d -o root -g guanghu-history -m 0750 /etc/guanghu install -m 0640 -o root -g guanghu-history \ "${source_root}/config/BS-SH-005.json" \ "${config_target}" +install -m 0644 -o root -g root \ + "${source_root}/config/BS-SH-005-publisher.json" \ + "${publisher_config_target}" +chown root:guanghu-history-publisher "${publisher_token}" +chmod 0640 "${publisher_token}" install -d -o guanghu-history -g guanghu-history -m 0750 "${state_root}" install -d -o guanghu-history -g guanghu-history -m 0750 "${state_root}/public" setfacl -m u:guanghu-history:--x /guanghu/gestation install -d -o root -g guanghu-history -m 0750 "${private_source_root}" install -d -o root -g root -m 0755 "${receipt_root}" +install -d -o guanghu-history-publisher -g guanghu-history-publisher -m 0750 \ + "${publisher_state_root}" ln -sfn "${target_root}/runtime" "${current_link}" install -m 0644 \ "${source_root}/packaging/guanghu-persona-history-recovery.service" \ "${unit_target}" +install -m 0644 \ + "${source_root}/packaging/guanghu-persona-history-publisher.service" \ + "${publisher_unit_target}" +install -m 0644 \ + "${source_root}/packaging/guanghu-persona-history-publisher.timer" \ + "${publisher_timer_target}" "${current_link}/guanghu_history_runtime.py" validate --config "${config_target}" systemctl daemon-reload systemctl enable guanghu-persona-history-recovery.service +systemctl enable guanghu-persona-history-publisher.timer systemctl restart guanghu-persona-history-recovery.service +systemctl start guanghu-persona-history-publisher.timer sleep 2 systemctl is-active --quiet guanghu-persona-history-recovery.service +systemctl is-active --quiet guanghu-persona-history-publisher.timer health=$(curl -fsS http://127.0.0.1:8089/healthz) echo "${health}" | grep -q '"status": "ok"' @@ -84,6 +120,9 @@ target_root: ${target_root} previous_target: ${previous_target} service: guanghu-persona-history-recovery.service service_state: active +publisher_service: guanghu-persona-history-publisher.service +publisher_timer: active +publisher_scope: REPO-014_AND_REPO-012_SANITIZED_WATERMARK_ONLY listen: 127.0.0.1:8089 private_source_root: ${private_source_root} public_state_root: ${state_root}/public