From bc1ba8e47d98c5a6e75f52e0c4d16375d3e863cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=86=B0=E6=9C=94?= <565183519@qq.com> Date: Sat, 1 Aug 2026 20:48:01 +0800 Subject: [PATCH] fix(history): stage metadata-only git mirrors --- .../runtime/guanghu_history_runtime.py | 129 ++++++++++++++++-- .../runtime/test_guanghu_history_runtime.py | 70 ++++++++++ 2 files changed, 189 insertions(+), 10 deletions(-) diff --git a/engineering/persona-history-runtime/runtime/guanghu_history_runtime.py b/engineering/persona-history-runtime/runtime/guanghu_history_runtime.py index 9245987..765544d 100644 --- a/engineering/persona-history-runtime/runtime/guanghu_history_runtime.py +++ b/engineering/persona-history-runtime/runtime/guanghu_history_runtime.py @@ -68,6 +68,24 @@ def sha256_file(path: pathlib.Path) -> str: return digest.hexdigest() +def sanitize_process_error_detail(value: str | bytes | None) -> str: + if value is None: + return "" + if isinstance(value, bytes): + value = value.decode("utf-8", errors="replace") + value = re.sub( + r"(?i)(https?://)[^/@\s]+@", + r"\1[REDACTED]@", + value, + ) + value = re.sub( + r"(?i)\b(authorization|token|password|secret)\s*[:=]\s*\S+", + r"\1=[REDACTED]", + value, + ) + return " ".join(value.split())[-800:] + + def classify_personas(text: str) -> list[str]: return [ persona @@ -966,26 +984,117 @@ class Runtime: def git_mirror(self, source: dict) -> pathlib.Path: mirror = self.state_root / "git" / f"{source['id']}.git" + pending = mirror.with_name(f".{mirror.name}.pending") mirror.parent.mkdir(parents=True, exist_ok=True) if mirror.exists(): - subprocess.run( - ["git", "-C", str(mirror), "fetch", "--all", "--prune"], - check=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.PIPE, - text=True, + self.run_git( + [ + "git", + "-C", + str(mirror), + "fetch", + "--all", + "--prune", + "--filter=blob:none", + ], timeout=600, ) - else: + return mirror + + if not pending.exists(): + self.run_git( + ["git", "init", "--bare", str(pending)], + timeout=60, + ) + self.run_git( + [ + "git", + "-C", + str(pending), + "remote", + "add", + "origin", + source["url"], + ], + timeout=60, + ) + for refspec in ( + "+refs/heads/*:refs/heads/*", + "+refs/tags/*:refs/tags/*", + ): + self.run_git( + [ + "git", + "-C", + str(pending), + "config", + "--add", + "remote.origin.fetch", + refspec, + ], + timeout=60, + ) + self.run_git( + [ + "git", + "-C", + str(pending), + "config", + "remote.origin.promisor", + "true", + ], + timeout=60, + ) + self.run_git( + [ + "git", + "-C", + str(pending), + "config", + "remote.origin.partialclonefilter", + "blob:none", + ], + timeout=60, + ) + + self.run_git( + [ + "git", + "-C", + str(pending), + "fetch", + "--prune", + "--filter=blob:none", + "origin", + ], + timeout=1800, + ) + os.replace(pending, mirror) + return mirror + + @staticmethod + def run_git(command: list[str], timeout: int) -> None: + try: subprocess.run( - ["git", "clone", "--mirror", source["url"], str(mirror)], + command, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True, - timeout=1800, + timeout=timeout, ) - return mirror + except subprocess.CalledProcessError as error: + detail = sanitize_process_error_detail(error.stderr) + suffix = f": {detail}" if detail else "" + raise RuntimeError( + f"git command failed with exit {error.returncode}{suffix}" + ) from error + except subprocess.TimeoutExpired as error: + detail = sanitize_process_error_detail(error.stderr) + suffix = f": {detail}" if detail else "" + raise RuntimeError( + f"git command timed out after {timeout} seconds{suffix}" + ) from error def process_git(self, source: dict) -> None: state = self.store.state(source["id"]) diff --git a/engineering/persona-history-runtime/runtime/test_guanghu_history_runtime.py b/engineering/persona-history-runtime/runtime/test_guanghu_history_runtime.py index 56d8ef7..a062da5 100644 --- a/engineering/persona-history-runtime/runtime/test_guanghu_history_runtime.py +++ b/engineering/persona-history-runtime/runtime/test_guanghu_history_runtime.py @@ -2,6 +2,7 @@ import io import hashlib import json import pathlib +import subprocess import tempfile import unittest @@ -57,6 +58,75 @@ class RuntimeTests(unittest.TestCase): ) self.assertTrue(runtime.retry_backoff_elapsed("invalid", 1800)) + def test_git_mirror_uses_bounded_metadata_only_staging(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + source_root = root / "source" + source_root.mkdir() + subprocess.run( + ["git", "init", "-q", "-b", "main"], + cwd=source_root, + check=True, + ) + subprocess.run( + ["git", "config", "user.name", "History Test"], + cwd=source_root, + check=True, + ) + subprocess.run( + ["git", "config", "user.email", "history@example.invalid"], + cwd=source_root, + check=True, + ) + (source_root / "history.txt").write_text("history\n") + subprocess.run( + ["git", "add", "history.txt"], + cwd=source_root, + check=True, + ) + subprocess.run( + ["git", "commit", "-q", "-m", "history"], + cwd=source_root, + check=True, + ) + config_path = root / "config.json" + config_path.write_text( + json.dumps( + { + "schema": "test", + "node_id": "BS-SH-005", + "state_root": str(root / "state"), + "private_source_root": str(root / "private"), + "listen": "127.0.0.1:0", + "sources": [], + } + ) + ) + service = runtime.Runtime(config_path) + source = { + "id": "GIT-TEST", + "url": str(source_root), + } + mirror = service.git_mirror(source) + self.assertTrue(mirror.is_dir()) + self.assertFalse( + mirror.with_name(f".{mirror.name}.pending").exists() + ) + result = subprocess.run( + ["git", "-C", str(mirror), "rev-list", "--all", "--count"], + check=True, + capture_output=True, + text=True, + ) + self.assertEqual(result.stdout.strip(), "1") + + def test_process_error_detail_redacts_credentials(self): + detail = runtime.sanitize_process_error_detail( + "fatal https://user:secret@example.invalid token=abc123" + ) + self.assertNotIn("user:secret", detail) + self.assertNotIn("abc123", detail) + def test_semantic_redaction_and_reality_boundary(self): redacted = runtime.redact_semantic_excerpt( "a@example.com token: sk-abcdefghijklmnop "