fix(history): stage metadata-only git mirrors
This commit is contained in:
parent
3adfa41dbc
commit
bc1ba8e47d
2 changed files with 189 additions and 10 deletions
|
|
@ -68,6 +68,24 @@ def sha256_file(path: pathlib.Path) -> str:
|
|||
return digest.hexdigest()
|
||||
|
||||
|
||||
def sanitize_process_error_detail(value: str | bytes | None) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, bytes):
|
||||
value = value.decode("utf-8", errors="replace")
|
||||
value = re.sub(
|
||||
r"(?i)(https?://)[^/@\s]+@",
|
||||
r"\1[REDACTED]@",
|
||||
value,
|
||||
)
|
||||
value = re.sub(
|
||||
r"(?i)\b(authorization|token|password|secret)\s*[:=]\s*\S+",
|
||||
r"\1=[REDACTED]",
|
||||
value,
|
||||
)
|
||||
return " ".join(value.split())[-800:]
|
||||
|
||||
|
||||
def classify_personas(text: str) -> list[str]:
|
||||
return [
|
||||
persona
|
||||
|
|
@ -966,26 +984,117 @@ class Runtime:
|
|||
|
||||
def git_mirror(self, source: dict) -> pathlib.Path:
|
||||
mirror = self.state_root / "git" / f"{source['id']}.git"
|
||||
pending = mirror.with_name(f".{mirror.name}.pending")
|
||||
mirror.parent.mkdir(parents=True, exist_ok=True)
|
||||
if mirror.exists():
|
||||
subprocess.run(
|
||||
["git", "-C", str(mirror), "fetch", "--all", "--prune"],
|
||||
check=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
self.run_git(
|
||||
[
|
||||
"git",
|
||||
"-C",
|
||||
str(mirror),
|
||||
"fetch",
|
||||
"--all",
|
||||
"--prune",
|
||||
"--filter=blob:none",
|
||||
],
|
||||
timeout=600,
|
||||
)
|
||||
else:
|
||||
return mirror
|
||||
|
||||
if not pending.exists():
|
||||
self.run_git(
|
||||
["git", "init", "--bare", str(pending)],
|
||||
timeout=60,
|
||||
)
|
||||
self.run_git(
|
||||
[
|
||||
"git",
|
||||
"-C",
|
||||
str(pending),
|
||||
"remote",
|
||||
"add",
|
||||
"origin",
|
||||
source["url"],
|
||||
],
|
||||
timeout=60,
|
||||
)
|
||||
for refspec in (
|
||||
"+refs/heads/*:refs/heads/*",
|
||||
"+refs/tags/*:refs/tags/*",
|
||||
):
|
||||
self.run_git(
|
||||
[
|
||||
"git",
|
||||
"-C",
|
||||
str(pending),
|
||||
"config",
|
||||
"--add",
|
||||
"remote.origin.fetch",
|
||||
refspec,
|
||||
],
|
||||
timeout=60,
|
||||
)
|
||||
self.run_git(
|
||||
[
|
||||
"git",
|
||||
"-C",
|
||||
str(pending),
|
||||
"config",
|
||||
"remote.origin.promisor",
|
||||
"true",
|
||||
],
|
||||
timeout=60,
|
||||
)
|
||||
self.run_git(
|
||||
[
|
||||
"git",
|
||||
"-C",
|
||||
str(pending),
|
||||
"config",
|
||||
"remote.origin.partialclonefilter",
|
||||
"blob:none",
|
||||
],
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
self.run_git(
|
||||
[
|
||||
"git",
|
||||
"-C",
|
||||
str(pending),
|
||||
"fetch",
|
||||
"--prune",
|
||||
"--filter=blob:none",
|
||||
"origin",
|
||||
],
|
||||
timeout=1800,
|
||||
)
|
||||
os.replace(pending, mirror)
|
||||
return mirror
|
||||
|
||||
@staticmethod
|
||||
def run_git(command: list[str], timeout: int) -> None:
|
||||
try:
|
||||
subprocess.run(
|
||||
["git", "clone", "--mirror", source["url"], str(mirror)],
|
||||
command,
|
||||
check=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
timeout=1800,
|
||||
timeout=timeout,
|
||||
)
|
||||
return mirror
|
||||
except subprocess.CalledProcessError as error:
|
||||
detail = sanitize_process_error_detail(error.stderr)
|
||||
suffix = f": {detail}" if detail else ""
|
||||
raise RuntimeError(
|
||||
f"git command failed with exit {error.returncode}{suffix}"
|
||||
) from error
|
||||
except subprocess.TimeoutExpired as error:
|
||||
detail = sanitize_process_error_detail(error.stderr)
|
||||
suffix = f": {detail}" if detail else ""
|
||||
raise RuntimeError(
|
||||
f"git command timed out after {timeout} seconds{suffix}"
|
||||
) from error
|
||||
|
||||
def process_git(self, source: dict) -> None:
|
||||
state = self.store.state(source["id"])
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import io
|
|||
import hashlib
|
||||
import json
|
||||
import pathlib
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
|
@ -57,6 +58,75 @@ class RuntimeTests(unittest.TestCase):
|
|||
)
|
||||
self.assertTrue(runtime.retry_backoff_elapsed("invalid", 1800))
|
||||
|
||||
def test_git_mirror_uses_bounded_metadata_only_staging(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = pathlib.Path(directory)
|
||||
source_root = root / "source"
|
||||
source_root.mkdir()
|
||||
subprocess.run(
|
||||
["git", "init", "-q", "-b", "main"],
|
||||
cwd=source_root,
|
||||
check=True,
|
||||
)
|
||||
subprocess.run(
|
||||
["git", "config", "user.name", "History Test"],
|
||||
cwd=source_root,
|
||||
check=True,
|
||||
)
|
||||
subprocess.run(
|
||||
["git", "config", "user.email", "history@example.invalid"],
|
||||
cwd=source_root,
|
||||
check=True,
|
||||
)
|
||||
(source_root / "history.txt").write_text("history\n")
|
||||
subprocess.run(
|
||||
["git", "add", "history.txt"],
|
||||
cwd=source_root,
|
||||
check=True,
|
||||
)
|
||||
subprocess.run(
|
||||
["git", "commit", "-q", "-m", "history"],
|
||||
cwd=source_root,
|
||||
check=True,
|
||||
)
|
||||
config_path = root / "config.json"
|
||||
config_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema": "test",
|
||||
"node_id": "BS-SH-005",
|
||||
"state_root": str(root / "state"),
|
||||
"private_source_root": str(root / "private"),
|
||||
"listen": "127.0.0.1:0",
|
||||
"sources": [],
|
||||
}
|
||||
)
|
||||
)
|
||||
service = runtime.Runtime(config_path)
|
||||
source = {
|
||||
"id": "GIT-TEST",
|
||||
"url": str(source_root),
|
||||
}
|
||||
mirror = service.git_mirror(source)
|
||||
self.assertTrue(mirror.is_dir())
|
||||
self.assertFalse(
|
||||
mirror.with_name(f".{mirror.name}.pending").exists()
|
||||
)
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(mirror), "rev-list", "--all", "--count"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
self.assertEqual(result.stdout.strip(), "1")
|
||||
|
||||
def test_process_error_detail_redacts_credentials(self):
|
||||
detail = runtime.sanitize_process_error_detail(
|
||||
"fatal https://user:secret@example.invalid token=abc123"
|
||||
)
|
||||
self.assertNotIn("user:secret", detail)
|
||||
self.assertNotIn("abc123", detail)
|
||||
|
||||
def test_semantic_redaction_and_reality_boundary(self):
|
||||
redacted = runtime.redact_semantic_excerpt(
|
||||
"a@example.com token: sk-abcdefghijklmnop "
|
||||
|
|
|
|||
Loading…
Reference in a new issue