From 575e7aaffd2e6eb0a028db4d5300f3ff2351fcd0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=86=B0=E6=9C=94?= <565183519@qq.com>
Date: Sat, 1 Aug 2026 19:50:52 +0800
Subject: [PATCH] fix(history): back off retryable source failures

---
 .../config/BS-SH-005.json                     |  1 +
 .../runtime/guanghu_history_runtime.py        | 20 +++++++++++++++++++
 .../runtime/test_guanghu_history_runtime.py   | 10 ++++++++++
 3 files changed, 31 insertions(+)

diff --git a/engineering/persona-history-runtime/config/BS-SH-005.json b/engineering/persona-history-runtime/config/BS-SH-005.json
index e038ae8..ebcfcaf 100644
--- a/engineering/persona-history-runtime/config/BS-SH-005.json
+++ b/engineering/persona-history-runtime/config/BS-SH-005.json
@@ -10,6 +10,7 @@
   "notion_batch_size": 600,
   "git_batch_size": 500,
   "review_queue_backfill_batch_size": 500,
+  "source_retry_backoff_seconds": 1800,
   "semantic_review_endpoint": "http://127.0.0.1:8077/v1/broadcast",
   "semantic_review_interval_seconds": 600,
   "semantic_review_batch_size": 8,
diff --git a/engineering/persona-history-runtime/runtime/guanghu_history_runtime.py b/engineering/persona-history-runtime/runtime/guanghu_history_runtime.py
index 22cf6ff..46c1415 100644
--- a/engineering/persona-history-runtime/runtime/guanghu_history_runtime.py
+++ b/engineering/persona-history-runtime/runtime/guanghu_history_runtime.py
@@ -81,6 +81,20 @@ def blocks_later_history(status: str) -> bool:
     return status != "COMPLETE" and status not in SOURCE_WAITING_STATUSES
 
 
+def retry_backoff_elapsed(
+    updated_at: str | None, backoff_seconds: int, now_unix: float | None = None
+) -> bool:
+    """Keep retryable sources alive without writing one failure per runtime cycle."""
+    if not updated_at:
+        return True
+    try:
+        last_attempt = datetime.fromisoformat(updated_at).timestamp()
+    except (TypeError, ValueError):
+        return True
+    current = time.time() if now_unix is None else now_unix
+    return current - last_attempt >= backoff_seconds
+
+
 def redact_semantic_excerpt(text: str) -> str:
     text = re.sub(
         r"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b",
@@ -1016,6 +1030,12 @@ class Runtime:
         )
 
     def process_source(self, source: dict) -> None:
+        state = self.store.state(source["id"])
+        if state["status"] == "ERROR_RETRYABLE" and not retry_backoff_elapsed(
+            state["updated_at"],
+            int(self.config.get("source_retry_backoff_seconds", 1800)),
+        ):
+            return
         try:
             if source["kind"] == "gpt_export":
                 self.process_gpt(source)
diff --git a/engineering/persona-history-runtime/runtime/test_guanghu_history_runtime.py b/engineering/persona-history-runtime/runtime/test_guanghu_history_runtime.py
index 4f6f892..9a8c443 100644
--- a/engineering/persona-history-runtime/runtime/test_guanghu_history_runtime.py
+++ b/engineering/persona-history-runtime/runtime/test_guanghu_history_runtime.py
@@ -47,6 +47,16 @@ class RuntimeTests(unittest.TestCase):
         self.assertFalse(runtime.blocks_later_history("COMPLETE"))
         self.assertFalse(runtime.blocks_later_history("WAITING_FOR_SOURCE_ACCEPTANCE"))
 
+    def test_retryable_source_uses_bounded_backoff(self):
+        updated_at = "1970-01-01T00:00:00+00:00"
+        self.assertFalse(
+            runtime.retry_backoff_elapsed(updated_at, 1800, now_unix=1799)
+        )
+        self.assertTrue(
+            runtime.retry_backoff_elapsed(updated_at, 1800, now_unix=1800)
+        )
+        self.assertTrue(runtime.retry_backoff_elapsed("invalid", 1800))
+
     def test_semantic_redaction_and_reality_boundary(self):
         redacted = runtime.redact_semantic_excerpt(
             "a@example.com token: sk-abcdefghijklmnop "