fix(history): back off retryable source failures

engineering/persona-history-runtime/runtime/guanghu_history_runtime.py

@@ -81,6 +81,20 @@ def blocks_later_history(status: str) -> bool:
     return status != "COMPLETE" and status not in SOURCE_WAITING_STATUSES
 
 
+def retry_backoff_elapsed(
+    updated_at: str | None, backoff_seconds: int, now_unix: float | None = None
+) -> bool:
+    """Keep retryable sources alive without writing one failure per runtime cycle."""
+    if not updated_at:
+        return True
+    try:
+        last_attempt = datetime.fromisoformat(updated_at).timestamp()
+    except (TypeError, ValueError):
+        return True
+    current = time.time() if now_unix is None else now_unix
+    return current - last_attempt >= backoff_seconds
+
+
 def redact_semantic_excerpt(text: str) -> str:
     text = re.sub(
         r"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b",
@@ -1016,6 +1030,12 @@ class Runtime:
         )
 
     def process_source(self, source: dict) -> None:
+        state = self.store.state(source["id"])
+        if state["status"] == "ERROR_RETRYABLE" and not retry_backoff_elapsed(
+            state["updated_at"],
+            int(self.config.get("source_retry_backoff_seconds", 1800)),
+        ):
+            return
         try:
             if source["kind"] == "gpt_export":
                 self.process_gpt(source)

engineering/persona-history-runtime/runtime/test_guanghu_history_runtime.py

@@ -47,6 +47,16 @@ class RuntimeTests(unittest.TestCase):
         self.assertFalse(runtime.blocks_later_history("COMPLETE"))
         self.assertFalse(runtime.blocks_later_history("WAITING_FOR_SOURCE_ACCEPTANCE"))
 
+    def test_retryable_source_uses_bounded_backoff(self):
+        updated_at = "1970-01-01T00:00:00+00:00"
+        self.assertFalse(
+            runtime.retry_backoff_elapsed(updated_at, 1800, now_unix=1799)
+        )
+        self.assertTrue(
+            runtime.retry_backoff_elapsed(updated_at, 1800, now_unix=1800)
+        )
+        self.assertTrue(runtime.retry_backoff_elapsed("invalid", 1800))
+
     def test_semantic_redaction_and_reality_boundary(self):
         redacted = runtime.redact_semantic_excerpt(
             "a@example.com token: sk-abcdefghijklmnop "