fix(history): back off retryable source failures

engineering/persona-history-runtime/runtime/guanghu_history_runtime.py

@@ -81,6 +81,20 @@ def blocks_later_history(status: str) -> bool:
     return status != "COMPLETE" and status not in SOURCE_WAITING_STATUSES
 
 
+def retry_backoff_elapsed(
+    updated_at: str | None, backoff_seconds: int, now_unix: float | None = None
+) -> bool:
+    """Keep retryable sources alive without writing one failure per runtime cycle."""
+    if not updated_at:
+        return True
+    try:
+        last_attempt = datetime.fromisoformat(updated_at).timestamp()
+    except (TypeError, ValueError):
+        return True
+    current = time.time() if now_unix is None else now_unix
+    return current - last_attempt >= backoff_seconds
+
+
 def redact_semantic_excerpt(text: str) -> str:
     text = re.sub(
         r"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b",
@@ -1016,6 +1030,12 @@ class Runtime:
         )
 
     def process_source(self, source: dict) -> None:
+        state = self.store.state(source["id"])
+        if state["status"] == "ERROR_RETRYABLE" and not retry_backoff_elapsed(
+            state["updated_at"],
+            int(self.config.get("source_retry_backoff_seconds", 1800)),
+        ):
+            return
         try:
             if source["kind"] == "gpt_export":
                 self.process_gpt(source)