scripts/halacha_drain_supervisor.py

@@ -22,10 +22,13 @@ log tails →
   • re-triggers the one-shot drain when idle and the queue is non-empty
   • restarts a HUNG run (online but no new chunk-checkpoint for > 25 min — the
     REAL liveness signal; the out-log only updates when a whole CASE finishes)
-  • backs off on rate-limit until quota resets — PRIMARY signal is the authoritative
-    OAuth usage endpoint (durable; the same util the Claude Code UI shows), with the
-    log 429 only as a fallback when that endpoint is unreachable. While limited it
-    STOPS the drain (no 429-hammering) and re-ignites once quota is back.
+  • backs off on rate-limit until quota resets. TWO signals, EITHER triggers a
+    cooldown: the authoritative OAuth usage endpoint (durable; the util the Claude
+    Code UI shows) AND a fresh CLI 429 in the logs. A fresh 429 is GROUND TRUTH —
+    the call is literally failing — so it overrides an endpoint that wrongly reports
+    the window available, and it VETOES the early resume. While limited it STOPS the
+    drain (no 429-hammering, no re-extracting/degrading completed cases) and
+    re-ignites once quota is back AND no fresh 429 remains.
   • verifies crash-safe per-chunk staging is committing (nothing lost)
 
 The night window is 23:00–05:00 IDT, with a bounded early-morning CATCH-UP band
@@ -68,6 +71,7 @@ VENV_PY = os.path.join(REPO, "mcp-server/.venv/bin/python")
 
 STUCK_SILENCE_SEC = 1500   # 25 min with no new chunk-checkpoint while online → hung
 WEEKLY_GAP_HOURS = 6       # reset further than this → treat as weekly, not 5h
+DEFAULT_COOLDOWN_MIN = 30  # cooldown when a fresh 429 has no parseable reset time
 IDT = timezone(timedelta(hours=3))   # Israel summer time (IDT, UTC+3) — display only
 NIGHT_START, NIGHT_END = 23, 5       # the drain's normal window (IDT hours)
 CATCHUP_END = 7        # soft window end (IDT) for early-morning catch-up — see fix B
@@ -441,26 +445,38 @@ def tick():
     # under the supervisor's own restart-churn, so an exhausted 5-hour window read
     # as 'hung' and got hammered with restarts. The endpoint can't scroll away.
     fresh = (age is not None and age < 1800)
-    log_rl = bool(rl_recent and fresh)
+    log_rl = bool(rl_recent and fresh)               # a FRESH CLI 429 in the logs
     auth = quota_exhausted()                 # (exhausted, reset_utc) | None if endpoint down
-    auth_says_ok = (auth is not None and not auth[0])
+    auth_exhausted = bool(auth is not None and auth[0])
+    auth_says_ok = bool(auth is not None and not auth[0])
+    # A fresh CLI 429 is GROUND TRUTH: the call is literally failing, so cooldown
+    # even when the usage endpoint reports the window available. Observed
+    # 2026-06-15: endpoint <100% while the CLI kept 429-ing ("session limit") —
+    # the old "endpoint is primary" logic then read it as 'hung' and restart-churned,
+    # re-extracting already-completed cases and DEGRADING them under the rate limit.
     cd_dt = None
-    if auth is not None and auth[0]:                 # authoritative: a window is exhausted
+    if auth_exhausted:
         cd_dt = auth[1] or reset_dt                  # prefer endpoint reset; fall back to parsed
-    elif log_rl and not auth_says_ok:                # endpoint down/silent → trust a fresh 429
-        cd_dt = reset_dt
+    if cd_dt is None and log_rl:                      # fresh 429 overrides an "available" endpoint
+        cd_dt = reset_dt or (now + timedelta(minutes=DEFAULT_COOLDOWN_MIN))
     if cd_dt is None and prev.get("cooldown_until"):  # persist a stored future reset
         try:
             cd_dt = datetime.fromisoformat(prev["cooldown_until"])
         except Exception:
             cd_dt = None
     in_cooldown = bool(cd_dt and now < cd_dt)
-    # Exit cooldown the instant quota is actually back — claude.ai usually frees up
-    # EARLIER than the reported reset. Authoritative all-clear is decisive; when the
-    # endpoint is down, fall back to the tiny live CLI probe (old behavior).
-    if in_cooldown and (auth_says_ok or (auth is None and quota_available())):
+    # Exit cooldown early ONLY when quota is actually back AND no fresh 429
+    # contradicts it. The endpoint can lie "available" while the CLI still 429s, so
+    # a fresh 429 VETOES the early resume (otherwise we'd bounce straight back into
+    # the churn). When there's no fresh 429: authoritative all-clear is decisive;
+    # if the endpoint is down, fall back to the tiny live CLI probe.
+    if (
+        in_cooldown
+        and not log_rl
+        and (auth_says_ok or (auth is None and quota_available()))
+    ):
         notes.append(
-            f"בדיקת-מכסה: המכסה זמינה — מתחדש מיד "
+            f"בדיקת-מכסה: המכסה זמינה (ואין 429 טרי) — מתחדש מיד "
             f"(לפני האיפוס המדווח {cd_dt.astimezone(IDT):%H:%M IDT}).")
         cd_dt, in_cooldown = None, False
     cooldown_until = cd_dt.isoformat() if cd_dt else None