"""Agent health taxonomy (TaskMaster #222) — one crisp state per stuck issue. Classifies each open, agent-assigned issue into a single health state so the stranded-child / recovery-loop cases we diagnose by hand today ([[reference_recovery_loop_stranded_child]], [[reference_paperclip_recovery_loops]]) surface automatically in the dashboard — the ``gt feed --problems`` idea from Gastown, grounded in *our* failure modes. Two layers, split for testability (same shape as web.agent_telemetry): - :func:`classify_issue_health` is a **pure function** over three primitives (live-run flag + wakeup counts). No DB, no platform symbols — trivially unit-tested and platform-agnostic (INV-G12). - The Paperclip-specific fetch that derives those primitives from ``heartbeat_runs`` / ``agent_wakeup_requests`` lives in the shell (``web.paperclip_client.get_agent_health``); this module only decides *how to label* what the fetch measured, so the taxonomy survives a platform swap. The wakeup/recovery signals it reads are the very ones #219 telemetry now emits (``agent.wakeup`` with recovery reasons) — this is the consumer that turns that stream into an at-a-glance verdict. """ from __future__ import annotations # Severity order (worst first) — the dashboard sorts problems by this. HEALTH_STATES: tuple[str, ...] = ("zombie", "stalled", "working", "idle") # Substrings of Paperclip wakeup ``reason`` values that mark a recovery-loop # wake (as opposed to a genuine new task). Sourced from the recovery-loop # forensics in memory: an issue that keeps getting these while agent-owned is a # zombie — assigned + "active" but nothing is really executing. RECOVERY_REASON_MARKERS: tuple[str, ...] = ( "source_scoped_recovery", "stranded_assigned", "issue_continuation", "issue_reopened", "missing_disposition", "successful_run_missing_state", "waiting_on_review", ) # ≥ this many wakeups inside the fetch window, with no recovery marker and no # live run, reads as "poked repeatedly but not progressing" = stalled. STALL_WAKEUP_THRESHOLD = 3 def is_recovery_reason(reason: str | None) -> bool: """True if a wakeup ``reason`` is a recovery-loop marker (not a new task).""" if not reason: return False return any(marker in reason for marker in RECOVERY_REASON_MARKERS) def classify_issue_health( *, has_live_run: bool, recovery_wakeups: int, total_wakeups: int, ) -> str: """Label one open, agent-assigned issue. Pure — no I/O. - ``working`` — a live run is executing right now. - ``zombie`` — no live run but recovery-loop wakeups are firing: the issue looks agent-owned + active yet nothing is really running (the stranded child we chase manually). - ``stalled`` — no live run, no recovery markers, but poked ``>=`` :data:`STALL_WAKEUP_THRESHOLD` times in-window (repeated wakes, no progress). - ``idle`` — open + assigned but quiet: benign waiting. Called only for issues that are already open and agent-assigned; unassigned or closed issues are not health-tracked (the fetch filters them out). """ if has_live_run: return "working" if recovery_wakeups >= 1: return "zombie" if total_wakeups >= STALL_WAKEUP_THRESHOLD: return "stalled" return "idle" # A zombie needs at least this many recovery-marker wakeups before the watchdog # auto-escalates it — filters a single transient recovery wake from a genuine, # budget-burning loop. Escalation reassigns the issue to the chair (via # escalate_issue), so an escalated issue is agent-unowned next sweep and never # re-escalates (idempotent by construction). ZOMBIE_ESCALATE_MIN_RECOVERY = 2 def zombie_escalation(item: dict) -> tuple[str, str] | None: """Decide whether a health item warrants auto-escalation to the chair. Pure (no I/O) so the watchdog's policy is unit-tested. Returns ``(severity, reason)`` for a persistent zombie, else ``None``. Severity scales with loop intensity — a long-running loop (``>=5`` recovery wakes) is ``high``, a fresh one ``medium``. Only ``zombie`` items escalate: a ``stalled``/``working``/``idle`` issue is not a recovery loop. """ if item.get("health") != "zombie": return None recovery = int(item.get("recovery_wakeups", 0)) if recovery < ZOMBIE_ESCALATE_MIN_RECOVERY: return None severity = "high" if recovery >= 5 else "medium" reason = ( f"auto-escalation (watchdog): זוהתה לולאת-recovery — {recovery} יקיצות-שחזור " f"על {item.get('identifier')} ({item.get('agent_name')}); ה-issue נמסר לחיים " f"במקום להמשיך לשרוף budget." ) return severity, reason