legal-ai/web/agent_health.py

"""Agent health taxonomy (TaskMaster #222) — one crisp state per stuck issue.

Classifies each open, agent-assigned issue into a single health state so the
stranded-child / recovery-loop cases we diagnose by hand today
([[reference_recovery_loop_stranded_child]], [[reference_paperclip_recovery_loops]])
surface automatically in the dashboard — the ``gt feed --problems`` idea from
Gastown, grounded in *our* failure modes.

Two layers, split for testability (same shape as web.agent_telemetry):
- :func:`classify_issue_health` is a **pure function** over three primitives
  (live-run flag + wakeup counts). No DB, no platform symbols — trivially
  unit-tested and platform-agnostic (INV-G12).
- The Paperclip-specific fetch that derives those primitives from
  ``heartbeat_runs`` / ``agent_wakeup_requests`` lives in the shell
  (``web.paperclip_client.get_agent_health``); this module only decides *how to
  label* what the fetch measured, so the taxonomy survives a platform swap.

The wakeup/recovery signals it reads are the very ones #219 telemetry now emits
(``agent.wakeup`` with recovery reasons) — this is the consumer that turns that
stream into an at-a-glance verdict.
"""
from __future__ import annotations

# Severity order (worst first) — the dashboard sorts problems by this.
HEALTH_STATES: tuple[str, ...] = ("zombie", "stalled", "working", "idle")

# Substrings of Paperclip wakeup ``reason`` values that mark a recovery-loop
# wake (as opposed to a genuine new task). Sourced from the recovery-loop
# forensics in memory: an issue that keeps getting these while agent-owned is a
# zombie — assigned + "active" but nothing is really executing.
RECOVERY_REASON_MARKERS: tuple[str, ...] = (
    "source_scoped_recovery",
    "stranded_assigned",
    "issue_continuation",
    "issue_reopened",
    "missing_disposition",
    "successful_run_missing_state",
    "waiting_on_review",
)

# ≥ this many wakeups inside the fetch window, with no recovery marker and no
# live run, reads as "poked repeatedly but not progressing" = stalled.
STALL_WAKEUP_THRESHOLD = 3


def is_recovery_reason(reason: str | None) -> bool:
    """True if a wakeup ``reason`` is a recovery-loop marker (not a new task)."""
    if not reason:
        return False
    return any(marker in reason for marker in RECOVERY_REASON_MARKERS)


def classify_issue_health(
    *,
    has_live_run: bool,
    recovery_wakeups: int,
    total_wakeups: int,
) -> str:
    """Label one open, agent-assigned issue. Pure — no I/O.

    - ``working`` — a live run is executing right now.
    - ``zombie`` — no live run but recovery-loop wakeups are firing: the issue
      looks agent-owned + active yet nothing is really running (the stranded
      child we chase manually).
    - ``stalled`` — no live run, no recovery markers, but poked ``>=``
      :data:`STALL_WAKEUP_THRESHOLD` times in-window (repeated wakes, no
      progress).
    - ``idle`` — open + assigned but quiet: benign waiting.

    Called only for issues that are already open and agent-assigned; unassigned
    or closed issues are not health-tracked (the fetch filters them out).
    """
    if has_live_run:
        return "working"
    if recovery_wakeups >= 1:
        return "zombie"
    if total_wakeups >= STALL_WAKEUP_THRESHOLD:
        return "stalled"
    return "idle"


# A zombie needs at least this many recovery-marker wakeups before the watchdog
# auto-escalates it — filters a single transient recovery wake from a genuine,
# budget-burning loop. Escalation reassigns the issue to the chair (via
# escalate_issue), so an escalated issue is agent-unowned next sweep and never
# re-escalates (idempotent by construction).
ZOMBIE_ESCALATE_MIN_RECOVERY = 2


def zombie_escalation(item: dict) -> tuple[str, str] | None:
    """Decide whether a health item warrants auto-escalation to the chair.

    Pure (no I/O) so the watchdog's policy is unit-tested. Returns
    ``(severity, reason)`` for a persistent zombie, else ``None``. Severity
    scales with loop intensity — a long-running loop (``>=5`` recovery wakes)
    is ``high``, a fresh one ``medium``. Only ``zombie`` items escalate: a
    ``stalled``/``working``/``idle`` issue is not a recovery loop.
    """
    if item.get("health") != "zombie":
        return None
    recovery = int(item.get("recovery_wakeups", 0))
    if recovery < ZOMBIE_ESCALATE_MIN_RECOVERY:
        return None
    severity = "high" if recovery >= 5 else "medium"
    reason = (
        f"auto-escalation (watchdog): זוהתה לולאת-recovery — {recovery} יקיצות-שחזור "
        f"על {item.get('identifier')} ({item.get('agent_name')}); ה-issue נמסר לחיים "
        f"במקום להמשיך לשרוף budget."
    )
    return severity, reason