Compare commits
2 Commits
a9db657541
...
8d0f8ad17b
| Author | SHA1 | Date | |
|---|---|---|---|
| 8d0f8ad17b | |||
| cf7c918bed |
@@ -29,12 +29,15 @@ import {
|
|||||||
useRunLog,
|
useRunLog,
|
||||||
useCancelRun,
|
useCancelRun,
|
||||||
useResetAgentSession,
|
useResetAgentSession,
|
||||||
|
useAgentHealth,
|
||||||
|
useRecentEscalations,
|
||||||
type OpsService,
|
type OpsService,
|
||||||
type OperationsSnapshot,
|
type OperationsSnapshot,
|
||||||
type PipelineStats,
|
type PipelineStats,
|
||||||
type AgentRun,
|
type AgentRun,
|
||||||
type SubscriptionUsage,
|
type SubscriptionUsage,
|
||||||
type UsageWindow,
|
type UsageWindow,
|
||||||
|
type AgentHealthState,
|
||||||
} from "@/lib/api/operations";
|
} from "@/lib/api/operations";
|
||||||
import { formatTime, israelParts } from "@/lib/format-date";
|
import { formatTime, israelParts } from "@/lib/format-date";
|
||||||
|
|
||||||
@@ -802,6 +805,154 @@ function LiveAgentsPanel() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Agent health taxonomy (#222) + recent escalations (#218) ───────────────
|
||||||
|
const HEALTH_LABEL: Record<AgentHealthState, string> = {
|
||||||
|
zombie: "זומבי",
|
||||||
|
stalled: "תקוע",
|
||||||
|
working: "עובד",
|
||||||
|
idle: "רגוע",
|
||||||
|
};
|
||||||
|
|
||||||
|
function AgentHealthPanel() {
|
||||||
|
const { data, isLoading } = useAgentHealth();
|
||||||
|
const { data: esc } = useRecentEscalations();
|
||||||
|
|
||||||
|
const counts = data?.counts;
|
||||||
|
// Only zombies/stalled are "problems" — the backend already sorts worst-first.
|
||||||
|
// An escalated issue is chair-owned, so it leaves this list; the escalations
|
||||||
|
// card below is the handled-loop history.
|
||||||
|
const problems = (data?.items ?? []).filter(
|
||||||
|
(i) => i.health === "zombie" || i.health === "stalled",
|
||||||
|
);
|
||||||
|
const healthyWorking = counts?.working ?? 0;
|
||||||
|
const healthyIdle = counts?.idle ?? 0;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="space-y-4">
|
||||||
|
<Card className="bg-surface border-rule shadow-sm">
|
||||||
|
<CardContent className="px-6 py-5">
|
||||||
|
{isLoading || !data ? (
|
||||||
|
<Skeleton className="h-24 w-full" />
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
<div className="flex flex-wrap gap-2 mb-4">
|
||||||
|
<StatTile
|
||||||
|
label="זומבי"
|
||||||
|
value={counts?.zombie ?? 0}
|
||||||
|
tone="red"
|
||||||
|
title="לולאת-recovery — משויך+פעיל-כביכול אך אין ריצה חיה. מוסלם אוטומטית."
|
||||||
|
/>
|
||||||
|
<StatTile
|
||||||
|
label="תקוע"
|
||||||
|
value={counts?.stalled ?? 0}
|
||||||
|
tone="amber"
|
||||||
|
title="ננער חוזר בלי התקדמות"
|
||||||
|
/>
|
||||||
|
<StatTile label="עובד" value={counts?.working ?? 0} tone="green" />
|
||||||
|
<StatTile label="רגוע" value={counts?.idle ?? 0} tone="muted" />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{problems.length === 0 ? (
|
||||||
|
<p className="text-sm text-emerald-600">
|
||||||
|
אין סוכן תקוע כרגע — כל הסוכנים תקינים.
|
||||||
|
</p>
|
||||||
|
) : (
|
||||||
|
<div className="grid gap-2">
|
||||||
|
{problems.map((p) => (
|
||||||
|
<div
|
||||||
|
key={p.issue_id}
|
||||||
|
className={
|
||||||
|
p.health === "zombie"
|
||||||
|
? "flex items-center justify-between gap-3 rounded-md border border-destructive/30 bg-destructive/5 px-3 py-2"
|
||||||
|
: "flex items-center justify-between gap-3 rounded-md border border-gold/40 bg-gold-wash px-3 py-2"
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<div className="flex items-center gap-2 flex-wrap min-w-0">
|
||||||
|
<Badge
|
||||||
|
variant={p.health === "zombie" ? "destructive" : "outline"}
|
||||||
|
className={
|
||||||
|
p.health === "stalled"
|
||||||
|
? "border-warn/50 text-warn font-normal"
|
||||||
|
: "font-normal"
|
||||||
|
}
|
||||||
|
>
|
||||||
|
{HEALTH_LABEL[p.health]}
|
||||||
|
</Badge>
|
||||||
|
<span dir="ltr" className="font-mono text-[0.82rem] text-navy font-semibold">
|
||||||
|
{p.identifier}
|
||||||
|
</span>
|
||||||
|
<span className="text-[0.82rem] text-navy font-semibold">
|
||||||
|
{p.agent_name}
|
||||||
|
</span>
|
||||||
|
<span className="text-[0.7rem] text-ink-muted">
|
||||||
|
{p.recovery_wakeups > 0
|
||||||
|
? `${p.recovery_wakeups} יקיצות-שחזור`
|
||||||
|
: `${p.wakeups} יקיצות`}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<span className="text-[0.7rem] text-ink-muted shrink-0">
|
||||||
|
{p.health === "zombie" ? "יוסלם אוטומטית" : "מנוטר"}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<p className="mt-3 text-[0.78rem] text-ink-muted">
|
||||||
|
<b className="text-navy">{healthyWorking + healthyIdle} סוכנים תקינים</b> —{" "}
|
||||||
|
{healthyWorking} עובדים, {healthyIdle} רגועים. מוצגות רק בעיות.
|
||||||
|
</p>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</CardContent>
|
||||||
|
</Card>
|
||||||
|
|
||||||
|
{/* handled-loop history — the watchdog's escalations, gold-wash like a gate */}
|
||||||
|
<Card className="bg-gold-wash border-gold/40 shadow-sm">
|
||||||
|
<CardContent className="px-5 py-4">
|
||||||
|
<h3 className="text-navy text-sm font-semibold mb-0.5">הסלמות אוטומטיות אחרונות</h3>
|
||||||
|
<p className="text-ink-muted text-[0.72rem] mb-3">
|
||||||
|
כל זומבי מתמשך (לולאת-recovery ≥2 יקיצות-שחזור) משויך אליך אוטומטית עם הערת-מערכת
|
||||||
|
— בלי להעיר סוכן מחדש.
|
||||||
|
</p>
|
||||||
|
{!esc?.items?.length ? (
|
||||||
|
<p className="text-sm text-ink-muted">אין הסלמות אחרונות.</p>
|
||||||
|
) : (
|
||||||
|
<div className="grid gap-1">
|
||||||
|
{esc.items.map((e) => (
|
||||||
|
<div
|
||||||
|
key={`${e.issue_id}-${e.created_at ?? ""}`}
|
||||||
|
className="flex items-start gap-2.5 border-t border-gold/15 first:border-0 pt-2 first:pt-0"
|
||||||
|
>
|
||||||
|
<Badge
|
||||||
|
variant={e.severity === "medium" ? "outline" : "destructive"}
|
||||||
|
className={
|
||||||
|
e.severity === "medium"
|
||||||
|
? "border-gold/50 text-gold-deep font-normal shrink-0"
|
||||||
|
: "font-normal shrink-0"
|
||||||
|
}
|
||||||
|
>
|
||||||
|
{e.severity}
|
||||||
|
</Badge>
|
||||||
|
<div className="min-w-0 flex-1">
|
||||||
|
<span dir="ltr" className="font-mono text-[0.8rem] text-navy font-semibold">
|
||||||
|
{e.identifier}
|
||||||
|
</span>
|
||||||
|
<div className="text-[0.78rem] text-ink-soft">{e.reason}</div>
|
||||||
|
</div>
|
||||||
|
<span className="text-[0.68rem] text-ink-muted shrink-0">
|
||||||
|
{e.created_at ? ago(Date.parse(e.created_at)) : ""}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</CardContent>
|
||||||
|
</Card>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export default function OperationsPage() {
|
export default function OperationsPage() {
|
||||||
const { data, isLoading, error } = useOperations();
|
const { data, isLoading, error } = useOperations();
|
||||||
|
|
||||||
@@ -840,6 +991,9 @@ export default function OperationsPage() {
|
|||||||
<SectionHeader>סוכנים פעילים</SectionHeader>
|
<SectionHeader>סוכנים פעילים</SectionHeader>
|
||||||
<LiveAgentsPanel />
|
<LiveAgentsPanel />
|
||||||
|
|
||||||
|
<SectionHeader>בריאות-הסוכנים</SectionHeader>
|
||||||
|
<AgentHealthPanel />
|
||||||
|
|
||||||
<SectionHeader>מתאמי-סוכנים</SectionHeader>
|
<SectionHeader>מתאמי-סוכנים</SectionHeader>
|
||||||
<AgentAdaptersPanel />
|
<AgentAdaptersPanel />
|
||||||
|
|
||||||
|
|||||||
@@ -190,6 +190,61 @@ export function useAgentRuns() {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Agent health taxonomy (#222) + recent escalations (#218) ───────────────
|
||||||
|
export type AgentHealthState = "zombie" | "stalled" | "working" | "idle";
|
||||||
|
|
||||||
|
export type AgentHealthItem = {
|
||||||
|
issue_id: string;
|
||||||
|
identifier: string;
|
||||||
|
status: string;
|
||||||
|
agent_id: string;
|
||||||
|
agent_name: string;
|
||||||
|
health: AgentHealthState;
|
||||||
|
wakeups: number;
|
||||||
|
recovery_wakeups: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type AgentHealthResponse = {
|
||||||
|
ok: boolean;
|
||||||
|
items: AgentHealthItem[]; // worst-first (zombie → stalled → working → idle)
|
||||||
|
counts: Record<AgentHealthState, number>;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Per-issue agent health — surfaces recovery-loop zombies automatically. */
|
||||||
|
export function useAgentHealth() {
|
||||||
|
return useQuery({
|
||||||
|
queryKey: ["operations", "agents", "health"],
|
||||||
|
queryFn: ({ signal }) =>
|
||||||
|
apiRequest<AgentHealthResponse>("/api/operations/agents/health", { signal }),
|
||||||
|
refetchInterval: 5000, // live alongside the running-agents view
|
||||||
|
staleTime: 3000,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export type EscalationItem = {
|
||||||
|
issue_id: string;
|
||||||
|
identifier: string;
|
||||||
|
severity: "critical" | "high" | "medium";
|
||||||
|
reason: string;
|
||||||
|
created_at: string | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type RecentEscalationsResponse = {
|
||||||
|
ok: boolean;
|
||||||
|
items: EscalationItem[]; // newest-first
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Recent chair escalations (watchdog + manual) — the handled-loop history. */
|
||||||
|
export function useRecentEscalations() {
|
||||||
|
return useQuery({
|
||||||
|
queryKey: ["operations", "agents", "escalations"],
|
||||||
|
queryFn: ({ signal }) =>
|
||||||
|
apiRequest<RecentEscalationsResponse>("/api/operations/agents/escalations", { signal }),
|
||||||
|
refetchInterval: 15000, // escalations change slowly (watchdog runs every 15m)
|
||||||
|
staleTime: 10000,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
/** Full output log of one run — fetched on demand (drawer open). */
|
/** Full output log of one run — fetched on demand (drawer open). */
|
||||||
export function useRunLog(runId: string | null) {
|
export function useRunLog(runId: string | null) {
|
||||||
return useQuery({
|
return useQuery({
|
||||||
|
|||||||
@@ -41,6 +41,7 @@ from web.paperclip_client import (
|
|||||||
create_project as pc_create_project,
|
create_project as pc_create_project,
|
||||||
create_workflow_issue as pc_create_workflow_issue,
|
create_workflow_issue as pc_create_workflow_issue,
|
||||||
get_agent_health as pc_get_agent_health,
|
get_agent_health as pc_get_agent_health,
|
||||||
|
get_recent_escalations as pc_get_recent_escalations,
|
||||||
get_predecessor_context as pc_get_predecessor_context,
|
get_predecessor_context as pc_get_predecessor_context,
|
||||||
get_agents_for_case as pc_get_agents_for_case,
|
get_agents_for_case as pc_get_agents_for_case,
|
||||||
get_agents_for_company as pc_get_agents,
|
get_agents_for_company as pc_get_agents,
|
||||||
@@ -180,6 +181,7 @@ __all__ = [
|
|||||||
"pc_escalate_issue",
|
"pc_escalate_issue",
|
||||||
# agent-run observability + control (live view + smart management)
|
# agent-run observability + control (live view + smart management)
|
||||||
"pc_get_agent_health",
|
"pc_get_agent_health",
|
||||||
|
"pc_get_recent_escalations",
|
||||||
"pc_get_predecessor_context",
|
"pc_get_predecessor_context",
|
||||||
"pc_list_live_runs",
|
"pc_list_live_runs",
|
||||||
"pc_get_run_log",
|
"pc_get_run_log",
|
||||||
|
|||||||
10
web/app.py
10
web/app.py
@@ -63,6 +63,7 @@ from web.agent_platform_port import (
|
|||||||
pc_create_workflow_issue,
|
pc_create_workflow_issue,
|
||||||
pc_escalate_issue,
|
pc_escalate_issue,
|
||||||
pc_get_agent_health,
|
pc_get_agent_health,
|
||||||
|
pc_get_recent_escalations,
|
||||||
pc_get_predecessor_context,
|
pc_get_predecessor_context,
|
||||||
pc_get_agents,
|
pc_get_agents,
|
||||||
pc_get_agents_for_case,
|
pc_get_agents_for_case,
|
||||||
@@ -7593,6 +7594,15 @@ async def operations_agent_health():
|
|||||||
return await pc_get_agent_health()
|
return await pc_get_agent_health()
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/operations/agents/escalations")
|
||||||
|
async def operations_agent_escalations(limit: int = 10, hours: int = 24):
|
||||||
|
"""Recent chair escalations — watchdog + manual (#218/#222).
|
||||||
|
|
||||||
|
Read-only history of what was handed to the chair, for the ops health panel.
|
||||||
|
"""
|
||||||
|
return await pc_get_recent_escalations(limit, hours)
|
||||||
|
|
||||||
|
|
||||||
@app.get("/api/operations/issues/{issue_id}/predecessor")
|
@app.get("/api/operations/issues/{issue_id}/predecessor")
|
||||||
async def operations_issue_predecessor(issue_id: str, limit: int = 3):
|
async def operations_issue_predecessor(issue_id: str, limit: int = 3):
|
||||||
"""Recent finished runs on an issue for session continuation (#220, "seance").
|
"""Recent finished runs on an issue for session continuation (#220, "seance").
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ from __future__ import annotations
|
|||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import uuid
|
import uuid
|
||||||
|
|
||||||
import asyncpg
|
import asyncpg
|
||||||
@@ -1132,6 +1133,57 @@ async def get_agent_health() -> dict:
|
|||||||
return {"ok": True, "items": items, "counts": counts}
|
return {"ok": True, "items": items, "counts": counts}
|
||||||
|
|
||||||
|
|
||||||
|
# The escalation note escalate_issue() writes as an author_type='system' comment.
|
||||||
|
# One prefix + a [severity] tag — parsed back here for the ops panel (#218/#222 UI).
|
||||||
|
_ESCALATION_BODY_PREFIX = "🚨 הסלמה"
|
||||||
|
_ESCALATION_SEVERITY_RE = re.compile(r"\[(critical|high|medium)\]")
|
||||||
|
|
||||||
|
|
||||||
|
async def get_recent_escalations(limit: int = 10, hours: int = 24) -> dict:
|
||||||
|
"""Recent chair escalations (watchdog + manual), newest-first (#218/#222 UI).
|
||||||
|
|
||||||
|
Read-only. Reconstructs escalations from the ``author_type='system'`` notes
|
||||||
|
escalate_issue() leaves (``🚨 הסלמה [severity] לחיים\\n\\n<reason>``) — severity
|
||||||
|
parsed from the tag, reason from the body, identifier joined from the issue.
|
||||||
|
Scoped to the two legal-ai companies; ``limit`` clamped 1..50, ``hours`` 1..168.
|
||||||
|
"""
|
||||||
|
limit = max(1, min(int(limit), 50))
|
||||||
|
hours = max(1, min(int(hours), 168))
|
||||||
|
company_ids = list(COMPANIES.values())
|
||||||
|
conn = await asyncpg.connect(PAPERCLIP_DB_URL)
|
||||||
|
try:
|
||||||
|
rows = await conn.fetch(
|
||||||
|
f"""SELECT ic.issue_id, ic.body, ic.created_at, i.identifier
|
||||||
|
FROM issue_comments ic
|
||||||
|
JOIN issues i ON i.id = ic.issue_id
|
||||||
|
WHERE ic.author_type = 'system'
|
||||||
|
AND ic.company_id = ANY($1::uuid[])
|
||||||
|
AND ic.body LIKE $2
|
||||||
|
AND ic.deleted_at IS NULL
|
||||||
|
AND ic.created_at > now() - interval '{hours} hours'
|
||||||
|
ORDER BY ic.created_at DESC
|
||||||
|
LIMIT $3""",
|
||||||
|
company_ids, f"{_ESCALATION_BODY_PREFIX}%", limit,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
await conn.close()
|
||||||
|
|
||||||
|
items = []
|
||||||
|
for r in rows:
|
||||||
|
body = r["body"] or ""
|
||||||
|
m = _ESCALATION_SEVERITY_RE.search(body)
|
||||||
|
severity = m.group(1) if m else "medium"
|
||||||
|
reason = body.split("\n\n", 1)[1].strip() if "\n\n" in body else body
|
||||||
|
items.append({
|
||||||
|
"issue_id": str(r["issue_id"]),
|
||||||
|
"identifier": r["identifier"],
|
||||||
|
"severity": severity,
|
||||||
|
"reason": reason,
|
||||||
|
"created_at": r["created_at"].isoformat() if r["created_at"] else None,
|
||||||
|
})
|
||||||
|
return {"ok": True, "items": items}
|
||||||
|
|
||||||
|
|
||||||
async def get_predecessor_context(issue_id: str, limit: int = 3) -> dict:
|
async def get_predecessor_context(issue_id: str, limit: int = 3) -> dict:
|
||||||
"""Recent finished runs on an issue, newest-first, for session continuation (#220).
|
"""Recent finished runs on an issue, newest-first, for session continuation (#220).
|
||||||
|
|
||||||
|
|||||||
@@ -143,3 +143,58 @@ def test_get_agent_health_empty(monkeypatch):
|
|||||||
monkeypatch.setattr(pc.asyncpg, "connect", _connect)
|
monkeypatch.setattr(pc.asyncpg, "connect", _connect)
|
||||||
result = asyncio.run(pc.get_agent_health())
|
result = asyncio.run(pc.get_agent_health())
|
||||||
assert result == {"ok": True, "items": [], "counts": {"zombie": 0, "stalled": 0, "working": 0, "idle": 0}}
|
assert result == {"ok": True, "items": [], "counts": {"zombie": 0, "stalled": 0, "working": 0, "idle": 0}}
|
||||||
|
|
||||||
|
|
||||||
|
# ── recent escalations (fake DB) — parses severity + reason from system notes ──
|
||||||
|
class _FetchConn:
|
||||||
|
def __init__(self, rows):
|
||||||
|
self._rows = rows
|
||||||
|
|
||||||
|
async def fetch(self, query, *args):
|
||||||
|
return self._rows
|
||||||
|
|
||||||
|
async def close(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_recent_escalations_parses_severity_and_reason(monkeypatch):
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
rows = [
|
||||||
|
{
|
||||||
|
"issue_id": "iss-1",
|
||||||
|
"identifier": "CMP-141",
|
||||||
|
"body": "🚨 הסלמה [high] לחיים\n\nזוהתה לולאת-recovery — 4 יקיצות-שחזור; נמסר לחיים.",
|
||||||
|
"created_at": datetime(2026, 7, 7, 3, 30, 0),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"issue_id": "iss-2",
|
||||||
|
"identifier": "CMPA-112",
|
||||||
|
"body": "🚨 הסלמה [medium] לחיים\n\nזוהתה לולאת-recovery — 2 יקיצות.",
|
||||||
|
"created_at": datetime(2026, 7, 7, 2, 49, 0),
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
async def _connect(_url):
|
||||||
|
return _FetchConn(rows)
|
||||||
|
|
||||||
|
monkeypatch.setattr(pc.asyncpg, "connect", _connect)
|
||||||
|
result = asyncio.run(pc.get_recent_escalations())
|
||||||
|
assert result["ok"] is True
|
||||||
|
a, b = result["items"]
|
||||||
|
assert a["identifier"] == "CMP-141" and a["severity"] == "high"
|
||||||
|
assert "4 יקיצות" in a["reason"] and "🚨" not in a["reason"]
|
||||||
|
assert a["created_at"] == "2026-07-07T03:30:00"
|
||||||
|
assert b["severity"] == "medium"
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_recent_escalations_defaults_severity_when_untagged(monkeypatch):
|
||||||
|
rows = [{"issue_id": "x", "identifier": "CMP-9", "body": "🚨 הסלמה לחיים", "created_at": None}]
|
||||||
|
|
||||||
|
async def _connect(_url):
|
||||||
|
return _FetchConn(rows)
|
||||||
|
|
||||||
|
monkeypatch.setattr(pc.asyncpg, "connect", _connect)
|
||||||
|
(item,) = asyncio.run(pc.get_recent_escalations())["items"]
|
||||||
|
assert item["severity"] == "medium" # fallback
|
||||||
|
assert item["created_at"] is None
|
||||||
|
|||||||
Reference in New Issue
Block a user