feat(ops): /operations dashboard — everything running in the background

A single live page for all the background work that downloads/analyses, so the
chair can see what's running instead of guessing.

- court_fetch_service: GET /pm2 (unauthenticated, host-only) → trimmed pm2 jlist
  for the legal-* services (status, restarts, mem, cron schedule).
- FastAPI GET /api/operations: aggregates the DB-backed pipelines (court_fetch
  jobs, metadata + halacha extraction queues, halacha review gate,
  missing_precedents, digests, recent court ingests) and proxies the host /pm2
  over the docker bridge (graceful if the host service is down).
- web-ui /operations page (+ src/lib/api/operations.ts hook, nav entry under
  admin): services grid (with Hebrew labels + schedules) + pipeline cards +
  recent-fetch / recent-ingest lists. Auto-refreshes every 5s.

tsc --noEmit clean; pm2 status carries nothing sensitive and the bind
(10.0.1.1) is host/container-only, so /pm2 needs no secret.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-08 07:28:41 +00:00
parent 64b9bd9d99
commit 34d80a39e5
5 changed files with 488 additions and 0 deletions

View File

@@ -55,6 +55,55 @@ async def health(request: web.Request) -> web.Response:
return web.json_response(info)
# Background services we surface on the /operations dashboard. pm2 jlist is a
# host-only capability (the legal-ai container can't run pm2), so the container's
# FastAPI proxies this read-only endpoint over the docker bridge. No secret:
# pm2 status (names/cpu/mem) carries nothing sensitive and the bind (10.0.1.1)
# is already host/container-only.
_PM2_PREFIXES = ("legal-", "paperclip")
async def pm2_status(request: web.Request) -> web.Response:
"""Return a trimmed ``pm2 jlist`` for the legal-ai background services."""
import asyncio as _asyncio
try:
proc = await _asyncio.create_subprocess_exec(
"pm2", "jlist",
stdout=_asyncio.subprocess.PIPE, stderr=_asyncio.subprocess.PIPE,
)
out, err = await _asyncio.wait_for(proc.communicate(), timeout=10)
if proc.returncode != 0:
return web.json_response(
{"error": f"pm2 jlist failed: {err.decode('utf-8','replace')[:200]}"},
status=502,
)
apps = json.loads(out.decode("utf-8", "replace"))
except FileNotFoundError:
return web.json_response({"error": "pm2 not found on PATH"}, status=502)
except Exception as e: # never throw
return web.json_response({"error": f"pm2 error: {e}"}, status=502)
services = []
for a in apps:
name = a.get("name", "")
if not any(name.startswith(p) for p in _PM2_PREFIXES):
continue
env = a.get("pm2_env", {}) or {}
services.append({
"name": name,
"status": env.get("status", ""),
"restarts": env.get("restart_time", 0),
"uptime_ms": env.get("pm_uptime", 0),
"cpu": (a.get("monit") or {}).get("cpu", 0),
"memory_bytes": (a.get("monit") or {}).get("memory", 0),
"cron": env.get("cron_restart") or "",
"autorestart": env.get("autorestart", True),
})
services.sort(key=lambda s: s["name"])
return web.json_response({"services": services})
def _check_bearer(request: web.Request) -> web.Response | None:
auth = request.headers.get("Authorization", "")
expected = "Bearer " + _SHARED_SECRET
@@ -106,6 +155,7 @@ async def fetch(request: web.Request) -> web.Response:
def build_app() -> web.Application:
app = web.Application(client_max_size=64 * 1024 * 1024)
app.router.add_get("/health", health)
app.router.add_get("/pm2", pm2_status)
app.router.add_post("/fetch", fetch)
return app