diff --git a/mcp-server/tests/test_panel_calibrate_captured.py b/mcp-server/tests/test_panel_calibrate_captured.py new file mode 100644 index 0000000..d70cbd1 --- /dev/null +++ b/mcp-server/tests/test_panel_calibrate_captured.py @@ -0,0 +1,75 @@ +"""Tests for #133 / FU-5 — captured-mode calibration of the halacha panel. + +Covers the PURE helpers in scripts/halacha_panel_calibrate.py +(summarize_calibration, bucket_by_round): from captured (panel ⋈ chair) pairs +they must report the split-rate and auto-precision the panel ACTUALLY delivered +against the chair's ground-truth, and break it down per round-day so the loop's +trend is visible. Fully OFFLINE (no DB, no LLM, no re-voting). +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "scripts")) +import halacha_panel_calibrate as cal # noqa: E402 + + +def _pair(chair, verdict, action, ts="2026-06-12T04:00:00Z"): + # judge votes are irrelevant to split-rate/precision here; keep them aligned + return { + "chair_keep": chair, "verdict": verdict, "applied_action": action, + "round_ts": ts, "rule_statement": "r", + "claude_vote": chair, "claude_reason": "", + "deepseek_vote": chair, "deepseek_reason": "", + "gemini_vote": chair, "gemini_reason": "", + } + + +def test_split_rate_and_precision(): + pairs = [ + _pair(True, "unanimous_yes", "approved"), # auto-correct + _pair(False, "unanimous_no", "rejected"), # auto-correct + _pair(False, "unanimous_yes", "approved"), # auto WRONG (false-keep) + _pair(True, "split", "chair"), # escalated + ] + s = cal.summarize_calibration(pairs) + assert s["n"] == 4 + assert s["escalated"] == 1 + assert s["auto_decided"] == 3 + assert s["split_rate"] == 0.25 + # 3 auto-decisions, 1 wrong (false-keep) → precision 2/3 + assert s["false_keep"] == 1 and s["false_drop"] == 0 + assert round(s["auto_precision"], 2) == 0.67 + + +def test_empty_pairs_safe(): + s = cal.summarize_calibration([]) + assert s["n"] == 0 + assert s["split_rate"] is None and s["auto_precision"] is None + + +def test_unlabeled_pairs_filtered(): + s = cal.summarize_calibration([_pair(None, "split", "chair")]) + assert s["n"] == 0 # chair=None contributes no calibration signal + + +def test_bucket_by_round_trend(): + pairs = [ + _pair(True, "unanimous_yes", "approved", ts="2026-06-10T04:00:00Z"), + _pair(False, "split", "chair", ts="2026-06-10T04:00:00Z"), + _pair(True, "unanimous_yes", "approved", ts="2026-06-12T05:00:00Z"), + ] + trend = cal.bucket_by_round(pairs) + days = [d for d, _ in trend] + assert days == ["2026-06-10", "2026-06-12"] # sorted by day + assert trend[0][1]["n"] == 2 and trend[0][1]["split_rate"] == 0.5 + assert trend[1][1]["n"] == 1 and trend[1][1]["split_rate"] == 0.0 + + +def test_missing_round_ts_bucketed_unknown(): + p = _pair(True, "unanimous_yes", "approved") + del p["round_ts"] + trend = cal.bucket_by_round([p]) + assert trend[0][0] == "unknown" diff --git a/scripts/SCRIPTS.md b/scripts/SCRIPTS.md index 19c9e0b..c27c1b4 100644 --- a/scripts/SCRIPTS.md +++ b/scripts/SCRIPTS.md @@ -62,7 +62,7 @@ | `_pipeline_runtime.py` | python | **runtime עמידות משותף (X16 / INV-DUR1)** ל-`final_halacha_pipeline` ו-`final_learning_pipeline` (מימוש אחד, G2). עוטף רשימת-צעדים async ב-LangGraph `StateGraph` ליניארי עם `AsyncSqliteSaver` (checkpoint לכל צעד; resume מדלג על צעדים שהושלמו). **degradation חיננית:** ללא langgraph (`pip install -e ".[durable]"`) — ריצה ליניארית כמו קודם (הכפתור לא נשבר). `Step(name, run)` + `run_pipeline(steps, thread_id, checkpoint_db, fresh)`. נבדק: `mcp-server/tests/test_pipeline_runtime.py`. | מיובא ע"י סקריפטי-המסלול-הסופי | | `curator_apply_pipeline_branch.py` | python | **מקור-אמת לחיווט-הכפתורים של הרמס.** prompt-ה-curator חי רק ב-Paperclip DB (`agents.adapter_config.promptTemplate`). הסקריפט מקדים branch כך שיקיצה עם reason `final_learning_*`/`final_halacha_*` מריצה את ה-pipeline המתאים (HOME/DOTENV/DATA_DIR מוחלטים → DeepSeek+Gemini keys + DATA_DIR נפתרים נכון) ועוצרת, אחרת §A/§B כרגיל. idempotent (מסיר branch קודם). מחיל על שני הסוכנים (CMP+CMPA). `--verify`. **להריץ אחרי reset/יצירה-מחדש של סוכן-curator.** | אחרי reset prompt של curator | | `halacha_panel_audit.py` | python | **רשת-ביטחון לפאנל** (selective-prediction monitoring) — דוגם הלכות שאושרו ע"י הפאנל (`reviewer LIKE 'panel:%'`), מריץ עליהן **שוב** את הצבעת-ה-KEEP של 3 השופטים, ומציף כל מקרה שכעת נוטה DROP (false-keep פוטנציאלי). report-only כברירת-מחדל; `--flag` מחזיר את ה-flips ל-`pending_review` לסקירת-יו"ר. `--sample N`/`--seed`. בסיס 2026-06-07: 0/15. מיועד להרצה תקופתית (שבועי). מייבא שופטים מ-`halacha_panel_approve`. **חובה מקומי**. | תקופתי (שבועי) — ניטור | -| `halacha_panel_calibrate.py` | python | **כיול מדיניות-ההצבעה של הפאנל** (Trust-or-Escalate, ICLR 2025). מריץ את שאלת-ה-KEEP של `halacha_panel_approve` על מדגם-הזהב ומודד מול `is_holding` (הציר-הגס) precision+coverage לכל מדיניות (unanimous/majority) + ספירת false-keep/false-drop. נותן את **אחוז-הטעות בפועל** לבחירת סף-סיכון α. מייבא שופטים מ-`halacha_panel_approve` (מקור-אמת יחיד). read-only, **חובה מקומי**. | ידני — לפני חיווט `--apply` | +| `halacha_panel_calibrate.py` | python | **כיול + מדידת הפאנל** (Trust-or-Escalate, ICLR 2025). `--source live` (ברירת-מחדל): מריץ את שאלת-ה-KEEP על מדגם-הזהב ומודד מול `is_holding` precision+coverage+**split-rate** לכל מדיניות + false-keep/false-drop (מייבא שופטים מ-`halacha_panel_approve`, **חובה מקומי**). **#133/FU-5** — `--source captured`: **אפס-עלות** (בלי re-vote/LLM) — מצליב סבבים שמורים (FU-1) מול הכרעות-יו"ר (FU-2) דרך `db.panel_rounds_vs_chair` ומדווח split-rate+auto-precision **לכל סבב** (מגמת הלולאה: ככל שהרובריקה משתפרת precision נשמר ו-split יורד); משתף את `analyze_pairs` של FU-4 (מקור-יחיד). שתי המדידות מדווחות **anon-stability** (מבחן-אנונימיזציה #81.7) כמטריקת-בריאות נגד echo-chamber. `--batch`/`--limit`/`--concurrency`. | ידני — לפני חיווט `--apply` (live) / תקופתי — מעקב-לולאה (captured) | | `halacha_rubric_distill.py` | python | **#133/FU-4 — זיקוק-רובריקה PROPOSE-ONLY.** מצליב `halacha_panel_rounds` (FU-1, הצבעות+נימוקים) מול הכרעות-היו"ר (FU-2, seeds ב-`halacha_goldset` batch `chair-live`) דרך `db.panel_rounds_vs_chair` (read-only), מנתח דטרמיניסטית **כשלים שיטתיים** (false-keep/false-drop, פיצולים-שהוכרעו, שיעור-מחלוקת-עם-היו"ר לכל שופט), ומציע `KEEP_SYSTEM` v2 + exemplars מופשטים (claude_session מקומי, אפס עלות) כ**דוח-diff** ל-`data/learning/rubric-proposal-.md`. **לעולם לא auto-apply** — אימוץ v2 = עריכה אנושית של הקבוע דרך PR (INV-LRN1); exemplars מופשטים בלבד (INV-LRN5); הסיגנל היחיד = הכרעת-יו"ר, לא הצבעות-פאנל (anti-echo). מתחת ל-12 זוגות → "אין מספיק נתונים". `--no-llm` (סטטיסטיקה בלבד) / `--limit N`. **חובה מקומי**. | תקופתי — אחרי שהצטברו הכרעות-יו"ר על מחלוקות-פאנל | | `halacha_rule_role_backfill.py` | python | **INV-DM7** — backfill חד-פעמי: מסווג-מחדש את ההלכות הישנות (`rule_type IN ('binding','persuasive')` — ערכי-סמכות שנשמרו במסווה תפקיד לפני פיצול הצירים) לאחד מחמשת **תפקידי-הכלל** (holding/interpretive/procedural/application/obiter) דרך claude_session המקומי (אפס עלות). **לא נוגע בסמכות** (נגזרת מ-`precedent_level`). `--apply` (ברירת-מחדל dry-run) / `--limit N` / `--concurrency`. כותב backup CSV ל-`data/audit/` תחילה. fail-safe (פריט שנכשל → נשמר ערך ישן). **חובה מקומי** (claude_session). | ידני חד-פעמי אחרי deploy של פיצול-הסמכות | | `halacha_batch_reconcile.py` | python | **#82.7** — dedup חוצה-פסקים offline (שמרני, **dry-run בלבד**). dedup-on-insert משווה רק תוך-פסק; כאן סף מחמיר (cosine ≥0.95, `--cosine`) ולא-הרסני: מאתר זוגות הלכות near-duplicate בין פסקים שונים (pgvector `<=>` exact) עם איתות לקסיקלי (Jaccard/Levenshtein) ומדווח ל-CSV ב-`data/audit/` לסקירת היו"ר. לא מדלג/ממזג/מוחק. `--include-pending`. **`--link`** רושם את הזוגות שנמצאו כ-`equivalent_halachot` (parallel authority, #84.2 — קישור-מקביל ברמת-הלכה, **לא** ציטוט; idempotent, לא-הרסני). רץ עם venv של mcp-server. אומת: 800 הלכות → 5 זוגות (קושרו). | ידני — דוח-סקירה / `--link` לקישור | diff --git a/scripts/halacha_panel_calibrate.py b/scripts/halacha_panel_calibrate.py index 8fb2898..7aab175 100644 --- a/scripts/halacha_panel_calibrate.py +++ b/scripts/halacha_panel_calibrate.py @@ -17,8 +17,20 @@ precision (vs is_holding) and coverage (how many it decides vs escalates): Pick the policy whose auto-error stays under your tolerance while covering the most items. Read-only. Local-only (claude_session needs the CLI). +FU-5 (#133) adds two things on top of the original live calibration: + - ``--source captured``: ZERO-COST measurement that reads the stored panel + rounds (FU-1) joined with the chair's rulings (FU-2) and reports, PER ROUND, + the split-rate + auto-precision the panel actually delivered against the + GROWING chair gold-set. This is how we see the active-learning loop working: + as the rubric improves (FU-4 → chair adopts v2), precision should hold while + the split-rate falls. + - anon-stability: the share of gold-set items whose verdict survives masking + the case names (#81.7 probe) — an echo-chamber health metric. A falling rate + means the panel is recognizing cases, not reasoning. + cd ~/legal-ai/mcp-server - .venv/bin/python ../scripts/halacha_panel_calibrate.py + .venv/bin/python ../scripts/halacha_panel_calibrate.py # live calibration + .venv/bin/python ../scripts/halacha_panel_calibrate.py --source captured # per-round trend (free) """ from __future__ import annotations @@ -32,6 +44,63 @@ from legal_mcp.services import db from halacha_panel_approve import ( # noqa: E402 KEEP_SYSTEM, _bool, _keep_user, judge_claude, judge_deepseek, judge_gemini, ) +# reuse FU-4's stats core so "what failed" is computed in ONE place (no drift) +from halacha_rubric_distill import _JUDGES, analyze_pairs # noqa: E402 + + +def summarize_calibration(pairs: list[dict]) -> dict: + """Pure: from captured (panel ⋈ chair) pairs, the calibration health metrics + the panel ACTUALLY delivered against the chair's ground-truth (#133/FU-5). + + split_rate — fraction the panel escalated to the chair (split/incomplete); + the cost of caution. auto_precision — of the auto-decisions, the fraction + the chair later agreed with. As the rubric improves (FU-4 → chair adopts), + precision should hold while split_rate falls. Reuses FU-4's analyze_pairs + so the failure buckets are defined once.""" + labeled = [p for p in pairs if p.get("chair_keep") is not None] + a = analyze_pairs(labeled) + n = a["n_pairs"] + escalated = a["n_splits_resolved"] + auto = n - escalated + wrong = a["n_false_keep"] + a["n_false_drop"] + return { + "n": n, + "auto_decided": auto, + "escalated": escalated, + "split_rate": round(escalated / n, 3) if n else None, + "auto_precision": round((auto - wrong) / auto, 3) if auto else None, + "false_keep": a["n_false_keep"], + "false_drop": a["n_false_drop"], + "judge_disagree": {j: a["judge_stats"][j]["disagree_rate"] for j in _JUDGES}, + } + + +def bucket_by_round(pairs: list[dict]) -> list[tuple[str, dict]]: + """Pure: split the pairs by the panel round's CALENDAR DAY and summarize each + bucket — the per-round trend on the GROWING chair gold-set. round_ts is the + captured round's ISO timestamp; the day prefix groups a run together.""" + buckets: dict[str, list[dict]] = {} + for p in pairs: + day = (p.get("round_ts") or "")[:10] or "unknown" + buckets.setdefault(day, []).append(p) + return [(day, summarize_calibration(buckets[day])) for day in sorted(buckets)] + + +async def _anon_stability(batch: str) -> dict: + """Echo-chamber health metric (#133/FU-5). The anon probe (#81.7) re-judges + gold-set items with case names masked; ``anon_stable`` is True when the + masked verdict still matches the consensus. A FALLING stable-rate means the + panel is recognizing cases rather than reasoning — the echo-chamber symptom + the loop must watch. Reads stored columns (populated by goldset_panel_label + for the 'default' batch); returns counts so callers can report or skip.""" + items = await db.goldset_list(batch) + probed = [it for it in items if it.get("anon_stable") is not None] + stable = sum(1 for it in probed if it["anon_stable"]) + return { + "probed": len(probed), + "stable": stable, + "stable_rate": round(stable / len(probed), 3) if probed else None, + } async def _votes(client, h) -> list[bool]: @@ -63,7 +132,49 @@ def _decide(votes: list[bool], policy: str) -> bool | None: return None # tie +def _fmt(x) -> str: + return "—" if x is None else (f"{x:.1%}" if isinstance(x, float) else str(x)) + + +async def _run_captured(args: argparse.Namespace) -> int: + """FU-5 — measure the panel's REAL performance against the growing chair + gold-set, from CAPTURED rounds (FU-1) ⋈ chair rulings (FU-2). Zero-cost: no + re-voting, no LLM — it reports what the panel actually delivered, per round. + """ + pairs = await db.panel_rounds_vs_chair(limit=args.limit or 5000) + overall = summarize_calibration(pairs) + print(f"captured calibration on {overall['n']} (panel ⋈ chair) pairs\n", flush=True) + if not overall["n"]: + print("no chair-resolved pairs yet — seeds accrue as the chair reviews " + "panel-judged halachot (FU-2). Nothing to measure.", flush=True) + return 0 + + print("=" * 70) + print(f"{'round-day':<12}{'pairs':>7}{'auto':>6}{'split%':>9}{'precision':>11}" + f"{'fKEEP':>7}{'fDROP':>7}") + print("-" * 70) + for day, s in bucket_by_round(pairs): + print(f"{day:<12}{s['n']:>7}{s['auto_decided']:>6}{_fmt(s['split_rate']):>9}" + f"{_fmt(s['auto_precision']):>11}{s['false_keep']:>7}{s['false_drop']:>7}") + print("-" * 70) + print(f"{'OVERALL':<12}{overall['n']:>7}{overall['auto_decided']:>6}" + f"{_fmt(overall['split_rate']):>9}{_fmt(overall['auto_precision']):>11}" + f"{overall['false_keep']:>7}{overall['false_drop']:>7}") + print("\nper-judge disagreement with the chair (lower = better aligned):") + for j in _JUDGES: + print(f" {j:<10} {_fmt(overall['judge_disagree'][j])}") + + anon = await _anon_stability("default") + print(f"\nanon-stability (echo-chamber health, batch 'default'): " + f"{_fmt(anon['stable_rate'])} over {anon['probed']} probed " + f"({'falling = memorization risk' if anon['probed'] else 'not populated — run goldset_panel_label'})") + return 0 + + async def main(args: argparse.Namespace) -> int: + if args.source == "captured": + return await _run_captured(args) + items = [it for it in await db.goldset_list(args.batch) if it.get("is_holding") is not None] if args.limit: items = items[: args.limit] @@ -82,8 +193,8 @@ async def main(args: argparse.Namespace) -> int: print(f" …{len(rows)}/{len(items)}", flush=True) print("\n" + "=" * 64) - print(f"{'policy':<11}{'auto':>6}{'escalate':>10}{'correct':>9}{'wrong':>7}{'precision':>11}{'coverage':>10}") - print("-" * 64) + print(f"{'policy':<11}{'auto':>6}{'escalate':>10}{'correct':>9}{'wrong':>7}{'precision':>11}{'coverage':>10}{'split':>10}") + print("-" * 74) for policy in ("unanimous", "majority"): auto = wrong = correct = 0 for r in rows: @@ -98,7 +209,8 @@ async def main(args: argparse.Namespace) -> int: esc = len(rows) - auto prec = correct / auto if auto else 0.0 cov = auto / len(rows) if rows else 0.0 - print(f"{policy:<11}{auto:>6}{esc:>10}{correct:>9}{wrong:>7}{prec:>10.1%}{cov:>10.1%}") + split = esc / len(rows) if rows else 0.0 # FU-5: escalation = split rate + print(f"{policy:<11}{auto:>6}{esc:>10}{correct:>9}{wrong:>7}{prec:>10.1%}{cov:>10.1%}{split:>10.1%}") # where do the WRONG auto-decisions fall? (false-keep is the costly one) print("\n=== costly errors: panel auto-KEEPS but human says NOT-holding (per policy) ===") @@ -106,12 +218,21 @@ async def main(args: argparse.Namespace) -> int: fk = sum(1 for r in rows if _decide(r["votes"], policy) is True and not r["truth"]) fd = sum(1 for r in rows if _decide(r["votes"], policy) is False and r["truth"]) print(f" {policy:<11} false-KEEP (bad rule approved): {fk} false-DROP (good rule rejected): {fd}") + + # FU-5: echo-chamber health — does masking case names flip the verdict? + anon = await _anon_stability(args.batch) + print(f"\nanon-stability (echo-chamber health): {_fmt(anon['stable_rate'])} " + f"over {anon['probed']} probed" + + ("" if anon["probed"] else " — not populated; run goldset_panel_label")) return 0 if __name__ == "__main__": - ap = argparse.ArgumentParser() - ap.add_argument("--batch", default="default") + ap = argparse.ArgumentParser(description="Calibrate / measure the halacha panel (Trust-or-Escalate; FU-5).") + ap.add_argument("--source", choices=("live", "captured"), default="live", + help="live: re-vote the gold-set now (needs CLI+keys). " + "captured: zero-cost — measure stored rounds (FU-1) vs chair rulings (FU-2), per round.") + ap.add_argument("--batch", default="default", help="gold-set batch for live mode + anon-stability") ap.add_argument("--limit", type=int, default=0) ap.add_argument("--concurrency", type=int, default=6) raise SystemExit(asyncio.run(main(ap.parse_args())))