diff --git a/mcp-server/src/legal_mcp/services/court_fetch_orchestrator.py b/mcp-server/src/legal_mcp/services/court_fetch_orchestrator.py index 4fbd403..2301e13 100644 --- a/mcp-server/src/legal_mcp/services/court_fetch_orchestrator.py +++ b/mcp-server/src/legal_mcp/services/court_fetch_orchestrator.py @@ -314,10 +314,14 @@ async def _record_failure( async def _open_gap(citation: str, *, reason: str) -> None: """Open a missing_precedent gap so the chair sees it (INV-CF2/CF3). - Best-effort + de-duplicated by the missing_precedents layer; a failure - here is logged, never raised (it must not mask the original outcome). + Best-effort + de-duplicated (designator-aware via citation_norm, #143); a + failure here is logged, never raised (it must not mask the original outcome). """ try: - await db.create_missing_precedent(citation=citation, notes=reason) + if await db.find_missing_precedent_by_citation(citation): + return + await db.create_missing_precedent( + citation=citation, notes=reason, discovery_source="court_fetch", + ) except Exception: logger.warning("could not open missing_precedent for %s", citation) diff --git a/mcp-server/src/legal_mcp/services/digest_library.py b/mcp-server/src/legal_mcp/services/digest_library.py index f69a8ac..c455738 100644 --- a/mcp-server/src/legal_mcp/services/digest_library.py +++ b/mcp-server/src/legal_mcp/services/digest_library.py @@ -83,38 +83,60 @@ async def try_autolink(digest_id: UUID | str, underlying_citation: str) -> str | logger.warning("digest try_autolink lookup failed for %r: %s", citation, e) return None if not match: - # Gap (INV-DIG3): the underlying ruling isn't in the corpus. If it's a - # court verdict (not ועדת-ערר), enqueue an X13 auto-fetch job so the gap - # is actionable instead of silently dropped (INV-CF2). Never raises. - await _enqueue_court_fetch(digest_id, citation) + # Gap (INV-DIG3): the underlying ruling isn't in the corpus. Surface it — + # never drop silently (INV-CF2). Court verdicts (supreme/admin) get an X13 + # auto-fetch job; ועדת-ערר / unknown — which נט-המשפט can't serve — get a + # missing_precedent the chair sees on /missing-precedents (#136). Never + # raises. + await _handle_unlinked_citation(digest_id, citation) return None await db.link_digest_to_case_law(digest_id, match["id"]) return str(match["id"]) -async def _enqueue_court_fetch(digest_id: UUID | str, citation: str) -> None: - """Queue an X13 court-verdict fetch for an unlinked digest citation. +async def _handle_unlinked_citation(digest_id: UUID | str, citation: str) -> None: + """Surface an unlinked digest citation — auto-fetch if possible, else record + a missing_precedent. Closes the silent-drop gap (#136, INV-DIG3/CF2). - Court rulings (supreme/admin) → a ``court_fetch_jobs`` row drained later by - ``court_fetch_drain``. ועדת-ערר (skip) is left alone — it needs Nevo and is - surfaced through the normal missing-precedent path, not auto-fetch. + Routing via the ONE canonical classifier (``court_citation.classify``): + * supreme/admin → ``court_fetch_jobs`` (drained by X13; on fetch failure the + orchestrator opens its own missing_precedent, so no double-record here). + * skip (ערר/בל"מ) / unknown → ``missing_precedents`` (needs Nevo / manual; + נט-המשפט can't serve it). Deduped designator-aware via citation_norm + (#143) so re-runs and overlaps don't pile up. """ try: from legal_mcp.services import court_citation cit = court_citation.classify(citation) - if cit.tier not in ("supreme", "admin"): + if cit.tier in ("supreme", "admin"): + await db.court_fetch_job_upsert( + case_number_norm=cit.case_number_norm, + citation_raw=citation, + tier=cit.tier, + court=cit.court_prefix, + digest_id=UUID(str(digest_id)), + ) + logger.info("digest %s: enqueued court-fetch for %r (tier=%s)", + digest_id, citation, cit.tier) return - await db.court_fetch_job_upsert( - case_number_norm=cit.case_number_norm, - citation_raw=citation, - tier=cit.tier, - court=cit.court_prefix, - digest_id=UUID(str(digest_id)), + # Non-fetchable (ערר/בל"מ/unknown) — open a missing_precedent gap so it's + # visible and actionable instead of vanishing. Dedup first (#143). + if await db.find_missing_precedent_by_citation(citation): + return + digest = await db.get_digest(digest_id) + yomon = (digest or {}).get("yomon_number") or "" + note = (f"זוהה דרך יומון מס' {yomon} (digest_id={digest_id})" if yomon + else f"זוהה דרך יומון (digest_id={digest_id})") + await db.create_missing_precedent( + citation=citation, + discovery_source="digest", + notes=note, ) - logger.info("digest %s: enqueued court-fetch for %r (tier=%s)", + logger.info("digest %s: opened missing_precedent for %r (tier=%s)", digest_id, citation, cit.tier) except Exception as e: # never break digest ingest - logger.warning("digest court-fetch enqueue failed for %r: %s", citation, e) + logger.warning("digest unlinked-citation handling failed for %r: %s", + citation, e) # ── Container-safe creation (web upload) — no LLM, no embedding ────── diff --git a/mcp-server/tests/test_digest_unlinked_citation.py b/mcp-server/tests/test_digest_unlinked_citation.py new file mode 100644 index 0000000..3755015 --- /dev/null +++ b/mcp-server/tests/test_digest_unlinked_citation.py @@ -0,0 +1,88 @@ +"""Regression test for #136 — an unlinked digest citation must never be dropped +silently. + +``_handle_unlinked_citation`` routes via the canonical classifier: + * supreme/admin → a court-fetch job (no missing_precedent here — the X13 + orchestrator opens its own on failure), + * skip (ערר/בל"מ) / unknown → a deduped missing_precedent (discovery_source + 'digest'), which previously vanished. + +Runs OFFLINE — monkeypatches the db calls and records what each routing did. +""" + +from __future__ import annotations + +import asyncio + +import pytest + +from legal_mcp.services import digest_library as dl +from legal_mcp.services import db + + +class _Spy: + def __init__(self): + self.court_fetch = [] + self.created_mp = [] + self.find_mp_returns = None + + def install(self, monkeypatch): + async def _job_upsert(**kw): + self.court_fetch.append(kw) + async def _find_mp(citation, case_id=None): + return self.find_mp_returns + async def _create_mp(**kw): + self.created_mp.append(kw) + return {"id": "mp"} + async def _get_digest(_id): + return {"yomon_number": "5167"} + monkeypatch.setattr(db, "court_fetch_job_upsert", _job_upsert) + monkeypatch.setattr(db, "find_missing_precedent_by_citation", _find_mp) + monkeypatch.setattr(db, "create_missing_precedent", _create_mp) + monkeypatch.setattr(db, "get_digest", _get_digest) + + +def _run(coro): + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(coro) + finally: + loop.close() + + +@pytest.fixture() +def spy(monkeypatch): + s = _Spy() + s.install(monkeypatch) + return s + + +_DID = "11111111-1111-1111-1111-111111111111" + + +def test_committee_citation_opens_missing_precedent(spy): + _run(dl._handle_unlinked_citation(_DID, "ערר 1198-12-25 זאטוס")) + assert spy.court_fetch == [] # ערר is never auto-fetched + assert len(spy.created_mp) == 1, spy.created_mp + mp = spy.created_mp[0] + assert mp["discovery_source"] == "digest" + assert "יומון" in (mp["notes"] or "") # provenance recorded + + +def test_court_verdict_enqueues_fetch_not_mp(spy): + _run(dl._handle_unlinked_citation(_DID, 'עע"מ 3975/22 פלוני')) + assert len(spy.court_fetch) == 1, spy.court_fetch + assert spy.created_mp == [] # fetchable → orchestrator owns its MP + + +def test_dedup_skips_existing_gap(spy): + spy.find_mp_returns = {"id": "existing"} # gap already recorded + _run(dl._handle_unlinked_citation(_DID, "ערר 1192/18")) + assert spy.created_mp == [] # no duplicate + + +def test_unknown_citation_opens_missing_precedent(spy): + _run(dl._handle_unlinked_citation(_DID, "משהו בלי ערכאה ברורה")) + # unknown tier is not fetchable → must still surface as a gap, never dropped. + assert spy.court_fetch == [] + assert len(spy.created_mp) == 1 diff --git a/scripts/SCRIPTS.md b/scripts/SCRIPTS.md index ed6ad5e..43dbf3d 100644 --- a/scripts/SCRIPTS.md +++ b/scripts/SCRIPTS.md @@ -35,6 +35,7 @@ | `reconcile_metadata_status.py` | python | **נרמול `metadata_extraction_status` תקוע (G1)** — שורות עם ברירת-המחדל `'pending'` שאינן בצנרת-Gemini נערמות כ-backlog-רפאים שהדריינר (סורק `*_requested_at IS NOT NULL`) לעולם לא מנקה ומנפח את מונה "ממתין" ב-/operations. מיישב כל שורה למצב-אמת במקור: `internal_committee`→`completed` (מטא דטרמיניסטי, מחוץ ל-Gemini), `external_upload` מלא→`completed`, `external_upload` עם טקסט וחסר שם/תקציר→חותם `requested_at` (הדריינר יטפל), `cited_only` (אין טקסט)→`skipped`. **מכסה את שני התורים (#140):** אותו `cited_only→skipped` מוחל גם על `halacha_extraction_status` (תור-תאום, G2). אידמפוטנטי. תיקון-המקור הנלווה ב-`db.create_internal_committee_decision` + מסנן `EXTRACTION_ELIGIBLE_PREDICATE` ב-`list_pending_extraction_requests`. הרצה: `mcp-server/.venv/bin/python scripts/reconcile_metadata_status.py`. | חד-פעמי / re-runnable כהגנת-drift | | `reconcile_under_extracted_halacha.py` | python | **#144 — שחזור פסיקה תת-מחולצת** שהושלמה אך עם 0 הלכות למרות ≥3 מקטעי-נימוק (legal_analysis/ruling/conclusion) — חתימת ה-checkpoint-הריק שנוצרה לפני תיקון limit-notice ב-claude_session. מאפס checkpoints + `request_halacha_extraction` (נתיב קנוני, G2) → הדריינר מחלץ מחדש. שמרני (≥3 מקטעים → לא מטפל ב-remand לגיטימי חסר-הלכה; אפס אובדן כי 0 הלכות ממילא). מחריג cited_only. אידמפוטנטי, dry-run כברירת-מחדל / `--apply`. הרצה: `HOME=/home/chaim mcp-server/.venv/bin/python scripts/reconcile_under_extracted_halacha.py --apply`. | חד-פעמי / re-runnable | | `derive_missing_from_cited_only.py` | python | **#143 — איחוד cited_only↔missing_precedents (G2)**: גוזר רשומת `missing_precedents` 'open' לכל stub `cited_only` (פסיקה מצוטטת ללא טקסט), כך ש-31 ה-stubs מופיעים בדף "פסיקה חסרה" (היו היו חפיפה≈0). (1) backfill `citation_norm` (מפתח-dedup designator-aware — `court_citation.citation_dedup_key`) ל-291 הקיימים; (2) לכל stub → `create_missing_precedent(discovery_source='cited_only', linked_case_law_id=stub, notes=מצטטים)` עם dedup. `linked_case_law_id`=זהות-קנונית-ידועה, `status='open'` עד העלאת-טקסט (→ promote-in-place דרך ON CONFLICT). אידמפוטנטי, dry-run / `--apply`. הרצה: `HOME=/home/chaim mcp-server/.venv/bin/python scripts/derive_missing_from_cited_only.py --apply`. | חד-פעמי / re-runnable | +| `backfill_digest_missing_precedents.py` | python | **#136 — חיבור יומונים-לא-מקושרים ל"פסיקה חסרה"**: לכל digest עם `underlying_citation` ו-`linked_case_law_id IS NULL` (461) מריץ את `digest_library.try_autolink` הקנוני (G2) — מקשר אם אפשר, אחרת פותח gap: ערר/בל"מ/unknown → `missing_precedent` (discovery_source='digest', dedup designator-aware), פס"ד בתי-משפט → `court_fetch_job` (X13). dry-run מציג פילוח-tier (369 ערר + 21 unknown → MP; 71 fetchable → court_fetch). אידמפוטנטי. הרצה: `HOME=/home/chaim mcp-server/.venv/bin/python scripts/backfill_digest_missing_precedents.py --apply`. | חד-פעמי / re-runnable | | `backfill_plans_registry.py` | python | **ייבוא מרשם-התכניות (V38) מקורפוס-ההחלטות** — סורק `data/cases/*/drafts/decision.md` + `data/training/cmp/*.md`, מאתר פסקאות-תוקף ("פורסמה למתן תוקף"), מחלץ רשומת-תכנית מובנית (`plans_extractor`, claude CLI מקומי) ועושה `upsert_plan(review_status='pending_review')` עם provenance. ה-SSOT לזהות+תוקף של תכנית, פעם-אחת במקום גזירה-מחדש מהשומות בכל תיק (G2). idempotent על plan_number מנורמל (G1/G3). `--dry-run` (ברירת-מחדל, כלום לא נכתב) / `--apply` / `--glob` (תת-קבוצה). אחרי הרצה: אישור-יו"ר ב-`plan_review`/תור-האישור (G10). הרץ: `mcp-server/.venv/bin/python scripts/backfill_plans_registry.py`. | ידני (חד-פעמי + לפי-צורך כשנוספות החלטות) | | `backfill_precedent_citations.py` | python | **#145** — backfill ל-`citation_formatted` (מראה-מקום) ברשומות `case_law` ריקות, באמצעות `db.format_precedent_citation` הדטרמיניסטי (X1 §3 / INV-ID2 — שדה-תצוגה נגזר, לא מעוצב ע"י LLM ש-הפיל אותו, #145). שני מעברים לכל שורה: (1) **ללא-LLM** — הרכבה מהשדות השמורים (ממלא שורות-ועדה עם parties+docket+date); (2) **LLM** — אם (1) נמנע ויש full_text, מריץ את מחלץ-המטא (extract_and_apply) שמחלץ רכיבים (parties, citation_prefix) ואז מרכיב — זה ממלא את 171 פסקי-בתי-המשפט מהכותרת. שורות בלי רובריקה (אין צדדים) נשארות ריקות ומדווחות, לא מנוחשות (INV-AH). idempotent — רק שדה ריק (G3). `--apply` / `--limit N` / `--no-llm`. הרץ: `HOME=/home/chaim mcp-server/.venv/bin/python scripts/backfill_precedent_citations.py`. | ידני (חד-פעמי + לפי-צורך) | | `auto-sync-cases.sh` | bash | סנכרון תיקי ערר ל-Gitea — רץ כל דקה | `* * * * *` (cron) | diff --git a/scripts/backfill_digest_missing_precedents.py b/scripts/backfill_digest_missing_precedents.py new file mode 100644 index 0000000..a954c74 --- /dev/null +++ b/scripts/backfill_digest_missing_precedents.py @@ -0,0 +1,68 @@ +"""Backfill missing_precedents / court-fetch from unlinked digests (#136). + +The digest pipeline used to silently drop an underlying citation it couldn't +autolink unless it was a fetchable court verdict — so ערר/בל"מ rulings mentioned +in the daily yomon never surfaced as gaps. After the fix, ``try_autolink`` opens +a missing_precedent for non-fetchable gaps (and a court-fetch job for fetchable). + +This re-runs that canonical path over every already-ingested digest that has an +``underlying_citation`` but no ``linked_case_law_id`` — so the historical +backlog surfaces too. Reuses ``digest_library.try_autolink`` (one code path, G2): +each digest is re-attempted (it may now link to a precedent added since) and, +failing that, a deduped gap is opened. + +Idempotent (dedup designator-aware via citation_norm). Dry-run by default — +classifies and counts without writing; ``--apply`` runs the autolink. +Host-only. Run: + HOME=/home/chaim mcp-server/.venv/bin/python scripts/backfill_digest_missing_precedents.py [--apply] +""" + +from __future__ import annotations + +import asyncio +import os +import sys +from collections import Counter + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "mcp-server", "src")) + +from legal_mcp.services import court_citation, db, digest_library + + +async def main(apply: bool) -> int: + pool = await db.get_pool() + rows = await pool.fetch( + "SELECT id, underlying_citation, yomon_number FROM digests " + "WHERE COALESCE(underlying_citation, '') <> '' " + "AND linked_case_law_id IS NULL" + ) + print(f"unlinked digests with a citation: {len(rows)}") + + tiers = Counter() + for r in rows: + tiers[court_citation.classify(r["underlying_citation"]).tier] += 1 + print("by tier:", dict(tiers), + "→ fetchable(supreme+admin)=%d, gap(skip+unknown)=%d" + % (tiers["supreme"] + tiers["admin"], tiers["skip"] + tiers["unknown"])) + + if not apply: + print("\n(dry-run — pass --apply to run autolink: links what it can, opens " + "deduped missing_precedents for ערר/unknown, court-fetch for verdicts)") + return 0 + + linked = gaps = 0 + for r in rows: + before = await db.find_missing_precedent_by_citation(r["underlying_citation"]) + result = await digest_library.try_autolink(r["id"], r["underlying_citation"]) + if result: + linked += 1 + elif before is None: + # a new gap (court-fetch job or missing_precedent) was opened + gaps += 1 + print(f"\nlinked now: {linked} new gaps opened: {gaps} " + f"(already-present deduped: {len(rows) - linked - gaps})") + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main("--apply" in sys.argv)))