Merge pull request 'feat(digests): יומון-לא-מקושר → "פסיקה חסרה" — סוף לבליעה-שקטה (#136)' (#270) from worktree-digest-missing-precedent into main

mcp-server/src/legal_mcp/services/court_fetch_orchestrator.py

@@ -314,10 +314,14 @@ async def _record_failure(
 async def _open_gap(citation: str, *, reason: str) -> None:
     """Open a missing_precedent gap so the chair sees it (INV-CF2/CF3).
 
-    Best-effort + de-duplicated by the missing_precedents layer; a failure
+    Best-effort + de-duplicated (designator-aware via citation_norm, #143); a
-    here is logged, never raised (it must not mask the original outcome).
+    failure here is logged, never raised (it must not mask the original outcome).
     """
     try:
-        await db.create_missing_precedent(citation=citation, notes=reason)
+        if await db.find_missing_precedent_by_citation(citation):
+            return
+        await db.create_missing_precedent(
+            citation=citation, notes=reason, discovery_source="court_fetch",
+        )
     except Exception:
         logger.warning("could not open missing_precedent for %s", citation)

mcp-server/src/legal_mcp/services/digest_library.py

@@ -83,27 +83,32 @@ async def try_autolink(digest_id: UUID | str, underlying_citation: str) -> str |
         logger.warning("digest try_autolink lookup failed for %r: %s", citation, e)
         return None
     if not match:
-        # Gap (INV-DIG3): the underlying ruling isn't in the corpus. If it's a
+        # Gap (INV-DIG3): the underlying ruling isn't in the corpus. Surface it —
-        # court verdict (not ועדת-ערר), enqueue an X13 auto-fetch job so the gap
+        # never drop silently (INV-CF2). Court verdicts (supreme/admin) get an X13
-        # is actionable instead of silently dropped (INV-CF2). Never raises.
+        # auto-fetch job; ועדת-ערר / unknown — which נט-המשפט can't serve — get a
-        await _enqueue_court_fetch(digest_id, citation)
+        # missing_precedent the chair sees on /missing-precedents (#136). Never
+        # raises.
+        await _handle_unlinked_citation(digest_id, citation)
         return None
     await db.link_digest_to_case_law(digest_id, match["id"])
     return str(match["id"])
 
 
-async def _enqueue_court_fetch(digest_id: UUID | str, citation: str) -> None:
+async def _handle_unlinked_citation(digest_id: UUID | str, citation: str) -> None:
-    """Queue an X13 court-verdict fetch for an unlinked digest citation.
+    """Surface an unlinked digest citation — auto-fetch if possible, else record
+    a missing_precedent. Closes the silent-drop gap (#136, INV-DIG3/CF2).
 
-    Court rulings (supreme/admin) → a ``court_fetch_jobs`` row drained later by
+    Routing via the ONE canonical classifier (``court_citation.classify``):
-    ``court_fetch_drain``. ועדת-ערר (skip) is left alone — it needs Nevo and is
+      * supreme/admin → ``court_fetch_jobs`` (drained by X13; on fetch failure the
-    surfaced through the normal missing-precedent path, not auto-fetch.
+        orchestrator opens its own missing_precedent, so no double-record here).
+      * skip (ערר/בל"מ) / unknown → ``missing_precedents`` (needs Nevo / manual;
+        נט-המשפט can't serve it). Deduped designator-aware via citation_norm
+        (#143) so re-runs and overlaps don't pile up.
     """
     try:
         from legal_mcp.services import court_citation
         cit = court_citation.classify(citation)
-        if cit.tier not in ("supreme", "admin"):
+        if cit.tier in ("supreme", "admin"):
-            return
             await db.court_fetch_job_upsert(
                 case_number_norm=cit.case_number_norm,
                 citation_raw=citation,
@@ -113,8 +118,25 @@ async def _enqueue_court_fetch(digest_id: UUID | str, citation: str) -> None:
             )
             logger.info("digest %s: enqueued court-fetch for %r (tier=%s)",
                         digest_id, citation, cit.tier)
+            return
+        # Non-fetchable (ערר/בל"מ/unknown) — open a missing_precedent gap so it's
+        # visible and actionable instead of vanishing. Dedup first (#143).
+        if await db.find_missing_precedent_by_citation(citation):
+            return
+        digest = await db.get_digest(digest_id)
+        yomon = (digest or {}).get("yomon_number") or ""
+        note = (f"זוהה דרך יומון מס' {yomon} (digest_id={digest_id})" if yomon
+                else f"זוהה דרך יומון (digest_id={digest_id})")
+        await db.create_missing_precedent(
+            citation=citation,
+            discovery_source="digest",
+            notes=note,
+        )
+        logger.info("digest %s: opened missing_precedent for %r (tier=%s)",
+                    digest_id, citation, cit.tier)
     except Exception as e:  # never break digest ingest
-        logger.warning("digest court-fetch enqueue failed for %r: %s", citation, e)
+        logger.warning("digest unlinked-citation handling failed for %r: %s",
+                       citation, e)
 
 
 # ── Container-safe creation (web upload) — no LLM, no embedding ──────

mcp-server/tests/test_digest_unlinked_citation.py

@@ -0,0 +1,88 @@
+"""Regression test for #136 — an unlinked digest citation must never be dropped
+silently.
+
+``_handle_unlinked_citation`` routes via the canonical classifier:
+  * supreme/admin → a court-fetch job (no missing_precedent here — the X13
+    orchestrator opens its own on failure),
+  * skip (ערר/בל"מ) / unknown → a deduped missing_precedent (discovery_source
+    'digest'), which previously vanished.
+
+Runs OFFLINE — monkeypatches the db calls and records what each routing did.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+import pytest
+
+from legal_mcp.services import digest_library as dl
+from legal_mcp.services import db
+
+
+class _Spy:
+    def __init__(self):
+        self.court_fetch = []
+        self.created_mp = []
+        self.find_mp_returns = None
+
+    def install(self, monkeypatch):
+        async def _job_upsert(**kw):
+            self.court_fetch.append(kw)
+        async def _find_mp(citation, case_id=None):
+            return self.find_mp_returns
+        async def _create_mp(**kw):
+            self.created_mp.append(kw)
+            return {"id": "mp"}
+        async def _get_digest(_id):
+            return {"yomon_number": "5167"}
+        monkeypatch.setattr(db, "court_fetch_job_upsert", _job_upsert)
+        monkeypatch.setattr(db, "find_missing_precedent_by_citation", _find_mp)
+        monkeypatch.setattr(db, "create_missing_precedent", _create_mp)
+        monkeypatch.setattr(db, "get_digest", _get_digest)
+
+
+def _run(coro):
+    loop = asyncio.new_event_loop()
+    try:
+        return loop.run_until_complete(coro)
+    finally:
+        loop.close()
+
+
+@pytest.fixture()
+def spy(monkeypatch):
+    s = _Spy()
+    s.install(monkeypatch)
+    return s
+
+
+_DID = "11111111-1111-1111-1111-111111111111"
+
+
+def test_committee_citation_opens_missing_precedent(spy):
+    _run(dl._handle_unlinked_citation(_DID, "ערר 1198-12-25 זאטוס"))
+    assert spy.court_fetch == []                       # ערר is never auto-fetched
+    assert len(spy.created_mp) == 1, spy.created_mp
+    mp = spy.created_mp[0]
+    assert mp["discovery_source"] == "digest"
+    assert "יומון" in (mp["notes"] or "")             # provenance recorded
+
+
+def test_court_verdict_enqueues_fetch_not_mp(spy):
+    _run(dl._handle_unlinked_citation(_DID, 'עע"מ 3975/22 פלוני'))
+    assert len(spy.court_fetch) == 1, spy.court_fetch
+    assert spy.created_mp == []                        # fetchable → orchestrator owns its MP
+
+
+def test_dedup_skips_existing_gap(spy):
+    spy.find_mp_returns = {"id": "existing"}           # gap already recorded
+    _run(dl._handle_unlinked_citation(_DID, "ערר 1192/18"))
+    assert spy.created_mp == []                        # no duplicate
+
+
+def test_unknown_citation_opens_missing_precedent(spy):
+    _run(dl._handle_unlinked_citation(_DID, "משהו בלי ערכאה ברורה"))
+    # unknown tier is not fetchable → must still surface as a gap, never dropped.
+    assert spy.court_fetch == []
+    assert len(spy.created_mp) == 1

scripts/SCRIPTS.md

@@ -35,6 +35,7 @@
 | `reconcile_metadata_status.py` | python | **נרמול `metadata_extraction_status` תקוע (G1)** — שורות עם ברירת-המחדל `'pending'` שאינן בצנרת-Gemini נערמות כ-backlog-רפאים שהדריינר (סורק `*_requested_at IS NOT NULL`) לעולם לא מנקה ומנפח את מונה "ממתין" ב-/operations. מיישב כל שורה למצב-אמת במקור: `internal_committee`→`completed` (מטא דטרמיניסטי, מחוץ ל-Gemini), `external_upload` מלא→`completed`, `external_upload` עם טקסט וחסר שם/תקציר→חותם `requested_at` (הדריינר יטפל), `cited_only` (אין טקסט)→`skipped`. **מכסה את שני התורים (#140):** אותו `cited_only→skipped` מוחל גם על `halacha_extraction_status` (תור-תאום, G2). אידמפוטנטי. תיקון-המקור הנלווה ב-`db.create_internal_committee_decision` + מסנן `EXTRACTION_ELIGIBLE_PREDICATE` ב-`list_pending_extraction_requests`. הרצה: `mcp-server/.venv/bin/python scripts/reconcile_metadata_status.py`. | חד-פעמי / re-runnable כהגנת-drift |
 | `reconcile_under_extracted_halacha.py` | python | **#144 — שחזור פסיקה תת-מחולצת** שהושלמה אך עם 0 הלכות למרות ≥3 מקטעי-נימוק (legal_analysis/ruling/conclusion) — חתימת ה-checkpoint-הריק שנוצרה לפני תיקון limit-notice ב-claude_session. מאפס checkpoints + `request_halacha_extraction` (נתיב קנוני, G2) → הדריינר מחלץ מחדש. שמרני (≥3 מקטעים → לא מטפל ב-remand לגיטימי חסר-הלכה; אפס אובדן כי 0 הלכות ממילא). מחריג cited_only. אידמפוטנטי, dry-run כברירת-מחדל / `--apply`. הרצה: `HOME=/home/chaim mcp-server/.venv/bin/python scripts/reconcile_under_extracted_halacha.py --apply`. | חד-פעמי / re-runnable |
 | `derive_missing_from_cited_only.py` | python | **#143 — איחוד cited_only↔missing_precedents (G2)**: גוזר רשומת `missing_precedents` 'open' לכל stub `cited_only` (פסיקה מצוטטת ללא טקסט), כך ש-31 ה-stubs מופיעים בדף "פסיקה חסרה" (היו היו חפיפה≈0). (1) backfill `citation_norm` (מפתח-dedup designator-aware — `court_citation.citation_dedup_key`) ל-291 הקיימים; (2) לכל stub → `create_missing_precedent(discovery_source='cited_only', linked_case_law_id=stub, notes=מצטטים)` עם dedup. `linked_case_law_id`=זהות-קנונית-ידועה, `status='open'` עד העלאת-טקסט (→ promote-in-place דרך ON CONFLICT). אידמפוטנטי, dry-run / `--apply`. הרצה: `HOME=/home/chaim mcp-server/.venv/bin/python scripts/derive_missing_from_cited_only.py --apply`. | חד-פעמי / re-runnable |
+| `backfill_digest_missing_precedents.py` | python | **#136 — חיבור יומונים-לא-מקושרים ל"פסיקה חסרה"**: לכל digest עם `underlying_citation` ו-`linked_case_law_id IS NULL` (461) מריץ את `digest_library.try_autolink` הקנוני (G2) — מקשר אם אפשר, אחרת פותח gap: ערר/בל"מ/unknown → `missing_precedent` (discovery_source='digest', dedup designator-aware), פס"ד בתי-משפט → `court_fetch_job` (X13). dry-run מציג פילוח-tier (369 ערר + 21 unknown → MP; 71 fetchable → court_fetch). אידמפוטנטי. הרצה: `HOME=/home/chaim mcp-server/.venv/bin/python scripts/backfill_digest_missing_precedents.py --apply`. | חד-פעמי / re-runnable |
 | `backfill_plans_registry.py` | python | **ייבוא מרשם-התכניות (V38) מקורפוס-ההחלטות** — סורק `data/cases/*/drafts/decision.md` + `data/training/cmp/*.md`, מאתר פסקאות-תוקף ("פורסמה למתן תוקף"), מחלץ רשומת-תכנית מובנית (`plans_extractor`, claude CLI מקומי) ועושה `upsert_plan(review_status='pending_review')` עם provenance. ה-SSOT לזהות+תוקף של תכנית, פעם-אחת במקום גזירה-מחדש מהשומות בכל תיק (G2). idempotent על plan_number מנורמל (G1/G3). `--dry-run` (ברירת-מחדל, כלום לא נכתב) / `--apply` / `--glob` (תת-קבוצה). אחרי הרצה: אישור-יו"ר ב-`plan_review`/תור-האישור (G10). הרץ: `mcp-server/.venv/bin/python scripts/backfill_plans_registry.py`. | ידני (חד-פעמי + לפי-צורך כשנוספות החלטות) |
 | `backfill_precedent_citations.py` | python | **#145** — backfill ל-`citation_formatted` (מראה-מקום) ברשומות `case_law` ריקות, באמצעות `db.format_precedent_citation` הדטרמיניסטי (X1 §3 / INV-ID2 — שדה-תצוגה נגזר, לא מעוצב ע"י LLM ש-הפיל אותו, #145). שני מעברים לכל שורה: (1) **ללא-LLM** — הרכבה מהשדות השמורים (ממלא שורות-ועדה עם parties+docket+date); (2) **LLM** — אם (1) נמנע ויש full_text, מריץ את מחלץ-המטא (extract_and_apply) שמחלץ רכיבים (parties, citation_prefix) ואז מרכיב — זה ממלא את 171 פסקי-בתי-המשפט מהכותרת. שורות בלי רובריקה (אין צדדים) נשארות ריקות ומדווחות, לא מנוחשות (INV-AH). idempotent — רק שדה ריק (G3). `--apply` / `--limit N` / `--no-llm`. הרץ: `HOME=/home/chaim mcp-server/.venv/bin/python scripts/backfill_precedent_citations.py`. | ידני (חד-פעמי + לפי-צורך) |
 | `auto-sync-cases.sh` | bash | סנכרון תיקי ערר ל-Gitea — רץ כל דקה | `* * * * *` (cron) |

scripts/backfill_digest_missing_precedents.py

@@ -0,0 +1,68 @@
+"""Backfill missing_precedents / court-fetch from unlinked digests (#136).
+
+The digest pipeline used to silently drop an underlying citation it couldn't
+autolink unless it was a fetchable court verdict — so ערר/בל"מ rulings mentioned
+in the daily yomon never surfaced as gaps. After the fix, ``try_autolink`` opens
+a missing_precedent for non-fetchable gaps (and a court-fetch job for fetchable).
+
+This re-runs that canonical path over every already-ingested digest that has an
+``underlying_citation`` but no ``linked_case_law_id`` — so the historical
+backlog surfaces too. Reuses ``digest_library.try_autolink`` (one code path, G2):
+each digest is re-attempted (it may now link to a precedent added since) and,
+failing that, a deduped gap is opened.
+
+Idempotent (dedup designator-aware via citation_norm). Dry-run by default —
+classifies and counts without writing; ``--apply`` runs the autolink.
+Host-only. Run:
+    HOME=/home/chaim mcp-server/.venv/bin/python scripts/backfill_digest_missing_precedents.py [--apply]
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import sys
+from collections import Counter
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "mcp-server", "src"))
+
+from legal_mcp.services import court_citation, db, digest_library
+
+
+async def main(apply: bool) -> int:
+    pool = await db.get_pool()
+    rows = await pool.fetch(
+        "SELECT id, underlying_citation, yomon_number FROM digests "
+        "WHERE COALESCE(underlying_citation, '') <> '' "
+        "AND linked_case_law_id IS NULL"
+    )
+    print(f"unlinked digests with a citation: {len(rows)}")
+
+    tiers = Counter()
+    for r in rows:
+        tiers[court_citation.classify(r["underlying_citation"]).tier] += 1
+    print("by tier:", dict(tiers),
+          "→ fetchable(supreme+admin)=%d, gap(skip+unknown)=%d"
+          % (tiers["supreme"] + tiers["admin"], tiers["skip"] + tiers["unknown"]))
+
+    if not apply:
+        print("\n(dry-run — pass --apply to run autolink: links what it can, opens "
+              "deduped missing_precedents for ערר/unknown, court-fetch for verdicts)")
+        return 0
+
+    linked = gaps = 0
+    for r in rows:
+        before = await db.find_missing_precedent_by_citation(r["underlying_citation"])
+        result = await digest_library.try_autolink(r["id"], r["underlying_citation"])
+        if result:
+            linked += 1
+        elif before is None:
+            # a new gap (court-fetch job or missing_precedent) was opened
+            gaps += 1
+    print(f"\nlinked now: {linked}   new gaps opened: {gaps}   "
+          f"(already-present deduped: {len(rows) - linked - gaps})")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(asyncio.run(main("--apply" in sys.argv)))