From 161e370a4c1a60dd05d4332c380bee19f66e1d29 Mon Sep 17 00:00:00 2001 From: Chaim Date: Mon, 15 Jun 2026 08:56:34 +0000 Subject: [PATCH] =?UTF-8?q?feat(precedents):=20=D7=90=D7=99=D7=97=D7=95?= =?UTF-8?q?=D7=93=20cited=5Fonly=E2=86=94missing=5Fprecedents=20=E2=80=94?= =?UTF-8?q?=20=D7=92=D7=96=D7=99=D7=A8=D7=AA=20=D7=A4=D7=A1=D7=99=D7=A7?= =?UTF-8?q?=D7=94-=D7=97=D7=A1=D7=A8=D7=94=20(#143,=20G2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit שתי מערכות מקבילות לאותו מושג ("פסיקה מצוטטת שטקסטה לא נקלט"): טבלת missing_precedents (תור-רכישה ידני של היו"ר) מול case_law source_kind='cited_only' (stubs מגרף-הציטוטים/X11). חפיפה≈0 → 31 ה-stubs לא הופיעו ב-/missing-precedents. הכרעה (G2): missing_precedents = SoT-לתור-יחיד; cited_only = מקור-גילוי נגזר (כמו יומונים מזינים radar). גוזרים רשומת missing_precedents 'open' לכל stub. תיקון: - court_citation.citation_dedup_key — מפתח-dedup **designator-aware** (`{designator}|{docket}`). **מתקן פגם בתוכנית-הניתוח:** dedup על מספר-בלבד היה ממזג בטעות אותו docket בערכאות שונות (בג"ץ 389/87 ≠ ע"א 389/87; 18 כאלה בקיים). - סכמה V40: missing_precedents מקבל citation_norm (מפתח-dedup) + discovery_source (manual|cited_only|digest|writer) + index. **בלי UNIQUE** — הקורפוס מחזיק לגיטימית אותו docket בערכאות שונות; ייחודיות נאכפת designator-aware בנתיב-היצירה. - create_missing_precedent: מחשב citation_norm בכתיבה (G1), מקבל discovery_source + linked_case_law_id. find_missing_precedent_by_citation: dedup דרך citation_norm (fallback ל-citation גולמי כשאין מספר). - scripts/derive_missing_from_cited_only.py: backfill citation_norm ל-291 + גזירת 31 (dry-run: 31 ייווצרו, 0 deduped). linked_case_law_id=stub, status=open → promote-in-place בהעלאת-טקסט דרך ON CONFLICT הקיים. אידמפוטנטי. תלוי-הקשר: #140 (הגדרת cited_only). מתואם עם #136 (digest→MP — אותו citation_norm + create path). תיקון-נתון יורץ אחרי הפריסה. בדיקות: test_dedup_key_is_designator_aware (בג"ץ≠ע"א, ערר≠בל"מ, גרסאות-format מתמזגות). כל 356 עוברות. guards נקיים. Invariants: G2 (SoT-לתור יחיד, cited_only נגזר), G1 (citation_norm מנורמל בכתיבה), G3 (idempotent upsert), G10 (שער-העלאה ידני נשמר), G12. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/legal_mcp/services/court_citation.py | 26 ++++ mcp-server/src/legal_mcp/services/db.py | 64 ++++++++-- mcp-server/tests/test_court_citation.py | 18 +++ scripts/SCRIPTS.md | 1 + scripts/derive_missing_from_cited_only.py | 113 ++++++++++++++++++ 5 files changed, 209 insertions(+), 13 deletions(-) create mode 100644 scripts/derive_missing_from_cited_only.py diff --git a/mcp-server/src/legal_mcp/services/court_citation.py b/mcp-server/src/legal_mcp/services/court_citation.py index 038640a..414ef86 100644 --- a/mcp-server/src/legal_mcp/services/court_citation.py +++ b/mcp-server/src/legal_mcp/services/court_citation.py @@ -134,6 +134,32 @@ def case_number_from_citation(citation: str) -> str: return classify(citation).case_number_norm +def _norm_designator(prefix: str) -> str: + """Collapse a court/proceeding prefix to a comparison token. + + Strips gershayim variants and whitespace so ``עע"מ`` / ``עע״מ`` / ``עעמ`` + all map to the same token, while DISTINCT courts stay distinct. + """ + return re.sub(r"[\s\"״׳']", "", prefix or "") + + +def citation_dedup_key(citation: str) -> str: + """Designator-aware dedup key for a citation, or ``''`` if no number. + + Returns ``f"{designator}|{case_number_norm}"`` — e.g. + ``בג"ץ 389/87`` → ``בגץ|389-87`` and ``ע"א 389/87`` → ``עא|389-87`` are + DISTINCT keys, even though both share docket ``389-87``. This is the safe + dedup key for ``missing_precedents`` (#143/#136): deduping on the bare number + alone (``case_number_from_citation``) would WRONGLY MERGE the same docket + across different courts (18 such collisions already exist in the corpus). A + prefix-less citation (bare נט-format / unknown court) yields ``|``. + """ + cit = classify(citation) + if not cit.case_number_norm: + return "" + return f"{_norm_designator(cit.court_prefix)}|{cit.case_number_norm}" + + def _split_filed(num_norm: str) -> tuple[str, str, str] | None: """Split a normalized NNNNN-MM-YY number into (file, month, year). diff --git a/mcp-server/src/legal_mcp/services/db.py b/mcp-server/src/legal_mcp/services/db.py index ddc6b7e..7575286 100644 --- a/mcp-server/src/legal_mcp/services/db.py +++ b/mcp-server/src/legal_mcp/services/db.py @@ -14,7 +14,7 @@ import asyncpg from pgvector.asyncpg import register_vector from legal_mcp import config -from legal_mcp.services import halacha_quality +from legal_mcp.services import court_citation, halacha_quality logger = logging.getLogger(__name__) @@ -1571,6 +1571,19 @@ SCHEMA_V39_SQL = """ ALTER TABLE case_law ADD COLUMN IF NOT EXISTS parties TEXT DEFAULT ''; """ +# V40 (#143): missing_precedents gains a designator-aware dedup key +# (citation_norm) and a provenance tag (discovery_source: manual | cited_only | +# digest | writer). No UNIQUE constraint — the corpus legitimately holds the same +# docket across different courts (e.g. בג"ץ 389/87 vs ע"א 389/87), so uniqueness +# is enforced designator-aware in the create path, not by a DB constraint that +# would over-merge distinct precedents. +SCHEMA_V40_SQL = """ +ALTER TABLE missing_precedents ADD COLUMN IF NOT EXISTS citation_norm TEXT DEFAULT ''; +ALTER TABLE missing_precedents ADD COLUMN IF NOT EXISTS discovery_source TEXT DEFAULT 'manual'; +CREATE INDEX IF NOT EXISTS idx_missing_precedents_citation_norm + ON missing_precedents(citation_norm); +""" + # Stable, arbitrary key for the session-level advisory lock that serialises # schema DDL across processes. Every short-lived process (cron drains, services) @@ -1589,7 +1602,7 @@ async def _run_schema_migrations(pool: asyncpg.Pool) -> None: await _apply_schema_ddl(conn) finally: await conn.execute("SELECT pg_advisory_unlock($1)", _MIGRATION_LOCK_KEY) - logger.info("Database schema initialized (v1-v38)") + logger.info("Database schema initialized (v1-v40)") async def _apply_schema_ddl(conn: asyncpg.Connection) -> None: @@ -1633,6 +1646,7 @@ async def _apply_schema_ddl(conn: asyncpg.Connection) -> None: await conn.execute(SCHEMA_V37_SQL) await conn.execute(SCHEMA_V38_SQL) await conn.execute(SCHEMA_V39_SQL) + await conn.execute(SCHEMA_V40_SQL) async def init_schema() -> None: @@ -7259,27 +7273,40 @@ async def create_missing_precedent( legal_issue: str | None = None, claim_quote: str | None = None, notes: str | None = None, + discovery_source: str = "manual", + linked_case_law_id: UUID | None = None, ) -> dict: - """Create a new missing-precedent row (status='open' by default).""" + """Create a new missing-precedent row (status='open' by default). + + ``citation_norm`` is computed at write-time (designator-aware, #143) so + duplicate gaps for the same court+docket dedup correctly without merging the + same docket across different courts. ``discovery_source`` records how the gap + surfaced (manual | cited_only | digest | writer); ``linked_case_law_id`` may + be set up-front when the canonical identity is already known (a cited_only + stub awaiting its text) while ``status`` stays 'open' until the text lands. + """ if not citation.strip(): raise ValueError("citation is required") if cited_by_party and cited_by_party not in ALLOWED_MP_PARTIES: raise ValueError( f"cited_by_party must be one of {sorted(ALLOWED_MP_PARTIES)}" ) + citation_norm = court_citation.citation_dedup_key(citation) pool = await get_pool() async with pool.acquire() as conn: row = await conn.fetchrow( """INSERT INTO missing_precedents ( citation, case_name, cited_in_case_id, cited_in_document_id, cited_by_party, cited_by_party_name, legal_topic, legal_issue, - claim_quote, notes + claim_quote, notes, citation_norm, discovery_source, + linked_case_law_id ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) RETURNING *""", citation.strip(), case_name, cited_in_case_id, cited_in_document_id, cited_by_party, cited_by_party_name, legal_topic, legal_issue, - claim_quote, notes, + claim_quote, notes, citation_norm, discovery_source, + linked_case_law_id, ) return _row_to_missing_precedent(row) @@ -7443,20 +7470,31 @@ async def find_missing_precedent_by_citation( citation: str, case_id: UUID | None = None, ) -> dict | None: - """Look up an existing row by citation string (exact match) and optionally - cited-in case_id. Used to deduplicate auto-creation by the researcher.""" + """Look up an existing gap by DESIGNATOR-AWARE normalized citation and + optionally cited-in case_id. Used to deduplicate auto-creation (researcher, + cited_only derivation #143, digest derivation #136). + + Matches on ``citation_norm`` (``court|docket``) so formatting variants of the + same court+docket collapse to one gap, while the same docket across different + courts stays distinct. Falls back to an exact ``citation`` string match when + the citation carries no parseable number (citation_norm is empty).""" + norm = court_citation.citation_dedup_key(citation) pool = await get_pool() async with pool.acquire() as conn: + if norm: + match_col, match_val = "citation_norm", norm + else: + match_col, match_val = "citation", citation.strip() if case_id is not None: row = await conn.fetchrow( - "SELECT * FROM missing_precedents " - "WHERE citation = $1 AND cited_in_case_id = $2 LIMIT 1", - citation.strip(), case_id, + f"SELECT * FROM missing_precedents " + f"WHERE {match_col} = $1 AND cited_in_case_id = $2 LIMIT 1", + match_val, case_id, ) else: row = await conn.fetchrow( - "SELECT * FROM missing_precedents WHERE citation = $1 LIMIT 1", - citation.strip(), + f"SELECT * FROM missing_precedents WHERE {match_col} = $1 LIMIT 1", + match_val, ) return _row_to_missing_precedent(row) if row else None diff --git a/mcp-server/tests/test_court_citation.py b/mcp-server/tests/test_court_citation.py index f172420..22f4ba6 100644 --- a/mcp-server/tests/test_court_citation.py +++ b/mcp-server/tests/test_court_citation.py @@ -4,11 +4,29 @@ from __future__ import annotations from legal_mcp.services.court_citation import ( case_number_from_citation, + citation_dedup_key, classify, normalize_case_number, ) +def test_dedup_key_is_designator_aware(): + """#143 — same docket across DIFFERENT courts must NOT collapse to one key + (deduping on the bare number would wrongly merge distinct precedents).""" + bagatz = citation_dedup_key('בג"ץ 389/87') + civil = citation_dedup_key('ע"א 389/87') + assert bagatz and civil and bagatz != civil, (bagatz, civil) + # ...while gershayim/format variants of the SAME court+docket DO collapse. + assert citation_dedup_key('עע"מ 9057/09') == citation_dedup_key("עע״מ 9057-09") + # committee proceeding-type is part of the key (ערר ≠ בל"מ). + assert citation_dedup_key("ערר 1192/18") != citation_dedup_key('בל"מ 1192/18') + + +def test_dedup_key_empty_without_number(): + assert citation_dedup_key("") == "" + assert citation_dedup_key("פסק דין בלי מספר") == "" + + def test_admin_filed_format_the_example(): """The plan's example: עת"מ 46111-12-22 → admin, parsed into (46111,12,22).""" c = classify('עת"מ 46111-12-22 יכין-אפק בע"מ נ\' הוועדה המחוזית') diff --git a/scripts/SCRIPTS.md b/scripts/SCRIPTS.md index ca31d07..ed6ad5e 100644 --- a/scripts/SCRIPTS.md +++ b/scripts/SCRIPTS.md @@ -34,6 +34,7 @@ | `legal-metadata-drain.config.cjs` | pm2/js | **תזמון כל 15 דק' של `drain_metadata_queue.py`** (cron `*/15 * * * *`, `METADATA_DRAIN_CRON` לעקיפה) — מונע סתימה של תור חילוץ-המטא ב-/precedents. דורש `GEMINI_API_KEY` ב-`~/.env`. התקנה: `pm2 start scripts/legal-metadata-drain.config.cjs && pm2 save`. | pm2 cron (host-side) | | `reconcile_metadata_status.py` | python | **נרמול `metadata_extraction_status` תקוע (G1)** — שורות עם ברירת-המחדל `'pending'` שאינן בצנרת-Gemini נערמות כ-backlog-רפאים שהדריינר (סורק `*_requested_at IS NOT NULL`) לעולם לא מנקה ומנפח את מונה "ממתין" ב-/operations. מיישב כל שורה למצב-אמת במקור: `internal_committee`→`completed` (מטא דטרמיניסטי, מחוץ ל-Gemini), `external_upload` מלא→`completed`, `external_upload` עם טקסט וחסר שם/תקציר→חותם `requested_at` (הדריינר יטפל), `cited_only` (אין טקסט)→`skipped`. **מכסה את שני התורים (#140):** אותו `cited_only→skipped` מוחל גם על `halacha_extraction_status` (תור-תאום, G2). אידמפוטנטי. תיקון-המקור הנלווה ב-`db.create_internal_committee_decision` + מסנן `EXTRACTION_ELIGIBLE_PREDICATE` ב-`list_pending_extraction_requests`. הרצה: `mcp-server/.venv/bin/python scripts/reconcile_metadata_status.py`. | חד-פעמי / re-runnable כהגנת-drift | | `reconcile_under_extracted_halacha.py` | python | **#144 — שחזור פסיקה תת-מחולצת** שהושלמה אך עם 0 הלכות למרות ≥3 מקטעי-נימוק (legal_analysis/ruling/conclusion) — חתימת ה-checkpoint-הריק שנוצרה לפני תיקון limit-notice ב-claude_session. מאפס checkpoints + `request_halacha_extraction` (נתיב קנוני, G2) → הדריינר מחלץ מחדש. שמרני (≥3 מקטעים → לא מטפל ב-remand לגיטימי חסר-הלכה; אפס אובדן כי 0 הלכות ממילא). מחריג cited_only. אידמפוטנטי, dry-run כברירת-מחדל / `--apply`. הרצה: `HOME=/home/chaim mcp-server/.venv/bin/python scripts/reconcile_under_extracted_halacha.py --apply`. | חד-פעמי / re-runnable | +| `derive_missing_from_cited_only.py` | python | **#143 — איחוד cited_only↔missing_precedents (G2)**: גוזר רשומת `missing_precedents` 'open' לכל stub `cited_only` (פסיקה מצוטטת ללא טקסט), כך ש-31 ה-stubs מופיעים בדף "פסיקה חסרה" (היו היו חפיפה≈0). (1) backfill `citation_norm` (מפתח-dedup designator-aware — `court_citation.citation_dedup_key`) ל-291 הקיימים; (2) לכל stub → `create_missing_precedent(discovery_source='cited_only', linked_case_law_id=stub, notes=מצטטים)` עם dedup. `linked_case_law_id`=זהות-קנונית-ידועה, `status='open'` עד העלאת-טקסט (→ promote-in-place דרך ON CONFLICT). אידמפוטנטי, dry-run / `--apply`. הרצה: `HOME=/home/chaim mcp-server/.venv/bin/python scripts/derive_missing_from_cited_only.py --apply`. | חד-פעמי / re-runnable | | `backfill_plans_registry.py` | python | **ייבוא מרשם-התכניות (V38) מקורפוס-ההחלטות** — סורק `data/cases/*/drafts/decision.md` + `data/training/cmp/*.md`, מאתר פסקאות-תוקף ("פורסמה למתן תוקף"), מחלץ רשומת-תכנית מובנית (`plans_extractor`, claude CLI מקומי) ועושה `upsert_plan(review_status='pending_review')` עם provenance. ה-SSOT לזהות+תוקף של תכנית, פעם-אחת במקום גזירה-מחדש מהשומות בכל תיק (G2). idempotent על plan_number מנורמל (G1/G3). `--dry-run` (ברירת-מחדל, כלום לא נכתב) / `--apply` / `--glob` (תת-קבוצה). אחרי הרצה: אישור-יו"ר ב-`plan_review`/תור-האישור (G10). הרץ: `mcp-server/.venv/bin/python scripts/backfill_plans_registry.py`. | ידני (חד-פעמי + לפי-צורך כשנוספות החלטות) | | `backfill_precedent_citations.py` | python | **#145** — backfill ל-`citation_formatted` (מראה-מקום) ברשומות `case_law` ריקות, באמצעות `db.format_precedent_citation` הדטרמיניסטי (X1 §3 / INV-ID2 — שדה-תצוגה נגזר, לא מעוצב ע"י LLM ש-הפיל אותו, #145). שני מעברים לכל שורה: (1) **ללא-LLM** — הרכבה מהשדות השמורים (ממלא שורות-ועדה עם parties+docket+date); (2) **LLM** — אם (1) נמנע ויש full_text, מריץ את מחלץ-המטא (extract_and_apply) שמחלץ רכיבים (parties, citation_prefix) ואז מרכיב — זה ממלא את 171 פסקי-בתי-המשפט מהכותרת. שורות בלי רובריקה (אין צדדים) נשארות ריקות ומדווחות, לא מנוחשות (INV-AH). idempotent — רק שדה ריק (G3). `--apply` / `--limit N` / `--no-llm`. הרץ: `HOME=/home/chaim mcp-server/.venv/bin/python scripts/backfill_precedent_citations.py`. | ידני (חד-פעמי + לפי-צורך) | | `auto-sync-cases.sh` | bash | סנכרון תיקי ערר ל-Gitea — רץ כל דקה | `* * * * *` (cron) | diff --git a/scripts/derive_missing_from_cited_only.py b/scripts/derive_missing_from_cited_only.py new file mode 100644 index 0000000..1e2ea5f --- /dev/null +++ b/scripts/derive_missing_from_cited_only.py @@ -0,0 +1,113 @@ +"""Derive missing_precedents 'open' gaps from cited_only stubs (#143, G2). + +Two parallel systems described the same concept — "a cited precedent whose text +isn't in the corpus": the ``missing_precedents`` queue (the chair's acquisition +list) and ``case_law`` rows with ``source_kind='cited_only'`` (citation-only +stubs seeded by the X11 / corpus-graph). Overlap was ~0, so the 31 cited_only +stubs never surfaced on /missing-precedents. + +This makes ``missing_precedents`` the single source-of-truth FOR THE QUEUE and +``cited_only`` a DERIVED discovery source (like digests feed the radar): + 1. Backfill ``citation_norm`` (designator-aware dedup key) for every existing + missing_precedent — required before the dedup below can match. + 2. For each cited_only stub, derive an 'open' missing_precedent (deduped on + citation_norm), with ``discovery_source='cited_only'``, + ``linked_case_law_id`` = the stub (its canonical identity is known; status + stays 'open' until the text is uploaded → promote-in-place), and notes + listing the precedents that cite it. + +Idempotent / re-runnable. Dry-run by default; ``--apply`` to write. +Host-only. Run: + HOME=/home/chaim mcp-server/.venv/bin/python scripts/derive_missing_from_cited_only.py [--apply] +""" + +from __future__ import annotations + +import asyncio +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "mcp-server", "src")) + +from legal_mcp.services import court_citation, db + + +async def _backfill_citation_norm(pool, apply: bool) -> int: + rows = await pool.fetch( + "SELECT id, citation FROM missing_precedents " + "WHERE COALESCE(citation_norm, '') = ''" + ) + n = 0 + for r in rows: + norm = court_citation.citation_dedup_key(r["citation"] or "") + if not norm: + continue + if apply: + await pool.execute( + "UPDATE missing_precedents SET citation_norm = $2 WHERE id = $1", + r["id"], norm, + ) + n += 1 + return n + + +async def _citing_precedents_note(pool, stub_id) -> str: + rows = await pool.fetch( + """SELECT DISTINCT cl.case_number + FROM precedent_internal_citations p + JOIN case_law cl ON cl.id = p.source_case_law_id + WHERE p.cited_case_law_id = $1 AND COALESCE(cl.case_number,'') <> '' + ORDER BY cl.case_number LIMIT 8""", + stub_id, + ) + citers = [r["case_number"] for r in rows] + base = "נגזר מ-cited_only (גרף-הציטוטים)" + if citers: + return f"{base}; מצוטט ע\"י: {', '.join(citers)}" + return base + + +async def main(apply: bool) -> int: + pool = await db.get_pool() + + backfilled = await _backfill_citation_norm(pool, apply) + print(f"citation_norm backfill (existing rows){'' if apply else ' [dry]'}: {backfilled}") + + stubs = await pool.fetch( + "SELECT id, case_number, case_name FROM case_law " + "WHERE source_kind = 'cited_only' ORDER BY case_number" + ) + print(f"cited_only stubs: {len(stubs)}") + + created = 0 + skipped = 0 + for s in stubs: + citation = (s["case_number"] or "").strip() + if not citation: + print(f" SKIP (no case_number) id={s['id']}") + continue + existing = await db.find_missing_precedent_by_citation(citation) + if existing: + skipped += 1 + continue + norm = court_citation.citation_dedup_key(citation) + print(f" + {citation:<22} norm={norm!r} name={(s['case_name'] or '')[:24]!r}") + if apply: + note = await _citing_precedents_note(pool, s["id"]) + await db.create_missing_precedent( + citation=citation, + case_name=s["case_name"] or None, + discovery_source="cited_only", + linked_case_law_id=s["id"], + notes=note, + ) + created += 1 + + print(f"\n{'created' if apply else 'would create'}: {created} already-present (deduped): {skipped}") + if not apply: + print("(dry-run — pass --apply to write)") + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main("--apply" in sys.argv))) -- 2.49.1