Chaim
4de555367d
Inline # comments in gitignore are not supported — they were silently breaking three patterns (data/checkpoints/, data/adapter-migration-state.json, .claude/agents/.generated/). Moved comments to their own lines and added missing entries for runtime dirs (data/audit/, data/logs/, etc.) and temp files (.interaction_tmp.json, .design-build/, .taskmaster bak files). Also tracks previously untracked legitimate files: scripts, tests, docs, skills references, .env.example, taskmaster templates. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
103 lines
5.7 KiB
Python
103 lines
5.7 KiB
Python
"""Batch ingest of appeals-committee decisions staged in data/precedents/incoming/.
|
|
|
|
Sequential (NOT concurrent — avoids the 2026-05-31 load-spike incident) ingest of
|
|
each .doc/.docx via the canonical internal pipeline, followed by metadata extraction
|
|
per case (the internal path does NOT auto-queue metadata — INV-ING3). Halacha is
|
|
auto-queued by ingest; drain it separately via MCP precedent_process_pending.
|
|
|
|
case_number canonical follows the filename/Nevo convention validated against the
|
|
corpus + missing_precedents list:
|
|
- מרכז/חיפה/ת"א committees number with month: NNNN/MM/YY → NNNN-MM-YY
|
|
- ירושלים/צפון committees number without month: NNNN/YY → NNNN-YY
|
|
decision_date / summary / subject_tags / appeal_subtype are left empty on purpose —
|
|
the metadata extractor fills them from the full text (more reliable than parsing here).
|
|
|
|
Run: mcp-server/.venv/bin/python scripts/ingest_incoming_batch.py
|
|
Config (POSTGRES_URL, VOYAGE_API_KEY, ANTHROPIC_API_KEY) auto-loads from ~/.env.
|
|
"""
|
|
|
|
import asyncio
|
|
import os
|
|
import sys
|
|
import traceback
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "mcp-server", "src"))
|
|
|
|
from legal_mcp.services import internal_decisions as svc
|
|
from legal_mcp.services import precedent_metadata_extractor as meta
|
|
|
|
INC = "/home/chaim/legal-ai/data/precedents/incoming"
|
|
|
|
# file, case_number(canonical-with-slashes), chair, district, court, practice_area
|
|
DECISIONS = [
|
|
("105-07.doc", "105/07", "דרור לביא-אפרת", "צפון", "rishuy_uvniya"),
|
|
("ARAR-17-105-44.doc", "105/17", "רונית אלפר", "מרכז", "betterment_levy"),
|
|
("ARAR-18-1029.doc", "1029/18", "אליעד וינשל", "ירושלים", "rishuy_uvniya"),
|
|
("ARAR-20-1018-44.doc", "1018/20", "שרית אריאלי בן שמחון", "ירושלים", "rishuy_uvniya"),
|
|
("ARAR-20-1023-55.doc", "1023/20", "שרית אריאלי בן שמחון", "ירושלים", "rishuy_uvniya"),
|
|
("ARAR-21-1080-55.doc", "1080/21", "נילי בן משה ידגר", "צפון", "rishuy_uvniya"),
|
|
("ARAR-21-11-1051.doc", "1051/11/21", "רונית אלפר", "מרכז", "rishuy_uvniya"),
|
|
("ARAR-22-01-1015.doc", "1015/01/22", "שרית אריאלי בן שמחון", "ירושלים", "rishuy_uvniya"),
|
|
("ARAR-22-06-1029.doc", "1029/06/22", "מאיה אשכנזי", "מרכז", "rishuy_uvniya"),
|
|
("ARAR-22-08-1044.doc", "1044/08/22", "מאיה אשכנזי", "מרכז", "rishuy_uvniya"),
|
|
("ARAR-22-10-1050.doc", "1050/10/22", "מאיה אשכנזי", "מרכז", "rishuy_uvniya"),
|
|
("ARAR-22-1079.doc", "1079/22", "שרית אריאלי בן שמחון", "ירושלים", "rishuy_uvniya"),
|
|
("ARAR-23-04-1010.doc", "1010/04/23", "מאיה אשכנזי", "מרכז", "rishuy_uvniya"),
|
|
("ARAR-23-08-1074-9.doc", "1074/08/23", "מיכל הלברשטם דגני", "חיפה", "rishuy_uvniya"),
|
|
("ARAR-23-1034.docx", "1006/23", "שרית אריאלי בן שמחון", "ירושלים", "rishuy_uvniya"),
|
|
("ARAR-23-1073.doc", "1073/23", "שרית אריאלי בן שמחון", "ירושלים", "rishuy_uvniya"),
|
|
("ARAR-23-1085.doc", "1085/23", "נילי בן משה ידגר", "צפון", "rishuy_uvniya"),
|
|
("ARAR-24-01-1009-5.docx","1009/01/24", "מיכל דגני הלברשטם", "תל אביב", "rishuy_uvniya"),
|
|
("ARAR-24-05-1044.doc", "1044/05/24", "שרית אריאלי בן שמחון", "ירושלים", "rishuy_uvniya"),
|
|
("ARAR-25-01-10072.docx", "1007/01/25", "יפעת בן אריה שטיינברג", "תל אביב", "rishuy_uvniya"),
|
|
]
|
|
|
|
|
|
async def main():
|
|
results = []
|
|
for fname, case_number, chair, district, parea in DECISIONS:
|
|
fp = Path(INC) / fname
|
|
rec = {"file": fname, "case_number": case_number}
|
|
if not fp.exists():
|
|
rec["error"] = "file-missing"
|
|
print(f"✗ {fname}: file missing", flush=True)
|
|
results.append(rec)
|
|
continue
|
|
try:
|
|
out = await svc.ingest_internal_decision(
|
|
file_path=fp,
|
|
case_number=case_number,
|
|
chair_name=chair,
|
|
district=district,
|
|
court=f"ועדת הערר לתכנון ובנייה — מחוז {district}",
|
|
practice_area=parea,
|
|
proceeding_type="ערר",
|
|
is_binding=False,
|
|
)
|
|
cid = out.get("case_law_id")
|
|
rec["case_law_id"] = cid
|
|
rec["chunks"] = out.get("chunks")
|
|
print(f"✓ ingest {case_number}: id={cid} chunks={out.get('chunks')}", flush=True)
|
|
# metadata (internal path does not auto-queue it)
|
|
m = await meta.extract_and_apply(cid)
|
|
rec["meta_status"] = m.get("status")
|
|
sug = m.get("suggested") or {}
|
|
rec["suggested_case_number"] = sug.get("case_number_clean")
|
|
rec["citation_formatted"] = sug.get("citation_formatted")
|
|
rec["meta_date"] = sug.get("decision_date_iso")
|
|
print(f" meta {case_number}: {m.get('status')} | clean={sug.get('case_number_clean')} | {sug.get('citation_formatted')}", flush=True)
|
|
except Exception as e:
|
|
rec["error"] = f"{type(e).__name__}: {e}"
|
|
print(f"✗ {fname} ({case_number}): {e}", flush=True)
|
|
traceback.print_exc()
|
|
results.append(rec)
|
|
|
|
print("\n===SUMMARY===", flush=True)
|
|
for r in results:
|
|
print(r, flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|