chore: fix .gitignore inline comments + add untracked code files
Inline # comments in gitignore are not supported — they were silently breaking three patterns (data/checkpoints/, data/adapter-migration-state.json, .claude/agents/.generated/). Moved comments to their own lines and added missing entries for runtime dirs (data/audit/, data/logs/, etc.) and temp files (.interaction_tmp.json, .design-build/, .taskmaster bak files). Also tracks previously untracked legitimate files: scripts, tests, docs, skills references, .env.example, taskmaster templates. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
102
scripts/ingest_incoming_batch.py
Normal file
102
scripts/ingest_incoming_batch.py
Normal file
@@ -0,0 +1,102 @@
|
||||
"""Batch ingest of appeals-committee decisions staged in data/precedents/incoming/.
|
||||
|
||||
Sequential (NOT concurrent — avoids the 2026-05-31 load-spike incident) ingest of
|
||||
each .doc/.docx via the canonical internal pipeline, followed by metadata extraction
|
||||
per case (the internal path does NOT auto-queue metadata — INV-ING3). Halacha is
|
||||
auto-queued by ingest; drain it separately via MCP precedent_process_pending.
|
||||
|
||||
case_number canonical follows the filename/Nevo convention validated against the
|
||||
corpus + missing_precedents list:
|
||||
- מרכז/חיפה/ת"א committees number with month: NNNN/MM/YY → NNNN-MM-YY
|
||||
- ירושלים/צפון committees number without month: NNNN/YY → NNNN-YY
|
||||
decision_date / summary / subject_tags / appeal_subtype are left empty on purpose —
|
||||
the metadata extractor fills them from the full text (more reliable than parsing here).
|
||||
|
||||
Run: mcp-server/.venv/bin/python scripts/ingest_incoming_batch.py
|
||||
Config (POSTGRES_URL, VOYAGE_API_KEY, ANTHROPIC_API_KEY) auto-loads from ~/.env.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "mcp-server", "src"))
|
||||
|
||||
from legal_mcp.services import internal_decisions as svc
|
||||
from legal_mcp.services import precedent_metadata_extractor as meta
|
||||
|
||||
INC = "/home/chaim/legal-ai/data/precedents/incoming"
|
||||
|
||||
# file, case_number(canonical-with-slashes), chair, district, court, practice_area
|
||||
DECISIONS = [
|
||||
("105-07.doc", "105/07", "דרור לביא-אפרת", "צפון", "rishuy_uvniya"),
|
||||
("ARAR-17-105-44.doc", "105/17", "רונית אלפר", "מרכז", "betterment_levy"),
|
||||
("ARAR-18-1029.doc", "1029/18", "אליעד וינשל", "ירושלים", "rishuy_uvniya"),
|
||||
("ARAR-20-1018-44.doc", "1018/20", "שרית אריאלי בן שמחון", "ירושלים", "rishuy_uvniya"),
|
||||
("ARAR-20-1023-55.doc", "1023/20", "שרית אריאלי בן שמחון", "ירושלים", "rishuy_uvniya"),
|
||||
("ARAR-21-1080-55.doc", "1080/21", "נילי בן משה ידגר", "צפון", "rishuy_uvniya"),
|
||||
("ARAR-21-11-1051.doc", "1051/11/21", "רונית אלפר", "מרכז", "rishuy_uvniya"),
|
||||
("ARAR-22-01-1015.doc", "1015/01/22", "שרית אריאלי בן שמחון", "ירושלים", "rishuy_uvniya"),
|
||||
("ARAR-22-06-1029.doc", "1029/06/22", "מאיה אשכנזי", "מרכז", "rishuy_uvniya"),
|
||||
("ARAR-22-08-1044.doc", "1044/08/22", "מאיה אשכנזי", "מרכז", "rishuy_uvniya"),
|
||||
("ARAR-22-10-1050.doc", "1050/10/22", "מאיה אשכנזי", "מרכז", "rishuy_uvniya"),
|
||||
("ARAR-22-1079.doc", "1079/22", "שרית אריאלי בן שמחון", "ירושלים", "rishuy_uvniya"),
|
||||
("ARAR-23-04-1010.doc", "1010/04/23", "מאיה אשכנזי", "מרכז", "rishuy_uvniya"),
|
||||
("ARAR-23-08-1074-9.doc", "1074/08/23", "מיכל הלברשטם דגני", "חיפה", "rishuy_uvniya"),
|
||||
("ARAR-23-1034.docx", "1006/23", "שרית אריאלי בן שמחון", "ירושלים", "rishuy_uvniya"),
|
||||
("ARAR-23-1073.doc", "1073/23", "שרית אריאלי בן שמחון", "ירושלים", "rishuy_uvniya"),
|
||||
("ARAR-23-1085.doc", "1085/23", "נילי בן משה ידגר", "צפון", "rishuy_uvniya"),
|
||||
("ARAR-24-01-1009-5.docx","1009/01/24", "מיכל דגני הלברשטם", "תל אביב", "rishuy_uvniya"),
|
||||
("ARAR-24-05-1044.doc", "1044/05/24", "שרית אריאלי בן שמחון", "ירושלים", "rishuy_uvniya"),
|
||||
("ARAR-25-01-10072.docx", "1007/01/25", "יפעת בן אריה שטיינברג", "תל אביב", "rishuy_uvniya"),
|
||||
]
|
||||
|
||||
|
||||
async def main():
|
||||
results = []
|
||||
for fname, case_number, chair, district, parea in DECISIONS:
|
||||
fp = Path(INC) / fname
|
||||
rec = {"file": fname, "case_number": case_number}
|
||||
if not fp.exists():
|
||||
rec["error"] = "file-missing"
|
||||
print(f"✗ {fname}: file missing", flush=True)
|
||||
results.append(rec)
|
||||
continue
|
||||
try:
|
||||
out = await svc.ingest_internal_decision(
|
||||
file_path=fp,
|
||||
case_number=case_number,
|
||||
chair_name=chair,
|
||||
district=district,
|
||||
court=f"ועדת הערר לתכנון ובנייה — מחוז {district}",
|
||||
practice_area=parea,
|
||||
proceeding_type="ערר",
|
||||
is_binding=False,
|
||||
)
|
||||
cid = out.get("case_law_id")
|
||||
rec["case_law_id"] = cid
|
||||
rec["chunks"] = out.get("chunks")
|
||||
print(f"✓ ingest {case_number}: id={cid} chunks={out.get('chunks')}", flush=True)
|
||||
# metadata (internal path does not auto-queue it)
|
||||
m = await meta.extract_and_apply(cid)
|
||||
rec["meta_status"] = m.get("status")
|
||||
sug = m.get("suggested") or {}
|
||||
rec["suggested_case_number"] = sug.get("case_number_clean")
|
||||
rec["citation_formatted"] = sug.get("citation_formatted")
|
||||
rec["meta_date"] = sug.get("decision_date_iso")
|
||||
print(f" meta {case_number}: {m.get('status')} | clean={sug.get('case_number_clean')} | {sug.get('citation_formatted')}", flush=True)
|
||||
except Exception as e:
|
||||
rec["error"] = f"{type(e).__name__}: {e}"
|
||||
print(f"✗ {fname} ({case_number}): {e}", flush=True)
|
||||
traceback.print_exc()
|
||||
results.append(rec)
|
||||
|
||||
print("\n===SUMMARY===", flush=True)
|
||||
for r in results:
|
||||
print(r, flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user