ezer-mishpati/legal-ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore(eval): add 9 chair-approved semantic queries to gold-set (FU-5)

Browse Source 
The gold-set was 77 known-item probes (query=case_name). Added 9 chair-approved
SEMANTIC queries (S1–S9) — a real legal question per row, relevant = the
precedents that should surface (drawn from subject_tags, chair-confirmed). These
test what matters: does retrieval answer a legal issue, not just find a case by
name. source='chair' (preserved across re-bootstrap). practice_area left empty
so the filter never excludes a cross-tagged precedent (s.197 rulings sit under
betterment_levy).

Baseline now 86 queries. Finding from the 9 semantic queries: MRR ≈ 1.0 — the
system surfaces a lead relevant precedent at rank 1 for nearly every question —
but R@10 ranges 0.5–1.0: for broad questions with many co-relevant precedents
(e.g. נטרול תמ"א 38 = 5 relevant → R@10 0.60; שמאי מכריע = 2 → 0.50) some
co-relevant rulings miss the top-10. Lead-precedent retrieval is strong;
exhaustive multi-precedent recall is the gap.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chaim
2026-05-31 15:57:45 +00:00
parent eef04b0f09
commit 7161c3d010
2 changed files with 29 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
data/eval/baseline.json
View File

@@ -1,5 +1,5 @@
{ {
"gold_size": 77, "gold_size": 86,
"retrieval_config": { "retrieval_config": {
"MULTIMODAL_ENABLED": true, "MULTIMODAL_ENABLED": true,
"VOYAGE_RERANK_ENABLED": true, "VOYAGE_RERANK_ENABLED": true,
@@ -9,13 +9,13 @@
"BM25_HYBRID_ENABLED": true "BM25_HYBRID_ENABLED": true
}, },
"overall": { "overall": {
"P@5": 0.1922, "P@5": 0.214,
"R@5": 0.9351, "R@5": 0.899,
"nDCG@5": 0.8545, "nDCG@5": 0.8311,
"P@10": 0.1013, "P@10": 0.1163,
"R@10": 0.987, "R@10": 0.9649,
"nDCG@10": 0.8718, "nDCG@10": 0.8554,
"MRR": 0.8367 "MRR": 0.8482
}, },
"by_corpus": { "by_corpus": {
"internal_decisions": { "internal_decisions": {
@@ -24,17 +24,17 @@
"nDCG@5": 0.887, "nDCG@5": 0.887,
"P@10": 0.1019, "P@10": 0.1019,
"R@10": 1.0, "R@10": 1.0,
"nDCG@10": 0.899, "nDCG@10": 0.8994,
"MRR": 0.871 "MRR": 0.8713
}, },
"precedent_library": { "precedent_library": {
"P@5": 0.1826, "P@5": 0.2438,
"R@5": 0.8696, "R@5": 0.7911,
"nDCG@5": 0.778, "nDCG@5": 0.7367,
"P@10": 0.1, "P@10": 0.1406,
"R@10": 0.9565, "R@10": 0.9057,
"nDCG@10": 0.808, "nDCG@10": 0.7813,
"MRR": 0.7562 "MRR": 0.8092
} }
}, },
"by_practice_area": { "by_practice_area": {
@@ -44,8 +44,8 @@
"nDCG@5": 0.8595, "nDCG@5": 0.8595,
"P@10": 0.1, "P@10": 0.1,
"R@10": 0.9744, "R@10": 0.9744,
"nDCG@10": 0.8761, "nDCG@10": 0.8766,
"MRR": 0.8432 "MRR": 0.8437
}, },
"compensation_197": { "compensation_197": {
"P@5": 0.2, "P@5": 0.2,
@@ -66,5 +66,5 @@
"MRR": 0.8346 "MRR": 0.8346
} }
}, },
"generated_at": "20260531T154736Z" "generated_at": "20260531T155717Z"
} }

9
data/eval/gold-set.jsonl
View File

@@ -75,3 +75,12 @@
{"id": "g-ae5917860b", "query": "סרוזברג ואח'", "practice_area": "", "corpus": "precedent_library", "relevant_case_law_ids": ["d9772726-9766-4509-8067-b20fa625a1a9"], "source": "bootstrap_known_item", "note": "known-item: search by case_name → expect the case itself (1 same-named)"} {"id": "g-ae5917860b", "query": "סרוזברג ואח'", "practice_area": "", "corpus": "precedent_library", "relevant_case_law_ids": ["d9772726-9766-4509-8067-b20fa625a1a9"], "source": "bootstrap_known_item", "note": "known-item: search by case_name → expect the case itself (1 same-named)"}
{"id": "g-e1e175248c", "query": "עמותת העצמאים באילת", "practice_area": "rishuy_uvniya", "corpus": "precedent_library", "relevant_case_law_ids": ["f59e74c2-6433-47c9-bd0e-580cf4171fbb"], "source": "bootstrap_known_item", "note": "known-item: search by case_name → expect the case itself (1 same-named)"} {"id": "g-e1e175248c", "query": "עמותת העצמאים באילת", "practice_area": "rishuy_uvniya", "corpus": "precedent_library", "relevant_case_law_ids": ["f59e74c2-6433-47c9-bd0e-580cf4171fbb"], "source": "bootstrap_known_item", "note": "known-item: search by case_name → expect the case itself (1 same-named)"}
{"id": "g-86116ced86", "query": "שמי אשקלוני", "practice_area": "betterment_levy", "corpus": "precedent_library", "relevant_case_law_ids": ["7352e510-c769-45e4-b4ef-d85271743506"], "source": "bootstrap_known_item", "note": "known-item: search by case_name → expect the case itself (1 same-named)"} {"id": "g-86116ced86", "query": "שמי אשקלוני", "practice_area": "betterment_levy", "corpus": "precedent_library", "relevant_case_law_ids": ["7352e510-c769-45e4-b4ef-d85271743506"], "source": "bootstrap_known_item", "note": "known-item: search by case_name → expect the case itself (1 same-named)"}
{"id": "g-7e9438b730", "query": "פטור מהיטל השבחה למוסד ציבורי לפי סעיף 19(ב)(4)", "practice_area": "", "corpus": "precedent_library", "relevant_case_law_ids": ["ced7ea50-689b-465d-bf79-99e22a72e0df", "aadedc2d-e990-4d6d-9dd1-8be4fa6dcbe2", "587381e4-d194-4d37-b00f-ccf7242ba228", "4bde8ca8-7862-4b19-9dd7-de2e31d82721", "4f85e3f1-237a-4dac-b949-87a43ee6f633"], "source": "chair", "note": "semantic query (chair-approved 2026-05-31)"}
{"id": "g-89bc8d6161", "query": "נטרול תרומת תמ\"א 38 בשומת \"מצב קודם\"", "practice_area": "", "corpus": "precedent_library", "relevant_case_law_ids": ["436efd48-c8ab-49f0-b3a9-52bf15ea806d", "b80d94a0-b836-44f5-8cc6-18d8cf26e41d", "57be0d1a-293f-481f-aa5b-bfa7dc73f99e", "7352e510-c769-45e4-b4ef-d85271743506", "53ccf47e-0fc7-4248-b486-02f57a9c689c"], "source": "chair", "note": "semantic query (chair-approved 2026-05-31)"}
{"id": "g-f4c06ec2f9", "query": "פטור מהיטל בתמ\"א 38 — מימוש במכר מול מימוש בהיתר", "practice_area": "", "corpus": "precedent_library", "relevant_case_law_ids": ["53ccf47e-0fc7-4248-b486-02f57a9c689c", "e57c4a6b-66a0-4d52-85af-5018f03cf295", "7352e510-c769-45e4-b4ef-d85271743506"], "source": "chair", "note": "semantic query (chair-approved 2026-05-31)"}
{"id": "g-8c8b82486c", "query": "נטרול ציפיות לתכנית עתידית בשווי מצב קודם (אקו-סיטי/לוסטרניק)", "practice_area": "", "corpus": "precedent_library", "relevant_case_law_ids": ["950d8c1b-4976-4a68-8b8e-7d0bdd056e1d", "7352e510-c769-45e4-b4ef-d85271743506", "436efd48-c8ab-49f0-b3a9-52bf15ea806d"], "source": "chair", "note": "semantic query (chair-approved 2026-05-31)"}
{"id": "g-bbe92ea5e3", "query": "היתר לשימוש חורג בקרקע חקלאית — סטייה ניכרת ומגמת תכנון", "practice_area": "", "corpus": "precedent_library", "relevant_case_law_ids": ["e08f81d3-6183-494c-aec3-f20d39e2755e", "e26f2fa2-50e5-407d-8724-8c707dcda51b", "b673d649-d162-4f81-a323-c7d89e8334ce", "f59e74c2-6433-47c9-bd0e-580cf4171fbb"], "source": "chair", "note": "semantic query (chair-approved 2026-05-31)"}
{"id": "g-19376b63de", "query": "זכות עמידה / זכות התנגדות לבקשה להיתר בנייה", "practice_area": "", "corpus": "precedent_library", "relevant_case_law_ids": ["48909f09-8a65-4a2d-8697-e2f50bf9a756", "9024da7b-f408-4b6f-808f-c514a83728e4"], "source": "chair", "note": "semantic query (chair-approved 2026-05-31)"}
{"id": "g-3d2f9fc270", "query": "היקף התערבות בית המשפט בשיקול דעת תכנוני של ועדה", "practice_area": "", "corpus": "precedent_library", "relevant_case_law_ids": ["41d5a21c-a28a-428f-a35e-bc7d0dc89539", "9024da7b-f408-4b6f-808f-c514a83728e4", "e26f2fa2-50e5-407d-8724-8c707dcda51b"], "source": "chair", "note": "semantic query (chair-approved 2026-05-31)"}
{"id": "g-9e96222cc5", "query": "אמת המידה להתערבות ועדת ערר בשומת שמאי מכריע", "practice_area": "", "corpus": "precedent_library", "relevant_case_law_ids": ["8bfcd217-cde3-4930-a058-c9a59182c338", "1847e97e-6e38-494f-b079-0fc59066788a"], "source": "chair", "note": "semantic query (chair-approved 2026-05-31)"}
{"id": "g-181b020ea9", "query": "חובת ועדת ערר להעביר השגות שמאיות לשמאי מייעץ (ס'197)", "practice_area": "", "corpus": "precedent_library", "relevant_case_law_ids": ["e18aa906-e0f5-452f-a17a-f1c299095340", "8bfcd217-cde3-4930-a058-c9a59182c338"], "source": "chair", "note": "semantic query (chair-approved 2026-05-31)"}