ezer-mishpati/legal-ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(principles): cap ranks consensus-first (votes, then score) — criterion A (#152)
All checks were successful
G12 Leak-Guard / leak-guard (pull_request) Successful in 4s
Details
Lint — undefined names / undefined-names (pull_request) Successful in 10s
Details

Browse Source 
chaim 2026-06-20: a unanimous 3-vote principle must outrank a 2-vote one
regardless of score (cross-model agreement is the more reliable keep signal).
apply_cap now sorts survivors by (votes, score), matching cluster_candidates.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chaim
2026-06-20 09:21:51 +00:00
parent bfc034b44c
commit 29b1da534c
2 changed files with 21 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
mcp-server/src/legal_mcp/services/panel_extraction.py
View File

@@ -136,7 +136,10 @@ def apply_cap(judged: list[dict], max_new: int | None = None) -> list[dict]:
""" """
max_new = config.HALACHA_PANEL_MAX_NEW if max_new is None else max_new max_new = config.HALACHA_PANEL_MAX_NEW if max_new is None else max_new
survivors = [j for j in judged if j.get("verdict") in ("approved", "pending_review")] survivors = [j for j in judged if j.get("verdict") in ("approved", "pending_review")]
survivors.sort(key=lambda j: j.get("score", 0.0), reverse=True) # Rank consensus-first, then confidence (chaim 2026-06-20): a unanimous 3-vote
# principle outranks a 2-vote one regardless of score — cross-model agreement is
# the more reliable keep signal (AC1=0.92). Mirrors cluster_candidates' ordering.
survivors.sort(key=lambda j: (j.get("votes", 0), j.get("score", 0.0)), reverse=True)
keep_ids = {id(j) for j in survivors[:max_new]} keep_ids = {id(j) for j in survivors[:max_new]}
out = [] out = []
for j in judged: for j in judged:

24
mcp-server/tests/test_panel_extraction.py
View File

@@ -106,17 +106,27 @@ def test_cluster_same_model_twice_counts_one_vote_keeps_best_score():
assert cl["rule_statement"] == "X" assert cl["rule_statement"] == "X"
def test_apply_cap_downgrades_over_cap_survivors_by_score(): def test_apply_cap_downgrades_over_cap_survivors_by_votes_then_score():
judged = [ judged = [
{"verdict": "approved", "score": 0.9}, {"verdict": "approved", "votes": 3, "score": 0.9},
{"verdict": "approved", "score": 0.7}, {"verdict": "approved", "votes": 3, "score": 0.7},
{"verdict": "pending_review", "score": 0.8}, {"verdict": "pending_review", "votes": 2, "score": 0.8},
{"verdict": "rejected", "score": 0.95}, # already rejected stays {"verdict": "rejected", "votes": 1, "score": 0.95}, # already rejected stays
] ]
out = pe.apply_cap(judged, max_new=2) out = pe.apply_cap(judged, max_new=2)
fv = [j["final_verdict"] for j in out] fv = [j["final_verdict"] for j in out]
# top-2 survivors by score = 0.9(approved) + 0.8(pending); 0.7 → over cap → rejected # top-2 by (votes,score) = both 3-vote (0.9, 0.7); the 2-vote/0.8 → over cap → rejected
assert fv == ["approved", "rejected", "pending_review", "rejected"] assert fv == ["approved", "approved", "rejected", "rejected"]
def test_apply_cap_votes_outrank_score():
# a 2-vote/0.95 must NOT beat a 3-vote/0.80 — consensus dominates confidence
judged = [
{"verdict": "approved", "votes": 2, "score": 0.95},
{"verdict": "approved", "votes": 3, "score": 0.80},
]
out = pe.apply_cap(judged, max_new=1)
assert [j["final_verdict"] for j in out] == ["rejected", "approved"]
def test_apply_cap_keeps_all_when_under_cap(): def test_apply_cap_keeps_all_when_under_cap():