From 29b1da534ca46a7430c7252bf340cc97d5a21b48 Mon Sep 17 00:00:00 2001 From: Chaim Date: Sat, 20 Jun 2026 09:21:51 +0000 Subject: [PATCH] =?UTF-8?q?fix(principles):=20cap=20ranks=20consensus-firs?= =?UTF-8?q?t=20(votes,=20then=20score)=20=E2=80=94=20criterion=20A=20(#152?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit chaim 2026-06-20: a unanimous 3-vote principle must outrank a 2-vote one regardless of score (cross-model agreement is the more reliable keep signal). apply_cap now sorts survivors by (votes, score), matching cluster_candidates. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../legal_mcp/services/panel_extraction.py | 5 +++- mcp-server/tests/test_panel_extraction.py | 24 +++++++++++++------ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/mcp-server/src/legal_mcp/services/panel_extraction.py b/mcp-server/src/legal_mcp/services/panel_extraction.py index e18a659..4f007c0 100644 --- a/mcp-server/src/legal_mcp/services/panel_extraction.py +++ b/mcp-server/src/legal_mcp/services/panel_extraction.py @@ -136,7 +136,10 @@ def apply_cap(judged: list[dict], max_new: int | None = None) -> list[dict]: """ max_new = config.HALACHA_PANEL_MAX_NEW if max_new is None else max_new survivors = [j for j in judged if j.get("verdict") in ("approved", "pending_review")] - survivors.sort(key=lambda j: j.get("score", 0.0), reverse=True) + # Rank consensus-first, then confidence (chaim 2026-06-20): a unanimous 3-vote + # principle outranks a 2-vote one regardless of score — cross-model agreement is + # the more reliable keep signal (AC1=0.92). Mirrors cluster_candidates' ordering. + survivors.sort(key=lambda j: (j.get("votes", 0), j.get("score", 0.0)), reverse=True) keep_ids = {id(j) for j in survivors[:max_new]} out = [] for j in judged: diff --git a/mcp-server/tests/test_panel_extraction.py b/mcp-server/tests/test_panel_extraction.py index 30e387f..7a31be6 100644 --- a/mcp-server/tests/test_panel_extraction.py +++ b/mcp-server/tests/test_panel_extraction.py @@ -106,17 +106,27 @@ def test_cluster_same_model_twice_counts_one_vote_keeps_best_score(): assert cl["rule_statement"] == "X" -def test_apply_cap_downgrades_over_cap_survivors_by_score(): +def test_apply_cap_downgrades_over_cap_survivors_by_votes_then_score(): judged = [ - {"verdict": "approved", "score": 0.9}, - {"verdict": "approved", "score": 0.7}, - {"verdict": "pending_review", "score": 0.8}, - {"verdict": "rejected", "score": 0.95}, # already rejected stays + {"verdict": "approved", "votes": 3, "score": 0.9}, + {"verdict": "approved", "votes": 3, "score": 0.7}, + {"verdict": "pending_review", "votes": 2, "score": 0.8}, + {"verdict": "rejected", "votes": 1, "score": 0.95}, # already rejected stays ] out = pe.apply_cap(judged, max_new=2) fv = [j["final_verdict"] for j in out] - # top-2 survivors by score = 0.9(approved) + 0.8(pending); 0.7 → over cap → rejected - assert fv == ["approved", "rejected", "pending_review", "rejected"] + # top-2 by (votes,score) = both 3-vote (0.9, 0.7); the 2-vote/0.8 → over cap → rejected + assert fv == ["approved", "approved", "rejected", "rejected"] + + +def test_apply_cap_votes_outrank_score(): + # a 2-vote/0.95 must NOT beat a 3-vote/0.80 — consensus dominates confidence + judged = [ + {"verdict": "approved", "votes": 2, "score": 0.95}, + {"verdict": "approved", "votes": 3, "score": 0.80}, + ] + out = pe.apply_cap(judged, max_new=1) + assert [j["final_verdict"] for j in out] == ["rejected", "approved"] def test_apply_cap_keeps_all_when_under_cap():