diff --git a/mcp-server/src/legal_mcp/services/case_citation_verification.py b/mcp-server/src/legal_mcp/services/case_citation_verification.py index bbeb0ff..88a1084 100644 --- a/mcp-server/src/legal_mcp/services/case_citation_verification.py +++ b/mcp-server/src/legal_mcp/services/case_citation_verification.py @@ -48,6 +48,25 @@ _SUGGEST_FLOOR = 0.45 #: deadlocks. Bounding the fan-out both fixes the failure and speeds it up. _MAX_CONCURRENT_LOOKUPS = 8 +def _passes_floor(hit: dict) -> bool: + """Is this hit similar enough to the argument to suggest to the chair? + + Reads ``relevance`` (always a cosine similarity), NOT ``score`` — ``score`` + becomes a rank-fusion value (~0.008-0.02) as soon as the lexical leg returns + rows, and comparing that against a cosine-calibrated floor rejected every + hit. That is why this tab showed no supporting precedent for any argument. + + ``relevance is None`` means the row came from the lexical leg only, so no + cosine was ever computed. It is KEPT: it earned its place by BM25 rank + inside an already-tiny top-k, and dropping it would silently hide exact + phrase/docket matches — the very hits a chair searches for by name. + """ + rel = hit.get("relevance") + if rel is None: + return True + return float(rel) >= _SUGGEST_FLOOR + + #: Wall-clock ceiling for the whole retrieval phase. The proxy gives up at 30s; #: cutting ourselves off earlier lets us return the suggestions that DID land #: instead of a 500 that shows the chair nothing. Partial results are labelled @@ -103,7 +122,7 @@ async def build_view(case_number: str) -> dict: logger.warning("citation_verification search failed (%s): %s", title[:30], e) # Resolve the authority breakdown for the hit set in one batched query. clids = [UUID(str(h["case_law_id"])) for h in hits - if h.get("case_law_id") and float(h.get("score", 0) or 0) >= _SUGGEST_FLOOR] + if h.get("case_law_id") and _passes_floor(h)] authority = await db.citation_authority(clids) if clids else {} return hits, authority @@ -157,7 +176,7 @@ async def build_view(case_number: str) -> dict: clid = str(h.get("case_law_id") or "") if not clid or clid in seen: continue - if float(h.get("score", 0) or 0) < _SUGGEST_FLOOR: + if not _passes_floor(h): continue seen.add(clid) att = attached_by_arg.get(aid, {}).get(clid) diff --git a/mcp-server/src/legal_mcp/services/db.py b/mcp-server/src/legal_mcp/services/db.py index be08245..bb56034 100644 --- a/mcp-server/src/legal_mcp/services/db.py +++ b/mcp-server/src/legal_mcp/services/db.py @@ -7979,6 +7979,14 @@ async def search_precedent_library_semantic( # Calibrated so the average (≈0.85) stays at +0.05 (legacy value). _conf = float(d.get("confidence") or 0.0) d["score"] = float(d["score"]) + max(_conf * 0.06, 0.0) + # Stable cosine-scale relevance, carried alongside ``score``. + # ``score`` is the RANKING signal and downstream fusion overwrites it + # with an RRF value (~0.008-0.02) whenever the lexical leg returns + # rows — a different scale entirely. Anything that THRESHOLDS must + # read ``relevance`` instead, which always means "cosine similarity + # to the query" no matter which fusion stages ran. See + # hybrid_search._merge_sem_lex. + d["relevance"] = d["score"] d["type"] = "halacha" # authority is DERIVED from the source, never stored (INV-DM7) d["authority"] = halacha_quality.derive_authority(d.get("precedent_level")) @@ -7990,6 +7998,7 @@ async def search_precedent_library_semantic( if d.get("decision_date") is not None: d["decision_date"] = d["decision_date"].isoformat() d["score"] = float(d["score"]) + d["relevance"] = d["score"] # cosine anchor — see the halacha branch above d["type"] = "passage" _maybe_swap_parent(d) results.append(d) diff --git a/mcp-server/src/legal_mcp/services/hybrid_search.py b/mcp-server/src/legal_mcp/services/hybrid_search.py index 001bf8b..0938482 100644 --- a/mcp-server/src/legal_mcp/services/hybrid_search.py +++ b/mcp-server/src/legal_mcp/services/hybrid_search.py @@ -303,6 +303,21 @@ def _merge_sem_lex( if key in lex_row_by_key else 0.0 d["lex_rank"] = lex_rank or 0 d["score"] = sem_term + lex_term + # ``score`` is now an RRF value (~0.008-0.02), NOT a cosine. Carry the + # cosine forward under ``relevance`` so thresholding callers keep a + # stable scale — without it, a caller comparing score >= 0.45 silently + # drops every result the moment the lexical leg returns anything, which + # is exactly how the citation-verification tab ended up showing no + # supporting precedent for any argument. + # + # A lexical-only row has no cosine at all. It gets ``relevance = None`` + # rather than 0.0: "we did not measure this" is not "measured as + # irrelevant", and the row earned its place by BM25 rank. Callers decide + # (see case_citation_verification) — but they must decide knowingly. + if key in sem_row_by_key: + d["relevance"] = float(sem_row_by_key[key].get("relevance", d["sem_score"])) + else: + d["relevance"] = None merged.append(d) merged.sort(key=lambda x: -float(x["score"])) diff --git a/mcp-server/tests/test_relevance_scale.py b/mcp-server/tests/test_relevance_scale.py new file mode 100644 index 0000000..3e7d9bf --- /dev/null +++ b/mcp-server/tests/test_relevance_scale.py @@ -0,0 +1,68 @@ +"""`score` and `relevance` are different things — thresholds must use `relevance`. + +Retrieval returns cosine similarities (~0.4-0.75) until the lexical leg returns +rows; then `_merge_sem_lex` replaces `score` with an RRF value (~0.008-0.02). +Both are legitimate *ranking* signals, but they are not on the same scale, so a +caller comparing `score >= 0.45` rejected every hit the moment BM25 matched +anything. That is how the citation-verification tab came to show no supporting +precedent for a single argument, on every case. + +`relevance` is the fix: always a cosine, or None when the row came from the +lexical leg alone and no cosine was ever computed. +""" + +import pytest + +from legal_mcp.services.case_citation_verification import _SUGGEST_FLOOR, _passes_floor +from legal_mcp.services.hybrid_search import _merge_sem_lex + + +def _sem(key: str, score: float) -> dict: + return {"chunk_id": key, "case_law_id": "c1", "score": score, "relevance": score} + + +def _lex(key: str, score: float) -> dict: + # The lexical leg emits ts_rank_cd, never a cosine — so no `relevance`. + return {"chunk_id": key, "case_law_id": "c1", "score": score} + + +def test_fusion_replaces_score_but_preserves_the_cosine(): + """The regression in one assertion.""" + merged = _merge_sem_lex([_sem("a", 0.73)], [_lex("a", 0.31)], limit=10) + row = merged[0] + assert row["score"] < 0.1, "fused score is an RRF value, not a cosine" + assert row["relevance"] == pytest.approx(0.73), "the cosine must survive fusion" + + +def test_lexical_only_row_has_no_fabricated_cosine(): + """None means 'not measured' — not 'measured as irrelevant'.""" + merged = _merge_sem_lex([], [_lex("b", 0.31)], limit=10) + assert merged[0]["relevance"] is None + + +def test_semantic_only_row_keeps_its_cosine(): + merged = _merge_sem_lex([_sem("c", 0.62)], [], limit=10) + assert merged[0]["relevance"] == pytest.approx(0.62) + + +def test_floor_would_have_rejected_everything_on_the_fused_score(): + """Guards the exact production symptom: fused scores are ~0.008, the floor + is 0.45, so score-based filtering wipes the result set.""" + merged = _merge_sem_lex( + [_sem(k, 0.70) for k in "abcd"], [_lex(k, 0.30) for k in "abcd"], limit=10) + assert all(r["score"] < _SUGGEST_FLOOR for r in merged) # the bug + assert all(_passes_floor(r) for r in merged) # the fix + + +def test_floor_still_rejects_genuinely_weak_hits(): + """The fix must not become 'accept everything'.""" + assert not _passes_floor({"relevance": 0.10}) + assert not _passes_floor({"relevance": _SUGGEST_FLOOR - 0.01}) + assert _passes_floor({"relevance": _SUGGEST_FLOOR}) + + +def test_lexical_only_hits_are_kept_deliberately(): + """An exact docket/phrase match reaches the top by BM25 rank with no cosine. + Dropping it would hide precisely what a chair searches for by name.""" + assert _passes_floor({"relevance": None}) + assert _passes_floor({}) # missing key behaves the same as None