Merge pull request 'feat(corpus): עיצוב-מחדש קורפוס-הפסיקה — ביטול תור-ההלכות, שכבת-מאומת-מאזכורים, דירוג-בזמן-אחזור (#153)' (#315) from worktree-canonical-synthesis into main

Merge PR #315: corpus redesign — no queue, verified-by-citation, rank-at-retrieval (#153)

mcp-server/src/legal_mcp/config.py

@@ -162,6 +162,23 @@ HALACHA_AUTO_APPROVE_THRESHOLD = float(
     os.environ.get("HALACHA_AUTO_APPROVE_THRESHOLD", "0.80")
 )
 
+# Corpus redesign (#153, chaim 2026-06-20): ELIMINATE the halacha review queue.
+# When on (default), extraction never produces 'pending_review' — every extracted
+# principle lands as 'approved' = available BACKGROUND (no human review, ever).
+# Trust/ranking comes from chair citation (halachot.verified/cite_count), not from
+# an approval gate. The nli-audit found the quality flags are 97% false-positive,
+# so gating on them only created a phantom backlog (2,402 items). Set false to
+# restore the legacy confidence+flags auto-approve gate.
+HALACHA_NO_REVIEW_QUEUE = os.environ.get("HALACHA_NO_REVIEW_QUEUE", "true").lower() == "true"
+
+# Corpus redesign (#153): retrieval ranks VERIFIED (chair-cited) principles above
+# the unranked BACKGROUND. Added to the halacha similarity score (cosine 0-1): a flat
+# boost if the source precedent was chair-cited, plus a small per-citation increment
+# (capped). 0 disables (pure similarity). Tunable; calibrate against the canon.
+HALACHA_VERIFIED_BOOST = float(os.environ.get("HALACHA_VERIFIED_BOOST", "0.12"))
+HALACHA_CITE_BOOST_PER = float(os.environ.get("HALACHA_CITE_BOOST_PER", "0.01"))
+HALACHA_CITE_BOOST_CAP = int(os.environ.get("HALACHA_CITE_BOOST_CAP", "10"))
+
 # ── Tri-model panel extraction regime (legal-principles-redesign, #152) ──────
 # chaim 2026-06-19: replace single-model auto-approve with a 3-model panel that
 # deep-analyzes each decision. 3 models (Claude local + DeepSeek + Gemini) each
@@ -179,10 +196,22 @@ HALACHA_PANEL_MAX_NEW = int(os.environ.get("HALACHA_PANEL_MAX_NEW", "5"))
 # a floor misses genuine cross-model agreement → undercounts votes → over-culls.
 # Calibrate against the gold-set in Phase C before the production cull.
 HALACHA_PANEL_MATCH_COSINE = float(os.environ.get("HALACHA_PANEL_MATCH_COSINE", "0.80"))
-# When on (default), extraction uses the decision-level 3-model panel regime above
-# instead of the legacy per-chunk single-model auto-approve. Set false to fall back
-# to the legacy path (e.g. if all three judges are unreachable).
-HALACHA_PANEL_REGIME_ENABLED = os.environ.get("HALACHA_PANEL_REGIME_ENABLED", "true").lower() == "true"
+# DEFAULT OFF (#153, chaim 2026-06-20). The panel regime caps extraction to MAX_NEW
+# and filters by novelty — empirically PROVEN destructive (8508/1049/1200 each lost
+# 22-30 genuine principles incl. the core Lustrenik rule). The corpus redesign keeps
+# ALL extracted principles as an unranked BACKGROUND layer (trust comes from chair
+# citation, not extraction); so extraction reverts to the legacy rich per-chunk path.
+# The panel code is retained (dormant) for optional dedup, never for capping.
+HALACHA_PANEL_REGIME_ENABLED = os.environ.get("HALACHA_PANEL_REGIME_ENABLED", "false").lower() == "true"
+
+# Importance layer (#153) — principle-level gold matching. OUR_CHAIR's citations
+# (tier-1 gold, protective) vs other chairs' (tier-2 weight). Match threshold: a
+# chair-citation's match_context (or a digest's headline_holding) is matched to the
+# cited precedent's principles by cosine; ≥ this → the principle is flagged. Tuned
+# toward RECALL (a false-negative buries a principle the chair relied on; a
+# false-positive merely over-protects). 0.72 = paraphrase floor on voyage-law-2.
+OUR_CHAIR_NAME = os.environ.get("OUR_CHAIR_NAME", "דפנה תמיר")
+HALACHA_GOLD_MATCH_THRESHOLD = float(os.environ.get("HALACHA_GOLD_MATCH_THRESHOLD", "0.72"))
 
 # Halacha dedup-on-insert — within-precedent semantic cosine ceiling. Before
 # storing a halacha, store_halachot_for_chunk skips it if its rule-embedding has

mcp-server/src/legal_mcp/services/db.py

@@ -1655,6 +1655,34 @@ ALTER TABLE halachot ALTER COLUMN embedding DROP NOT NULL;
 CREATE INDEX IF NOT EXISTS idx_halachot_canonical ON halachot(canonical_id);
 CREATE INDEX IF NOT EXISTS idx_halachot_instance_type ON halachot(instance_type);
 
+-- Importance layer (#153, component 1): principle-level gold/chair-cited flags.
+-- gold_chair  = our chair (דפנה) cited THIS specific principle (tier-1, protective).
+-- gold_digest = a digest's headline_holding matches this principle (tier-1).
+-- chair_cited = another committee chair cited it (tier-2, weight not protection).
+-- gold_match_score = best cosine of the matched signal (audit/tuning, G9).
+-- All set by scripts/compute_principle_gold.py (embedding match, no LLM).
+ALTER TABLE halachot
+    ADD COLUMN IF NOT EXISTS gold_chair BOOLEAN NOT NULL DEFAULT false,
+    ADD COLUMN IF NOT EXISTS gold_digest BOOLEAN NOT NULL DEFAULT false,
+    ADD COLUMN IF NOT EXISTS chair_cited BOOLEAN NOT NULL DEFAULT false,
+    ADD COLUMN IF NOT EXISTS gold_match_score REAL;
+CREATE INDEX IF NOT EXISTS idx_halachot_gold ON halachot(gold_chair, gold_digest)
+    WHERE gold_chair OR gold_digest;
+
+-- Corpus redesign (#153, chaim 2026-06-20: "trusted = citation, not review").
+-- Two-layer retrieval model:
+--   • verified  = the principle's SOURCE precedent was actually cited by a chair
+--                 (precedent_internal_citations). The ONLY trust signal — never
+--                 from human review (the halacha review queue is eliminated).
+--   • cite_count = # distinct chair decisions citing the source precedent → the
+--                 importance/ranking signal (verified ≫ background at retrieval).
+-- Refreshed by db.refresh_verified_layer() (scripts/build_verified_layer.py), and
+-- grows automatically as new chair decisions are ingested (active-learning).
+ALTER TABLE halachot
+    ADD COLUMN IF NOT EXISTS verified BOOLEAN NOT NULL DEFAULT false,
+    ADD COLUMN IF NOT EXISTS cite_count INT NOT NULL DEFAULT 0;
+CREATE INDEX IF NOT EXISTS idx_halachot_verified ON halachot(verified) WHERE verified;
+
 -- halacha_citation_corroboration (X11) gains canonical_id so the signal
 -- aggregates at the principle level rather than the per-instance level.
 -- Backfill: UPDATE halacha_citation_corroboration SET canonical_id =
@@ -5222,7 +5250,8 @@ async def store_halachot(case_law_id: UUID, halachot: list[dict]) -> int:
     async with pool.acquire() as conn:
         for i, h in enumerate(halachot):
             confidence = float(h.get("confidence", 0.0))
-            auto_approve = confidence >= threshold
+            # #153: no review queue → everything is available background (approved).
+            auto_approve = config.HALACHA_NO_REVIEW_QUEUE or confidence >= threshold
             review_status = "approved" if auto_approve else "pending_review"
             reviewer = (
                 f"auto-approved (confidence ≥ {threshold:.2f})"
@@ -5427,7 +5456,9 @@ async def store_halachot_for_chunk(
                         instance_type = "citation"
 
                 confidence = float(h.get("confidence", 0.0))
-                auto_approve = confidence >= threshold and not flags
+                # #153: no review queue → everything is available background (approved);
+                # quality flags become ranking signals, not an approval gate (nli 97% FP).
+                auto_approve = config.HALACHA_NO_REVIEW_QUEUE or (confidence >= threshold and not flags)
                 review_status = "approved" if auto_approve else "pending_review"
                 reviewer = (
                     f"auto-approved (confidence ≥ {threshold:.2f})"
@@ -6388,6 +6419,141 @@ async def apply_canonical_synthesis(
     return result.split()[-1] != "0"
 
 
+# ── Importance layer (#153) — principle-level gold matching ──────────────────
+
+async def gold_chair_citations() -> list[dict]:
+    """Committee-sourced citations for gold matching (#153).
+
+    Each row: the cited precedent, the citing chair's name, and the match_context
+    (text around the citation — the holding the chair invoked). `is_our_chair`
+    distinguishes tier-1 (OUR_CHAIR → gold_chair) from tier-2 (other chair →
+    chair_cited).
+    """
+    pool = await get_pool()
+    rows = await pool.fetch(
+        "SELECT pic.cited_case_law_id, pic.match_context, "
+        "       COALESCE(src.chair_name,'') AS citing_chair, "
+        "       (src.chair_name = $1) AS is_our_chair "
+        "FROM precedent_internal_citations pic "
+        "JOIN case_law src ON src.id = pic.source_case_law_id "
+        "WHERE pic.cited_case_law_id IS NOT NULL "
+        "  AND length(COALESCE(pic.match_context,'')) > 20",
+        config.OUR_CHAIR_NAME,
+    )
+    return [dict(r) for r in rows]
+
+
+async def gold_digest_holdings() -> list[dict]:
+    """Digest→precedent links with the headline holding the digest highlights (#153)."""
+    pool = await get_pool()
+    rows = await pool.fetch(
+        "SELECT linked_case_law_id, "
+        "       COALESCE(NULLIF(headline_holding,''), summary, '') AS holding_text "
+        "FROM digests "
+        "WHERE linked_case_law_id IS NOT NULL "
+        "  AND length(COALESCE(NULLIF(headline_holding,''), summary, '')) > 20",
+    )
+    return [dict(r) for r in rows]
+
+
+async def nearest_original_principle(
+    case_law_id: "UUID", vec: list[float],
+) -> "tuple[str, float] | None":
+    """Nearest live 'original' principle of a precedent to `vec`, with cosine sim.
+
+    Scoped to one precedent (the cited/linked source) so a chair's citation is
+    matched only against principles actually extracted from THAT decision. Skips
+    rejected instances. Returns (halacha_id, sim) or None if the precedent has no
+    embedded live principle.
+    """
+    pool = await get_pool()
+    row = await pool.fetchrow(
+        "SELECT id::text AS id, 1 - (embedding <=> $2) AS sim "
+        "FROM halachot "
+        "WHERE case_law_id = $1 AND instance_type = 'original' "
+        "  AND embedding IS NOT NULL AND review_status <> 'rejected' "
+        "ORDER BY embedding <=> $2 LIMIT 1",
+        case_law_id, vec,
+    )
+    return (row["id"], float(row["sim"])) if row else None
+
+
+async def set_principle_gold(
+    halacha_id: "UUID", *, gold_chair: bool = False, gold_digest: bool = False,
+    chair_cited: bool = False, score: float | None = None,
+) -> None:
+    """OR-merge gold/chair-cited flags onto a principle (a principle may be both
+    chair-cited AND in a digest). Keeps the MAX match score seen (#153)."""
+    pool = await get_pool()
+    await pool.execute(
+        "UPDATE halachot SET "
+        "  gold_chair = gold_chair OR $2, "
+        "  gold_digest = gold_digest OR $3, "
+        "  chair_cited = chair_cited OR $4, "
+        "  gold_match_score = GREATEST(COALESCE(gold_match_score, 0), COALESCE($5, 0)), "
+        "  updated_at = now() "
+        "WHERE id = $1",
+        halacha_id, gold_chair, gold_digest, chair_cited, score,
+    )
+
+
+async def reset_principle_gold() -> int:
+    """Clear all gold/chair-cited flags (idempotent re-run of the matcher). #153."""
+    pool = await get_pool()
+    res = await pool.execute(
+        "UPDATE halachot SET gold_chair=false, gold_digest=false, chair_cited=false, "
+        "gold_match_score=NULL WHERE gold_chair OR gold_digest OR chair_cited",
+    )
+    return int(res.split()[-1]) if res.split()[-1].isdigit() else 0
+
+
+async def gold_coverage_stats() -> dict:
+    """Counts for the gold-matching coverage report (#153)."""
+    pool = await get_pool()
+    row = await pool.fetchrow(
+        "SELECT "
+        " count(*) FILTER (WHERE instance_type='original' AND review_status<>'rejected') AS live_original, "
+        " count(*) FILTER (WHERE gold_chair) AS gold_chair, "
+        " count(*) FILTER (WHERE gold_digest) AS gold_digest, "
+        " count(*) FILTER (WHERE chair_cited) AS chair_cited, "
+        " count(*) FILTER (WHERE gold_chair OR gold_digest) AS protected "
+        "FROM halachot",
+    )
+    return dict(row)
+
+
+async def refresh_verified_layer() -> dict:
+    """Recompute the verified/cite_count layer from chair citations (#153).
+
+    'verified' = the principle's SOURCE precedent was cited by a chair (any
+    committee decision). 'cite_count' = # distinct chair decisions citing it. This
+    is the ONLY trust signal — never human review. Idempotent (full recompute).
+    Returns {verified_principles, verified_precedents}.
+    """
+    pool = await get_pool()
+    async with pool.acquire() as conn:
+        async with conn.transaction():
+            await conn.execute(
+                "UPDATE halachot SET verified=false, cite_count=0 "
+                "WHERE verified OR cite_count>0")
+            await conn.execute(
+                "WITH cc AS ("
+                "  SELECT pic.cited_case_law_id AS id, "
+                "         count(DISTINCT pic.source_case_law_id) AS n "
+                "  FROM precedent_internal_citations pic "
+                "  JOIN case_law src ON src.id = pic.source_case_law_id "
+                "  WHERE src.source_kind='internal_committee' "
+                "    AND pic.cited_case_law_id IS NOT NULL "
+                "  GROUP BY pic.cited_case_law_id) "
+                "UPDATE halachot h SET verified=true, cite_count=cc.n, updated_at=now() "
+                "FROM cc WHERE h.case_law_id = cc.id")
+            row = await conn.fetchrow(
+                "SELECT count(*) FILTER (WHERE verified) AS vp, "
+                "       count(DISTINCT case_law_id) FILTER (WHERE verified) AS vc "
+                "FROM halachot")
+    return {"verified_principles": row["vp"], "verified_precedents": row["vc"]}
+
+
 async def list_canonical_instances(canonical_id: "UUID") -> list[dict]:
     """List all halachot (instances) sharing a canonical_id — used by the UI accordion."""
     pool = await get_pool()
@@ -6914,7 +7080,7 @@ async def search_precedent_library_semantic(
     """
     pool = await get_pool()
     halacha_filters = [
-        "h.review_status IN ('approved', 'published')",
+        "h.review_status <> 'rejected'",  # #153: include background; rank verified higher
         f"cl.source_kind = '{source_kind}'",
         "cl.searchable = true",
     ]
@@ -6978,18 +7144,22 @@ async def search_precedent_library_semantic(
         c_params.append(chair_name)
         c_idx += 1
 
+    # #153: verified (chair-cited) principles float above background.
+    vboost = (f"(CASE WHEN h.verified THEN {config.HALACHA_VERIFIED_BOOST} ELSE 0 END "
+              f"+ LEAST(h.cite_count, {config.HALACHA_CITE_BOOST_CAP}) * {config.HALACHA_CITE_BOOST_PER})")
     halacha_sql = f"""
         SELECT h.id AS halacha_id, h.case_law_id, h.rule_statement,
                h.reasoning_summary, h.supporting_quote, h.page_reference,
                h.practice_areas, h.subject_tags, h.confidence, h.rule_type,
                cl.case_number, cl.case_name, cl.court, cl.date AS decision_date,
                cl.precedent_level, cl.chair_name, cl.district,
-               1 - (h.embedding <=> $1) AS score
+               h.verified, h.cite_count,
+               (1 - (h.embedding <=> $1)) + {vboost} AS score
         FROM halachot h
         JOIN case_law cl ON cl.id = h.case_law_id
         WHERE {' AND '.join(halacha_filters)}
           AND h.embedding IS NOT NULL
-        ORDER BY h.embedding <=> $1
+        ORDER BY score DESC
         LIMIT $2
     """
 
@@ -7153,7 +7323,7 @@ async def search_precedent_library_lexical(
 
     pool = await get_pool()
     halacha_filters = [
-        "h.review_status IN ('approved', 'published')",
+        "h.review_status <> 'rejected'",  # #153: include background; rank verified higher
         f"cl.source_kind = '{source_kind}'",
         "cl.searchable = true",
     ]
@@ -7218,18 +7388,23 @@ async def search_precedent_library_lexical(
         c_params.append(chair_name)
         c_idx += 1
 
+    # #153: verified (chair-cited) principles float above background.
+    vboost = (f"(CASE WHEN h.verified THEN {config.HALACHA_VERIFIED_BOOST} ELSE 0 END "
+              f"+ LEAST(h.cite_count, {config.HALACHA_CITE_BOOST_CAP}) * {config.HALACHA_CITE_BOOST_PER})")
     halacha_sql = f"""
         SELECT h.id AS halacha_id, h.case_law_id, h.rule_statement,
                h.reasoning_summary, h.supporting_quote, h.page_reference,
                h.practice_areas, h.subject_tags, h.confidence, h.rule_type,
                cl.case_number, cl.case_name, cl.court, cl.date AS decision_date,
                cl.precedent_level, cl.chair_name, cl.district,
+               h.verified, h.cite_count,
                GREATEST(
                    ts_rank_cd(h.rule_tsv, plainto_tsquery('simple', $1)),
                    ts_rank_cd(cl.meta_tsv, plainto_tsquery('simple', $1))
                )
                + CASE WHEN cl.meta_tsv @@ plainto_tsquery('simple', $1)
-                      THEN 1.0 ELSE 0.0 END AS score
+                      THEN 1.0 ELSE 0.0 END
+               + {vboost} AS score
         FROM halachot h
         JOIN case_law cl ON cl.id = h.case_law_id
         WHERE {' AND '.join(halacha_filters)}

mcp-server/src/legal_mcp/services/principle_gold.py

@@ -0,0 +1,83 @@
+"""Principle-level gold matching (importance layer #153, component 1).
+
+Flags the SPECIFIC principle a chair relied on — not the whole precedent. A chair
+cites a precedent for ONE holding; we embed the citation's ``match_context`` (or a
+digest's ``headline_holding``) and cosine-match it to the principles extracted from
+that precedent. The best match ≥ threshold gets the flag:
+
+  • OUR_CHAIR (דפנה) cited it      → ``gold_chair``   (tier-1, protective in the cull)
+  • another committee chair cited it → ``chair_cited`` (tier-2, weight not protection)
+  • a digest highlights it          → ``gold_digest``  (tier-1)
+
+Embedding-only — NO LLM, so it is cheap and re-runnable (the periodic refresh job).
+Tuned toward recall (HALACHA_GOLD_MATCH_THRESHOLD): missing a chair-relied principle
+(buried by the cull) is worse than over-protecting one.
+"""
+from __future__ import annotations
+
+import logging
+from uuid import UUID
+
+from legal_mcp import config
+from legal_mcp.services import db, embeddings
+
+logger = logging.getLogger(__name__)
+
+
+async def _embed_all(texts: list[str]) -> list[list[float]]:
+    """Embed in Voyage-sized batches (embed_texts already chunks at 128)."""
+    return await embeddings.embed_texts(texts, input_type="query") if texts else []
+
+
+async def match_all(threshold: float | None = None, apply: bool = False) -> dict:
+    """Run gold matching over all chair citations + digests.
+
+    Returns a stats dict; when ``apply`` writes the flags (idempotent — clears
+    prior flags first). Dry-run computes everything but writes nothing.
+    """
+    threshold = config.HALACHA_GOLD_MATCH_THRESHOLD if threshold is None else threshold
+    if apply:
+        cleared = await db.reset_principle_gold()
+        logger.info("principle_gold: cleared %d prior flags before re-match", cleared)
+
+    citations = await db.gold_chair_citations()
+    digests = await db.gold_digest_holdings()
+
+    cit_vecs = await _embed_all([c["match_context"] for c in citations])
+    dig_vecs = await _embed_all([d["holding_text"] for d in digests])
+
+    stats = {"threshold": threshold, "applied": apply,
+             "chair_citations": len(citations), "digests": len(digests),
+             "gold_chair": 0, "chair_cited": 0, "gold_digest": 0,
+             "chair_no_match": 0, "digest_no_match": 0, "samples": []}
+
+    async def _flag(case_law_id, vec, *, gold_chair=False, gold_digest=False,
+                    chair_cited=False, label=""):
+        match = await db.nearest_original_principle(case_law_id, list(vec))
+        if match is None or match[1] < threshold:
+            return None
+        hid, sim = match
+        if apply:
+            await db.set_principle_gold(
+                UUID(hid), gold_chair=gold_chair, gold_digest=gold_digest,
+                chair_cited=chair_cited, score=sim)
+        if len(stats["samples"]) < 12:
+            stats["samples"].append({"label": label, "halacha_id": hid, "sim": round(sim, 3)})
+        return hid
+
+    for c, vec in zip(citations, cit_vecs):
+        cid = c["cited_case_law_id"]
+        if c["is_our_chair"]:
+            hid = await _flag(cid, vec, gold_chair=True, label="chair:דפנה")
+            stats["gold_chair" if hid else "chair_no_match"] += 1
+        else:
+            hid = await _flag(cid, vec, chair_cited=True, label=f"chair:{c['citing_chair']}")
+            stats["chair_cited" if hid else "chair_no_match"] += 1
+
+    for d, vec in zip(digests, dig_vecs):
+        hid = await _flag(d["linked_case_law_id"], vec, gold_digest=True, label="digest")
+        stats["gold_digest" if hid else "digest_no_match"] += 1
+
+    if apply:
+        stats["coverage"] = await db.gold_coverage_stats()
+    return stats