Merge pull request 'feat(corpus): עיצוב-מחדש קורפוס-הפסיקה — ביטול תור-ההלכות, שכבת-מאומת-מאזכורים, דירוג-בזמן-אחזור (#153)' (#315) from worktree-canonical-synthesis into main

Merge PR #315: corpus redesign — no queue, verified-by-citation, rank-at-retrieval (#153)

mcp-server/src/legal_mcp/services/db.py

@@ -1655,6 +1655,34 @@ ALTER TABLE halachot ALTER COLUMN embedding DROP NOT NULL;
 CREATE INDEX IF NOT EXISTS idx_halachot_canonical ON halachot(canonical_id);
 CREATE INDEX IF NOT EXISTS idx_halachot_instance_type ON halachot(instance_type);
 
+-- Importance layer (#153, component 1): principle-level gold/chair-cited flags.
+-- gold_chair  = our chair (דפנה) cited THIS specific principle (tier-1, protective).
+-- gold_digest = a digest's headline_holding matches this principle (tier-1).
+-- chair_cited = another committee chair cited it (tier-2, weight not protection).
+-- gold_match_score = best cosine of the matched signal (audit/tuning, G9).
+-- All set by scripts/compute_principle_gold.py (embedding match, no LLM).
+ALTER TABLE halachot
+    ADD COLUMN IF NOT EXISTS gold_chair BOOLEAN NOT NULL DEFAULT false,
+    ADD COLUMN IF NOT EXISTS gold_digest BOOLEAN NOT NULL DEFAULT false,
+    ADD COLUMN IF NOT EXISTS chair_cited BOOLEAN NOT NULL DEFAULT false,
+    ADD COLUMN IF NOT EXISTS gold_match_score REAL;
+CREATE INDEX IF NOT EXISTS idx_halachot_gold ON halachot(gold_chair, gold_digest)
+    WHERE gold_chair OR gold_digest;
+
+-- Corpus redesign (#153, chaim 2026-06-20: "trusted = citation, not review").
+-- Two-layer retrieval model:
+--   • verified  = the principle's SOURCE precedent was actually cited by a chair
+--                 (precedent_internal_citations). The ONLY trust signal — never
+--                 from human review (the halacha review queue is eliminated).
+--   • cite_count = # distinct chair decisions citing the source precedent → the
+--                 importance/ranking signal (verified ≫ background at retrieval).
+-- Refreshed by db.refresh_verified_layer() (scripts/build_verified_layer.py), and
+-- grows automatically as new chair decisions are ingested (active-learning).
+ALTER TABLE halachot
+    ADD COLUMN IF NOT EXISTS verified BOOLEAN NOT NULL DEFAULT false,
+    ADD COLUMN IF NOT EXISTS cite_count INT NOT NULL DEFAULT 0;
+CREATE INDEX IF NOT EXISTS idx_halachot_verified ON halachot(verified) WHERE verified;
+
 -- halacha_citation_corroboration (X11) gains canonical_id so the signal
 -- aggregates at the principle level rather than the per-instance level.
 -- Backfill: UPDATE halacha_citation_corroboration SET canonical_id =
@@ -5222,7 +5250,8 @@ async def store_halachot(case_law_id: UUID, halachot: list[dict]) -> int:
     async with pool.acquire() as conn:
         for i, h in enumerate(halachot):
             confidence = float(h.get("confidence", 0.0))
-            auto_approve = confidence >= threshold
+            # #153: no review queue → everything is available background (approved).
+            auto_approve = config.HALACHA_NO_REVIEW_QUEUE or confidence >= threshold
             review_status = "approved" if auto_approve else "pending_review"
             reviewer = (
                 f"auto-approved (confidence ≥ {threshold:.2f})"
@@ -5427,7 +5456,9 @@ async def store_halachot_for_chunk(
                         instance_type = "citation"
 
                 confidence = float(h.get("confidence", 0.0))
-                auto_approve = confidence >= threshold and not flags
+                # #153: no review queue → everything is available background (approved);
+                # quality flags become ranking signals, not an approval gate (nli 97% FP).
+                auto_approve = config.HALACHA_NO_REVIEW_QUEUE or (confidence >= threshold and not flags)
                 review_status = "approved" if auto_approve else "pending_review"
                 reviewer = (
                     f"auto-approved (confidence ≥ {threshold:.2f})"
@@ -6388,6 +6419,141 @@ async def apply_canonical_synthesis(
     return result.split()[-1] != "0"
 
 
+# ── Importance layer (#153) — principle-level gold matching ──────────────────
+
+async def gold_chair_citations() -> list[dict]:
+    """Committee-sourced citations for gold matching (#153).
+
+    Each row: the cited precedent, the citing chair's name, and the match_context
+    (text around the citation — the holding the chair invoked). `is_our_chair`
+    distinguishes tier-1 (OUR_CHAIR → gold_chair) from tier-2 (other chair →
+    chair_cited).
+    """
+    pool = await get_pool()
+    rows = await pool.fetch(
+        "SELECT pic.cited_case_law_id, pic.match_context, "
+        "       COALESCE(src.chair_name,'') AS citing_chair, "
+        "       (src.chair_name = $1) AS is_our_chair "
+        "FROM precedent_internal_citations pic "
+        "JOIN case_law src ON src.id = pic.source_case_law_id "
+        "WHERE pic.cited_case_law_id IS NOT NULL "
+        "  AND length(COALESCE(pic.match_context,'')) > 20",
+        config.OUR_CHAIR_NAME,
+    )
+    return [dict(r) for r in rows]
+
+
+async def gold_digest_holdings() -> list[dict]:
+    """Digest→precedent links with the headline holding the digest highlights (#153)."""
+    pool = await get_pool()
+    rows = await pool.fetch(
+        "SELECT linked_case_law_id, "
+        "       COALESCE(NULLIF(headline_holding,''), summary, '') AS holding_text "
+        "FROM digests "
+        "WHERE linked_case_law_id IS NOT NULL "
+        "  AND length(COALESCE(NULLIF(headline_holding,''), summary, '')) > 20",
+    )
+    return [dict(r) for r in rows]
+
+
+async def nearest_original_principle(
+    case_law_id: "UUID", vec: list[float],
+) -> "tuple[str, float] | None":
+    """Nearest live 'original' principle of a precedent to `vec`, with cosine sim.
+
+    Scoped to one precedent (the cited/linked source) so a chair's citation is
+    matched only against principles actually extracted from THAT decision. Skips
+    rejected instances. Returns (halacha_id, sim) or None if the precedent has no
+    embedded live principle.
+    """
+    pool = await get_pool()
+    row = await pool.fetchrow(
+        "SELECT id::text AS id, 1 - (embedding <=> $2) AS sim "
+        "FROM halachot "
+        "WHERE case_law_id = $1 AND instance_type = 'original' "
+        "  AND embedding IS NOT NULL AND review_status <> 'rejected' "
+        "ORDER BY embedding <=> $2 LIMIT 1",
+        case_law_id, vec,
+    )
+    return (row["id"], float(row["sim"])) if row else None
+
+
+async def set_principle_gold(
+    halacha_id: "UUID", *, gold_chair: bool = False, gold_digest: bool = False,
+    chair_cited: bool = False, score: float | None = None,
+) -> None:
+    """OR-merge gold/chair-cited flags onto a principle (a principle may be both
+    chair-cited AND in a digest). Keeps the MAX match score seen (#153)."""
+    pool = await get_pool()
+    await pool.execute(
+        "UPDATE halachot SET "
+        "  gold_chair = gold_chair OR $2, "
+        "  gold_digest = gold_digest OR $3, "
+        "  chair_cited = chair_cited OR $4, "
+        "  gold_match_score = GREATEST(COALESCE(gold_match_score, 0), COALESCE($5, 0)), "
+        "  updated_at = now() "
+        "WHERE id = $1",
+        halacha_id, gold_chair, gold_digest, chair_cited, score,
+    )
+
+
+async def reset_principle_gold() -> int:
+    """Clear all gold/chair-cited flags (idempotent re-run of the matcher). #153."""
+    pool = await get_pool()
+    res = await pool.execute(
+        "UPDATE halachot SET gold_chair=false, gold_digest=false, chair_cited=false, "
+        "gold_match_score=NULL WHERE gold_chair OR gold_digest OR chair_cited",
+    )
+    return int(res.split()[-1]) if res.split()[-1].isdigit() else 0
+
+
+async def gold_coverage_stats() -> dict:
+    """Counts for the gold-matching coverage report (#153)."""
+    pool = await get_pool()
+    row = await pool.fetchrow(
+        "SELECT "
+        " count(*) FILTER (WHERE instance_type='original' AND review_status<>'rejected') AS live_original, "
+        " count(*) FILTER (WHERE gold_chair) AS gold_chair, "
+        " count(*) FILTER (WHERE gold_digest) AS gold_digest, "
+        " count(*) FILTER (WHERE chair_cited) AS chair_cited, "
+        " count(*) FILTER (WHERE gold_chair OR gold_digest) AS protected "
+        "FROM halachot",
+    )
+    return dict(row)
+
+
+async def refresh_verified_layer() -> dict:
+    """Recompute the verified/cite_count layer from chair citations (#153).
+
+    'verified' = the principle's SOURCE precedent was cited by a chair (any
+    committee decision). 'cite_count' = # distinct chair decisions citing it. This
+    is the ONLY trust signal — never human review. Idempotent (full recompute).
+    Returns {verified_principles, verified_precedents}.
+    """
+    pool = await get_pool()
+    async with pool.acquire() as conn:
+        async with conn.transaction():
+            await conn.execute(
+                "UPDATE halachot SET verified=false, cite_count=0 "
+                "WHERE verified OR cite_count>0")
+            await conn.execute(
+                "WITH cc AS ("
+                "  SELECT pic.cited_case_law_id AS id, "
+                "         count(DISTINCT pic.source_case_law_id) AS n "
+                "  FROM precedent_internal_citations pic "
+                "  JOIN case_law src ON src.id = pic.source_case_law_id "
+                "  WHERE src.source_kind='internal_committee' "
+                "    AND pic.cited_case_law_id IS NOT NULL "
+                "  GROUP BY pic.cited_case_law_id) "
+                "UPDATE halachot h SET verified=true, cite_count=cc.n, updated_at=now() "
+                "FROM cc WHERE h.case_law_id = cc.id")
+            row = await conn.fetchrow(
+                "SELECT count(*) FILTER (WHERE verified) AS vp, "
+                "       count(DISTINCT case_law_id) FILTER (WHERE verified) AS vc "
+                "FROM halachot")
+    return {"verified_principles": row["vp"], "verified_precedents": row["vc"]}
+
+
 async def list_canonical_instances(canonical_id: "UUID") -> list[dict]:
     """List all halachot (instances) sharing a canonical_id — used by the UI accordion."""
     pool = await get_pool()
@@ -6914,7 +7080,7 @@ async def search_precedent_library_semantic(
     """
     pool = await get_pool()
     halacha_filters = [
-        "h.review_status IN ('approved', 'published')",
+        "h.review_status <> 'rejected'",  # #153: include background; rank verified higher
         f"cl.source_kind = '{source_kind}'",
         "cl.searchable = true",
     ]
@@ -6978,18 +7144,22 @@ async def search_precedent_library_semantic(
         c_params.append(chair_name)
         c_idx += 1
 
+    # #153: verified (chair-cited) principles float above background.
+    vboost = (f"(CASE WHEN h.verified THEN {config.HALACHA_VERIFIED_BOOST} ELSE 0 END "
+              f"+ LEAST(h.cite_count, {config.HALACHA_CITE_BOOST_CAP}) * {config.HALACHA_CITE_BOOST_PER})")
     halacha_sql = f"""
         SELECT h.id AS halacha_id, h.case_law_id, h.rule_statement,
                h.reasoning_summary, h.supporting_quote, h.page_reference,
                h.practice_areas, h.subject_tags, h.confidence, h.rule_type,
                cl.case_number, cl.case_name, cl.court, cl.date AS decision_date,
                cl.precedent_level, cl.chair_name, cl.district,
-               1 - (h.embedding <=> $1) AS score
+               h.verified, h.cite_count,
+               (1 - (h.embedding <=> $1)) + {vboost} AS score
         FROM halachot h
         JOIN case_law cl ON cl.id = h.case_law_id
         WHERE {' AND '.join(halacha_filters)}
           AND h.embedding IS NOT NULL
-        ORDER BY h.embedding <=> $1
+        ORDER BY score DESC
         LIMIT $2
     """
 
@@ -7153,7 +7323,7 @@ async def search_precedent_library_lexical(
 
     pool = await get_pool()
     halacha_filters = [
-        "h.review_status IN ('approved', 'published')",
+        "h.review_status <> 'rejected'",  # #153: include background; rank verified higher
         f"cl.source_kind = '{source_kind}'",
         "cl.searchable = true",
     ]
@@ -7218,18 +7388,23 @@ async def search_precedent_library_lexical(
         c_params.append(chair_name)
         c_idx += 1
 
+    # #153: verified (chair-cited) principles float above background.
+    vboost = (f"(CASE WHEN h.verified THEN {config.HALACHA_VERIFIED_BOOST} ELSE 0 END "
+              f"+ LEAST(h.cite_count, {config.HALACHA_CITE_BOOST_CAP}) * {config.HALACHA_CITE_BOOST_PER})")
     halacha_sql = f"""
         SELECT h.id AS halacha_id, h.case_law_id, h.rule_statement,
                h.reasoning_summary, h.supporting_quote, h.page_reference,
                h.practice_areas, h.subject_tags, h.confidence, h.rule_type,
                cl.case_number, cl.case_name, cl.court, cl.date AS decision_date,
                cl.precedent_level, cl.chair_name, cl.district,
+               h.verified, h.cite_count,
                GREATEST(
                    ts_rank_cd(h.rule_tsv, plainto_tsquery('simple', $1)),
                    ts_rank_cd(cl.meta_tsv, plainto_tsquery('simple', $1))
                )
                + CASE WHEN cl.meta_tsv @@ plainto_tsquery('simple', $1)
-                      THEN 1.0 ELSE 0.0 END AS score
+                      THEN 1.0 ELSE 0.0 END
+               + {vboost} AS score
         FROM halachot h
         JOIN case_law cl ON cl.id = h.case_law_id
         WHERE {' AND '.join(halacha_filters)}