fix(retrieval): 104 החלטות ועדות-ערר היו בלתי-נראות בחיפוש — סינון ולא דירוג (#232)
`search_precedent_library` — הכלי שסוכני הכתיבה קוראים לו בפועל — לא חשף `source_kind` כלל, וכל שכבה מתחתיו נפלה לברירת-מחדל `external_upload`. התוצאה: `WHERE cl.source_kind = 'external_upload'` חתך 104 החלטות ועדות-ערר (29% מהקורפוס) עוד לפני הדירוג. ערר (חיפה) 83/16 — האסמכתה הישירה ביותר בקורפוס לטענת-סף ס-2 ב-1069-04-26 — לא הוחזרה גם בשאילתה כמעט-מילולית. ההיפותזה שנרשמה ב-#232 (הטיית verified/cite_count מדירה החלטות חדשות) **נבדקה ונפסלה**: כיסוי ה-verified כמעט זהה בשני הקורפוסים (55% מול 49%), ותקרת-ההטיה 0.22 לא הסבירה פער מול התאמה כמעט-מילולית. הסיבה הייתה סינום קשיח, לא דירוג. מה שונה - `_source_kind_clause()` — הגדרה אחת לאוצר-המילים של הסלקטור (G2), במקום שכפולו בכל אתר חיפוש/רשימה. `""`/`"all"` = כל הקורפוס; סלקטור לא-מוכר מרים ValueError במקום להגיע ל-SQL (הערך מוזרק ב-f-string, אז ה-whitelist הוא גם מה ששומר על זה בטוח). - ברירת-המחדל בכל שרשרת-החיפוש והרשימה: `""` = הקורפוס כולו. - `search_precedent_library` + `precedent_library_list` חושפים `source_kind` לסינון מפורש; `/api/precedent-library/search` מקבל אותו גם הוא כדי ש-UI ו-MCP לא יתפצלו. - נרמול מפריד במספר-תיק: `83/16` ו-`83-16` מחזירים את אותה שורה. אימות מול ה-DB החי — 4/4 החלטות ועדה חוזרות במקום **1** בשאילתה בלשון הלכה מאושרת (83-16, 1029-18, 1085-23, 1094-09-19 פדילה). 83/16 עלתה ל-0.711 מול 0.663 של 3213/97 שחסמה אותה קודם. רשימת-הקורפוס: 259 → 386. invariants: G1 (נרמול במקור, לא תיקון-תסמין בקריאה) · G2 (הגדרה אחת לסלקטור; UI ו-MCP על אותו מסלול) · §6 (סלקטור שגוי מתפוצץ, לא נבלע) טסטים: 5 חדשים (tests/test_source_kind_selector.py), אחד מהם נועל את ברירות-המחדל של 8 נקודות-הכניסה — זה בדיוק הבאג. 514 עוברים.
This commit is contained in:
@@ -5324,21 +5324,19 @@ async def list_external_case_law(
|
||||
search: str = "",
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
source_kind: str = "external_upload",
|
||||
source_kind: str = "",
|
||||
) -> list[dict]:
|
||||
"""List chair-uploaded precedents, with simple filters.
|
||||
|
||||
source_kind="" (default) = the whole corpus — court rulings *and*
|
||||
appeals-committee decisions. The old ``external_upload`` default hid
|
||||
every committee decision from a plain listing (#232).
|
||||
source_kind="all_committees" expands to: source_kind='internal_committee'
|
||||
OR (source_kind='external_upload' AND source_type='appeals_committee').
|
||||
"""
|
||||
pool = await get_pool()
|
||||
if source_kind == "all_committees":
|
||||
conditions = [
|
||||
"(source_kind = 'internal_committee' OR "
|
||||
"(source_kind = 'external_upload' AND source_type = 'appeals_committee'))"
|
||||
]
|
||||
else:
|
||||
conditions = [f"source_kind = '{source_kind}'"]
|
||||
sk_clause = _source_kind_clause(source_kind)
|
||||
conditions = [sk_clause] if sk_clause else []
|
||||
params: list = []
|
||||
idx = 1
|
||||
if practice_area:
|
||||
@@ -5358,13 +5356,19 @@ async def list_external_case_law(
|
||||
params.append(source_type)
|
||||
idx += 1
|
||||
if search:
|
||||
# Case-number separator normalisation (#232, trap 2): committee numbers
|
||||
# are stored hyphenated ("83-16") while people write them with a slash
|
||||
# ("83/16"). Fold both sides to '/' so either form finds the row —
|
||||
# matching at the point of comparison rather than asking every caller
|
||||
# to guess the stored form.
|
||||
conditions.append(
|
||||
f"(case_number ILIKE ${idx} OR case_name ILIKE ${idx} "
|
||||
f"OR summary ILIKE ${idx} OR headnote ILIKE ${idx})"
|
||||
f"OR summary ILIKE ${idx} OR headnote ILIKE ${idx} "
|
||||
f"OR replace(case_number, '-', '/') ILIKE replace(${idx}, '-', '/'))"
|
||||
)
|
||||
params.append(f"%{search}%")
|
||||
idx += 1
|
||||
where_sql = " AND ".join(conditions)
|
||||
where_sql = " AND ".join(conditions) if conditions else "TRUE"
|
||||
params.extend([limit, offset])
|
||||
sql = f"""
|
||||
SELECT id, case_number, case_name, court, date, practice_area,
|
||||
@@ -7775,6 +7779,45 @@ async def list_corroboration_for_halacha(halacha_id: UUID) -> list[dict]:
|
||||
]
|
||||
|
||||
|
||||
#: Accepted ``source_kind`` selectors. ``""``/``"all"`` mean *no filter* —
|
||||
#: the whole corpus, court rulings and appeals-committee decisions alike.
|
||||
_SOURCE_KIND_SELECTORS = frozenset(
|
||||
{"", "all", "all_committees", "external_upload", "internal_committee", "cited_only"}
|
||||
)
|
||||
|
||||
|
||||
def _source_kind_clause(source_kind: str, column_prefix: str = "") -> str:
|
||||
"""Return the SQL predicate for a ``source_kind`` selector, or "" for none.
|
||||
|
||||
Single definition for every caller (G2) — the selector vocabulary was
|
||||
previously restated at each search/list site, which is how the
|
||||
``external_upload`` default silently hid 104 appeals-committee
|
||||
decisions from ``search_precedent_library`` (#232).
|
||||
|
||||
``column_prefix`` is the table alias plus dot (e.g. ``"cl."``) or "" when
|
||||
the query selects from ``case_law`` directly.
|
||||
|
||||
Raises ValueError on an unknown selector rather than interpolating it —
|
||||
the value reaches SQL by f-string, so the whitelist is also what keeps
|
||||
that safe.
|
||||
"""
|
||||
sk = (source_kind or "").strip()
|
||||
if sk not in _SOURCE_KIND_SELECTORS:
|
||||
raise ValueError(
|
||||
f"source_kind לא מוכר: {source_kind!r}. "
|
||||
f"ערכים חוקיים: {sorted(_SOURCE_KIND_SELECTORS - {''})} או '' (הכל)"
|
||||
)
|
||||
if sk in ("", "all"):
|
||||
return ""
|
||||
p = column_prefix
|
||||
if sk == "all_committees":
|
||||
return (
|
||||
f"({p}source_kind = 'internal_committee' OR "
|
||||
f"({p}source_kind = 'external_upload' AND {p}source_type = 'appeals_committee'))"
|
||||
)
|
||||
return f"{p}source_kind = '{sk}'"
|
||||
|
||||
|
||||
async def search_precedent_library_semantic(
|
||||
query_embedding: list[float],
|
||||
practice_area: str = "",
|
||||
@@ -7785,14 +7828,15 @@ async def search_precedent_library_semantic(
|
||||
subject_tag: str = "",
|
||||
limit: int = 10,
|
||||
include_halachot: bool = True,
|
||||
source_kind: str = "external_upload",
|
||||
source_kind: str = "",
|
||||
district: str = "",
|
||||
chair_name: str = "",
|
||||
) -> list[dict]:
|
||||
"""Semantic search over precedents filtered by source_kind.
|
||||
"""Semantic search over precedents, optionally filtered by source_kind.
|
||||
|
||||
source_kind='external_upload' → court rulings (default)
|
||||
source_kind='internal_committee' → appeals-committee decisions
|
||||
source_kind='' → the whole corpus (default, #232)
|
||||
source_kind='external_upload' → court rulings only
|
||||
source_kind='internal_committee' → appeals-committee decisions only
|
||||
|
||||
Returns merged halachot + chunks. Halachot are pre-distilled rules, so
|
||||
they get a small score boost. Only ``approved`` / ``published`` halachot
|
||||
@@ -7800,12 +7844,15 @@ async def search_precedent_library_semantic(
|
||||
of halacha review status.
|
||||
"""
|
||||
pool = await get_pool()
|
||||
sk_clause = _source_kind_clause(source_kind, "cl.")
|
||||
halacha_filters = [
|
||||
"h.review_status <> 'rejected'", # #153: include background; rank verified higher
|
||||
f"cl.source_kind = '{source_kind}'",
|
||||
"cl.searchable = true",
|
||||
]
|
||||
chunk_filters = [f"cl.source_kind = '{source_kind}'", "cl.searchable = true"]
|
||||
chunk_filters = ["cl.searchable = true"]
|
||||
if sk_clause:
|
||||
halacha_filters.append(sk_clause)
|
||||
chunk_filters.append(sk_clause)
|
||||
h_params: list = [query_embedding, limit]
|
||||
c_params: list = [query_embedding, limit]
|
||||
h_idx = 3
|
||||
@@ -8017,7 +8064,7 @@ async def search_precedent_library_lexical(
|
||||
appeal_subtype: str = "",
|
||||
is_binding: bool | None = None,
|
||||
subject_tag: str = "",
|
||||
source_kind: str = "external_upload",
|
||||
source_kind: str = "",
|
||||
district: str = "",
|
||||
chair_name: str = "",
|
||||
limit: int = 30,
|
||||
@@ -8043,12 +8090,15 @@ async def search_precedent_library_lexical(
|
||||
return []
|
||||
|
||||
pool = await get_pool()
|
||||
sk_clause = _source_kind_clause(source_kind, "cl.")
|
||||
halacha_filters = [
|
||||
"h.review_status <> 'rejected'", # #153: include background; rank verified higher
|
||||
f"cl.source_kind = '{source_kind}'",
|
||||
"cl.searchable = true",
|
||||
]
|
||||
chunk_filters = [f"cl.source_kind = '{source_kind}'", "cl.searchable = true"]
|
||||
chunk_filters = ["cl.searchable = true"]
|
||||
if sk_clause:
|
||||
halacha_filters.append(sk_clause)
|
||||
chunk_filters.append(sk_clause)
|
||||
# $1 = query, $2 = limit. Filters append starting at $3.
|
||||
h_params: list = [query, limit]
|
||||
c_params: list = [query, limit]
|
||||
|
||||
@@ -98,15 +98,16 @@ async def search_precedent_library_hybrid(
|
||||
is_binding: bool | None = None,
|
||||
subject_tag: str = "",
|
||||
include_halachot: bool = True,
|
||||
source_kind: str = "external_upload",
|
||||
source_kind: str = "",
|
||||
district: str = "",
|
||||
chair_name: str = "",
|
||||
max_per_case_law: int = 2,
|
||||
) -> list[dict]:
|
||||
"""Hybrid wrapper for precedent-library search.
|
||||
|
||||
source_kind='external_upload' → court rulings (default)
|
||||
source_kind='internal_committee' → appeals-committee decisions
|
||||
source_kind='' → the whole corpus (default, #232)
|
||||
source_kind='external_upload' → court rulings only
|
||||
source_kind='internal_committee' → appeals-committee decisions only
|
||||
max_per_case_law: MMR-style diversity cap — at most N hits per
|
||||
case_law_id in the final ranked list (default 2). Prevents a
|
||||
single precedent from monopolizing the result list when many of
|
||||
|
||||
@@ -477,7 +477,7 @@ async def list_precedents(
|
||||
precedent_level: str = "",
|
||||
source_type: str = "",
|
||||
search: str = "",
|
||||
source_kind: str = "external_upload",
|
||||
source_kind: str = "",
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
) -> list[dict]:
|
||||
@@ -503,12 +503,18 @@ async def search_library(
|
||||
subject_tag: str = "",
|
||||
limit: int = 10,
|
||||
include_halachot: bool = True,
|
||||
source_kind: str = "",
|
||||
) -> list[dict]:
|
||||
"""Semantic search merging halachot (rule-level) and chunks (passage-level).
|
||||
|
||||
Only ``approved`` / ``published`` halachot are returned, per chair-review
|
||||
policy. Chunks are returned regardless of halacha review status.
|
||||
|
||||
``source_kind=""`` (default) covers the whole corpus — court rulings and
|
||||
appeals-committee decisions together. It used to be hard-wired to
|
||||
``external_upload`` here, which made 104 committee decisions
|
||||
unreachable through this entry point (#232).
|
||||
|
||||
When ``VOYAGE_RERANK_ENABLED`` is set, results are passed through
|
||||
voyage rerank-2 (cross-encoder). The +0.05 halacha boost from
|
||||
``search_precedent_library_semantic`` is preserved before rerank
|
||||
@@ -529,4 +535,5 @@ async def search_library(
|
||||
is_binding=is_binding,
|
||||
subject_tag=subject_tag,
|
||||
include_halachot=include_halachot,
|
||||
source_kind=source_kind,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user