feat(learning): FU-1 — לכידת סבבי-פאנל להלכות (#133)
All checks were successful
G12 Leak-Guard / leak-guard (pull_request) Successful in 7s
All checks were successful
G12 Leak-Guard / leak-guard (pull_request) Successful in 7s
לולאת ה-active-learning זקוקה לסיגנל ללמוד ממנו, אבל הפאנל (halacha_panel_approve.py) זרק עד כה את הצבעות-3-השופטים ואת ההנמקות — שרד רק review_status הסופי על halachot. בלי ההצבעות+הנימוקים אין דרך לזקק rubric משופר. FU-1: - טבלה חדשה halacha_panel_rounds (SCHEMA_V35) — שורה לכל (הלכה, סבב): הצבעה+נימוק לכל לינאז' (claude/deepseek/gemini), ה-verdict, ומה הריצה עשתה (applied_action), apply_mode. במתכונת עמודות-הפאנל של halacha_goldset. - db.insert_panel_round() — helper כתיבה (capture-only). - halacha_panel_approve.py: שומר את התשובות הגולמיות (במקום לזרוק את הנימוק), מוסיף reason ל-NLI_SYSTEM, וכותב סבב לכל פריט בשני המצבים (dry-run ו---apply). --no-capture לדילוג. capture-only: לעולם לא נוגע ב-halachot — שער-היו"ר ב-/precedents נשאר מקור-האמת היחיד (INV-G10). ה-seed ללמידה נוצר בהצלבה מול הכרעת-היו"ר המאוחרת על אותה הלכה (FU-2). Invariants: מקיים INV-G10 (capture-only, שער-יו"ר יחיד), INV-LRN1/3 (לכידה-מבנית; propose-only — אין auto-commit), G1 (לכידה-במקור), G2 (יכולת חדשה, לא מסלול-מקביל), G12 (לא נוגע ב-Paperclip port). חלק מ-#133. smoke (dry-run --limit 8): 6 nli captured, errors=0, נימוקים מלאים מ-3 השופטים. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -1455,6 +1455,39 @@ CREATE INDEX IF NOT EXISTS idx_decision_lessons_review
|
||||
ON decision_lessons(review_status);
|
||||
"""
|
||||
|
||||
SCHEMA_V35_SQL = """
|
||||
-- halacha_panel_rounds (#133 / FU-1): captures EVERY 3-judge panel adjudication
|
||||
-- so the active-learning loop has something to learn from. Until now the panel
|
||||
-- (halacha_panel_approve.py) threw the per-judge votes and rationales away — only
|
||||
-- the final review_status survived on `halachot`. Without the votes+reasons there
|
||||
-- is no signal to mine ("panel said X, chair said Y") and no way to distil a better
|
||||
-- decision rubric. One row per (halacha, round): the three lineages' vote+reason,
|
||||
-- the derived verdict, and what the run did about it. This is a CAPTURE/audit table,
|
||||
-- NOT a decision — it never changes a halacha's review_status (the chair gate on
|
||||
-- /precedents stays the single source of truth, INV-G10). The learning seed is
|
||||
-- formed later by joining this against the chair's decision on `halachot`
|
||||
-- (reviewed_at > round_ts, reviewer='דפנה'). Modeled on halacha_goldset's panel
|
||||
-- columns. question = which axis was judged ('keep' for clean bucket, 'entailed'
|
||||
-- for nli). apply_mode=false means a dry-run produced the row (still kept — every
|
||||
-- analysis is a learning datapoint); true means --apply acted on it.
|
||||
CREATE TABLE IF NOT EXISTS halacha_panel_rounds (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
halacha_id UUID NOT NULL REFERENCES halachot(id) ON DELETE CASCADE,
|
||||
round_ts TIMESTAMPTZ NOT NULL, -- one stamp shared by a whole run
|
||||
question TEXT NOT NULL, -- 'keep' | 'entailed'
|
||||
bucket TEXT NOT NULL DEFAULT '', -- clean | nli | defect | other
|
||||
claude_vote BOOLEAN, claude_reason TEXT NOT NULL DEFAULT '',
|
||||
deepseek_vote BOOLEAN, deepseek_reason TEXT NOT NULL DEFAULT '',
|
||||
gemini_vote BOOLEAN, gemini_reason TEXT NOT NULL DEFAULT '',
|
||||
verdict TEXT NOT NULL DEFAULT '', -- unanimous_yes|unanimous_no|split|incomplete
|
||||
applied_action TEXT NOT NULL DEFAULT '', -- approved|rejected|nli_cleared|chair|'' (dry-run)
|
||||
apply_mode BOOLEAN NOT NULL DEFAULT false,
|
||||
created_at TIMESTAMPTZ DEFAULT now()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_panel_rounds_halacha ON halacha_panel_rounds(halacha_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_panel_rounds_ts ON halacha_panel_rounds(round_ts);
|
||||
"""
|
||||
|
||||
|
||||
async def _run_schema_migrations(pool: asyncpg.Pool) -> None:
|
||||
async with pool.acquire() as conn:
|
||||
@@ -1493,7 +1526,8 @@ async def _run_schema_migrations(pool: asyncpg.Pool) -> None:
|
||||
await conn.execute(SCHEMA_V32_SQL)
|
||||
await conn.execute(SCHEMA_V33_SQL)
|
||||
await conn.execute(SCHEMA_V34_SQL)
|
||||
logger.info("Database schema initialized (v1-v33)")
|
||||
await conn.execute(SCHEMA_V35_SQL)
|
||||
logger.info("Database schema initialized (v1-v35)")
|
||||
|
||||
|
||||
async def init_schema() -> None:
|
||||
@@ -5015,6 +5049,42 @@ async def goldset_set_panel_label(
|
||||
)
|
||||
|
||||
|
||||
async def insert_panel_round(
|
||||
halacha_id: UUID, *, round_ts: datetime, question: str, bucket: str,
|
||||
claude: dict | None, deepseek: dict | None, gemini: dict | None,
|
||||
vote_key: str, verdict: str, applied_action: str = "", apply_mode: bool = False,
|
||||
) -> None:
|
||||
"""Persist ONE 3-judge panel adjudication of one halacha (#133 / FU-1).
|
||||
|
||||
Capture-only: writes to halacha_panel_rounds and never touches `halachot`
|
||||
(the chair gate stays the single source of truth, INV-G10). Each per-model
|
||||
dict is the judge's raw JSON reply ({"<vote_key>": bool, "reason": str}) or
|
||||
None when that judge failed. vote_key is 'keep' (clean bucket) or 'entailed'
|
||||
(nli). round_ts is shared across a whole run so a round can be reconstructed.
|
||||
The learning seed is formed later by joining this against the chair's later
|
||||
decision on the same halacha.
|
||||
"""
|
||||
def _v(d):
|
||||
if not isinstance(d, dict) or vote_key not in d:
|
||||
return None
|
||||
x = d[vote_key]
|
||||
return x if isinstance(x, bool) else str(x).strip().lower() in ("true", "1", "yes", "כן")
|
||||
|
||||
def _r(d):
|
||||
return str(d.get("reason") or "")[:500] if isinstance(d, dict) else ""
|
||||
|
||||
pool = await get_pool()
|
||||
await pool.execute(
|
||||
"INSERT INTO halacha_panel_rounds (halacha_id, round_ts, question, bucket, "
|
||||
"claude_vote, claude_reason, deepseek_vote, deepseek_reason, "
|
||||
"gemini_vote, gemini_reason, verdict, applied_action, apply_mode) "
|
||||
"VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13)",
|
||||
halacha_id, round_ts, question, bucket,
|
||||
_v(claude), _r(claude), _v(deepseek), _r(deepseek),
|
||||
_v(gemini), _r(gemini), verdict, applied_action, apply_mode,
|
||||
)
|
||||
|
||||
|
||||
async def goldset_tag(
|
||||
goldset_id: UUID, *, is_holding: bool | None = None,
|
||||
correct_type: str | None = None, quote_complete: bool | None = None,
|
||||
|
||||
@@ -55,7 +55,7 @@
|
||||
| `goldset_panel_label.py` | python | **#81.7 — תיוג ה-gold-set בקונצנזוס תלת-מודלי (ללא man-in-the-loop, הנחיית-יו"ר 2026-06-11).** מריץ את שלושת השופטים העצמאיים (Opus/claude_session · DeepSeek · Gemini, מיובאים מ-`halacha_panel_approve`) עם ה-prompt העשיר (`is_holding`+`type`+נימוק מ-`goldset_ai_recommend`) על כל פריט; **רוב 2/3 נכתב ל-`is_holding`/`correct_type`** עם `tagged_by='panel:opus+deepseek+gemini'` (פיצול→NULL→יו"ר, INV-G10). מודד **Fleiss κ** (3 מעריכים) ומריץ **מבחן-אנונימיזציה** (שמות-תיק ממוסכים→שיפוט-מחדש; flip=שינון). לא מעגלי — הוולידטורים הנמדדים rule-based. כותב per-model+consensus+anon ל-DB ודוח ל-`data/audit/`. **מחליף** תיוג-ידני; `goldset_ai_recommend`/`goldset_independent_judge` נשארים כבדיקות single-model. `--limit`/`--no-anon`/`--force`. **חובה מקומי**. | ידני — לאחר יצירת/הרחבת batch |
|
||||
| `goldset_ai_recommend.py` | python | **#81.7 QA (single-model, נבלע ב-panel)** — חוות-דעת claude בלבד ל-`ai_*`. כעת לינאז' 1/3 בתוך `goldset_panel_label`; נשאר כבדיקת-claude עצמאית/חידוש נקודתי. `--force`/`--limit`. **חובה מקומי**. | ידני — בדיקה נקודתית |
|
||||
| `goldset_independent_judge.py` | python | **INV-DM7 ולידציה** — שופט-תפקיד **עצמאי שני** ממודל אחר (DeepSeek API ישיר, OpenAI-compatible) ששובר את עיגון-ה-AI: מסווג rule_role **בעיוור** (בלי לראות תיוג-אדם או המלצת-claude) ומחשב מטריצת-הסכמה (deepseek↔אדם מול ai↔אדם) + ציר-גס (כלל-בר-הכללה מול application/obiter). **ממצא (2026-06-07):** ai↔אדם=100% (מעוגן), deepseek↔אדם=50% מדויק אך **92% גס** → תת-הסוג holding/interpretive/procedural עמום-מטבעו (לא לשער עליו); הציר-הגס אמין חוצה-מודלים. read-only על הזהב. `--model`/`--limit`/`--concurrency`. מפתח מ-`~/.hermes/profiles/deepseek/.env`. raw→`/tmp/goldset_judge_raw.json`. | ידני — ולידציית אמינות-תוויות |
|
||||
| `halacha_panel_approve.py` | python | **פאנל-אישור הלכות (Trust-or-Escalate, dry-run).** 3 שופטים בלתי-תלויי-לינאז' (Opus/claude_session · DeepSeek · Gemini-2.5-flash) מצביעים על ה**ציר-הגס האמין** (92% חוצה-מודלים): נקיות→"הלכה לשמירה?"; nli_unsupported→"הציטוט תומך בכלל?" (שיפוט-מחדש); פגומות→re-extraction. רק ורדיקט מוסכם פועל אוטומטית, **פיצול מסלים ליו"ר** (INV-G10). `--apply` **מחווט** (clean: רוב 2/3; nli: פה-אחד-entailed מנקה flag) — הפיך, מגבה ל-`data/audit/` קודם. מפתחות: DeepSeek מ-`~/.hermes/...`, Gemini מ-`~/.env`. **חובה מקומי**. dry-run 2026-06-07: 197→103 אוטו (פה-אחד) / ~15 (רוב). | ידני / שלב-אימות-הלכות במסלול-הסופי |
|
||||
| `halacha_panel_approve.py` | python | **פאנל-אישור הלכות (Trust-or-Escalate, dry-run).** 3 שופטים בלתי-תלויי-לינאז' (Opus/claude_session · DeepSeek · Gemini-2.5-flash) מצביעים על ה**ציר-הגס האמין** (92% חוצה-מודלים): נקיות→"הלכה לשמירה?"; nli_unsupported→"הציטוט תומך בכלל?" (שיפוט-מחדש); פגומות→re-extraction. רק ורדיקט מוסכם פועל אוטומטית, **פיצול מסלים ליו"ר** (INV-G10). `--apply` **מחווט** (clean: רוב 2/3; nli: פה-אחד-entailed מנקה flag) — הפיך, מגבה ל-`data/audit/` קודם. מפתחות: DeepSeek מ-`~/.hermes/...`, Gemini מ-`~/.env`. **חובה מקומי**. dry-run 2026-06-07: 197→103 אוטו (פה-אחד) / ~15 (רוב). **FU-1 (#133):** כל סבב — הצבעות **+נימוקי-כל-שופט** — נשמר ל-`halacha_panel_rounds` בשני המצבים (capture-only, לא נוגע ב-`halachot`; `apply_mode` מתעד dry-run מול apply); ה-seed ללמידה נוצר בהצלבה מול הכרעת-היו"ר המאוחרת על אותה הלכה. `--no-capture` לדילוג. | ידני / שלב-אימות-הלכות במסלול-הסופי |
|
||||
| `style_lesson_panel.py` | python | **פאנל-סגנון דו-סוכני (למידה כפולה).** על-גבי דיסטילציית-ה-Opus (draft↔final ב-`draft_final_pairs.analysis`), שני שופטים בלתי-תלויים — DeepSeek + Gemini-2.5-flash — מצביעים לכל לקח על השאלה הגסה "האם זו הנחיית-סגנון מופשטת ובת-הכללה (INV-LRN5 — קול ולא מהות)?". הסכמה 2/2-keep → נכתב כ-`decision_lesson` (`source=panel:deepseek+gemini`); 2/2-drop → לא נכתב; פיצול/substance → מוסלם ליו"ר. `--apply` הפיך, מגבה ל-`data/audit/`. הטמעה ל-SKILL.md/lessons.md נשארת שער-יו"ר ידני (INV-G10). מפתחות כמו פאנל-ההלכות. **חובה מקומי**. `--case <num>` / `--pair-id <uuid>`. | שלב-למידה במסלול-הסופי |
|
||||
| `final_learning_pipeline.py` | python | **תזמור שלב-הלמידה (פקודה אחת).** מופעל ע"י הרמס כשלוחצים "הרץ למידת-קול" במסלול-הסופי. דטרמיניסטי: (1) `ingest_final_version` עם נתיב-הסופי, (2) רישום לקורפוס-הסגנון (idempotent), (3) `style_lesson_panel --apply`. **עמיד (X16/INV-DUR1):** 3 הצעדים רצים דרך `_pipeline_runtime.py` (משותף עם halacha) עם checkpoint לכל תיק — קריסה בפאנל [3] ממשיכה מ-[3] במקום לשלם שוב על דיסטילציית-Opus [1]. ברירת-מחדל auto-resume; `--fresh` ריצה נקייה. idempotent. **חובה מקומי**. `--case <num>` / `--force` / `--fresh`. | אוטו (כפתור run-learning) / ידני |
|
||||
| `final_halacha_pipeline.py` | python | **תזמור שלב-אימות-ההלכות (פקודה אחת).** מופעל ע"י הרמס כשלוחצים "הרץ אימות-הלכות". דטרמיניסטי: (0) `precedent_extract_halachot` (החלטה), (1) `extract_internal_citations(chair)`, (2) `corroboration.build_all()`, (3) `halacha_panel_approve --apply`. **עמיד (X16/INV-DUR1):** 4 הצעדים רצים דרך `_pipeline_runtime.py` עם checkpoint לכל תיק — קריסה בפאנל [3] ממשיכה מ-[3]. ברירת-מחדל auto-resume; `--fresh` ריצה נקייה. **חובה מקומי**. `--case <num>` / `--limit N` / `--fresh`. | אוטו (כפתור run-halacha) / ידני |
|
||||
|
||||
@@ -22,10 +22,17 @@ Three buckets of pending_review:
|
||||
3. other quality flags (quote_unverified/truncated/thin) → genuine extraction
|
||||
defects → flagged for re-extraction, never auto-approved.
|
||||
|
||||
DRY-RUN writes NOTHING. --apply acts on the agreed verdicts (clean: 2/3 majority;
|
||||
DRY-RUN writes no DECISIONS. --apply acts on the agreed verdicts (clean: 2/3 majority;
|
||||
nli: unanimous-entailed clears the flag) — reversible, backed up to data/audit/ first.
|
||||
Splits/defects stay pending_review for the chair. Local-only (claude_session needs CLI).
|
||||
|
||||
FU-1 (#133, active-learning): EVERY adjudication — votes AND per-judge rationale — is
|
||||
persisted to halacha_panel_rounds in BOTH modes (a dry-run analysis is still a learning
|
||||
datapoint; apply_mode records which). This is capture-only and never touches `halachot`
|
||||
(the chair gate stays the single source of truth, INV-G10). The learning seed is formed
|
||||
later by joining a round against the chair's own later decision on the same halacha. Pass
|
||||
--no-capture to skip.
|
||||
|
||||
cd ~/legal-ai/mcp-server
|
||||
.venv/bin/python ../scripts/halacha_panel_approve.py --limit 12 # smoke
|
||||
.venv/bin/python ../scripts/halacha_panel_approve.py # full dry-run
|
||||
@@ -75,7 +82,8 @@ KEEP_SYSTEM = (
|
||||
NLI_SYSTEM = (
|
||||
"אתה בודק היסק משפטי. בהינתן כלל וציטוט-תומך, הכרע האם הציטוט באמת תומך בכלל "
|
||||
"ואינו מרחיב מעבר למה שכתוב בו (entailed=true), או שהכלל מרחיב/חורג מהציטוט "
|
||||
'(entailed=false). החזר JSON בלבד: {"entailed": true/false}. ללא markdown, ללא הסבר.'
|
||||
'(entailed=false). החזר JSON בלבד: {"entailed": true/false, "reason": "<משפט קצר>"}. '
|
||||
"ללא markdown."
|
||||
)
|
||||
|
||||
|
||||
@@ -161,6 +169,8 @@ async def panel_vote(client, system, user, key) -> dict:
|
||||
votes["_verdict"] = ("unanimous_yes" if unanimous_yes else
|
||||
"unanimous_no" if unanimous_no else
|
||||
"split" if len(valid) >= 2 else "incomplete")
|
||||
# keep the raw replies so the per-judge rationale can be persisted (FU-1)
|
||||
votes["_raw"] = {"claude": c, "deepseek": ds, "gemini": gm}
|
||||
return votes
|
||||
|
||||
|
||||
@@ -189,6 +199,9 @@ async def main(args: argparse.Namespace) -> int:
|
||||
buckets[bucket(h)].append(h)
|
||||
print("queue:", {k: len(v) for k, v in buckets.items()}, "\n", flush=True)
|
||||
|
||||
# one stamp shared by the whole run, so a round is reconstructable later (FU-1)
|
||||
round_ts = datetime.now(timezone.utc)
|
||||
|
||||
sem = asyncio.Semaphore(args.concurrency)
|
||||
results = {"clean": [], "nli": []}
|
||||
|
||||
@@ -259,10 +272,6 @@ async def main(args: argparse.Namespace) -> int:
|
||||
# NLI → asymmetric: unanimous-entailed → clear nli flag (+approve if clean),
|
||||
# majority not-entailed → rejected, else → chair
|
||||
# DEFECT → untouched (needs re-extraction)
|
||||
if not args.apply:
|
||||
print("\n(dry-run — pass --apply to write the approved policy)")
|
||||
return 0
|
||||
|
||||
def majority(v: dict) -> bool | None:
|
||||
vs = [v[k] for k in ("claude", "deepseek", "gemini") if v[k] is not None]
|
||||
if len(vs) < 2:
|
||||
@@ -270,63 +279,89 @@ async def main(args: argparse.Namespace) -> int:
|
||||
y, n = sum(vs), len(vs) - sum(vs)
|
||||
return True if y > n else (False if n > y else None)
|
||||
|
||||
ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||||
audit = Path(__file__).resolve().parent.parent / "data" / "audit"
|
||||
audit.mkdir(parents=True, exist_ok=True)
|
||||
backup = audit / f"halacha-panel-apply-backup-{ts}.csv"
|
||||
with backup.open("w", encoding="utf-8", newline="") as f:
|
||||
w = csv.writer(f)
|
||||
w.writerow(["id", "review_status", "quality_flags"])
|
||||
for r in clean + nli:
|
||||
h = r["_h"]
|
||||
w.writerow([h["id"], h["review_status"], "|".join(h.get("quality_flags") or [])])
|
||||
if args.apply:
|
||||
ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||||
audit = Path(__file__).resolve().parent.parent / "data" / "audit"
|
||||
audit.mkdir(parents=True, exist_ok=True)
|
||||
backup = audit / f"halacha-panel-apply-backup-{ts}.csv"
|
||||
with backup.open("w", encoding="utf-8", newline="") as f:
|
||||
w = csv.writer(f)
|
||||
w.writerow(["id", "review_status", "quality_flags"])
|
||||
for r in clean + nli:
|
||||
h = r["_h"]
|
||||
w.writerow([h["id"], h["review_status"], "|".join(h.get("quality_flags") or [])])
|
||||
|
||||
pool = await db.get_pool()
|
||||
REV = "panel:opus+deepseek+gemini"
|
||||
approved = rejected = cleared = chair = 0
|
||||
pool = await db.get_pool()
|
||||
REV = "panel:opus+deepseek+gemini"
|
||||
approved = rejected = cleared = chair = 0
|
||||
|
||||
for r in clean:
|
||||
d = majority(r)
|
||||
if d is True:
|
||||
await pool.execute("UPDATE halachot SET review_status='approved', "
|
||||
"reviewed_at=now(), reviewer=$2, updated_at=now() WHERE id=$1",
|
||||
r["_h"]["id"], REV + " 2/3-keep")
|
||||
approved += 1
|
||||
elif d is False:
|
||||
await pool.execute("UPDATE halachot SET review_status='rejected', "
|
||||
"reviewed_at=now(), reviewer=$2, updated_at=now() WHERE id=$1",
|
||||
r["_h"]["id"], REV + " 2/3-drop")
|
||||
rejected += 1
|
||||
else:
|
||||
chair += 1
|
||||
for r in clean:
|
||||
d = majority(r)
|
||||
if d is True:
|
||||
await pool.execute("UPDATE halachot SET review_status='approved', "
|
||||
"reviewed_at=now(), reviewer=$2, updated_at=now() WHERE id=$1",
|
||||
r["_h"]["id"], REV + " 2/3-keep")
|
||||
approved += 1; r["_action"] = "approved"
|
||||
elif d is False:
|
||||
await pool.execute("UPDATE halachot SET review_status='rejected', "
|
||||
"reviewed_at=now(), reviewer=$2, updated_at=now() WHERE id=$1",
|
||||
r["_h"]["id"], REV + " 2/3-drop")
|
||||
rejected += 1; r["_action"] = "rejected"
|
||||
else:
|
||||
chair += 1; r["_action"] = "chair"
|
||||
|
||||
for r in nli:
|
||||
vs = [r[k] for k in ("claude", "deepseek", "gemini") if r[k] is not None]
|
||||
unanimous_yes = len(vs) == 3 and all(vs)
|
||||
maj_no = len(vs) >= 2 and sum(vs) < len(vs) - sum(vs)
|
||||
if unanimous_yes:
|
||||
rest = [x for x in (r["_h"].get("quality_flags") or []) if x != "nli_unsupported"]
|
||||
if rest: # other flags remain → clear nli but keep in queue
|
||||
await pool.execute("UPDATE halachot SET quality_flags=$2, updated_at=now() "
|
||||
"WHERE id=$1", r["_h"]["id"], rest)
|
||||
cleared += 1; chair += 1
|
||||
else: # nli was the only blocker → clear + approve
|
||||
await pool.execute("UPDATE halachot SET quality_flags='{}', "
|
||||
"review_status='approved', reviewed_at=now(), reviewer=$2, "
|
||||
"updated_at=now() WHERE id=$1", r["_h"]["id"], REV + " 3/3-entailed")
|
||||
approved += 1; cleared += 1
|
||||
elif maj_no:
|
||||
await pool.execute("UPDATE halachot SET review_status='rejected', "
|
||||
"reviewed_at=now(), reviewer=$2, updated_at=now() WHERE id=$1",
|
||||
r["_h"]["id"], REV + " maj-not-entailed")
|
||||
rejected += 1
|
||||
else:
|
||||
chair += 1
|
||||
for r in nli:
|
||||
vs = [r[k] for k in ("claude", "deepseek", "gemini") if r[k] is not None]
|
||||
unanimous_yes = len(vs) == 3 and all(vs)
|
||||
maj_no = len(vs) >= 2 and sum(vs) < len(vs) - sum(vs)
|
||||
if unanimous_yes:
|
||||
rest = [x for x in (r["_h"].get("quality_flags") or []) if x != "nli_unsupported"]
|
||||
if rest: # other flags remain → clear nli but keep in queue
|
||||
await pool.execute("UPDATE halachot SET quality_flags=$2, updated_at=now() "
|
||||
"WHERE id=$1", r["_h"]["id"], rest)
|
||||
cleared += 1; chair += 1; r["_action"] = "nli_cleared"
|
||||
else: # nli was the only blocker → clear + approve
|
||||
await pool.execute("UPDATE halachot SET quality_flags='{}', "
|
||||
"review_status='approved', reviewed_at=now(), reviewer=$2, "
|
||||
"updated_at=now() WHERE id=$1", r["_h"]["id"], REV + " 3/3-entailed")
|
||||
approved += 1; cleared += 1; r["_action"] = "approved"
|
||||
elif maj_no:
|
||||
await pool.execute("UPDATE halachot SET review_status='rejected', "
|
||||
"reviewed_at=now(), reviewer=$2, updated_at=now() WHERE id=$1",
|
||||
r["_h"]["id"], REV + " maj-not-entailed")
|
||||
rejected += 1; r["_action"] = "rejected"
|
||||
else:
|
||||
chair += 1; r["_action"] = "chair"
|
||||
|
||||
print(f"\nAPPLIED (reversible): approved {approved} · rejected {rejected} · "
|
||||
f"nli-flag-cleared {cleared} · left to chair {chair + len(buckets['defect'])} "
|
||||
f"(incl. {len(buckets['defect'])} defects for re-extraction)")
|
||||
print(f"backup → {backup}")
|
||||
print(f"\nAPPLIED (reversible): approved {approved} · rejected {rejected} · "
|
||||
f"nli-flag-cleared {cleared} · left to chair {chair + len(buckets['defect'])} "
|
||||
f"(incl. {len(buckets['defect'])} defects for re-extraction)")
|
||||
print(f"backup → {backup}")
|
||||
else:
|
||||
print("\n(dry-run — pass --apply to write the approved policy)")
|
||||
|
||||
# ── FU-1 (#133): persist EVERY adjudication so active-learning has a signal.
|
||||
# Capture-only — writes to halacha_panel_rounds, never touches `halachot`
|
||||
# (chair gate stays the single source of truth, INV-G10). Runs in BOTH modes:
|
||||
# a dry-run analysis is still a learning datapoint (apply_mode records which).
|
||||
if not args.no_capture:
|
||||
captured = errs = 0
|
||||
for tag, q in (("clean", "keep"), ("nli", "entailed")):
|
||||
for r in results[tag]:
|
||||
raw = r.get("_raw") or {}
|
||||
try:
|
||||
await db.insert_panel_round(
|
||||
r["_h"]["id"], round_ts=round_ts, question=q, bucket=tag,
|
||||
claude=raw.get("claude"), deepseek=raw.get("deepseek"),
|
||||
gemini=raw.get("gemini"), vote_key=q, verdict=r["_verdict"],
|
||||
applied_action=r.get("_action", ""), apply_mode=args.apply,
|
||||
)
|
||||
captured += 1
|
||||
except Exception as e:
|
||||
errs += 1
|
||||
print(f" capture-error {r['_h']['id']}: {e}", flush=True)
|
||||
print(f"captured {captured} panel rounds → halacha_panel_rounds "
|
||||
f"(apply_mode={args.apply}, errors={errs})")
|
||||
return 0
|
||||
|
||||
|
||||
@@ -337,4 +372,6 @@ if __name__ == "__main__":
|
||||
ap.add_argument("--concurrency", type=int, default=6)
|
||||
ap.add_argument("--apply", action="store_true",
|
||||
help="write the agreed verdicts (reversible, CSV-backed); default dry-run")
|
||||
ap.add_argument("--no-capture", action="store_true",
|
||||
help="skip persisting per-judge votes+reasons to halacha_panel_rounds (FU-1, #133)")
|
||||
raise SystemExit(asyncio.run(main(ap.parse_args())))
|
||||
|
||||
Reference in New Issue
Block a user