feat(goldset): interactive gold-set tagging page (#81.7/#81.8)
Replaces the CSV-edit workflow with an in-app tagging page so the chair/Dafna
can label the extraction-quality gold-set by clicking, and see validator
precision/recall live.
Schema (V29): halacha_goldset — a stratified, human-tagged evaluation batch
(is_holding / correct_type / quote_complete, NULL until tagged).
db.py:
- goldset_create_sample (stratified round-robin over case×rule_type, idempotent),
- goldset_list (items + halacha content + the machine's own labels),
- goldset_tag (partial — one field at a time for keyboard tagging),
- goldset_score (ports the script's P/R/F1: each validator scored as a
not-a-holding detector against the human tags — the #81.8 input).
API: GET /api/goldset, POST /api/goldset/sample, GET /api/goldset/score,
PATCH /api/goldset/{id}.
web-ui:
- lib/api/goldset.ts (hooks),
- components/goldset/goldset-panel.tsx — card-per-item, keyboard-first
(J/K nav, H/N holding, C/X quote), progress bar, hide-tagged toggle, and a
collapsible live score table,
- app/goldset/page.tsx + nav link "מדגם-זהב" under ידע ולמידה.
Methodology guard kept explicit in UI + docstrings: tags are HUMAN ground truth,
no AI pre-fill (circular bias). Populated a 150-item stratified batch.
Verified: backend create/list/tag/score against the live DB; tsc --noEmit 0;
py_compile ok. (Local Turbopack build blocked by worktree symlink — CI builds clean.)
Invariants: G1 (eval set modeled at source in its own table); G2 (reuses the same
halacha_quality validators the extractor runs — no parallel scoring logic).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -51,6 +51,7 @@ const NAV_GROUPS: NavGroup[] = [
|
||||
items: [
|
||||
{ href: "/precedents", label: "ספריית פסיקה" },
|
||||
{ href: "/missing-precedents", label: "פסיקה חסרה" },
|
||||
{ href: "/goldset", label: "מדגם-זהב" },
|
||||
{ href: "/training", label: "אימון סגנון" },
|
||||
{ href: "/methodology", label: "מתודולוגיה" },
|
||||
],
|
||||
|
||||
283
web-ui/src/components/goldset/goldset-panel.tsx
Normal file
283
web-ui/src/components/goldset/goldset-panel.tsx
Normal file
@@ -0,0 +1,283 @@
|
||||
"use client";
|
||||
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import { Check, X, ChevronDown, ChevronLeft } from "lucide-react";
|
||||
import { toast } from "sonner";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import {
|
||||
useGoldset, useGoldsetScore, useTagGoldset, useCreateGoldsetSample,
|
||||
type GoldsetItem,
|
||||
} from "@/lib/api/goldset";
|
||||
|
||||
const TYPES: { value: string; label: string }[] = [
|
||||
{ value: "binding", label: "מחייבת" },
|
||||
{ value: "interpretive", label: "פרשני" },
|
||||
{ value: "application", label: "יישום" },
|
||||
{ value: "obiter", label: "אמרת-אגב" },
|
||||
{ value: "procedural", label: "פרוצדורלי" },
|
||||
{ value: "persuasive", label: "משכנע" },
|
||||
];
|
||||
|
||||
const FLAG_LABELS: Record<string, string> = {
|
||||
non_decision: "אי-הכרעה", truncated_quote: "ציטוט קטוע", thin_restatement: "ניסוח דק",
|
||||
quote_unverified: "ציטוט לא מאומת", nli_unsupported: "כלל לא נגזר", application: "יישום",
|
||||
near_duplicate: "כפילות-קרובה", nevo_preamble_leak: "דליפת רציו",
|
||||
};
|
||||
|
||||
function cleanCitation(s: string | null | undefined): string {
|
||||
if (!s) return "—";
|
||||
return s.replace(/[--]/g, "").trim();
|
||||
}
|
||||
|
||||
function isTagged(it: GoldsetItem): boolean {
|
||||
return it.is_holding !== null;
|
||||
}
|
||||
|
||||
// ─── Score panel ──────────────────────────────────────────────────────────────
|
||||
|
||||
function ScorePanel({ batch }: { batch: string }) {
|
||||
const { data } = useGoldsetScore(batch);
|
||||
const [open, setOpen] = useState(false);
|
||||
if (!data || data.labeled === 0) return null;
|
||||
const rows = Object.entries(data.validators);
|
||||
return (
|
||||
<div className="rounded-lg border border-rule bg-surface">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setOpen((v) => !v)}
|
||||
className="w-full flex items-center gap-2 px-4 py-2.5 text-sm hover:bg-gold-wash/30"
|
||||
aria-expanded={open}
|
||||
>
|
||||
{open ? <ChevronDown className="w-4 h-4" /> : <ChevronLeft className="w-4 h-4" />}
|
||||
<span className="font-semibold text-navy">ציון הוולידטורים מול התיוג</span>
|
||||
<span className="text-ink-muted">({data.labeled} מתויגות)</span>
|
||||
</button>
|
||||
{open && (
|
||||
<div className="px-4 pb-3 overflow-x-auto">
|
||||
<table className="w-full text-sm tabular-nums">
|
||||
<thead>
|
||||
<tr className="text-ink-muted text-[0.72rem] border-b border-rule">
|
||||
<th className="text-start py-1 ps-1">ולידטור</th>
|
||||
<th className="text-start">Precision</th>
|
||||
<th className="text-start">Recall</th>
|
||||
<th className="text-start">F1</th>
|
||||
<th className="text-start">tp/fp/fn/tn</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{rows.map(([name, v]) => (
|
||||
<tr key={name} className="border-b border-rule-soft last:border-0">
|
||||
<td className="py-1 ps-1 text-navy">{name}</td>
|
||||
<td>{v.precision.toFixed(2)}</td>
|
||||
<td>{v.recall.toFixed(2)}</td>
|
||||
<td className="font-semibold">{v.f1.toFixed(2)}</td>
|
||||
<td className="text-ink-muted text-[0.72rem]">
|
||||
{v.tp}/{v.fp}/{v.fn}/{v.tn}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ─── Tag card ─────────────────────────────────────────────────────────────────
|
||||
|
||||
function TagCard({
|
||||
it, focused, onTag,
|
||||
}: {
|
||||
it: GoldsetItem;
|
||||
focused: boolean;
|
||||
onTag: (tag: { is_holding?: boolean; correct_type?: string; quote_complete?: boolean }) => void;
|
||||
}) {
|
||||
const tagged = isTagged(it);
|
||||
return (
|
||||
<div
|
||||
data-goldset-id={it.id}
|
||||
className={`rounded-lg border bg-surface p-4 space-y-3 transition-colors
|
||||
${focused ? "border-gold ring-2 ring-gold/40 shadow-md" : tagged ? "border-rule-soft opacity-70" : "border-rule"}`}
|
||||
>
|
||||
<div className="flex items-center gap-2 text-[0.72rem] text-ink-muted flex-wrap">
|
||||
<span className="font-semibold text-navy">{cleanCitation(it.case_number)}</span>
|
||||
<Badge variant="outline" className="text-[0.65rem]">מכונה: {it.rule_type}</Badge>
|
||||
{it.confidence != null && (
|
||||
<Badge variant="outline" className="text-[0.65rem] tabular-nums">ביטחון {it.confidence.toFixed(2)}</Badge>
|
||||
)}
|
||||
{(it.quality_flags ?? []).map((f) => (
|
||||
<Badge key={f} variant="outline" className="text-[0.65rem] bg-danger-bg text-danger border-danger/40">
|
||||
{FLAG_LABELS[f] ?? f}
|
||||
</Badge>
|
||||
))}
|
||||
{tagged && (
|
||||
<Badge variant="outline" className="text-[0.65rem] bg-gold-wash text-gold-deep border-gold/40 ms-auto">
|
||||
<Check className="w-3 h-3 me-1" /> תויג
|
||||
</Badge>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<p className="text-navy font-medium leading-relaxed" dir="rtl">{it.rule_statement}</p>
|
||||
<blockquote className="text-ink-soft text-sm leading-relaxed border-r-2 border-gold pr-3" dir="rtl">
|
||||
“{it.supporting_quote}”
|
||||
</blockquote>
|
||||
|
||||
<div className="grid gap-3 sm:grid-cols-3 pt-1 border-t border-rule-soft">
|
||||
{/* is_holding */}
|
||||
<div>
|
||||
<div className="text-[0.7rem] text-ink-muted mb-1">האם זו הלכה אמיתית?</div>
|
||||
<div className="flex gap-1">
|
||||
<Button size="sm" variant={it.is_holding === true ? "default" : "ghost"}
|
||||
className={it.is_holding === true ? "bg-gold text-navy hover:bg-gold-deep" : ""}
|
||||
onClick={() => onTag({ is_holding: true })}>הלכה (H)</Button>
|
||||
<Button size="sm" variant={it.is_holding === false ? "default" : "ghost"}
|
||||
className={it.is_holding === false ? "bg-danger text-parchment hover:bg-danger" : "text-danger"}
|
||||
onClick={() => onTag({ is_holding: false })}>לא (N)</Button>
|
||||
</div>
|
||||
</div>
|
||||
{/* correct_type */}
|
||||
<div>
|
||||
<div className="text-[0.7rem] text-ink-muted mb-1">הסוג הנכון</div>
|
||||
<div className="flex gap-1 flex-wrap">
|
||||
{TYPES.map((t) => (
|
||||
<Button key={t.value} size="sm"
|
||||
variant={it.correct_type === t.value ? "default" : "ghost"}
|
||||
className={`text-[0.7rem] px-2 ${it.correct_type === t.value ? "bg-navy text-parchment hover:bg-navy-soft" : ""}`}
|
||||
onClick={() => onTag({ correct_type: t.value })}>{t.label}</Button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
{/* quote_complete */}
|
||||
<div>
|
||||
<div className="text-[0.7rem] text-ink-muted mb-1">הציטוט שלם?</div>
|
||||
<div className="flex gap-1">
|
||||
<Button size="sm" variant={it.quote_complete === true ? "default" : "ghost"}
|
||||
className={it.quote_complete === true ? "bg-gold text-navy hover:bg-gold-deep" : ""}
|
||||
onClick={() => onTag({ quote_complete: true })}>שלם (C)</Button>
|
||||
<Button size="sm" variant={it.quote_complete === false ? "default" : "ghost"}
|
||||
className={it.quote_complete === false ? "bg-danger text-parchment hover:bg-danger" : "text-danger"}
|
||||
onClick={() => onTag({ quote_complete: false })}>קטוע (X)</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ─── Main panel ───────────────────────────────────────────────────────────────
|
||||
|
||||
export function GoldsetPanel() {
|
||||
const batch = "default";
|
||||
const { data, isPending, error } = useGoldset(batch);
|
||||
const tag = useTagGoldset(batch);
|
||||
const createSample = useCreateGoldsetSample(batch);
|
||||
const [focusedId, setFocusedId] = useState<string | null>(null);
|
||||
const [hideTagged, setHideTagged] = useState(false);
|
||||
|
||||
const items = useMemo(() => data?.items ?? [], [data]);
|
||||
const taggedCount = items.filter(isTagged).length;
|
||||
const visible = useMemo(
|
||||
() => (hideTagged ? items.filter((i) => !isTagged(i)) : items),
|
||||
[items, hideTagged],
|
||||
);
|
||||
|
||||
const focused = focusedId ? visible.find((i) => i.id === focusedId) ?? null : null;
|
||||
|
||||
useEffect(() => {
|
||||
if (focusedId && visible.some((i) => i.id === focusedId)) return;
|
||||
setFocusedId(visible[0]?.id ?? null);
|
||||
}, [focusedId, visible]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!focusedId) return;
|
||||
document.querySelector(`[data-goldset-id="${focusedId}"]`)
|
||||
?.scrollIntoView({ block: "nearest", behavior: "smooth" });
|
||||
}, [focusedId]);
|
||||
|
||||
const move = (delta: 1 | -1) => {
|
||||
if (!visible.length) return;
|
||||
const idx = focusedId ? visible.findIndex((i) => i.id === focusedId) : -1;
|
||||
const next = idx < 0 ? (delta > 0 ? 0 : visible.length - 1)
|
||||
: Math.max(0, Math.min(visible.length - 1, idx + delta));
|
||||
setFocusedId(visible[next].id);
|
||||
};
|
||||
|
||||
const doTag = async (
|
||||
it: GoldsetItem,
|
||||
t: { is_holding?: boolean; correct_type?: string; quote_complete?: boolean },
|
||||
) => {
|
||||
try {
|
||||
await tag.mutateAsync({ id: it.id, tag: t });
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : "שגיאה");
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
const onKey = (e: KeyboardEvent) => {
|
||||
const tagName = (e.target as HTMLElement)?.tagName?.toLowerCase();
|
||||
if (tagName === "input" || tagName === "textarea" || tagName === "select") return;
|
||||
if (e.key === "j") { e.preventDefault(); move(1); }
|
||||
else if (e.key === "k") { e.preventDefault(); move(-1); }
|
||||
else if (focused && (e.key === "h" || e.key === "H")) { e.preventDefault(); doTag(focused, { is_holding: true }); }
|
||||
else if (focused && (e.key === "n" || e.key === "N")) { e.preventDefault(); doTag(focused, { is_holding: false }); }
|
||||
else if (focused && (e.key === "c" || e.key === "C")) { e.preventDefault(); doTag(focused, { quote_complete: true }); }
|
||||
else if (focused && (e.key === "x" || e.key === "X")) { e.preventDefault(); doTag(focused, { quote_complete: false }); }
|
||||
};
|
||||
window.addEventListener("keydown", onKey);
|
||||
return () => window.removeEventListener("keydown", onKey);
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [focused, visible]);
|
||||
|
||||
if (error) {
|
||||
return <div className="rounded bg-danger-bg border border-danger/40 px-6 py-5 text-danger text-center">{error.message}</div>;
|
||||
}
|
||||
if (isPending) {
|
||||
return <div className="space-y-3">{[...Array(3)].map((_, i) => <Skeleton key={i} className="h-40 w-full" />)}</div>;
|
||||
}
|
||||
if (!items.length) {
|
||||
return (
|
||||
<div className="text-center text-ink-muted py-16 space-y-4">
|
||||
<p className="text-lg">אין מדגם-זהב עדיין.</p>
|
||||
<Button disabled={createSample.isPending}
|
||||
onClick={() => createSample.mutate(150)}
|
||||
className="bg-gold text-navy hover:bg-gold-deep">
|
||||
צור מדגם של 150 הלכות לתיוג
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const pct = items.length ? Math.round((taggedCount / items.length) * 100) : 0;
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<ScorePanel batch={batch} />
|
||||
|
||||
<div className="flex items-center gap-3 flex-wrap text-sm">
|
||||
<span className="text-navy font-semibold tabular-nums">{taggedCount}/{items.length} תויגו</span>
|
||||
<div className="h-2 w-40 rounded-full bg-rule-soft overflow-hidden">
|
||||
<div className="h-full bg-gold" style={{ width: `${pct}%` }} />
|
||||
</div>
|
||||
<span className="text-ink-muted text-[0.72rem]">
|
||||
מקלדת: <kbd className="bg-rule-soft px-1.5 rounded">J</kbd>/<kbd className="bg-rule-soft px-1.5 rounded">K</kbd>
|
||||
{" "}· הלכה <kbd className="bg-rule-soft px-1.5 rounded">H</kbd> / לא <kbd className="bg-rule-soft px-1.5 rounded">N</kbd>
|
||||
{" "}· ציטוט שלם <kbd className="bg-rule-soft px-1.5 rounded">C</kbd> / קטוע <kbd className="bg-rule-soft px-1.5 rounded">X</kbd>
|
||||
</span>
|
||||
<Button size="sm" variant="ghost" className="ms-auto" onClick={() => setHideTagged((v) => !v)}>
|
||||
{hideTagged ? "הצג הכל" : "הסתר מתויגים"}
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<div className="space-y-3">
|
||||
{visible.map((it) => (
|
||||
<TagCard key={it.id} it={it} focused={it.id === focusedId}
|
||||
onTag={(t) => doTag(it, t)} />
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user