diff --git a/mcp-server/src/legal_mcp/server.py b/mcp-server/src/legal_mcp/server.py index aadfd9c..574566e 100644 --- a/mcp-server/src/legal_mcp/server.py +++ b/mcp-server/src/legal_mcp/server.py @@ -410,6 +410,12 @@ async def search_digests( ) +@mcp.tool() +async def digest_process_pending(limit: int = 20) -> str: + """ריקון תור היומונים שהועלו מה-UI וממתינים לעיבוד-LLM מקומי. מריץ חילוץ-מטא-דאטה + embedding + autolink על כל יומון 'pending' (מקומית עם CLI). חלופת-MCP ל-scripts/ingest_digests_batch.py.""" + return await digest_tools.digest_process_pending(_clamp_limit(limit)) + + @mcp.tool() async def halacha_review( halacha_id: str, diff --git a/mcp-server/src/legal_mcp/services/db.py b/mcp-server/src/legal_mcp/services/db.py index 36e9301..e5d1589 100644 --- a/mcp-server/src/legal_mcp/services/db.py +++ b/mcp-server/src/legal_mcp/services/db.py @@ -3785,6 +3785,18 @@ async def list_digests( return [_row_to_digest(r) for r in rows] +async def list_pending_digests(limit: int = 20) -> list[dict]: + """Digests awaiting local LLM enrichment (web-upload queue, X12). The + drainer (digest_library.process_pending_digests) picks these up.""" + pool = await get_pool() + rows = await pool.fetch( + f"SELECT {_DIGEST_COLS} FROM digests WHERE extraction_status = 'pending' " + f"ORDER BY created_at LIMIT $1", + limit, + ) + return [_row_to_digest(r) for r in rows] + + async def search_digests_semantic( query_embedding: list[float], practice_area: str = "", diff --git a/mcp-server/src/legal_mcp/services/digest_library.py b/mcp-server/src/legal_mcp/services/digest_library.py index dc7488e..6f37799 100644 --- a/mcp-server/src/legal_mcp/services/digest_library.py +++ b/mcp-server/src/legal_mcp/services/digest_library.py @@ -2,18 +2,23 @@ A digest ("כל יום" daily one-pager) is a SECONDARY source that POINTS at a ruling — it is never cited in a decision (INV-DIG1) and never enters the -precedent/halacha pipeline (INV-DIG2). Ingest is therefore a short, standalone -path that reuses only ATOMIC services (extract_text, embeddings), NOT the -canonical ``ingest.ingest_document`` (which is bound to case_law): +precedent/halacha pipeline (INV-DIG2). Ingest reuses only ATOMIC services +(extract_text, embeddings), NOT the canonical ``ingest.ingest_document``. - file → extract_text → content_hash (idempotent) → LLM metadata extract - → create_digest → single embedding (concept+headline+summary+analysis) - → try_autolink(underlying_citation → case_law) [INV-DIG3] - → extraction_status='completed' +Two intake paths share one enrichment core: + + - ``ingest_digest`` (local/MCP, e.g. batch script) — does everything + synchronously: stage → extract_text → create → + LLM enrich → embed → autolink → completed. + - ``create_pending_digest`` (CONTAINER-SAFE — the web upload) — stage → + extract_text → create row with status='pending'. + No LLM, no embedding. ``process_pending_digests`` + (local/MCP) drains the queue and enriches. claude_session rule: ``digest_metadata_extractor`` (local CLI) is imported -LAZILY inside ``ingest_digest`` only, so this module is import-safe from the -FastAPI container for the search/list/link/delete paths (DB + voyage only). +LAZILY inside the enrichment core only, so this module stays import-safe from +the FastAPI container for create_pending / search / list / link / delete +(DB + voyage only — voyage embedding only runs in the local enrich path). """ from __future__ import annotations @@ -42,13 +47,26 @@ async def _noop_progress(_status: str, _percent: int, _msg: str) -> None: return None -def _embedding_text(fields: dict) -> str: +def _coerce_date(v) -> date | None: + if v is None or v == "": + return None + if isinstance(v, date): + return v + if isinstance(v, str): + try: + return date.fromisoformat(v[:10]) + except ValueError: + return None + return None + + +def _embedding_text(row: dict) -> str: """The single vector indexes the digest as an atomic discovery unit.""" parts = [ - fields.get("concept_tag", ""), - fields.get("headline_holding", ""), - fields.get("summary", ""), - fields.get("analysis_text", ""), + row.get("concept_tag", ""), + row.get("headline_holding", ""), + row.get("summary", ""), + row.get("analysis_text", ""), ] return "\n".join(p for p in parts if p).strip() @@ -70,6 +88,161 @@ async def try_autolink(digest_id: UUID | str, underlying_citation: str) -> str | return str(match["id"]) +# ── Container-safe creation (web upload) — no LLM, no embedding ────── + +async def create_pending_digest( + *, + file_path: str | Path, + yomon_number: str = "", + digest_date: date | str | None = None, + practice_area: str = "", + appeal_subtype: str = "", + subject_tags: list[str] | None = None, + progress: ProgressCb | None = None, +) -> dict: + """Stage the file, extract text (PyMuPDF — container-safe), and create a + digest row with extraction_status='pending'. The LLM metadata extraction, + embedding, and autolink are deferred to ``process_pending_digests`` (local). + + Returns {status, digest_id, extraction_status} or {status:'exists', ...}. + Idempotent on content_hash (INV-G3). + """ + progress = progress or _noop_progress + if practice_area and practice_area not in _VALID_PRACTICE_AREAS: + raise ValueError(f"invalid practice_area: {practice_area!r}") + src = Path(file_path) + if not src.exists(): + raise ValueError(f"file not found: {file_path}") + + await progress("staging", 10, "מעתיק קובץ") + staged = ingest._stage_file(src, DIGEST_LIBRARY_DIR, "incoming") + rel_path = str(staged.relative_to(config.DATA_DIR)) \ + if str(staged).startswith(str(config.DATA_DIR)) else str(staged) + + await progress("extracting_text", 50, "מחלץ טקסט") + raw_text, _pc, _off = await extractor.extract_text(str(staged)) + raw_text = (raw_text or "").strip() + if not raw_text: + raise ValueError("no text extracted from digest") + + content_hash = db._content_hash(raw_text) + existing = await db.get_digest_by_content_hash(content_hash) + if existing: + await progress("completed", 100, "יומון זהה כבר קיים") + return {"status": "exists", "digest_id": existing["id"], + "extraction_status": existing.get("extraction_status")} + + record = await db.create_digest( + analysis_text=raw_text, + yomon_number=yomon_number.strip(), + digest_date=_coerce_date(digest_date), + practice_area=practice_area, + appeal_subtype=appeal_subtype.strip(), + subject_tags=list(subject_tags) if subject_tags else [], + source_document_path=rel_path, + extraction_status="pending", + ) + await progress("queued", 100, "ממתין לעיבוד מקומי (LLM)") + return {"status": "pending", "digest_id": record["id"], + "extraction_status": "pending"} + + +# ── Local enrichment core (LLM + embed + autolink) ────────────────── + +async def enrich_digest(digest_id: UUID | str, progress: ProgressCb | None = None) -> dict: + """Run LLM metadata extraction over a digest's analysis_text, fill ONLY + empty fields (preserve user-supplied values), embed, autolink, complete. + + **MCP-tool-only path** (uses the local LLM extractor). Idempotent. + """ + progress = progress or _noop_progress + row = await db.get_digest(digest_id) + if not row: + raise ValueError("digest not found") + analysis = (row.get("analysis_text") or "").strip() + if not analysis: + await db.update_digest(digest_id, extraction_status="failed") + return {"status": "no_text", "digest_id": str(digest_id)} + + await db.update_digest(digest_id, extraction_status="processing") + await progress("extracting_metadata", 40, "מחלץ מטא-דאטה (LLM)") + from legal_mcp.services import digest_metadata_extractor + extracted = await digest_metadata_extractor.extract(analysis) + + # Fill only empty fields (preserve user-supplied values from the form). + fields: dict = {} + for key in ("yomon_number", "concept_tag", "headline_holding", "summary", + "underlying_citation", "underlying_court", "underlying_judge", + "practice_area", "appeal_subtype"): + if not (row.get(key) or "").strip() and extracted.get(key): + fields[key] = extracted[key] + if row.get("digest_date") is None and extracted.get("digest_date"): + fields["digest_date"] = extracted["digest_date"] + if row.get("underlying_date") is None and extracted.get("underlying_date"): + fields["underlying_date"] = extracted["underlying_date"] + if not (row.get("subject_tags") or []) and extracted.get("subject_tags"): + fields["subject_tags"] = extracted["subject_tags"] + + if fields: + await db.update_digest(digest_id, **fields) + merged = await db.get_digest(digest_id) + + await progress("embedding", 75, "מחשב embedding") + emb_text = _embedding_text(merged) + if emb_text: + try: + vecs = await embeddings.embed_texts([emb_text], input_type="document") + if vecs: + await db.store_digest_embedding(digest_id, vecs[0]) + except Exception as e: # surfaced, not swallowed (§6) + logger.warning("digest embedding failed for %s: %s", digest_id, e) + + await progress("linking", 90, "מנסה לקשר לפסק המקורי") + linked_id = None + if not merged.get("linked_case_law_id"): + linked_id = await try_autolink(digest_id, merged.get("underlying_citation", "")) + + await db.update_digest(digest_id, extraction_status="completed") + await progress("completed", 100, "הושלם") + return { + "status": "completed", + "digest_id": str(digest_id), + "yomon_number": merged.get("yomon_number", ""), + "underlying_citation": merged.get("underlying_citation", ""), + "linked_case_law_id": merged.get("linked_case_law_id") or linked_id, + "fields_filled": sorted(fields.keys()), + } + + +async def process_pending_digests(limit: int = 20) -> dict: + """Drain the digest extraction queue (rows stamped extraction_status='pending' + by the web upload). Local/MCP only — runs the LLM enrichment per row. + Sequential (avoids LLM rate-limit storms), mirrors process_pending_extractions.""" + pending = await db.list_pending_digests(limit=limit) + if not pending: + return {"status": "no_pending", "processed": 0, "results": []} + results = [] + processed = 0 + for row in pending: + did = row["id"] + try: + res = await enrich_digest(did) + processed += 1 + results.append({"digest_id": str(did), "status": res.get("status"), + "linked": bool(res.get("linked_case_law_id"))}) + except Exception as e: + logger.exception("process_pending_digests failed for %s: %s", did, e) + try: + await db.update_digest(did, extraction_status="failed") + except Exception: + logger.exception("could not mark digest %s failed", did) + results.append({"digest_id": str(did), "status": "failed", "error": str(e)}) + return {"status": "completed", "processed": processed, + "total_pending": len(pending), "results": results} + + +# ── Full synchronous ingest (local/MCP, e.g. batch script) ────────── + async def ingest_digest( *, file_path: str | Path, @@ -80,109 +253,25 @@ async def ingest_digest( subject_tags: list[str] | None = None, progress: ProgressCb | None = None, ) -> dict: - """Ingest one digest. **MCP-tool-only** (uses the local LLM extractor). + """Ingest one digest synchronously. **MCP-tool-only** (uses the LLM). - User-supplied args win over LLM-extracted values for the same field - (the chair typed them deliberately); empty args are filled from the LLM. - Idempotent on yomon_number / content_hash (INV-G3). + Creates the row (with any user-supplied values) then enriches in place. + Idempotent on content_hash (INV-G3). """ progress = progress or _noop_progress - if practice_area and practice_area not in _VALID_PRACTICE_AREAS: - raise ValueError(f"invalid practice_area: {practice_area!r}") + created = await create_pending_digest( + file_path=file_path, yomon_number=yomon_number, digest_date=digest_date, + practice_area=practice_area, appeal_subtype=appeal_subtype, + subject_tags=subject_tags, progress=progress, + ) + if created.get("status") == "exists": + return created + digest_id = created["digest_id"] + enriched = await enrich_digest(digest_id, progress=progress) + return enriched - src = Path(file_path) - if not src.exists(): - raise ValueError(f"file not found: {file_path}") - - await progress("staging", 5, "מעתיק קובץ") - staged = ingest._stage_file(src, DIGEST_LIBRARY_DIR, "incoming") - rel_path = str(staged.relative_to(config.DATA_DIR)) \ - if str(staged).startswith(str(config.DATA_DIR)) else str(staged) - - await progress("extracting_text", 20, "מחלץ טקסט") - raw_text, _page_count, _offsets = await extractor.extract_text(str(staged)) - raw_text = (raw_text or "").strip() - if not raw_text: - raise ValueError("no text extracted from digest") - - # Idempotency: identical text already ingested → return existing row. - content_hash = db._content_hash(raw_text) - existing = await db.get_digest_by_content_hash(content_hash) - if existing: - await progress("completed", 100, "יומון זהה כבר קיים — לא נוצר כפל") - return { - "status": "exists", - "digest_id": existing["id"], - "yomon_number": existing.get("yomon_number", ""), - "linked_case_law_id": existing.get("linked_case_law_id"), - } - - # LLM metadata extraction (lazy import — keeps this module container-safe). - await progress("extracting_metadata", 45, "מחלץ מטא-דאטה (LLM)") - from legal_mcp.services import digest_metadata_extractor - extracted = await digest_metadata_extractor.extract(raw_text) - - def _coerce_date(v) -> date | None: - if v is None or v == "": - return None - if isinstance(v, date): - return v - if isinstance(v, str): - try: - return date.fromisoformat(v[:10]) - except ValueError: - return None - return None - - # Merge: explicit user args win; otherwise fall back to LLM extraction. - fields = { - "analysis_text": raw_text, - "yomon_number": yomon_number.strip() or extracted.get("yomon_number", ""), - "digest_date": _coerce_date(digest_date) or extracted.get("digest_date"), - "concept_tag": extracted.get("concept_tag", ""), - "headline_holding": extracted.get("headline_holding", ""), - "summary": extracted.get("summary", ""), - "underlying_citation": extracted.get("underlying_citation", ""), - "underlying_court": extracted.get("underlying_court", ""), - "underlying_date": extracted.get("underlying_date"), - "underlying_judge": extracted.get("underlying_judge", ""), - "practice_area": practice_area or extracted.get("practice_area", ""), - "appeal_subtype": appeal_subtype.strip() or extracted.get("appeal_subtype", ""), - "subject_tags": list(subject_tags) if subject_tags else extracted.get("subject_tags", []), - "source_document_path": rel_path, - "extraction_status": "processing", - } - - await progress("storing", 70, "שומר רשומה") - record = await db.create_digest(**fields) - digest_id = record["id"] - - # Single embedding for the whole digest (atomic discovery unit — X12 §6). - await progress("embedding", 85, "מחשב embedding") - emb_text = _embedding_text(fields) - if emb_text: - try: - vecs = await embeddings.embed_texts([emb_text], input_type="document") - if vecs: - await db.store_digest_embedding(digest_id, vecs[0]) - except Exception as e: # surfaced, not swallowed (§6) - logger.warning("digest embedding failed for %s: %s", digest_id, e) - - # Bridge to the underlying ruling if it is already in the library (INV-DIG3). - await progress("linking", 95, "מנסה לקשר לפסק המקורי") - linked_id = await try_autolink(digest_id, fields["underlying_citation"]) - - await db.update_digest(digest_id, extraction_status="completed") - await progress("completed", 100, "הושלם") - return { - "status": "completed", - "digest_id": digest_id, - "yomon_number": fields["yomon_number"], - "underlying_citation": fields["underlying_citation"], - "linked_case_law_id": linked_id, - "fields_extracted": sorted(extracted.keys()), - } +# ── Linking (INV-DIG3) ────────────────────────────────────────────── async def link_digest(digest_id: UUID | str, case_law_id: UUID | str) -> dict: """Manually link a digest to an underlying ruling (INV-DIG3). Idempotent.""" @@ -205,8 +294,7 @@ async def link_digest(digest_id: UUID | str, case_law_id: UUID | str) -> dict: async def relink_digest(digest_id: UUID | str) -> dict: - """Re-run autolink for a digest whose underlying ruling may now be in the - library. No-op if already linked or no match found.""" + """Re-run autolink for an unlinked digest. No-op if already linked / no match.""" digest = await db.get_digest(digest_id) if not digest: raise ValueError("digest not found") @@ -222,6 +310,16 @@ async def relink_digest(digest_id: UUID | str) -> dict: } +async def unlink_digest(digest_id: UUID | str) -> dict: + """Clear a digest's link to the underlying ruling.""" + updated = await db.link_digest_to_case_law(digest_id, None) + if updated is None: + raise ValueError("digest not found") + return {"unlinked": True, "digest_id": str(digest_id)} + + +# ── Read / search (container-safe: DB + voyage) ───────────────────── + async def search_digests( query: str, practice_area: str = "", @@ -255,14 +353,14 @@ async def list_digests( offset: int = 0, ) -> list[dict]: return await db.list_digests( - practice_area=practice_area, - concept_tag=concept_tag, - linked=linked, - search=search, - limit=limit, - offset=offset, + practice_area=practice_area, concept_tag=concept_tag, linked=linked, + search=search, limit=limit, offset=offset, ) +async def update_digest(digest_id: UUID | str, **fields) -> dict | None: + return await db.update_digest(digest_id, **fields) + + async def delete_digest(digest_id: UUID | str) -> bool: return await db.delete_digest(digest_id) diff --git a/mcp-server/src/legal_mcp/tools/digests.py b/mcp-server/src/legal_mcp/tools/digests.py index 0ff0b19..468e2ef 100644 --- a/mcp-server/src/legal_mcp/tools/digests.py +++ b/mcp-server/src/legal_mcp/tools/digests.py @@ -159,3 +159,14 @@ async def search_digests( if not results: return empty("לא נמצאו יומונים תואמים.") return _ok(results) + + +async def digest_process_pending(limit: int = 20) -> str: + """ריקון תור היומונים שהועלו מה-UI וממתינים לעיבוד-LLM מקומי. מריץ חילוץ- + מטא-דאטה + embedding + autolink על כל יומון בסטטוס 'pending', מקומית עם + ה-CLI (claude_session local-only). מנקה לסטטוס 'completed'.""" + try: + result = await digest_library.process_pending_digests(limit=limit) + except Exception as e: + return _err(str(e)) + return _ok(result) diff --git a/web-ui/src/app/digests/page.tsx b/web-ui/src/app/digests/page.tsx new file mode 100644 index 0000000..ee55e43 --- /dev/null +++ b/web-ui/src/app/digests/page.tsx @@ -0,0 +1,82 @@ +"use client"; + +import Link from "next/link"; +import { AppShell } from "@/components/app-shell"; +import { Card, CardContent } from "@/components/ui/card"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { Badge } from "@/components/ui/badge"; +import { DigestListPanel } from "@/components/digests/digest-list-panel"; +import { DigestSearchPanel } from "@/components/digests/digest-search-panel"; +import { useDigestPending } from "@/lib/api/digests"; + +/** + * Digests radar page (X12) — a SECONDARY discovery layer ABOVE the citation + * corpora. Deliberately a SEPARATE page from /precedents to keep the + * authoritative/secondary boundary visible: a digest POINTS at a ruling, it is + * never cited in a decision (INV-DIG1) and never extracts halachot (INV-DIG2). + * + * Two tabs: + * - יומונים — browse + upload + link to the underlying ruling + * - חיפוש — semantic radar search + */ + +function PendingBadge() { + const { data } = useDigestPending(); + const n = data?.count ?? 0; + if (!n) return null; + return ( + + {n} ממתינים + + ); +} + +export default function DigestsPage() { + return ( + +
+
+ +

יומונים — רדאר פסיקה

+

+ סיכומי "כל יום" (עפר טויסטר) של פסקי דין והחלטות עדכניים. + שכבת-גילוי בלבד: כל יומון מצביע על פסק הדין המקורי — + הוא אינו מצוטט בהחלטה ואינו מחלץ הלכות. כשהפסק רלוונטי, מעלים אותו + לספריית הפסיקה ומצטטים משם. +

+
+ +
+ + + + + + + יומונים + + + חיפוש + + + + + + + + + + + + +
+ + ); +} diff --git a/web-ui/src/components/app-shell.tsx b/web-ui/src/components/app-shell.tsx index 872ff7d..4eeb385 100644 --- a/web-ui/src/components/app-shell.tsx +++ b/web-ui/src/components/app-shell.tsx @@ -50,6 +50,7 @@ const NAV_GROUPS: NavGroup[] = [ id: "knowledge", items: [ { href: "/precedents", label: "ספריית פסיקה" }, + { href: "/digests", label: "יומונים" }, { href: "/missing-precedents", label: "פסיקה חסרה" }, { href: "/goldset", label: "מדגם-זהב" }, { href: "/training", label: "אימון סגנון" }, diff --git a/web-ui/src/components/digests/digest-card.tsx b/web-ui/src/components/digests/digest-card.tsx new file mode 100644 index 0000000..f10de94 --- /dev/null +++ b/web-ui/src/components/digests/digest-card.tsx @@ -0,0 +1,103 @@ +"use client"; + +import Link from "next/link"; +import type { ReactNode } from "react"; +import { Badge } from "@/components/ui/badge"; +import { practiceAreaLabel } from "@/components/precedents/practice-area"; +import type { Digest } from "@/lib/api/digests"; + +function formatDate(iso: string | null) { + if (!iso) return "—"; + try { + return new Date(iso).toLocaleDateString("he-IL"); + } catch { + return iso; + } +} + +/** + * Presentational card for a single digest ("כל יום" radar entry). + * + * A digest is a SECONDARY source that POINTS at a ruling — the card makes the + * pointer-not-citation nature explicit: the underlying ruling's citation is + * shown with a link-status badge (linked → the ruling is in the precedent + * library; unlinked → an open knowledge gap, INV-DIG3). + */ +export function DigestCard({ + digest, + score, + actions, +}: { + digest: Digest; + score?: number; + actions?: ReactNode; +}) { + const linked = Boolean(digest.linked_case_law_id); + return ( +
+
+ {digest.concept_tag && ( + {digest.concept_tag} + )} + {digest.yomon_number && ( + + יומון {digest.yomon_number} + + )} + {digest.digest_date && · {formatDate(digest.digest_date)}} + {digest.practice_area && ( + · {practiceAreaLabel(digest.practice_area)} + )} + {digest.extraction_status !== "completed" && ( + + {digest.extraction_status === "pending" ? "ממתין לעיבוד" : digest.extraction_status} + + )} + {typeof score === "number" && ( + דירוג {score.toFixed(2)} + )} +
+ + {digest.headline_holding && ( +

+ {digest.headline_holding} +

+ )} + {digest.summary && ( +

+ {digest.summary} +

+ )} + +
+ פסק מקורי: + + {digest.underlying_citation || "—"} + + {linked ? ( + + + מקושר לפסק ↗ + + + ) : ( + + הפסק טרם בקורפוס + + )} +
+ + {digest.subject_tags?.length > 0 && ( +
+ {digest.subject_tags.map((t) => ( + + {t} + + ))} +
+ )} + + {actions &&
{actions}
} +
+ ); +} diff --git a/web-ui/src/components/digests/digest-list-panel.tsx b/web-ui/src/components/digests/digest-list-panel.tsx new file mode 100644 index 0000000..af32d2f --- /dev/null +++ b/web-ui/src/components/digests/digest-list-panel.tsx @@ -0,0 +1,136 @@ +"use client"; + +import { useState } from "react"; +import { Link2, Trash2 } from "lucide-react"; +import { toast } from "sonner"; +import { Button } from "@/components/ui/button"; +import { Skeleton } from "@/components/ui/skeleton"; +import { + Select, SelectContent, SelectItem, SelectTrigger, SelectValue, +} from "@/components/ui/select"; +import { + useDigests, useDeleteDigest, useRelinkDigest, type DigestListFilters, +} from "@/lib/api/digests"; +import type { PracticeArea } from "@/lib/api/precedent-library"; +import { PRACTICE_AREAS } from "@/components/precedents/practice-area"; +import { DigestCard } from "./digest-card"; +import { DigestUploadDialog } from "./digest-upload-dialog"; + +type LinkedFilter = "all" | "linked" | "unlinked"; + +export function DigestListPanel() { + const [practiceArea, setPracticeArea] = useState(""); + const [linked, setLinked] = useState("all"); + + const filters: DigestListFilters = { + practiceArea: practiceArea || undefined, + linked: linked === "all" ? undefined : linked === "linked", + limit: 200, + }; + const { data, isLoading, error } = useDigests(filters); + const del = useDeleteDigest(); + const relink = useRelinkDigest(); + + const onRelink = (id: string) => + relink.mutate(id, { + onSuccess: (r) => + r.linked + ? toast.success("קושר לפסק המקורי") + : toast.info("הפסק המקורי עדיין לא בקורפוס — העלה אותו לספריית הפסיקה"), + onError: (e) => toast.error(e.message), + }); + + const onDelete = (id: string) => { + if (!confirm("למחוק את היומון מקורפוס-הגילוי?")) return; + del.mutate(id, { + onSuccess: () => toast.success("נמחק"), + onError: (e) => toast.error(e.message), + }); + }; + + return ( +
+
+
+ + +
+
+ + +
+
+ +
+
+ + {error ? ( +
+ {error.message} +
+ ) : isLoading ? ( +
+ {[...Array(3)].map((_, i) => )} +
+ ) : !data?.items.length ? ( +
+ אין יומונים עדיין. העלה יומון "כל יום", או הרץ + scripts/ingest_digests_batch.py. +
+ ) : ( +
+

{data.count} יומונים

+ {data.items.map((d) => ( + + {!d.linked_case_law_id && ( + + )} + + + } + /> + ))} +
+ )} +
+ ); +} diff --git a/web-ui/src/components/digests/digest-search-panel.tsx b/web-ui/src/components/digests/digest-search-panel.tsx new file mode 100644 index 0000000..80057f8 --- /dev/null +++ b/web-ui/src/components/digests/digest-search-panel.tsx @@ -0,0 +1,95 @@ +"use client"; + +import { useState } from "react"; +import { Search } from "lucide-react"; +import { Input } from "@/components/ui/input"; +import { Button } from "@/components/ui/button"; +import { Skeleton } from "@/components/ui/skeleton"; +import { + Select, SelectContent, SelectItem, SelectTrigger, SelectValue, +} from "@/components/ui/select"; +import { useDigestSearch } from "@/lib/api/digests"; +import type { PracticeArea } from "@/lib/api/precedent-library"; +import { PRACTICE_AREAS } from "@/components/precedents/practice-area"; +import { DigestCard } from "./digest-card"; + +export function DigestSearchPanel() { + const [draft, setDraft] = useState(""); + const [query, setQuery] = useState(""); + const [practiceArea, setPracticeArea] = useState(""); + + const { data, isFetching, error } = useDigestSearch(query, { + practiceArea: practiceArea || undefined, + limit: 20, + }); + + const onSubmit = (e: React.FormEvent) => { + e.preventDefault(); + setQuery(draft.trim()); + }; + + return ( +
+
+
+ + setDraft(e.target.value)} + placeholder="שיקול דעת הוועדה המחוזית בהקלה" + dir="rtl" + /> +
+
+ + +
+ +
+ +

+ חיפוש סמנטי ב-radar היומונים. כל תוצאה מצביעה על פסק דין מקורי — + הצטט מהפסק עצמו (ספריית הפסיקה), לא מהיומון. +

+ + {!query.trim() ? ( +
+ הקלד שאילתא כדי לחפש ביומונים. החיפוש סמנטי — לא טקסטואלי. +
+ ) : error ? ( +
+ {error.message} +
+ ) : isFetching ? ( +
+ {[...Array(3)].map((_, i) => )} +
+ ) : !data?.items.length ? ( +
+ לא נמצאו יומונים תואמים. נסה ניסוח אחר. +
+ ) : ( +
+

{data.count} תוצאות

+ {data.items.map((hit) => ( + + ))} +
+ )} +
+ ); +} diff --git a/web-ui/src/components/digests/digest-upload-dialog.tsx b/web-ui/src/components/digests/digest-upload-dialog.tsx new file mode 100644 index 0000000..f82387d --- /dev/null +++ b/web-ui/src/components/digests/digest-upload-dialog.tsx @@ -0,0 +1,152 @@ +"use client"; + +import { useState } from "react"; +import { Upload } from "lucide-react"; +import { toast } from "sonner"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { + Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, + DialogTitle, DialogTrigger, +} from "@/components/ui/dialog"; +import { + Select, SelectContent, SelectItem, SelectTrigger, SelectValue, +} from "@/components/ui/select"; +import { useUploadDigest, type DigestUploadInput } from "@/lib/api/digests"; +import type { PracticeArea } from "@/lib/api/precedent-library"; +import { PRACTICE_AREAS } from "@/components/precedents/practice-area"; + +/** + * Upload a "כל יום" digest PDF. The endpoint is container-safe: it only + * stages + extracts text, creating a row with status='pending'. The LLM + * enrichment (concept/headline/citation + embedding + autolink) runs locally + * via the MCP drainer ``digest_process_pending`` — so the toast tells the + * user the digest is queued, not yet searchable. + */ +export function DigestUploadDialog() { + const [open, setOpen] = useState(false); + const [file, setFile] = useState(null); + const [yomonNumber, setYomonNumber] = useState(""); + const [digestDate, setDigestDate] = useState(""); + const [practiceArea, setPracticeArea] = useState(""); + const upload = useUploadDigest(); + + const reset = () => { + setFile(null); + setYomonNumber(""); + setDigestDate(""); + setPracticeArea(""); + }; + + const onSubmit = (e: React.FormEvent) => { + e.preventDefault(); + if (!file) { + toast.error("בחר קובץ יומון (PDF)"); + return; + } + const input: DigestUploadInput = { file }; + if (yomonNumber.trim()) input.yomon_number = yomonNumber.trim(); + if (digestDate) input.digest_date = digestDate; + if (practiceArea) input.practice_area = practiceArea; + upload.mutate(input, { + onSuccess: (res) => { + if (res.status === "exists") { + toast.info("יומון זהה כבר קיים — לא נוצר כפל"); + } else { + toast.success( + "היומון נקלט וממתין לעיבוד מקומי (LLM). הרץ digest_process_pending " + + "או scripts/ingest_digests_batch.py כדי להשלים חילוץ + חיפוש.", + ); + } + reset(); + setOpen(false); + }, + onError: (err) => { + toast.error(`שגיאה בהעלאה: ${err.message}`); + }, + }); + }; + + return ( + + + + + + + העלאת יומון "כל יום" + + סיכום-עמוד של פסק דין. המערכת תחלץ אוטומטית את תג-המושג, ההלכה, ומראה-המקום + של הפסק המקורי. היומון משמש כ-radar בלבד — אינו מצוטט בהחלטה. + + + +
+
+ + setFile(e.target.files?.[0] ?? null)} + /> +
+
+
+ + setYomonNumber(e.target.value)} + placeholder="5163" + dir="ltr" + /> +
+
+ + setDigestDate(e.target.value)} + /> +
+
+
+ + +
+ + + + +
+ + + ); +} diff --git a/web-ui/src/components/header-context.ts b/web-ui/src/components/header-context.ts index a7f09d8..e74a41d 100644 --- a/web-ui/src/components/header-context.ts +++ b/web-ui/src/components/header-context.ts @@ -21,6 +21,7 @@ export function headerSubtitle(pathname: string): string { if (pathname.startsWith("/archive")) return "ארכיון"; if (pathname.startsWith("/training")) return "אימון סגנון"; if (pathname.startsWith("/precedents")) return "ספריית פסיקה"; + if (pathname.startsWith("/digests")) return "יומונים"; if (pathname.startsWith("/methodology")) return "מתודולוגיה"; if (pathname.startsWith("/skills")) return "מיומנויות"; if (pathname.startsWith("/diagnostics")) return "אבחון"; diff --git a/web-ui/src/lib/api/digests.ts b/web-ui/src/lib/api/digests.ts new file mode 100644 index 0000000..dcb2e89 --- /dev/null +++ b/web-ui/src/lib/api/digests.ts @@ -0,0 +1,284 @@ +/** + * Digests radar hooks (X12). + * + * A digest ("כל יום" daily one-pager, Ofer Toister) is a SECONDARY, discovery- + * layer source that POINTS at a ruling — never cited in a decision (INV-DIG1), + * never extracts halachot (INV-DIG2). Distinct from: + * - /api/precedent-library (authoritative case-law corpus, citable) + * - /api/training (Daphna's style corpus) + * + * Endpoints (all under /api/digests): + * - POST /upload (multipart) → { status, digest_id } (pending → local enrich) + * - GET / (filters) → list + * - GET /search → semantic radar search + * - GET /queue/pending → digests awaiting local LLM enrichment + * - GET /{id} → detail + * - PATCH /{id} → metadata edit + * - DELETE /{id} → remove + * - POST /{id}/link {case_law_id} → bridge to the underlying ruling + * - POST /{id}/relink → re-run autolink + * - DELETE /{id}/link → clear link + */ + +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { ApiError, apiRequest } from "./client"; +import type { PracticeArea } from "./precedent-library"; + +export type Digest = { + id: string; + yomon_number: string; + digest_date: string | null; + publication: string; + source_firm: string; + concept_tag: string; + headline_holding: string; + analysis_text: string; + summary: string; + underlying_citation: string; + underlying_court: string; + underlying_date: string | null; + underlying_judge: string; + practice_area: PracticeArea | ""; + appeal_subtype: string; + subject_tags: string[]; + linked_case_law_id: string | null; + source_document_path: string; + content_hash: string; + extraction_status: string; + created_at: string; + updated_at: string; +}; + +/** A search hit = a Digest plus the joined linked-ruling fields + score. */ +export type DigestSearchHit = Digest & { + linked_case_number: string | null; + linked_case_name: string | null; + linked_searchable: boolean | null; + score: number; + type: "digest"; +}; + +export type DigestListFilters = { + practiceArea?: PracticeArea; + conceptTag?: string; + /** undefined = all; true = linked to a ruling; false = unlinked (open gap) */ + linked?: boolean; + search?: string; + limit?: number; + offset?: number; +}; + +export const digestKeys = { + all: ["digests"] as const, + list: (filters: DigestListFilters) => + [...digestKeys.all, "list", filters] as const, + detail: (id: string) => [...digestKeys.all, "detail", id] as const, + search: (q: string, filters: Record) => + [...digestKeys.all, "search", q, filters] as const, + pending: () => [...digestKeys.all, "pending"] as const, +}; + +/** A digest is "active" while it awaits or is mid local LLM enrichment. */ +export function isDigestActive(d: Digest): boolean { + return d.extraction_status === "pending" || d.extraction_status === "processing"; +} + +export function useDigests(filters: DigestListFilters = {}) { + return useQuery({ + queryKey: digestKeys.list(filters), + queryFn: ({ signal }) => { + const p = new URLSearchParams(); + if (filters.practiceArea) p.set("practice_area", filters.practiceArea); + if (filters.conceptTag) p.set("concept_tag", filters.conceptTag); + if (filters.linked !== undefined) p.set("linked", String(filters.linked)); + if (filters.search) p.set("search", filters.search); + if (filters.limit) p.set("limit", String(filters.limit)); + if (filters.offset) p.set("offset", String(filters.offset)); + const qs = p.toString(); + return apiRequest<{ items: Digest[]; count: number }>( + `/api/digests${qs ? `?${qs}` : ""}`, + { signal }, + ); + }, + staleTime: 30_000, + // Poll while any row is awaiting/mid local enrichment; stop once settled. + refetchInterval: (query) => { + const data = query.state.data; + if (!data) return false; + return data.items.some((d) => isDigestActive(d)) ? 5000 : false; + }, + }); +} + +export function useDigest(id: string | null) { + return useQuery({ + queryKey: digestKeys.detail(id ?? ""), + queryFn: ({ signal }) => + apiRequest(`/api/digests/${encodeURIComponent(id!)}`, { signal }), + enabled: Boolean(id), + staleTime: 30_000, + }); +} + +export type DigestSearchFilters = { + practiceArea?: PracticeArea; + subjectTag?: string; + conceptTag?: string; + limit?: number; +}; + +export function useDigestSearch(query: string, filters: DigestSearchFilters = {}) { + const params: Record = {}; + if (filters.practiceArea) params.practice_area = filters.practiceArea; + if (filters.subjectTag) params.subject_tag = filters.subjectTag; + if (filters.conceptTag) params.concept_tag = filters.conceptTag; + + return useQuery({ + queryKey: digestKeys.search(query, params), + queryFn: ({ signal }) => { + const p = new URLSearchParams({ q: query }); + for (const [k, v] of Object.entries(params)) p.set(k, v); + if (filters.limit) p.set("limit", String(filters.limit)); + return apiRequest<{ items: DigestSearchHit[]; count: number }>( + `/api/digests/search?${p.toString()}`, + { signal }, + ); + }, + enabled: query.trim().length >= 2, + staleTime: 10_000, + placeholderData: (prev) => prev, + }); +} + +/** Digests awaiting local LLM enrichment (drained by `digest_process_pending`). */ +export function useDigestPending() { + return useQuery({ + queryKey: digestKeys.pending(), + queryFn: ({ signal }) => + apiRequest<{ items: Digest[]; count: number }>( + "/api/digests/queue/pending", + { signal }, + ), + staleTime: 15_000, + }); +} + +export type DigestUploadInput = { + file: File; + yomon_number?: string; + digest_date?: string; + practice_area?: PracticeArea; + appeal_subtype?: string; + subject_tags?: string[]; +}; + +export function useUploadDigest() { + const qc = useQueryClient(); + return useMutation({ + mutationFn: async (input: DigestUploadInput) => { + const fd = new FormData(); + fd.append("file", input.file); + if (input.yomon_number) fd.append("yomon_number", input.yomon_number); + if (input.digest_date) fd.append("digest_date", input.digest_date); + if (input.practice_area) fd.append("practice_area", input.practice_area); + if (input.appeal_subtype) fd.append("appeal_subtype", input.appeal_subtype); + if (input.subject_tags && input.subject_tags.length) + fd.append("subject_tags", JSON.stringify(input.subject_tags)); + + const res = await fetch("/api/digests/upload", { method: "POST", body: fd }); + const parsed = await res.json().catch(() => null); + if (!res.ok) { + throw new ApiError(`Upload failed with ${res.status}`, res.status, parsed); + } + return parsed as { status: string; digest_id: string; extraction_status: string }; + }, + onSuccess: () => { + qc.invalidateQueries({ queryKey: digestKeys.all }); + }, + }); +} + +export type DigestPatch = Partial<{ + yomon_number: string; + digest_date: string; + concept_tag: string; + headline_holding: string; + summary: string; + underlying_citation: string; + underlying_court: string; + underlying_date: string; + underlying_judge: string; + practice_area: PracticeArea; + appeal_subtype: string; + subject_tags: string[]; +}>; + +export function useUpdateDigest() { + const qc = useQueryClient(); + return useMutation({ + mutationFn: ({ id, patch }: { id: string; patch: DigestPatch }) => + apiRequest(`/api/digests/${encodeURIComponent(id)}`, { + method: "PATCH", + body: patch, + }), + onSuccess: () => { + qc.invalidateQueries({ queryKey: digestKeys.all }); + }, + }); +} + +export function useDeleteDigest() { + const qc = useQueryClient(); + return useMutation({ + mutationFn: (id: string) => + apiRequest<{ deleted: boolean }>( + `/api/digests/${encodeURIComponent(id)}`, + { method: "DELETE" }, + ), + onSuccess: () => { + qc.invalidateQueries({ queryKey: digestKeys.all }); + }, + }); +} + +export function useLinkDigest() { + const qc = useQueryClient(); + return useMutation({ + mutationFn: ({ id, caseLawId }: { id: string; caseLawId: string }) => + apiRequest<{ linked: boolean; case_number: string }>( + `/api/digests/${encodeURIComponent(id)}/link`, + { method: "POST", body: { case_law_id: caseLawId } }, + ), + onSuccess: () => { + qc.invalidateQueries({ queryKey: digestKeys.all }); + }, + }); +} + +export function useRelinkDigest() { + const qc = useQueryClient(); + return useMutation({ + mutationFn: (id: string) => + apiRequest<{ linked: boolean; case_law_id: string | null; changed: boolean }>( + `/api/digests/${encodeURIComponent(id)}/relink`, + { method: "POST" }, + ), + onSuccess: () => { + qc.invalidateQueries({ queryKey: digestKeys.all }); + }, + }); +} + +export function useUnlinkDigest() { + const qc = useQueryClient(); + return useMutation({ + mutationFn: (id: string) => + apiRequest<{ unlinked: boolean }>( + `/api/digests/${encodeURIComponent(id)}/link`, + { method: "DELETE" }, + ), + onSuccess: () => { + qc.invalidateQueries({ queryKey: digestKeys.all }); + }, + }); +} diff --git a/web/app.py b/web/app.py index 50b1d3b..ced4781 100644 --- a/web/app.py +++ b/web/app.py @@ -5827,6 +5827,210 @@ async def precedent_queue_pending(kind: str = "metadata", limit: int = 20): return {"items": items, "count": len(items)} +# ── Digests radar (X12) — secondary discovery layer ───────────────── +# A digest is a SECONDARY source that POINTS at a ruling (INV-DIG1/2): never +# cited, never extracts halachot. These endpoints are DB/voyage-only and run +# in the container; the LLM enrichment of an uploaded digest is deferred to the +# local MCP drainer (digest_process_pending) — see the upload endpoint. +from legal_mcp.services import digest_library as digest_service # noqa: E402 + + +class DigestUpdateRequest(BaseModel): + yomon_number: str | None = None + digest_date: str | None = None + concept_tag: str | None = None + headline_holding: str | None = None + summary: str | None = None + underlying_citation: str | None = None + underlying_court: str | None = None + underlying_date: str | None = None + underlying_judge: str | None = None + practice_area: str | None = None + appeal_subtype: str | None = None + subject_tags: list[str] | None = None + + +class DigestLinkRequest(BaseModel): + case_law_id: str + + +@app.post("/api/digests/upload") +async def digest_upload( + file: UploadFile = File(...), + yomon_number: str = Form(""), + digest_date: str = Form(""), + practice_area: str = Form(""), + appeal_subtype: str = Form(""), + subject_tags: str = Form("[]"), +): + """Upload a "כל יום" digest. Container-safe: stages the file and extracts + text, creating a row with extraction_status='pending'. LLM enrichment + (concept/headline/citation + embedding + autolink) is deferred to the local + MCP drainer ``digest_process_pending`` (claude CLI not in container).""" + if practice_area and practice_area not in _PRACTICE_AREAS: + raise HTTPException(400, "practice_area לא תקין") + suffix = Path(file.filename or "").suffix.lower() + if suffix not in ALLOWED_EXTENSIONS: + raise HTTPException(400, f"סוג קובץ לא נתמך: {suffix}") + try: + tags = json.loads(subject_tags) if subject_tags else [] + if not isinstance(tags, list): + tags = [] + except json.JSONDecodeError: + tags = [] + + UPLOAD_DIR.mkdir(parents=True, exist_ok=True) + staged = UPLOAD_DIR / f"digest_{uuid4().hex[:8]}_{file.filename}" + size = 0 + with staged.open("wb") as out: + while chunk := await file.read(1024 * 1024): + size += len(chunk) + if size > MAX_FILE_SIZE: + staged.unlink(missing_ok=True) + raise HTTPException(413, "קובץ גדול מדי") + out.write(chunk) + try: + result = await digest_service.create_pending_digest( + file_path=staged, + yomon_number=yomon_number.strip(), + digest_date=digest_date or None, + practice_area=practice_area, + appeal_subtype=appeal_subtype.strip(), + subject_tags=tags, + ) + except ValueError as e: + raise HTTPException(400, str(e)) + finally: + staged.unlink(missing_ok=True) + return result + + +@app.get("/api/digests") +async def digest_list( + practice_area: str = "", + concept_tag: str = "", + linked: bool | None = None, + search: str = "", + limit: int = 100, + offset: int = 0, +): + rows = await digest_service.list_digests( + practice_area=practice_area, concept_tag=concept_tag, linked=linked, + search=search, limit=limit, offset=offset, + ) + return {"items": rows, "count": len(rows)} + + +@app.get("/api/digests/search") +async def digest_search( + q: str, + practice_area: str = "", + subject_tag: str = "", + concept_tag: str = "", + limit: int = 10, +): + if not q or len(q.strip()) < 2: + return {"items": [], "count": 0} + results = await digest_service.search_digests( + query=q.strip(), practice_area=practice_area, + subject_tag=subject_tag, concept_tag=concept_tag, limit=limit, + ) + return {"items": results, "count": len(results)} + + +@app.get("/api/digests/queue/pending") +async def digest_queue_pending(limit: int = 20): + """Digests awaiting local LLM enrichment (UI badge 'N ממתינים לעיבוד').""" + items = await db.list_pending_digests(limit=limit) + return {"items": items, "count": len(items)} + + +@app.get("/api/digests/{digest_id}") +async def digest_get(digest_id: str): + try: + cid = UUID(digest_id) + except ValueError: + raise HTTPException(400, "digest_id לא תקין") + record = await digest_service.get_digest(cid) + if not record: + raise HTTPException(404, "יומון לא נמצא") + return record + + +@app.patch("/api/digests/{digest_id}") +async def digest_update(digest_id: str, req: DigestUpdateRequest): + try: + cid = UUID(digest_id) + except ValueError: + raise HTTPException(400, "digest_id לא תקין") + fields = {k: v for k, v in req.model_dump(exclude_unset=True).items() if v is not None} + if "practice_area" in fields and fields["practice_area"] not in _PRACTICE_AREAS: + raise HTTPException(400, "practice_area לא תקין") + for dk in ("digest_date", "underlying_date"): + if dk in fields and fields[dk]: + try: + from datetime import date as date_type + fields[dk] = date_type.fromisoformat(str(fields[dk])[:10]) + except ValueError: + raise HTTPException(400, f"{dk} לא תקין") + record = await digest_service.update_digest(cid, **fields) + if not record: + raise HTTPException(404, "יומון לא נמצא") + return record + + +@app.delete("/api/digests/{digest_id}") +async def digest_delete(digest_id: str): + try: + cid = UUID(digest_id) + except ValueError: + raise HTTPException(400, "digest_id לא תקין") + ok = await digest_service.delete_digest(cid) + if not ok: + raise HTTPException(404, "יומון לא נמצא") + return {"deleted": True, "digest_id": digest_id} + + +@app.post("/api/digests/{digest_id}/link") +async def digest_link(digest_id: str, req: DigestLinkRequest): + """Link a digest to its underlying ruling in the precedent library (INV-DIG3).""" + try: + UUID(digest_id) + UUID(req.case_law_id) + except ValueError: + raise HTTPException(400, "מזהה לא תקין") + try: + return await digest_service.link_digest(digest_id, req.case_law_id) + except ValueError as e: + raise HTTPException(404, str(e)) + + +@app.post("/api/digests/{digest_id}/relink") +async def digest_relink(digest_id: str): + """Re-run autolink: link to the underlying ruling if it is now in the library.""" + try: + UUID(digest_id) + except ValueError: + raise HTTPException(400, "digest_id לא תקין") + try: + return await digest_service.relink_digest(digest_id) + except ValueError as e: + raise HTTPException(404, str(e)) + + +@app.delete("/api/digests/{digest_id}/link") +async def digest_unlink(digest_id: str): + """Clear a digest's link to the underlying ruling.""" + try: + UUID(digest_id) + except ValueError: + raise HTTPException(400, "digest_id לא תקין") + try: + return await digest_service.unlink_digest(digest_id) + except ValueError as e: + raise HTTPException(404, str(e)) + + from legal_mcp.services import internal_decisions as int_decisions_service # noqa: E402