legal-ai/scripts/reconcile_under_extracted_halacha.py

"""Recover halacha precedents that completed extraction but are under-extracted (#144).

Before the claude_session limit-notice fix, a chunk that hit a usage/rate limit
could exit 0 with a refusal NOTICE as its "result" → the extractor read it as an
empty answer and checkpointed the chunk as done-with-0-halachot. A ``completed``
precedent could thus carry substantial reasoning yet ZERO holdings, and a resume
would skip the empty-checkpointed chunks forever.

This finds the strongly-degraded survivors — ``status='completed'``,
``source_kind <> 'cited_only'``, with >= MIN_REASONING_CHUNKS reasoning chunks
(legal_analysis / ruling / conclusion) yet 0 halachot — and re-queues them for a
clean re-extraction: clear their chunk checkpoints, set status 'pending', stamp
``halacha_extraction_requested_at`` so the nightly drain re-runs them.

CONSERVATIVE by design: requires >= 3 reasoning chunks so a genuine remand (a
1-2 chunk ruling that just sends the case back, legitimately holding-free) is
NOT churned into a pointless re-extraction loop. Nothing is lost (these have 0
halachot already). Chair-approved/published rows are untouched (there are none
on a 0-halachot precedent, but the drain's reset preserves them regardless).

Idempotent / re-runnable. Dry-run by default; pass ``--apply`` to write.
Host-only (reads POSTGRES_URL from ~/.env). Run:
    HOME=/home/chaim mcp-server/.venv/bin/python scripts/reconcile_under_extracted_halacha.py [--apply]
"""

from __future__ import annotations

import asyncio
import os
import sys

sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "mcp-server", "src"))

from legal_mcp.services import db

MIN_REASONING_CHUNKS = 3
_REASONING = ("legal_analysis", "ruling", "conclusion")

_CANDIDATES_SQL = """
SELECT cl.id, cl.case_number,
       (SELECT count(*) FROM precedent_chunks pc
          WHERE pc.case_law_id = cl.id
            AND pc.section_type = ANY($1::text[])) AS reasoning_chunks
FROM case_law cl
WHERE cl.halacha_extraction_status = 'completed'
  AND cl.source_kind <> 'cited_only'
  AND (SELECT count(*) FROM halachot h WHERE h.case_law_id = cl.id) = 0
  AND (SELECT count(*) FROM precedent_chunks pc
         WHERE pc.case_law_id = cl.id
           AND pc.section_type = ANY($1::text[])) >= $2
ORDER BY reasoning_chunks DESC
"""


async def main(apply: bool) -> int:
    pool = await db.get_pool()
    rows = await pool.fetch(_CANDIDATES_SQL, list(_REASONING), MIN_REASONING_CHUNKS)

    if not rows:
        print("no under-extracted completed precedents found — nothing to do")
        return 0

    print(f"under-extracted candidates (completed, 0 halachot, >= "
          f"{MIN_REASONING_CHUNKS} reasoning chunks): {len(rows)}")
    for r in rows:
        print(f"  {r['case_number']:<22} reasoning_chunks={r['reasoning_chunks']}  id={r['id']}")

    if not apply:
        print("\n(dry-run — pass --apply to re-queue these for clean re-extraction)")
        return 0

    requeued = 0
    for r in rows:
        cid = r["id"]
        async with pool.acquire() as conn:
            async with conn.transaction():
                # Clear checkpoints so the resume re-extracts every chunk (these
                # carry 0 halachot, so there is nothing to preserve/lose).
                await conn.execute(
                    "UPDATE precedent_chunks SET halacha_extracted_at = NULL "
                    "WHERE case_law_id = $1", cid,
                )
        # Status→pending + stamp requested_at via the canonical request path so
        # the drain picks it up (G2: same enqueue path as everything else).
        await db.request_halacha_extraction(cid)
        requeued += 1

    print(f"\n✓ re-queued {requeued} precedent(s) — the halacha drain will re-extract them")
    return 0


if __name__ == "__main__":
    sys.exit(asyncio.run(main("--apply" in sys.argv)))