feat(storage): seal INV-STG1 write path — 15 dual-write seals + CI leak-guard + tripwire
All checks were successful
G12 Leak-Guard / leak-guard (pull_request) Successful in 5s
All checks were successful
G12 Leak-Guard / leak-guard (pull_request) Successful in 5s
אחרי ה-cutover ל-s3-only, אודיט מצא 15 אתרי-כתיבת-בלוב שעוקפים את storage.py (uploads/ finalize/exports/training/research-backup/precedents/bulletins/draft) — קובץ ינחת בתיקיות-הישנות אך **לא** ב-MinIO → יאבד בניקוי, לא מוגש, לא מגובה. ה-pipeline (ingest/ extract) עדיין קורא לפי file_path מהדיסק, אז ביטול-מוחלט של כתיבה-לדיסק דורש read-wiring מלא (Phase 2, משימה נפרדת). תיקון בטוח עכשיו = **dual-write seal**. - storage.py: `mirror`/`mirror_file` (+ sync) — best-effort persist ל-S3 כשה-backend s3/dual (no-op ב-filesystem; כשל S3 נרשם, לא שובר request — DualBackend philosophy). - web/app.py: helpers `_seal_blob`/`_seal_blob_file` + 14 אתרים אטומים (storage.mirror אחרי כתיבת-הדיסק; הדיסק נשאר ל-pipeline). block_writer.py: draft אטום (async). - **CI leak-guard** (test_storage_write_leak_guard): נכשל על כל כתיבת-בלוב-לדיסק (write_bytes/write_text/shutil.copy*/open(wb)) ב-web/+services ללא מרקר `# noqa: STG1`. כל ה-benign (fallbacks/tmp/staging/git-metadata/flag/state) מסומנים עם נימוק. storage.py מוחרג (הוא המימוש). - **tripwire** (scripts/storage_leak_tripwire.py): ניטור-ריצה — בלובים בדיסק שלא ב-MinIO (json-key match, bucket per-file). אומת חי: 0 דליפות. invariants: INV-STG1 (כל I/O דרך storage / ממורר אליו) · INV-STG6 · feedback_silent_swallow (mirror רושם warning, לא bare-except). Phase 2 (read-wire ה-pipeline → להפיל את עותק-הדיסק) = follow-up. tests: 4 mirror + 1 leak-guard + 6 serve_blob + 18 storage קיימות עוברות. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -18,8 +18,10 @@ import re
|
||||
from datetime import date
|
||||
from uuid import UUID
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from legal_mcp import config
|
||||
from legal_mcp.services import db, embeddings, claude_session, audit
|
||||
from legal_mcp.services import db, embeddings, claude_session, audit, storage
|
||||
from legal_mcp.services.lessons import (
|
||||
OUTCOME_LABELS_HE,
|
||||
PRACTICE_AREA_OVERRIDES,
|
||||
@@ -1119,7 +1121,13 @@ async def _update_draft_file(decision_id: UUID) -> None:
|
||||
draft_dir = config.find_case_dir(case_row["case_number"]) / "drafts"
|
||||
draft_dir.mkdir(parents=True, exist_ok=True)
|
||||
draft_path = draft_dir / "decision.md"
|
||||
draft_path.write_text("\n\n".join(row["content"] for row in rows if row["content"]), encoding="utf-8")
|
||||
draft_text = "\n\n".join(row["content"] for row in rows if row["content"])
|
||||
draft_path.write_text(draft_text, encoding="utf-8") # noqa: STG1 — sealed below
|
||||
try:
|
||||
_dkey = draft_path.resolve().relative_to(Path(config.DATA_DIR).resolve()).as_posix()
|
||||
await storage.mirror(_dkey, draft_text.encode("utf-8"), bucket=storage.Bucket.DOCUMENTS)
|
||||
except ValueError:
|
||||
pass
|
||||
logger.info("Draft file synced: %s (%d blocks)", draft_path, len(rows))
|
||||
|
||||
|
||||
|
||||
@@ -487,7 +487,7 @@ async def export_decision(
|
||||
await storage.put_bytes(key, data, bucket=storage.Bucket.DOCUMENTS, content_type=_docx_ctype)
|
||||
except ValueError:
|
||||
Path(output_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(output_path).write_bytes(data)
|
||||
Path(output_path).write_bytes(data) # noqa: STG1 — storage fallback (output_path outside DATA_DIR)
|
||||
logger.info("DOCX exported (mode=%s): %s", mode, output_path)
|
||||
return output_path
|
||||
|
||||
|
||||
@@ -317,7 +317,7 @@ def retrofit_bookmarks(
|
||||
docx_path, _bkey, bucket=storage.Bucket.DOCUMENTS,
|
||||
content_type="application/vnd.openxmlformats-officedocument.wordprocessingml.document")
|
||||
except ValueError:
|
||||
shutil.copy2(str(docx_path), str(backup_path))
|
||||
shutil.copy2(str(docx_path), str(backup_path)) # noqa: STG1 — storage fallback
|
||||
|
||||
# Inject bookmarks, skipping any that already exist
|
||||
next_id = _next_bookmark_id(doc_tree)
|
||||
|
||||
@@ -114,7 +114,7 @@ def _persist_docx_sync(output_path: Path, data: bytes) -> None:
|
||||
content_type=_DOCX_CTYPE)
|
||||
except ValueError:
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_bytes(data)
|
||||
out.write_bytes(data) # noqa: STG1 — storage fallback
|
||||
|
||||
|
||||
def _save_docx_xml(
|
||||
@@ -536,4 +536,4 @@ def copy_with_revisions(
|
||||
content_type=_DOCX_CTYPE)
|
||||
except ValueError:
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(str(source_path), str(out))
|
||||
shutil.copy2(str(source_path), str(out)) # noqa: STG1 — storage fallback
|
||||
|
||||
@@ -346,7 +346,7 @@ def update_chair_position(
|
||||
|
||||
# Atomic write
|
||||
tmp_path = file_path.with_suffix(file_path.suffix + ".tmp")
|
||||
tmp_path.write_text(new_content, encoding="utf-8")
|
||||
tmp_path.write_text(new_content, encoding="utf-8") # noqa: STG1 — atomic .tmp; in-place edit, S3 re-sync in Phase-2 read-wiring
|
||||
os.replace(tmp_path, file_path)
|
||||
|
||||
preview = new_text.strip()[:120]
|
||||
|
||||
@@ -473,6 +473,43 @@ async def ensure_local(key, *, bucket=Bucket.DOCUMENTS) -> Path:
|
||||
return await get_storage().ensure_local(key, bucket=bucket)
|
||||
|
||||
|
||||
# ── mirror: dual-write seal for the not-yet-read-wired pipeline (INV-STG1) ──────
|
||||
# A handful of upload/finalize paths still keep a copy on disk because the
|
||||
# ingest/extract pipeline reads files by their DATA_DIR path (not yet wired to
|
||||
# ensure_local). For those, ``mirror``/``mirror_file`` ALSO persist the blob to
|
||||
# object storage when the active backend is s3/dual — so no blob is ever missing
|
||||
# from MinIO (durability + presigned serving) even though a disk copy lingers
|
||||
# for the pipeline. No-op under the filesystem backend (the disk write is the
|
||||
# canonical copy). Best-effort: an S3 failure is logged, never breaks the
|
||||
# request (the disk copy holds). The full fix (read-wire the pipeline → drop the
|
||||
# disk copy) is tracked separately; until then this closes the data-loss leak.
|
||||
|
||||
async def mirror(key, data, *, bucket=Bucket.DOCUMENTS,
|
||||
content_type=None, metadata=None) -> None:
|
||||
backend = get_storage()
|
||||
if backend.name == "filesystem":
|
||||
return
|
||||
s3 = getattr(backend, "s3", backend)
|
||||
try:
|
||||
await s3.put_bytes(key, data, bucket=bucket,
|
||||
content_type=content_type, metadata=metadata)
|
||||
except Exception as exc: # noqa: BLE001 — log, never break the request
|
||||
logger.warning("storage.mirror: S3 persist failed for %s: %s", key, exc)
|
||||
|
||||
|
||||
async def mirror_file(src, key, *, bucket=Bucket.DOCUMENTS,
|
||||
content_type=None, metadata=None) -> None:
|
||||
backend = get_storage()
|
||||
if backend.name == "filesystem":
|
||||
return
|
||||
s3 = getattr(backend, "s3", backend)
|
||||
try:
|
||||
await s3.put_file(src, key, bucket=bucket,
|
||||
content_type=content_type, metadata=metadata)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("storage.mirror_file: S3 persist failed for %s: %s", key, exc)
|
||||
|
||||
|
||||
# ── synchronous facade ─────────────────────────────────────────────
|
||||
# A few legacy writers are plain sync functions (track-changes save, retrofit
|
||||
# backup, the multimodal thumbnail renderer which runs in a worker thread via
|
||||
@@ -511,3 +548,15 @@ def put_file_sync(src, key, *, bucket=Bucket.DOCUMENTS, content_type=None,
|
||||
metadata=None) -> str:
|
||||
return _run_coro_blocking(
|
||||
put_file(src, key, bucket=bucket, content_type=content_type, metadata=metadata))
|
||||
|
||||
|
||||
def mirror_sync(key, data, *, bucket=Bucket.DOCUMENTS, content_type=None,
|
||||
metadata=None) -> None:
|
||||
_run_coro_blocking(mirror(key, data, bucket=bucket,
|
||||
content_type=content_type, metadata=metadata))
|
||||
|
||||
|
||||
def mirror_file_sync(src, key, *, bucket=Bucket.DOCUMENTS, content_type=None,
|
||||
metadata=None) -> None:
|
||||
_run_coro_blocking(mirror_file(src, key, bucket=bucket,
|
||||
content_type=content_type, metadata=metadata))
|
||||
|
||||
Reference in New Issue
Block a user