From 10e05700cc82670a0f7d1ddc3b75de9da359ece5 Mon Sep 17 00:00:00 2001
From: Chaim <chaim@marcus-law.co.il>
Date: Tue, 28 Jul 2026 07:47:05 +0000
Subject: [PATCH] =?UTF-8?q?feat(eval):=20--instructions=20=D7=9C-harness?=
 =?UTF-8?q?=20=E2=80=94=20A/B=20=D7=A9=D7=9C=20=D7=95=D7=A8=D7=99=D7=90?=
 =?UTF-8?q?=D7=A0=D7=98-=D7=A4=D7=A8=D7=95=D7=9E=D7=A4=D7=98?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

מוחל על כל המודלים בריצה (אחרת השוואת-מודלים הופכת בשקט להשוואת-פרומפטים),
ונרשם ב-grid_summary + בכותרת הדוח כדי שריצת-וריאנט לא תושווה בטעות לבסיס.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 scripts/calibrate_effort.py | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/scripts/calibrate_effort.py b/scripts/calibrate_effort.py
index b7bf422..4b2b430 100644
--- a/scripts/calibrate_effort.py
+++ b/scripts/calibrate_effort.py
@@ -421,11 +421,16 @@ async def _finals_for_calibration(case_filter: str | None) -> list[dict]:
 
 async def _score_cell(case_id, block_id: str, effort: str, final_section: str,
                       final_total_words: int, outcome: str, repeats: int,
-                      model: str | None = None) -> dict:
+                      model: str | None = None, instructions: str = "") -> dict:
     """Generate `block_id` at `effort` `repeats` times; score each vs the final section.
 
     `model` (optional) A/Bs the generation model via write_block(model_override=…).
     None ⇒ the pinned GENERATION_MODEL, i.e. the production path unchanged.
+
+    `instructions` (optional) is appended to the block prompt for EVERY cell in
+    the run — a prompt-variant A/B (e.g. an explicit formatting rule). It is
+    applied to all models so the comparison stays a model comparison rather
+    than silently becoming a prompt comparison.
     """
     from legal_mcp.services import block_writer
     from legal_mcp.services.style_distance import block_distance_to_final
@@ -433,7 +438,8 @@ async def _score_cell(case_id, block_id: str, effort: str, final_section: str,
     models_used: list[str] = []
     for _ in range(repeats):
         res = await block_writer.write_block(
-            case_id, block_id, effort_override=effort, model_override=model,
+            case_id, block_id, instructions=instructions,
+            effort_override=effort, model_override=model,
         )
         # Record what the CLI was actually asked to run, so a silent fallback to
         # a different build is visible in the report rather than mis-attributed.
@@ -488,6 +494,9 @@ async def _run(args, ts: str) -> dict:
         "finals": [c["case_number"] for c in cases_meta],
         "blocks": blocks, "efforts": efforts, "repeats": args.repeats,
         "models": models,
+        # Provenance: a prompt-variant run is NOT comparable to a baseline run,
+        # so the instruction text is recorded in the report, not just the shell.
+        "instructions": getattr(args, "instructions", "") or "",
         "total_generations": total_cells,
         "per_block_n": {b: len(plan[b]) for b in blocks},
     }
@@ -533,7 +542,7 @@ async def _run_blocks_for_model(model, blocks, efforts, plan, args, ts, grid_sum
                     cell = await _score_cell(
                         UUID(c["case_id"]), block_id, effort, final_section,
                         c["final_total_words"], c["outcome"], args.repeats,
-                        model=model,
+                        model=model, instructions=getattr(args, "instructions", "") or "",
                     )
                 except Exception as exc:  # noqa: BLE001 — harness must survive any cell failure
                     logger.warning(
@@ -626,6 +635,8 @@ def _write_report(result: dict, ts: str) -> tuple[Path, Path]:
         f"- efforts: {', '.join(g['efforts'])} · repeats/cell: {g['repeats']}",
         f"- models: {', '.join(m or 'pinned-default' for m in g.get('models', [None]))}",
         f"- סך ייצורי-מודל: {g['total_generations']}",
+        (f"- ⚠️ **וריאנט-פרומפט** (לא בר-השוואה לריצת-בסיס): `{g['instructions']}`"
+         if g.get("instructions") else "- וריאנט-פרומפט: — (פרומפט ייצור כפי-שהוא)"),
         "",
     ]
     if result.get("dry_run"):
@@ -717,6 +728,9 @@ async def main() -> int:
     ap.add_argument("--models", default="",
                     help="comma generation-model ids to A/B (e.g. claude-opus-4-8,claude-opus-5). "
                          "Empty (default) = the pinned GENERATION_MODEL, i.e. production unchanged.")
+    ap.add_argument("--instructions", default="",
+                    help="extra prompt instruction appended to EVERY cell (prompt-variant A/B). "
+                         "Applied to all models — the run stays a model comparison. Recorded in the report.")
     args = ap.parse_args()
 
     logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")