diff --git a/scripts/calibrate_effort.py b/scripts/calibrate_effort.py index b7bf422..4b2b430 100644 --- a/scripts/calibrate_effort.py +++ b/scripts/calibrate_effort.py @@ -421,11 +421,16 @@ async def _finals_for_calibration(case_filter: str | None) -> list[dict]: async def _score_cell(case_id, block_id: str, effort: str, final_section: str, final_total_words: int, outcome: str, repeats: int, - model: str | None = None) -> dict: + model: str | None = None, instructions: str = "") -> dict: """Generate `block_id` at `effort` `repeats` times; score each vs the final section. `model` (optional) A/Bs the generation model via write_block(model_override=…). None ⇒ the pinned GENERATION_MODEL, i.e. the production path unchanged. + + `instructions` (optional) is appended to the block prompt for EVERY cell in + the run — a prompt-variant A/B (e.g. an explicit formatting rule). It is + applied to all models so the comparison stays a model comparison rather + than silently becoming a prompt comparison. """ from legal_mcp.services import block_writer from legal_mcp.services.style_distance import block_distance_to_final @@ -433,7 +438,8 @@ async def _score_cell(case_id, block_id: str, effort: str, final_section: str, models_used: list[str] = [] for _ in range(repeats): res = await block_writer.write_block( - case_id, block_id, effort_override=effort, model_override=model, + case_id, block_id, instructions=instructions, + effort_override=effort, model_override=model, ) # Record what the CLI was actually asked to run, so a silent fallback to # a different build is visible in the report rather than mis-attributed. @@ -488,6 +494,9 @@ async def _run(args, ts: str) -> dict: "finals": [c["case_number"] for c in cases_meta], "blocks": blocks, "efforts": efforts, "repeats": args.repeats, "models": models, + # Provenance: a prompt-variant run is NOT comparable to a baseline run, + # so the instruction text is recorded in the report, not just the shell. + "instructions": getattr(args, "instructions", "") or "", "total_generations": total_cells, "per_block_n": {b: len(plan[b]) for b in blocks}, } @@ -533,7 +542,7 @@ async def _run_blocks_for_model(model, blocks, efforts, plan, args, ts, grid_sum cell = await _score_cell( UUID(c["case_id"]), block_id, effort, final_section, c["final_total_words"], c["outcome"], args.repeats, - model=model, + model=model, instructions=getattr(args, "instructions", "") or "", ) except Exception as exc: # noqa: BLE001 — harness must survive any cell failure logger.warning( @@ -626,6 +635,8 @@ def _write_report(result: dict, ts: str) -> tuple[Path, Path]: f"- efforts: {', '.join(g['efforts'])} · repeats/cell: {g['repeats']}", f"- models: {', '.join(m or 'pinned-default' for m in g.get('models', [None]))}", f"- סך ייצורי-מודל: {g['total_generations']}", + (f"- ⚠️ **וריאנט-פרומפט** (לא בר-השוואה לריצת-בסיס): `{g['instructions']}`" + if g.get("instructions") else "- וריאנט-פרומפט: — (פרומפט ייצור כפי-שהוא)"), "", ] if result.get("dry_run"): @@ -717,6 +728,9 @@ async def main() -> int: ap.add_argument("--models", default="", help="comma generation-model ids to A/B (e.g. claude-opus-4-8,claude-opus-5). " "Empty (default) = the pinned GENERATION_MODEL, i.e. production unchanged.") + ap.add_argument("--instructions", default="", + help="extra prompt instruction appended to EVERY cell (prompt-variant A/B). " + "Applied to all models — the run stays a model comparison. Recorded in the report.") args = ap.parse_args() logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")