From 86e66cc5bd7f12ccfd0d82f5acfcb21b3d5a67cd Mon Sep 17 00:00:00 2001 From: Chaim Date: Tue, 28 Jul 2026 11:17:35 +0000 Subject: [PATCH] =?UTF-8?q?feat(eval):=20=D7=A4=D7=99=D7=9C=D7=95=D7=97=20?= =?UTF-8?q?=D7=90=D7=A0=D7=98=D7=99-=D7=93=D7=A4=D7=95=D7=A1=D7=99=D7=9D?= =?UTF-8?q?=20per-=D7=A8=D7=99=D7=A6=D7=94=20=E2=80=94=20"=D7=90=D7=99?= =?UTF-8?q?=D7=96=D7=94=20=D7=9B=D7=9C=D7=9C=20=D7=94=D7=95=D7=A4=D7=A8",?= =?UTF-8?q?=20=D7=9C=D7=90=20=D7=A8=D7=A7=20=D7=9B=D7=9E=D7=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `block_distance_to_final` החזיר `anti_pattern_total` בלבד. ריצת-כיול שמדווחת "anti=4" אינה יכולה לומר מה לתקן — באבחון ה-A/B של 2026-07-28 נאלצנו להסיק את הדפוס האשם מהקשר במקום למדוד אותו. - `anti_by_pattern` (שם-דפוס → מספר-פגיעות) נוסף לתא-המדידה, מ- `count_anti_patterns` הקיים — אין ספירה מקבילה. - `_mean_by_pattern` ממצע על **כל** הריצות: דפוס שלא נורה בריצה נספר כ-0 ולא מושמט, אחרת הממוצע היה מוטה כלפי מעלה. - הדוח מקבל טבלת "פילוח אנטי-דפוסים (איזה כלל הופר)" per block×effort×model. invariants: INV-G8 (eval-harness) · G2 (מרונדר מ-count_anti_patterns/ ANTI_PATTERNS הקנוניים — מקור אחד). אימות: self-test ALL PASS · 470 passed · בדיקת-שפיות ישירה — טקסט עם 1 כותרת + 2 תבליטים + 1 פיצול-מיני מפולח נכון ל- {markdown_headers:1, bullet_lists:2, inline_numbered_fragments:1}. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/legal_mcp/services/style_distance.py | 9 ++++- scripts/calibrate_effort.py | 33 ++++++++++++++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/mcp-server/src/legal_mcp/services/style_distance.py b/mcp-server/src/legal_mcp/services/style_distance.py index 843e661..95fca60 100644 --- a/mcp-server/src/legal_mcp/services/style_distance.py +++ b/mcp-server/src/legal_mcp/services/style_distance.py @@ -176,7 +176,13 @@ def block_distance_to_final( outcome = canonical_outcome(outcome) diff = compute_diff_stats(regenerated_text or "", final_section_text or "") change_percent = diff["change_percent"] - anti_total = count_anti_patterns(regenerated_text or "")["total"] + anti = count_anti_patterns(regenerated_text or "") + anti_total = anti["total"] + # Per-pattern breakdown, not just the total: a calibration run that only + # reports "anti=4" cannot tell you WHICH rule was broken, so it cannot say + # what to fix. (Diagnosing the 2026-07-28 model A/B needed exactly this and + # had to fall back on inference.) + anti_by_pattern = {name: h["count"] for name, h in anti["by_pattern"].items()} section = _BLOCK_TO_SECTION.get(block_id) regen_words = len((regenerated_text or "").split()) @@ -205,6 +211,7 @@ def block_distance_to_final( "final_words": final_words, "change_percent": change_percent, "anti_pattern_total": anti_total, + "anti_by_pattern": anti_by_pattern, "golden_ratio_deviation_pp": ratio_dev, "distance": distance, } diff --git a/scripts/calibrate_effort.py b/scripts/calibrate_effort.py index 4b2b430..772026a 100644 --- a/scripts/calibrate_effort.py +++ b/scripts/calibrate_effort.py @@ -166,17 +166,33 @@ def aggregate_cell(per_run: list[dict]) -> dict: """Mean each metric across repeated generations of the same (case, block, effort).""" if not per_run: return {"distance": 1.0, "anti_pattern_total": 0.0, "change_percent": 100.0, - "golden_ratio_deviation_pp": None, "n": 0} + "golden_ratio_deviation_pp": None, "anti_by_pattern": {}, "n": 0} ratios = [r["golden_ratio_deviation_pp"] for r in per_run if r.get("golden_ratio_deviation_pp") is not None] return { "distance": round(mean(r["distance"] for r in per_run), 4), "anti_pattern_total": round(mean(r["anti_pattern_total"] for r in per_run), 2), "change_percent": round(mean(r["change_percent"] for r in per_run), 2), "golden_ratio_deviation_pp": round(mean(ratios), 2) if ratios else None, + "anti_by_pattern": _mean_by_pattern(per_run), "n": len(per_run), } +def _mean_by_pattern(per_run: list[dict]) -> dict: + """Mean hits PER anti-pattern name across runs — the 'which rule broke' view. + + A pattern absent from a run counts as 0 (count_anti_patterns omits zero-hit + keys), so the mean is over ALL runs, not only the ones that tripped it. + """ + names: set[str] = set() + for r in per_run: + names |= set((r.get("anti_by_pattern") or {}).keys()) + return { + name: round(mean((r.get("anti_by_pattern") or {}).get(name, 0) for r in per_run), 2) + for name in sorted(names) + } + + def _current_default(block_id: str) -> str | None: from legal_mcp.services.block_writer import BLOCK_CONFIG, DEFAULT_EFFORT cfg = BLOCK_CONFIG.get(block_id, {}) @@ -569,6 +585,7 @@ async def _run_blocks_for_model(model, blocks, efforts, plan, args, ts, grid_sum "anti_pattern_total": round(mean(r["anti_pattern_total"] for r in rows), 2), "change_percent": round(mean(r["change_percent"] for r in rows), 2), "golden_ratio_deviation_pp": round(mean(ratios), 2) if ratios else None, + "anti_by_pattern": _mean_by_pattern(rows), "n": len(rows), }) rec = recommend_effort(effort_rows) @@ -696,6 +713,20 @@ def _write_report(result: dict, ts: str) -> tuple[Path, Path]: f"{r['change_percent']} | {ratio if ratio is not None else '—'} | " f"{r['distance']:.4f} | {r['n']} |") lines.append("") + + # WHICH rule broke — a total alone can't tell you what to fix. + bd_rows = [(b, eff, m, r) for b in g["blocks"] for eff in g["efforts"] + for m, bb in by_model.items() + for r in (bb.get(b) or {}).get("efforts", []) if r["effort"] == eff] + if any(r.get("anti_by_pattern") for *_, r in bd_rows): + names = sorted({n for *_, r in bd_rows for n in (r.get("anti_by_pattern") or {})}) + lines += ["### פילוח אנטי-דפוסים (איזה כלל הופר)\n", + "| block | effort | model | " + " | ".join(names) + " |", + "|---|---|---|" + "---|" * len(names)] + for b, eff, m, r in bd_rows: + cells = " | ".join(str((r.get("anti_by_pattern") or {}).get(n, 0)) for n in names) + lines.append(f"| {b} | {eff} | {m} | {cells} |") + lines.append("") # A silent CLI fallback would make the whole comparison meaningless — surface it. for m, bb in by_model.items(): for b, bd in bb.items():