feat(eval): פילוח אנטי-דפוסים per-ריצה — "איזה כלל הופר", לא רק כמה
All checks were successful
G12 Leak-Guard / leak-guard (pull_request) Successful in 3s
Lint — undefined names / undefined-names (pull_request) Successful in 10s

`block_distance_to_final` החזיר `anti_pattern_total` בלבד. ריצת-כיול שמדווחת
"anti=4" אינה יכולה לומר מה לתקן — באבחון ה-A/B של 2026-07-28 נאלצנו להסיק
את הדפוס האשם מהקשר במקום למדוד אותו.

- `anti_by_pattern` (שם-דפוס → מספר-פגיעות) נוסף לתא-המדידה, מ-
  `count_anti_patterns` הקיים — אין ספירה מקבילה.
- `_mean_by_pattern` ממצע על **כל** הריצות: דפוס שלא נורה בריצה נספר כ-0
  ולא מושמט, אחרת הממוצע היה מוטה כלפי מעלה.
- הדוח מקבל טבלת "פילוח אנטי-דפוסים (איזה כלל הופר)" per block×effort×model.

invariants: INV-G8 (eval-harness) · G2 (מרונדר מ-count_anti_patterns/
ANTI_PATTERNS הקנוניים — מקור אחד).

אימות: self-test ALL PASS · 470 passed · בדיקת-שפיות ישירה —
טקסט עם 1 כותרת + 2 תבליטים + 1 פיצול-מיני מפולח נכון ל-
{markdown_headers:1, bullet_lists:2, inline_numbered_fragments:1}.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-07-28 11:17:35 +00:00
parent 10e05700cc
commit 86e66cc5bd
2 changed files with 40 additions and 2 deletions

View File

@@ -176,7 +176,13 @@ def block_distance_to_final(
outcome = canonical_outcome(outcome) outcome = canonical_outcome(outcome)
diff = compute_diff_stats(regenerated_text or "", final_section_text or "") diff = compute_diff_stats(regenerated_text or "", final_section_text or "")
change_percent = diff["change_percent"] change_percent = diff["change_percent"]
anti_total = count_anti_patterns(regenerated_text or "")["total"] anti = count_anti_patterns(regenerated_text or "")
anti_total = anti["total"]
# Per-pattern breakdown, not just the total: a calibration run that only
# reports "anti=4" cannot tell you WHICH rule was broken, so it cannot say
# what to fix. (Diagnosing the 2026-07-28 model A/B needed exactly this and
# had to fall back on inference.)
anti_by_pattern = {name: h["count"] for name, h in anti["by_pattern"].items()}
section = _BLOCK_TO_SECTION.get(block_id) section = _BLOCK_TO_SECTION.get(block_id)
regen_words = len((regenerated_text or "").split()) regen_words = len((regenerated_text or "").split())
@@ -205,6 +211,7 @@ def block_distance_to_final(
"final_words": final_words, "final_words": final_words,
"change_percent": change_percent, "change_percent": change_percent,
"anti_pattern_total": anti_total, "anti_pattern_total": anti_total,
"anti_by_pattern": anti_by_pattern,
"golden_ratio_deviation_pp": ratio_dev, "golden_ratio_deviation_pp": ratio_dev,
"distance": distance, "distance": distance,
} }

View File

@@ -166,17 +166,33 @@ def aggregate_cell(per_run: list[dict]) -> dict:
"""Mean each metric across repeated generations of the same (case, block, effort).""" """Mean each metric across repeated generations of the same (case, block, effort)."""
if not per_run: if not per_run:
return {"distance": 1.0, "anti_pattern_total": 0.0, "change_percent": 100.0, return {"distance": 1.0, "anti_pattern_total": 0.0, "change_percent": 100.0,
"golden_ratio_deviation_pp": None, "n": 0} "golden_ratio_deviation_pp": None, "anti_by_pattern": {}, "n": 0}
ratios = [r["golden_ratio_deviation_pp"] for r in per_run if r.get("golden_ratio_deviation_pp") is not None] ratios = [r["golden_ratio_deviation_pp"] for r in per_run if r.get("golden_ratio_deviation_pp") is not None]
return { return {
"distance": round(mean(r["distance"] for r in per_run), 4), "distance": round(mean(r["distance"] for r in per_run), 4),
"anti_pattern_total": round(mean(r["anti_pattern_total"] for r in per_run), 2), "anti_pattern_total": round(mean(r["anti_pattern_total"] for r in per_run), 2),
"change_percent": round(mean(r["change_percent"] for r in per_run), 2), "change_percent": round(mean(r["change_percent"] for r in per_run), 2),
"golden_ratio_deviation_pp": round(mean(ratios), 2) if ratios else None, "golden_ratio_deviation_pp": round(mean(ratios), 2) if ratios else None,
"anti_by_pattern": _mean_by_pattern(per_run),
"n": len(per_run), "n": len(per_run),
} }
def _mean_by_pattern(per_run: list[dict]) -> dict:
"""Mean hits PER anti-pattern name across runs — the 'which rule broke' view.
A pattern absent from a run counts as 0 (count_anti_patterns omits zero-hit
keys), so the mean is over ALL runs, not only the ones that tripped it.
"""
names: set[str] = set()
for r in per_run:
names |= set((r.get("anti_by_pattern") or {}).keys())
return {
name: round(mean((r.get("anti_by_pattern") or {}).get(name, 0) for r in per_run), 2)
for name in sorted(names)
}
def _current_default(block_id: str) -> str | None: def _current_default(block_id: str) -> str | None:
from legal_mcp.services.block_writer import BLOCK_CONFIG, DEFAULT_EFFORT from legal_mcp.services.block_writer import BLOCK_CONFIG, DEFAULT_EFFORT
cfg = BLOCK_CONFIG.get(block_id, {}) cfg = BLOCK_CONFIG.get(block_id, {})
@@ -569,6 +585,7 @@ async def _run_blocks_for_model(model, blocks, efforts, plan, args, ts, grid_sum
"anti_pattern_total": round(mean(r["anti_pattern_total"] for r in rows), 2), "anti_pattern_total": round(mean(r["anti_pattern_total"] for r in rows), 2),
"change_percent": round(mean(r["change_percent"] for r in rows), 2), "change_percent": round(mean(r["change_percent"] for r in rows), 2),
"golden_ratio_deviation_pp": round(mean(ratios), 2) if ratios else None, "golden_ratio_deviation_pp": round(mean(ratios), 2) if ratios else None,
"anti_by_pattern": _mean_by_pattern(rows),
"n": len(rows), "n": len(rows),
}) })
rec = recommend_effort(effort_rows) rec = recommend_effort(effort_rows)
@@ -696,6 +713,20 @@ def _write_report(result: dict, ts: str) -> tuple[Path, Path]:
f"{r['change_percent']} | {ratio if ratio is not None else ''} | " f"{r['change_percent']} | {ratio if ratio is not None else ''} | "
f"{r['distance']:.4f} | {r['n']} |") f"{r['distance']:.4f} | {r['n']} |")
lines.append("") lines.append("")
# WHICH rule broke — a total alone can't tell you what to fix.
bd_rows = [(b, eff, m, r) for b in g["blocks"] for eff in g["efforts"]
for m, bb in by_model.items()
for r in (bb.get(b) or {}).get("efforts", []) if r["effort"] == eff]
if any(r.get("anti_by_pattern") for *_, r in bd_rows):
names = sorted({n for *_, r in bd_rows for n in (r.get("anti_by_pattern") or {})})
lines += ["### פילוח אנטי-דפוסים (איזה כלל הופר)\n",
"| block | effort | model | " + " | ".join(names) + " |",
"|---|---|---|" + "---|" * len(names)]
for b, eff, m, r in bd_rows:
cells = " | ".join(str((r.get("anti_by_pattern") or {}).get(n, 0)) for n in names)
lines.append(f"| {b} | {eff} | {m} | {cells} |")
lines.append("")
# A silent CLI fallback would make the whole comparison meaningless — surface it. # A silent CLI fallback would make the whole comparison meaningless — surface it.
for m, bb in by_model.items(): for m, bb in by_model.items():
for b, bd in bb.items(): for b, bd in bb.items():