feat(eval): פילוח אנטי-דפוסים per-ריצה — "איזה כלל הופר", לא רק כמה
`block_distance_to_final` החזיר `anti_pattern_total` בלבד. ריצת-כיול שמדווחת
"anti=4" אינה יכולה לומר מה לתקן — באבחון ה-A/B של 2026-07-28 נאלצנו להסיק
את הדפוס האשם מהקשר במקום למדוד אותו.
- `anti_by_pattern` (שם-דפוס → מספר-פגיעות) נוסף לתא-המדידה, מ-
`count_anti_patterns` הקיים — אין ספירה מקבילה.
- `_mean_by_pattern` ממצע על **כל** הריצות: דפוס שלא נורה בריצה נספר כ-0
ולא מושמט, אחרת הממוצע היה מוטה כלפי מעלה.
- הדוח מקבל טבלת "פילוח אנטי-דפוסים (איזה כלל הופר)" per block×effort×model.
invariants: INV-G8 (eval-harness) · G2 (מרונדר מ-count_anti_patterns/
ANTI_PATTERNS הקנוניים — מקור אחד).
אימות: self-test ALL PASS · 470 passed · בדיקת-שפיות ישירה —
טקסט עם 1 כותרת + 2 תבליטים + 1 פיצול-מיני מפולח נכון ל-
{markdown_headers:1, bullet_lists:2, inline_numbered_fragments:1}.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -176,7 +176,13 @@ def block_distance_to_final(
|
||||
outcome = canonical_outcome(outcome)
|
||||
diff = compute_diff_stats(regenerated_text or "", final_section_text or "")
|
||||
change_percent = diff["change_percent"]
|
||||
anti_total = count_anti_patterns(regenerated_text or "")["total"]
|
||||
anti = count_anti_patterns(regenerated_text or "")
|
||||
anti_total = anti["total"]
|
||||
# Per-pattern breakdown, not just the total: a calibration run that only
|
||||
# reports "anti=4" cannot tell you WHICH rule was broken, so it cannot say
|
||||
# what to fix. (Diagnosing the 2026-07-28 model A/B needed exactly this and
|
||||
# had to fall back on inference.)
|
||||
anti_by_pattern = {name: h["count"] for name, h in anti["by_pattern"].items()}
|
||||
|
||||
section = _BLOCK_TO_SECTION.get(block_id)
|
||||
regen_words = len((regenerated_text or "").split())
|
||||
@@ -205,6 +211,7 @@ def block_distance_to_final(
|
||||
"final_words": final_words,
|
||||
"change_percent": change_percent,
|
||||
"anti_pattern_total": anti_total,
|
||||
"anti_by_pattern": anti_by_pattern,
|
||||
"golden_ratio_deviation_pp": ratio_dev,
|
||||
"distance": distance,
|
||||
}
|
||||
|
||||
@@ -166,17 +166,33 @@ def aggregate_cell(per_run: list[dict]) -> dict:
|
||||
"""Mean each metric across repeated generations of the same (case, block, effort)."""
|
||||
if not per_run:
|
||||
return {"distance": 1.0, "anti_pattern_total": 0.0, "change_percent": 100.0,
|
||||
"golden_ratio_deviation_pp": None, "n": 0}
|
||||
"golden_ratio_deviation_pp": None, "anti_by_pattern": {}, "n": 0}
|
||||
ratios = [r["golden_ratio_deviation_pp"] for r in per_run if r.get("golden_ratio_deviation_pp") is not None]
|
||||
return {
|
||||
"distance": round(mean(r["distance"] for r in per_run), 4),
|
||||
"anti_pattern_total": round(mean(r["anti_pattern_total"] for r in per_run), 2),
|
||||
"change_percent": round(mean(r["change_percent"] for r in per_run), 2),
|
||||
"golden_ratio_deviation_pp": round(mean(ratios), 2) if ratios else None,
|
||||
"anti_by_pattern": _mean_by_pattern(per_run),
|
||||
"n": len(per_run),
|
||||
}
|
||||
|
||||
|
||||
def _mean_by_pattern(per_run: list[dict]) -> dict:
|
||||
"""Mean hits PER anti-pattern name across runs — the 'which rule broke' view.
|
||||
|
||||
A pattern absent from a run counts as 0 (count_anti_patterns omits zero-hit
|
||||
keys), so the mean is over ALL runs, not only the ones that tripped it.
|
||||
"""
|
||||
names: set[str] = set()
|
||||
for r in per_run:
|
||||
names |= set((r.get("anti_by_pattern") or {}).keys())
|
||||
return {
|
||||
name: round(mean((r.get("anti_by_pattern") or {}).get(name, 0) for r in per_run), 2)
|
||||
for name in sorted(names)
|
||||
}
|
||||
|
||||
|
||||
def _current_default(block_id: str) -> str | None:
|
||||
from legal_mcp.services.block_writer import BLOCK_CONFIG, DEFAULT_EFFORT
|
||||
cfg = BLOCK_CONFIG.get(block_id, {})
|
||||
@@ -569,6 +585,7 @@ async def _run_blocks_for_model(model, blocks, efforts, plan, args, ts, grid_sum
|
||||
"anti_pattern_total": round(mean(r["anti_pattern_total"] for r in rows), 2),
|
||||
"change_percent": round(mean(r["change_percent"] for r in rows), 2),
|
||||
"golden_ratio_deviation_pp": round(mean(ratios), 2) if ratios else None,
|
||||
"anti_by_pattern": _mean_by_pattern(rows),
|
||||
"n": len(rows),
|
||||
})
|
||||
rec = recommend_effort(effort_rows)
|
||||
@@ -696,6 +713,20 @@ def _write_report(result: dict, ts: str) -> tuple[Path, Path]:
|
||||
f"{r['change_percent']} | {ratio if ratio is not None else '—'} | "
|
||||
f"{r['distance']:.4f} | {r['n']} |")
|
||||
lines.append("")
|
||||
|
||||
# WHICH rule broke — a total alone can't tell you what to fix.
|
||||
bd_rows = [(b, eff, m, r) for b in g["blocks"] for eff in g["efforts"]
|
||||
for m, bb in by_model.items()
|
||||
for r in (bb.get(b) or {}).get("efforts", []) if r["effort"] == eff]
|
||||
if any(r.get("anti_by_pattern") for *_, r in bd_rows):
|
||||
names = sorted({n for *_, r in bd_rows for n in (r.get("anti_by_pattern") or {})})
|
||||
lines += ["### פילוח אנטי-דפוסים (איזה כלל הופר)\n",
|
||||
"| block | effort | model | " + " | ".join(names) + " |",
|
||||
"|---|---|---|" + "---|" * len(names)]
|
||||
for b, eff, m, r in bd_rows:
|
||||
cells = " | ".join(str((r.get("anti_by_pattern") or {}).get(n, 0)) for n in names)
|
||||
lines.append(f"| {b} | {eff} | {m} | {cells} |")
|
||||
lines.append("")
|
||||
# A silent CLI fallback would make the whole comparison meaningless — surface it.
|
||||
for m, bb in by_model.items():
|
||||
for b, bd in bb.items():
|
||||
|
||||
Reference in New Issue
Block a user