| """Plot measured evaluation summaries and collect comparable seed-level results.""" |
|
|
| from pathlib import Path |
| import argparse, json, csv |
| import numpy as np |
| import matplotlib |
|
|
| matplotlib.use("Agg") |
| import matplotlib.pyplot as plt |
| from matplotlib import font_manager |
|
|
| ROOT = Path(__file__).resolve().parents[1] |
| for f in (ROOT / "assets/fonts").glob("*.ttf"): |
| font_manager.fontManager.addfont(str(f)) |
| plt.rcParams.update( |
| {"font.family": "Ubuntu", "mathtext.fontset": "cm", "pdf.fonttype": 42} |
| ) |
|
|
|
|
| def plot(paths, output): |
| records = [json.loads(Path(p).read_text()) for p in paths] |
| rows = [] |
| for d in records: |
| if d.get("protocol") != "split-first-v1": |
| raise ValueError("Use corrected evaluation outputs") |
| for search, metrics in d["summary"]["inverse"].items(): |
| rows.append( |
| { |
| "method": d["method"], |
| "search": search, |
| "seed": d["seed"], |
| **{k: v["mean"] for k, v in metrics.items()}, |
| } |
| ) |
| if not rows: |
| raise ValueError("No inverse results for this catalog and target split") |
| out = Path(output) |
| out.mkdir(parents=True, exist_ok=True) |
| with open(out / "summary.csv", "w") as f: |
| writer = csv.DictWriter(f, fieldnames=list(rows[0])) |
| writer.writeheader() |
| writer.writerows(rows) |
| fig, axes = plt.subplots(1, 2, figsize=(10, 4), layout="constrained") |
| labels = [ |
| r["method"] + "\n" + r["search"] + "\nseed " + str(r["seed"]) for r in rows |
| ] |
| for ax, k, title in zip( |
| axes, |
| ["top5", "measured_regret"], |
| [ |
| "Exact Top-5 recovery (higher is better)", |
| "Measured regret (lower is better)", |
| ], |
| ): |
| ax.bar(np.arange(len(rows)), [r[k] for r in rows], color="#62AEDD", width=0.6) |
| ax.set_xticks(np.arange(len(rows)), labels, fontsize=8) |
| ax.set_title(title, fontsize=11) |
| ax.spines[["top", "right"]].set_visible(False) |
| ax.grid(axis="y", alpha=0.2) |
| ax.set_axisbelow(True) |
| for ext in ["pdf", "png"]: |
| fig.savefig(out / f"evaluation.{ext}", dpi=220) |
| plt.close(fig) |
|
|
|
|
| if __name__ == "__main__": |
| p = argparse.ArgumentParser() |
| p.add_argument("results", nargs="+") |
| p.add_argument("--output", default="plots") |
| a = p.parse_args() |
| plot(a.results, a.output) |
|
|
