Evaluating Multimodal Vision Models with Moonshot PerceptionBench Using Robust Data Loading and Automated Judging

def run_eval(records, backend):
   print(f"n[eval] backend = {backend.name} on {len(records)} questions")
   t0 = time.time()
   preds = backend.predict_batch(records)
   rows = []
   for rec, raw in zip(records, preds):
       if CFG["JUDGE"] == "llm" and CFG["API_KEY"]:
           ok, how = llm_judge(raw, rec["answer"], rec["problem"])
       else:
           ok, how = rule_judge(raw, rec["answer"], CFG["NUM_REL_TOL"])
       rows.append(dict(index=rec["index"], code=rec["code"], category=rec["category"],
                        source_bmk=rec["source_bmk"], n_images=rec["n_images"],
                        q_chars=rec["q_chars"], max_side=rec["max_side"],
                        ans_type=answer_type(rec["answer"]),
                        gold=rec["answer"], pred=extract_answer(raw),
                        raw=str(raw)[:2000], correct=ok, how=how))
   print(f"[eval] done in {time.time()-t0:.1f}s")
   return pd.DataFrame(rows)
def bootstrap_ci(vals, n_boot=4000, seed=0):
   a = np.asarray(vals, dtype=float)
   if a.size == 0:
       return (float("nan"), float("nan"))
   rng = np.random.default_rng(seed)
   means = a[rng.integers(0, a.size, (n_boot, a.size))].mean(axis=1)
   return tuple(np.percentile(means, [2.5, 97.5]) * 100)
def report(res, label):
   print("n" + "=" * 78)
   print(f"§9  RESULTS — {label}")
   print("=" * 78)
   lo, hi = bootstrap_ci(res.correct)
   print(f"nOVERALL accuracy: {res.correct.mean()*100:5.1f}%   "
         f"95% CI [{lo:.1f}, {hi:.1f}]   (n={len(res)})")
   print("(card: no frontier model exceeds 60% overall)n")
   print("-- per atomic capability --")
   tab = []
   for code, g in res.groupby("code"):
       l, h = bootstrap_ci(g.correct)
       tab.append(dict(code=code, capability=CODE_FULL.get(code, code),
                       n=len(g), acc=g.correct.mean() * 100, lo=l, hi=h))
   t = pd.DataFrame(tab).sort_values("acc", ascending=False)
   print(t.to_string(index=False, float_format=lambda x: f"{x:6.1f}"))
   print("n-- difficulty slices --")
   res = res.copy()
   res["img_bucket"] = np.where(res.n_images > 1, "multi-image", "single-image")
   res["res_bucket"] = pd.cut(res.max_side, [0, 800, 1600, 10**6],
                              labels=["<800px", "800-1600px", ">1600px"])
   for col in ["img_bucket", "res_bucket", "ans_type"]:
       s = res.groupby(col, observed=True).correct.agg(["size", "mean"])
       s["mean"] = (s["mean"] * 100).round(1)
       print(f"n  by {col}:n{s.rename(columns={'size':'n','mean':'acc%'}).to_string()}")
   print("n-- judge decision breakdown --")
   print(res.how.value_counts().to_string())
   errs = res[res.correct == 0]
   if len(errs):
       print("n-- sample failures --")
       for _, r in errs.head(5).iterrows():
           print(f"  [{r.code}] gold={r.gold!r:>14}  pred={r['pred']!r:>20}  ({r.how})")
   return t
BACKEND = make_backend()
RES = run_eval(RECORDS, BACKEND)
PER_CAP = report(RES, BACKEND.name)
LEADERBOARD = {
   "GPT-5.6-Sol":      [59.7, 69.7, 62.4, 62.1, 55.5, 76.7, 67.0, 55.9, 60.0, 54.9, 26.9],
   "Kimi K3":          [58.5, 68.2, 59.7, 59.4, 52.4, 70.3, 59.1, 55.9, 53.3, 61.2, 41.7],
   "Claude-Fable-5":   [57.2, 58.5, 52.9, 60.9, 51.5, 70.4, 56.1, 51.6, 59.8, 64.3, 45.0],
   "Gemini-3.1-Pro":   [56.2, 58.8, 56.9, 61.8, 50.0, 52.7, 61.7, 54.8, 61.2, 64.3, 40.6],
   "Seed-2.1-Pro":     [55.0, 57.6, 51.2, 58.2, 43.6, 50.0, 59.5, 56.6, 60.4, 66.7, 49.8],
   "Qwen3.5-397B-A17B":[47.5, 55.2, 49.1, 53.0, 44.6, 46.7, 49.8, 44.8, 50.2, 52.9, 26.9],
   "Gemma-4-31B":      [40.7, 42.7, 33.9, 40.3, 39.1, 44.9, 43.7, 39.0, 45.9, 46.7, 32.1],
   "GLM-4.6V":         [32.5, 35.2, 31.8, 35.2, 29.1, 30.6, 34.8, 29.3, 33.7, 39.2, 26.9],
}
LB = pd.DataFrame(LEADERBOARD, index=["Overall"] + CODE_ORDER).T
print("n" + "=" * 78)
print("§10  OFFICIAL LEADERBOARD (subset, accuracy %)")
print("=" * 78)
print(LB.to_string())
print("nNote the structural finding from the card: Hallu is the weakest column "
     "almost everywhere,nand models with near-identical Overall scores have "
     "very different capability profiles.")
def radar(per_cap_df, label, compare=("GPT-5.6-Sol", "Gemma-4-31B")):
   codes = [c for c in CODE_ORDER if c in set(per_cap_df.code)]
   if len(codes) < 3:
       print("[radar] need >=3 capabilities"); return
   vals = per_cap_df.set_index("code").acc.reindex(codes).fillna(0).tolist()
   ang = np.linspace(0, 2 * np.pi, len(codes), endpoint=False).tolist()
   close = lambda v: v + v[:1]
   fig, ax = plt.subplots(figsize=(6.4, 6.4), subplot_kw=dict(polar=True))
   ax.plot(close(ang), close(vals), lw=2.4, color="#C44E52", label=label)
   ax.fill(close(ang), close(vals), alpha=.18, color="#C44E52")
   for m in compare:
       if m in LB.index:
           v = LB.loc[m, codes].tolist()
           ax.plot(close(ang), close(v), lw=1.3, ls="--", alpha=.85, label=m)
   ax.set_xticks(ang)
   ax.set_xticklabels(codes)
   ax.set_ylim(0, 100)
   ax.set_yticks([20, 40, 60, 80])
   ax.set_title("PerceptionBench capability profile", pad=24)
   ax.legend(loc="upper right", bbox_to_anchor=(1.32, 1.12), fontsize=8)
   plt.tight_layout(); plt.show()
if CFG["SHOW_PLOTS"]:
   radar(PER_CAP, BACKEND.name)
   fig, ax = plt.subplots(figsize=(7, 3.2))
   s = LB["Overall"].sort_values()
   ax.barh(s.index, s.values, color="#8C8C8C")
   ax.barh([BACKEND.name], [RES.correct.mean() * 100], color="#C44E52")
   ax.axvline(60, ls="--", c="k", lw=1)
   ax.text(60.5, -.4, "60% ceiling: unbeaten", fontsize=8)
   ax.set_xlabel("Overall accuracy (%)"); ax.set_title("Your run vs. the leaderboard")
   plt.tight_layout(); plt.show()
tag = re.sub(r"[^A-Za-z0-9_.-]", "_", BACKEND.name)
p_pred = os.path.join(CFG["OUT_DIR"], f"predictions_{tag}.jsonl")
p_cap  = os.path.join(CFG["OUT_DIR"], f"per_capability_{tag}.csv")
p_meta = os.path.join(CFG["OUT_DIR"], f"run_meta_{tag}.json")
with open(p_pred, "w") as f:
   for _, r in RES.iterrows():
       f.write(json.dumps(r.to_dict(), default=str) + "n")
PER_CAP.to_csv(p_cap, index=False)
json.dump({"config": {k: v for k, v in CFG.items() if "KEY" not in k},
          "backend": BACKEND.name, "n_questions": len(RES),
          "rows_scanned": N_SCANNED,
          "overall_acc": float(RES.correct.mean() * 100),
          "ci95": list(bootstrap_ci(RES.correct)),
          "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S")},
         open(p_meta, "w"), indent=2)
print(f"n[export] {p_pred}n[export] {p_cap}n[export] {p_meta}")
print("n" + "=" * 78)
print("DONE.  Next steps:")
print("  1) CFG['BACKEND']='api'  + PB_API_KEY/PB_API_BASE/PB_API_MODEL  -> score a real MLLM")
print("  2) CFG['BACKEND']='local' on a GPU runtime -> score an open 2-3B VLM")
print("  3) CFG['JUDGE']='llm' -> reproduce the paper's LLM-as-judge protocol")
print("  4) Raise N_PER_CATEGORY / MAX_SCAN, or LOAD_MODE='full' for all 3,000 rows")
print("  5) Ablations worth running: crop-to-region vs. full image, image resolution")
print("     sweep (MAX_IMAGE_SIDE 512/1024/2048), and CoT-on vs. CoT-off prompts")
print("=" * 78)