"""Compute all scores and write the comparison site.""" from __future__ import annotations import datetime as dt import html import json import shutil from pathlib import Path import numpy as np from .loading import Source, load_challenges, load_submissions, public_fields from .metrics import compute_metric, metric_names from .reference import collect_order, select_reference from .wav import read_wav, resolve_output def score_challenge(challenge: dict, submissions: list[dict], out_audio: Path | None = None) -> dict: """Load every output for one challenge, compute pairwise errors, choose the reference, and return a results dict.""" spec = challenge["output"] names = metric_names(challenge) primary = challenge["metrics"]["primary"] signals: dict[str, np.ndarray] = {} rows: dict[str, dict] = {} for sub in submissions: src: Source = sub["_source"] for sol in sub["solutions"]: key = f"{sub['_key']}/{sol['solution_id']}" p = resolve_output(sol["output"], sub["_path"].parent) signals[key] = read_wav(p, spec) audio_rel = None if out_audio is not None: dest = out_audio / sub["_key"] / f"{sol['solution_id']}.wav" dest.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(p, dest) audio_rel = f"audio/{sub['_key']}/{sol['solution_id']}.wav" fields_url = None if "fields" in sol: fpath = sol["fields"]["path"] fields_url = fpath if fpath.startswith("http") else src.blob_url(sub["_path"].parent / fpath) rows[key] = { "key": key, "source": src.label, "source_url": src.url, "submission_id": sub["submission_id"], "submission_url": src.blob_url(sub["_path"]), "solution_id": sol["solution_id"], "submitter": sub["submitter"]["name"], "date": sub.get("date"), "method": sol.get("method"), "parameters": sol.get("parameters"), "timing_seconds": sol.get("timing_seconds"), "hardware": sol.get("hardware"), "code_url": sol.get("code_url") or sub.get("code_url"), "viz_url": sol.get("viz_url") or sub.get("viz_url"), "fields_url": fields_url, "more_accurate_than": [f"{sub['_key']}/{w}" for w in sol.get("more_accurate_than", [])], "notes": sol.get("notes"), "audio": audio_rel, "peak": float(np.max(np.abs(signals[key]))), } keys = list(signals) pairwise = {name: {a: {} for a in keys} for name in names} for name in names: params = challenge["metrics"][name] for a in keys: for r in keys: pairwise[name][a][r] = 0.0 if a == r else compute_metric(name, signals[a], signals[r], params) info = collect_order(submissions) explicit = challenge.get("reference_solution") if explicit is not None and explicit not in keys: raise KeyError(f"reference_solution {explicit!r} is not among the solutions") selection = select_reference(info, pairwise[primary], explicit) ref = selection["reference"] for key, row in rows.items(): row["errors"] = {name: pairwise[name][key][ref] for name in names} if ref else {} row["is_reference"] = key == ref violations = [] if ref: for worse, better, _sid in info.pairs: for name in names: if not pairwise[name][better][ref] < pairwise[name][worse][ref]: violations.append({"metric": name, "worse": worse, "better": better}) ordered = sorted(rows.values(), key=lambda r: (r["errors"].get(primary, float("inf")), r["key"])) return { "challenge_id": challenge["challenge_id"], "title": challenge.get("title"), "generated": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"), "metrics": names, "primary_metric": primary, "reference": ref, "reference_selection": selection, "order_violations_vs_reference": violations, "solutions": ordered, "pairwise": pairwise, "submissions": [ {**{k: v for k, v in public_fields(sub).items() if k != "solutions"}, "key": sub["_key"], "source": sub["_source"].label, "source_url": sub["_source"].url} for sub in submissions ], } # ---------------------------------------------------------------- HTML _CSS = """ :root { --fg: #1b1b1b; --bg: #ffffff; --muted: #666; --line: #ddd; --accent: #2a6db0; --ref: #e8f4e8; } @media (prefers-color-scheme: dark) { :root { --fg: #e6e6e6; --bg: #151515; --muted: #999; --line: #333; --accent: #7ab3e8; --ref: #1f3320; } } body { font-family: system-ui, sans-serif; color: var(--fg); background: var(--bg); margin: 0; padding: 16px; line-height: 1.45; } main { max-width: 1300px; margin: 0 auto; } a { color: var(--accent); } table { border-collapse: collapse; width: 100%; font-size: 0.92em; } th, td { border-bottom: 1px solid var(--line); padding: 6px 8px; text-align: left; vertical-align: top; } th { position: sticky; top: 0; background: var(--bg); } td.num { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; } tr.ref { background: var(--ref); } .muted { color: var(--muted); } .wrap { overflow-x: auto; } details { margin: 1em 0; } code { font-size: 0.9em; } """ def _fmt(x, digits=4): if x is None: return "" if isinstance(x, float): if x == 0: return "0" return f"{x:.{digits}g}" return html.escape(str(x)) def _link(url, text): if not url: return "" return f'{html.escape(text)}' def render_challenge_page(res: dict, challenge: dict, links: dict, scene: bool = False) -> str: names = res["metrics"] primary = res["primary_metric"] head = "".join(f"{html.escape(n)}{' (primary)' if n == primary else ''}" for n in names) body = [] for r in res["solutions"]: errs = "".join(f'{_fmt(r["errors"].get(n))}' for n in names) params = html.escape(json.dumps(r["parameters"])) if r["parameters"] else "" mat = ", ".join(html.escape(k.rsplit("/", 1)[1]) for k in r["more_accurate_than"]) audio = f'
' if r["audio"] else "" linkbits = " ".join(x for x in [ _link(r["audio"], "wav"), _link(r["fields_url"], "fields"), _link(r["code_url"], "code"), _link(r["viz_url"], "viz"), ] if x) sub_cell = _link(r["submission_url"], r["submission_id"]) if r["submission_url"] else html.escape(r["submission_id"]) src_cell = _link(r["source_url"], r["source"]) if r["source"] else "" body.append( f'' f'{sub_cell}
{src_cell}{" ยท " if src_cell else ""}{html.escape(r["submitter"])}' f'{html.escape(r["solution_id"])}{" (reference)" if r["is_reference"] else ""}' f'{html.escape(r["method"] or "")}
{params}' f"{errs}" f'{_fmt(r["timing_seconds"], 3)}' f'{html.escape(r["hardware"] or "")}' f'{mat}' f"{audio}
{linkbits}" "" ) sel = res["reference_selection"] cand_rows = "".join( f"{html.escape(c['candidate'])}{c['pairs_preserved']}/{c['pairs_total']}" f"{_fmt(c['convergence_gap'])}{_fmt(c['mean_error_to_other_candidates'])}" f"{html.escape(', '.join(f'{w} vs {b}' for w, b in c['violated_pairs']))}" for c in sel.get("candidates", []) ) viol = res["order_violations_vs_reference"] viol_html = ( "

All asserted accuracy orderings are preserved by both metrics under the chosen reference.

" if not viol else "" ) pair_tables = [] keys = [r["key"] for r in res["solutions"]] for n in names: hdr = "".join(f"{html.escape(k)}" for k in keys) rws = "".join( f"{html.escape(a)}" + "".join(f"{_fmt(res['pairwise'][n][a][b], 3)}" for b in keys) + "" for a in keys ) pair_tables.append(f"

{html.escape(n)}

{hdr}{rws}
") spec_link = links.get("spec") playback_gain = float((challenge.get("playback") or {}).get("gain_db", 20)) return f""" {html.escape(res["challenge_id"])} results

All challenges

{html.escape(res["challenge_id"])}

{html.escape(res.get("title") or "")}. {"Challenge specification. " if spec_link else ""}results.json. Generated {html.escape(res["generated"])}.

Errors are relative to the reference solution {html.escape(res["reference"] or "none")}, chosen by rule {html.escape(sel["rule"])}. Rows are sorted by the primary metric. Submissions are named as repository/submission.

{"

Geometry

" if scene else ""}
{head} {"".join(body)}
SubmissionSolutionMethodTime (s)HardwareClaimed more accurate thanOutput
Reference selection

Candidates are the maximal elements of each submission's asserted order. The score is the number of asserted pairs from other submissions whose order is preserved by the primary error to this candidate. Ties are broken first by the convergence gap (the primary error between the candidate and the nearest solution of its own submission asserted to be less accurate; smaller means a tighter convergence claim), then by the mean primary error to the other candidates.

{cand_rows}
CandidatePairs preservedConvergence gapMean error to other candidatesViolated pairs

Order check against the chosen reference

{viol_html}
Pairwise errors (row vs column as reference){"".join(pair_tables)}
{"" if scene else ""} """ def render_index(results: list[dict], sources: list[Source], links: dict) -> str: items = "".join( f'
  • {html.escape(r["challenge_id"])}: {html.escape(r.get("title") or "")} ' f'({len(r["solutions"])} solutions, reference {html.escape(r["reference"] or "none")})' f'{" 3D model" if r.get("scene") else ""}
  • ' for r in results ) srcs = "".join( f"
  • {_link(s.url, s.repo or s.label or str(s.dir)) if s.url else html.escape(str(s.dir))}" f"{' (' + html.escape(s.branch) + ')' if s.branch else ''}" f"{': ' + html.escape(s.description) if s.description else ''}
  • " for s in sources ) repo = links.get("challenge_repo") return f""" Challenge results

    Challenge results

    {"Specifications and registry: " + html.escape(repo) + "." if repo else ""}

    Challenges

    Registered solution repositories

    """ def build(challenges_dir: Path, sources: list[Source], out: Path, challenge_repo_url: str | None = None) -> list[dict]: challenges = load_challenges(challenges_dir) submissions = load_submissions(sources) out.mkdir(parents=True, exist_ok=True) results = [] for cid, ch in challenges.items(): subs = [s for s in submissions if s["challenge_id"] == cid] cdir = out / cid cdir.mkdir(parents=True, exist_ok=True) res = score_challenge(ch, subs, cdir / "audio") if subs else { "challenge_id": cid, "title": ch.get("title"), "generated": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"), "metrics": metric_names(ch), "primary_metric": ch["metrics"]["primary"], "reference": None, "reference_selection": {"rule": "none", "candidates": []}, "order_violations_vs_reference": [], "solutions": [], "pairwise": {}, "submissions": [], } with open(cdir / "results.json", "w") as f: json.dump(res, f, indent=1) links = {"spec": f"{challenge_repo_url}/blob/main/challenges/{cid}/README.md" if challenge_repo_url else None} scene = challenges_dir / cid / "scene.json" if scene.exists(): shutil.copyfile(scene, cdir / "scene.json") res["scene"] = "scene.json" (cdir / "index.html").write_text(render_challenge_page(res, ch, links, scene=scene.exists())) results.append(res) (out / "index.html").write_text(render_index(results, sources, {"challenge_repo": challenge_repo_url})) (out / "player.js").write_text(player_js()) (out / "viewer.js").write_text(asset_text("viewer.js")) return results def asset_text(name: str) -> str: """Contents of a static asset shipped with the package (player.js, viewer.js).""" return (Path(__file__).parent / name).read_text() def asset_version(name: str) -> str: """Short hash of an asset, used as a cache-busting query on its script tag.""" import hashlib return hashlib.sha1(asset_text(name).encode()).hexdigest()[:10] def player_version() -> str: return asset_version("player.js") def player_js() -> str: return asset_text("player.js") def viewer_js() -> str: return asset_text("viewer.js")