"""Compute all scores and write the results site.""" from __future__ import annotations import datetime as dt import html import json import shutil from pathlib import Path import numpy as np from .loading import load_challenges, load_submissions from .metrics import compute_metric, metric_names from .reference import collect_order, select_reference from .wav import read_wav, resolve_output def score_challenge(challenge: dict, submissions: list[dict], out_audio: Path | None = None) -> dict: """Load every output for one challenge, compute pairwise errors, choose the reference, and return a results dict.""" spec = challenge["output"] names = metric_names(challenge) primary = challenge["metrics"]["primary"] signals: dict[str, np.ndarray] = {} rows: dict[str, dict] = {} for sub in submissions: for sol in sub["solutions"]: key = f"{sub['submission_id']}/{sol['solution_id']}" p = resolve_output(sol["output"], sub["_path"].parent) signals[key] = read_wav(p, spec) audio_rel = None if out_audio is not None: dest = out_audio / sub["submission_id"] / f"{sol['solution_id']}.wav" dest.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(p, dest) audio_rel = f"audio/{sub['submission_id']}/{sol['solution_id']}.wav" rows[key] = { "key": key, "submission_id": sub["submission_id"], "solution_id": sol["solution_id"], "submitter": sub["submitter"]["name"], "date": sub.get("date"), "method": sol.get("method"), "parameters": sol.get("parameters"), "timing_seconds": sol.get("timing_seconds"), "hardware": sol.get("hardware"), "code_url": sol.get("code_url") or sub.get("code_url"), "viz_url": sol.get("viz_url") or sub.get("viz_url"), "more_accurate_than": [f"{sub['submission_id']}/{w}" for w in sol.get("more_accurate_than", [])], "notes": sol.get("notes"), "audio": audio_rel, "peak": float(np.max(np.abs(signals[key]))), } keys = list(signals) pairwise = {name: {a: {} for a in keys} for name in names} for name in names: params = challenge["metrics"][name] for a in keys: for r in keys: pairwise[name][a][r] = 0.0 if a == r else compute_metric(name, signals[a], signals[r], params) info = collect_order(submissions) explicit = challenge.get("reference_solution") if explicit is not None and explicit not in keys: raise KeyError(f"reference_solution {explicit!r} is not among the solutions") selection = select_reference(info, pairwise[primary], explicit) ref = selection["reference"] for key, row in rows.items(): row["errors"] = {name: pairwise[name][key][ref] for name in names} if ref else {} row["is_reference"] = key == ref # Check every asserted pair against the chosen reference, for the report. violations = [] if ref: for worse, better, _sid in info.pairs: for name in names: if not pairwise[name][better][ref] < pairwise[name][worse][ref]: violations.append({"metric": name, "worse": worse, "better": better}) ordered = sorted(rows.values(), key=lambda r: (r["errors"].get(primary, float("inf")), r["key"])) return { "challenge_id": challenge["challenge_id"], "title": challenge.get("title"), "generated": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"), "metrics": names, "primary_metric": primary, "reference": ref, "reference_selection": selection, "order_violations_vs_reference": violations, "solutions": ordered, "pairwise": pairwise, "submissions": [ {k: v for k, v in sub.items() if not k.startswith("_") and k != "solutions"} for sub in submissions ], } # ---------------------------------------------------------------- HTML _CSS = """ :root { --fg: #1b1b1b; --bg: #ffffff; --muted: #666; --line: #ddd; --accent: #2a6db0; --ref: #e8f4e8; } @media (prefers-color-scheme: dark) { :root { --fg: #e6e6e6; --bg: #151515; --muted: #999; --line: #333; --accent: #7ab3e8; --ref: #1f3320; } } body { font-family: system-ui, sans-serif; color: var(--fg); background: var(--bg); margin: 0; padding: 16px; line-height: 1.45; } main { max-width: 1200px; margin: 0 auto; } a { color: var(--accent); } table { border-collapse: collapse; width: 100%; font-size: 0.92em; } th, td { border-bottom: 1px solid var(--line); padding: 6px 8px; text-align: left; vertical-align: top; } th { position: sticky; top: 0; background: var(--bg); } td.num { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; } tr.ref { background: var(--ref); } .muted { color: var(--muted); } .wrap { overflow-x: auto; } audio { width: 180px; height: 28px; } details { margin: 1em 0; } code { font-size: 0.9em; } """ def _fmt(x, digits=4): if x is None: return "" if isinstance(x, float): if x == 0: return "0" return f"{x:.{digits}g}" return html.escape(str(x)) def _link(url, text): if not url: return "" return f'{html.escape(text)}' def render_challenge_page(res: dict, challenge: dict, links: dict) -> str: names = res["metrics"] primary = res["primary_metric"] head = "".join( f"{html.escape(n)}{' (primary)' if n == primary else ''}" for n in names ) body = [] for r in res["solutions"]: errs = "".join(f'{_fmt(r["errors"].get(n))}' for n in names) params = html.escape(json.dumps(r["parameters"])) if r["parameters"] else "" mat = ", ".join(html.escape(k.split("/", 1)[1]) for k in r["more_accurate_than"]) audio = f'' if r["audio"] else "" linkbits = " ".join(x for x in [_link(r["code_url"], "code"), _link(r["viz_url"], "viz"), _link(r["audio"], "wav")] if x) body.append( f'' f'{html.escape(r["submission_id"])}
{html.escape(r["submitter"])}' f'{html.escape(r["solution_id"])}{" (reference)" if r["is_reference"] else ""}' f'{html.escape(r["method"] or "")}
{params}' f"{errs}" f'{_fmt(r["timing_seconds"], 3)}' f'{html.escape(r["hardware"] or "")}' f'{mat}' f"{audio}
{linkbits}" "" ) sel = res["reference_selection"] cand_rows = "".join( f"{html.escape(c['candidate'])}{c['pairs_preserved']}/{c['pairs_total']}" f"{_fmt(c['convergence_gap'])}{_fmt(c['mean_error_to_other_candidates'])}" f"{html.escape(', '.join(f'{w} vs {b}' for w, b in c['violated_pairs']))}" for c in sel.get("candidates", []) ) viol = res["order_violations_vs_reference"] viol_html = ( "

All asserted accuracy orderings are preserved by both metrics under the chosen reference.

" if not viol else "" ) pair_tables = [] keys = [r["key"] for r in res["solutions"]] for n in names: hdr = "".join(f"{html.escape(k)}" for k in keys) rws = "".join( f"{html.escape(a)}" + "".join(f"{_fmt(res['pairwise'][n][a][b], 3)}" for b in keys) + "" for a in keys ) pair_tables.append(f"

{html.escape(n)}

{hdr}{rws}
") spec_link = links.get("spec") return f""" {html.escape(res["challenge_id"])} results

All challenges

{html.escape(res["challenge_id"])}

{html.escape(res.get("title") or "")}. {"Challenge specification. " if spec_link else ""}results.json. Generated {html.escape(res["generated"])}.

Errors are relative to the reference solution {html.escape(res["reference"] or "none")}, chosen by rule {html.escape(sel["rule"])}. Rows are sorted by the primary metric.

{head} {"".join(body)}
SubmissionSolutionMethodTime (s)HardwareClaimed more accurate thanOutput
Reference selection

Candidates are the maximal elements of each submission's asserted order. The score is the number of asserted pairs from other submissions whose order is preserved by the primary error to this candidate. Ties are broken first by the convergence gap (the primary error between the candidate and the nearest solution of its own submission asserted to be less accurate; smaller means a tighter convergence claim), then by the mean primary error to the other candidates.

{cand_rows}
CandidatePairs preservedConvergence gapMean error to other candidatesViolated pairs

Order check against the chosen reference

{viol_html}
Pairwise errors (row vs column as reference){"".join(pair_tables)}
""" def render_index(results: list[dict], links: dict) -> str: items = "".join( f'
  • {html.escape(r["challenge_id"])}: {html.escape(r.get("title") or "")} ' f'({len(r["solutions"])} solutions, reference {html.escape(r["reference"] or "none")})
  • ' for r in results ) repo = links.get("challenge_repo") sols = links.get("solutions_repo") return f""" Challenge results

    Challenge results

    {"Specifications: " + html.escape(repo) + ". " if repo else ""}{"Solutions: " + html.escape(sols) + "." if sols else ""}

    """ def build(challenges_dir: Path, solutions_dir: Path, out: Path, challenge_repo_url: str | None = None, solutions_repo_url: str | None = None) -> list[dict]: challenges = load_challenges(challenges_dir) submissions = load_submissions(solutions_dir) out.mkdir(parents=True, exist_ok=True) results = [] for cid, ch in challenges.items(): subs = [s for s in submissions if s["challenge_id"] == cid] cdir = out / cid cdir.mkdir(parents=True, exist_ok=True) res = score_challenge(ch, subs, cdir / "audio") if subs else { "challenge_id": cid, "title": ch.get("title"), "generated": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"), "metrics": metric_names(ch), "primary_metric": ch["metrics"]["primary"], "reference": None, "reference_selection": {"rule": "none", "candidates": []}, "order_violations_vs_reference": [], "solutions": [], "pairwise": {}, "submissions": [], } with open(cdir / "results.json", "w") as f: json.dump(res, f, indent=1) links = {"spec": f"{challenge_repo_url}/blob/main/challenges/{cid}/README.md" if challenge_repo_url else None} (cdir / "index.html").write_text(render_challenge_page(res, ch, links)) results.append(res) (out / "index.html").write_text(render_index(results, {"challenge_repo": challenge_repo_url, "solutions_repo": solutions_repo_url})) return results