"""Compute all scores and write the comparison site."""
from __future__ import annotations
import datetime as dt
import html
import json
import shutil
from pathlib import Path
import numpy as np
from .loading import Source, load_challenges, load_submissions, public_fields
from .metrics import compute_metric, metric_names
from .reference import collect_order, select_reference
from .wav import read_wav, resolve_output
def score_challenge(challenge: dict, submissions: list[dict], out_audio: Path | None = None) -> dict:
"""Load every output for one challenge, compute pairwise errors, choose the reference, and return a results dict."""
spec = challenge["output"]
names = metric_names(challenge)
primary = challenge["metrics"]["primary"]
signals: dict[str, np.ndarray] = {}
rows: dict[str, dict] = {}
for sub in submissions:
src: Source = sub["_source"]
for sol in sub["solutions"]:
key = f"{sub['_key']}/{sol['solution_id']}"
p = resolve_output(sol["output"], sub["_path"].parent)
signals[key] = read_wav(p, spec)
audio_rel = None
if out_audio is not None:
dest = out_audio / sub["_key"] / f"{sol['solution_id']}.wav"
dest.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(p, dest)
audio_rel = f"audio/{sub['_key']}/{sol['solution_id']}.wav"
fields_url = None
if "fields" in sol:
fpath = sol["fields"]["path"]
fields_url = fpath if fpath.startswith("http") else src.blob_url(sub["_path"].parent / fpath)
rows[key] = {
"key": key,
"source": src.label,
"source_url": src.url,
"submission_id": sub["submission_id"],
"submission_url": src.blob_url(sub["_path"]),
"solution_id": sol["solution_id"],
"submitter": sub["submitter"]["name"],
"date": sub.get("date"),
"method": sol.get("method"),
"parameters": sol.get("parameters"),
"timing_seconds": sol.get("timing_seconds"),
"hardware": sol.get("hardware"),
"code_url": sol.get("code_url") or sub.get("code_url"),
"viz_url": sol.get("viz_url") or sub.get("viz_url"),
"fields_url": fields_url,
"more_accurate_than": [f"{sub['_key']}/{w}" for w in sol.get("more_accurate_than", [])],
"notes": sol.get("notes"),
"audio": audio_rel,
"peak": float(np.max(np.abs(signals[key]))),
}
keys = list(signals)
pairwise = {name: {a: {} for a in keys} for name in names}
for name in names:
params = challenge["metrics"][name]
for a in keys:
for r in keys:
pairwise[name][a][r] = 0.0 if a == r else compute_metric(name, signals[a], signals[r], params)
info = collect_order(submissions)
explicit = challenge.get("reference_solution")
if explicit is not None and explicit not in keys:
raise KeyError(f"reference_solution {explicit!r} is not among the solutions")
selection = select_reference(info, pairwise[primary], explicit)
ref = selection["reference"]
for key, row in rows.items():
row["errors"] = {name: pairwise[name][key][ref] for name in names} if ref else {}
row["is_reference"] = key == ref
violations = []
if ref:
for worse, better, _sid in info.pairs:
for name in names:
if not pairwise[name][better][ref] < pairwise[name][worse][ref]:
violations.append({"metric": name, "worse": worse, "better": better})
ordered = sorted(rows.values(), key=lambda r: (r["errors"].get(primary, float("inf")), r["key"]))
return {
"challenge_id": challenge["challenge_id"],
"title": challenge.get("title"),
"generated": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
"metrics": names,
"primary_metric": primary,
"reference": ref,
"reference_selection": selection,
"order_violations_vs_reference": violations,
"solutions": ordered,
"pairwise": pairwise,
"submissions": [
{**{k: v for k, v in public_fields(sub).items() if k != "solutions"}, "key": sub["_key"], "source": sub["_source"].label, "source_url": sub["_source"].url}
for sub in submissions
],
}
# ---------------------------------------------------------------- HTML
_CSS = """
:root { --fg: #1b1b1b; --bg: #ffffff; --muted: #666; --line: #ddd; --accent: #2a6db0; --ref: #e8f4e8; }
@media (prefers-color-scheme: dark) { :root { --fg: #e6e6e6; --bg: #151515; --muted: #999; --line: #333; --accent: #7ab3e8; --ref: #1f3320; } }
body { font-family: system-ui, sans-serif; color: var(--fg); background: var(--bg); margin: 0; padding: 16px; line-height: 1.45; }
main { max-width: 1300px; margin: 0 auto; }
a { color: var(--accent); }
table { border-collapse: collapse; width: 100%; font-size: 0.92em; }
th, td { border-bottom: 1px solid var(--line); padding: 6px 8px; text-align: left; vertical-align: top; }
th { position: sticky; top: 0; background: var(--bg); }
td.num { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; }
tr.ref { background: var(--ref); }
.muted { color: var(--muted); }
.wrap { overflow-x: auto; }
details { margin: 1em 0; }
code { font-size: 0.9em; }
"""
def _fmt(x, digits=4):
if x is None:
return ""
if isinstance(x, float):
if x == 0:
return "0"
return f"{x:.{digits}g}"
return html.escape(str(x))
def _link(url, text):
if not url:
return ""
return f'{html.escape(text)}'
def render_challenge_page(res: dict, challenge: dict, links: dict) -> str:
names = res["metrics"]
primary = res["primary_metric"]
head = "".join(f"
{html.escape(n)}{' (primary)' if n == primary else ''} | " for n in names)
body = []
for r in res["solutions"]:
errs = "".join(f'{_fmt(r["errors"].get(n))} | ' for n in names)
params = html.escape(json.dumps(r["parameters"])) if r["parameters"] else ""
mat = ", ".join(html.escape(k.rsplit("/", 1)[1]) for k in r["more_accurate_than"])
audio = f'' if r["audio"] else ""
linkbits = " ".join(x for x in [
_link(r["audio"], "wav"), _link(r["fields_url"], "fields"), _link(r["code_url"], "code"), _link(r["viz_url"], "viz"),
] if x)
sub_cell = _link(r["submission_url"], r["submission_id"]) if r["submission_url"] else html.escape(r["submission_id"])
src_cell = _link(r["source_url"], r["source"]) if r["source"] else ""
body.append(
f''
f'{sub_cell} {src_cell}{" ยท " if src_cell else ""}{html.escape(r["submitter"])} | '
f'{html.escape(r["solution_id"])}{" (reference)" if r["is_reference"] else ""} | '
f'{html.escape(r["method"] or "")}
{params} | '
f"{errs}"
f'{_fmt(r["timing_seconds"], 3)} | '
f'{html.escape(r["hardware"] or "")} | '
f'{mat} | '
f"{audio} {linkbits} | "
"
"
)
sel = res["reference_selection"]
cand_rows = "".join(
f"| {html.escape(c['candidate'])} | {c['pairs_preserved']}/{c['pairs_total']} | "
f"{_fmt(c['convergence_gap'])} | {_fmt(c['mean_error_to_other_candidates'])} | "
f"{html.escape(', '.join(f'{w} vs {b}' for w, b in c['violated_pairs']))} |
"
for c in sel.get("candidates", [])
)
viol = res["order_violations_vs_reference"]
viol_html = (
"All asserted accuracy orderings are preserved by both metrics under the chosen reference.
"
if not viol
else "" + "".join(
f"- {html.escape(v['metric'])}: {html.escape(v['better'])} asserted more accurate than {html.escape(v['worse'])}, but its error is not smaller
"
for v in viol
) + "
"
)
pair_tables = []
keys = [r["key"] for r in res["solutions"]]
for n in names:
hdr = "".join(f"{html.escape(k)} | " for k in keys)
rws = "".join(
f"| {html.escape(a)} | " + "".join(f"{_fmt(res['pairwise'][n][a][b], 3)} | " for b in keys) + "
"
for a in keys
)
pair_tables.append(f"{html.escape(n)}
")
spec_link = links.get("spec")
playback_gain = float((challenge.get("playback") or {}).get("gain_db", 20))
return f"""
{html.escape(res["challenge_id"])} results
All challenges
{html.escape(res["challenge_id"])}
{html.escape(res.get("title") or "")}. {"Challenge specification. " if spec_link else ""}results.json. Generated {html.escape(res["generated"])}.
Errors are relative to the reference solution {html.escape(res["reference"] or "none")}, chosen by rule {html.escape(sel["rule"])}. Rows are sorted by the primary metric. Submissions are named as repository/submission.
| Submission | Solution | Method | {head}Time (s) | Hardware | Claimed more accurate than | Output |
{"".join(body)}
Reference selection
Candidates are the maximal elements of each submission's asserted order. The score is the number of asserted pairs from other submissions whose order is preserved by the primary error to this candidate. Ties are broken first by the convergence gap (the primary error between the candidate and the nearest solution of its own submission asserted to be less accurate; smaller means a tighter convergence claim), then by the mean primary error to the other candidates.
| Candidate | Pairs preserved | Convergence gap | Mean error to other candidates | Violated pairs |
{cand_rows}
Order check against the chosen reference
{viol_html}
Pairwise errors (row vs column as reference)
{"".join(pair_tables)}
"""
def render_index(results: list[dict], sources: list[Source], links: dict) -> str:
items = "".join(
f'{html.escape(r["challenge_id"])}: {html.escape(r.get("title") or "")} '
f'({len(r["solutions"])} solutions, reference {html.escape(r["reference"] or "none")})'
for r in results
)
srcs = "".join(
f"{_link(s.url, s.repo or s.label or str(s.dir)) if s.url else html.escape(str(s.dir))}"
f"{' (' + html.escape(s.branch) + ')' if s.branch else ''}"
f"{': ' + html.escape(s.description) if s.description else ''}"
for s in sources
)
repo = links.get("challenge_repo")
return f"""
Challenge results
Challenge results
{"Specifications and registry: " + html.escape(repo) + "." if repo else ""}
Challenges
Registered solution repositories
"""
def build(challenges_dir: Path, sources: list[Source], out: Path, challenge_repo_url: str | None = None) -> list[dict]:
challenges = load_challenges(challenges_dir)
submissions = load_submissions(sources)
out.mkdir(parents=True, exist_ok=True)
results = []
for cid, ch in challenges.items():
subs = [s for s in submissions if s["challenge_id"] == cid]
cdir = out / cid
cdir.mkdir(parents=True, exist_ok=True)
res = score_challenge(ch, subs, cdir / "audio") if subs else {
"challenge_id": cid, "title": ch.get("title"), "generated": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
"metrics": metric_names(ch), "primary_metric": ch["metrics"]["primary"], "reference": None,
"reference_selection": {"rule": "none", "candidates": []}, "order_violations_vs_reference": [], "solutions": [], "pairwise": {}, "submissions": [],
}
with open(cdir / "results.json", "w") as f:
json.dump(res, f, indent=1)
links = {"spec": f"{challenge_repo_url}/blob/main/challenges/{cid}/README.md" if challenge_repo_url else None}
(cdir / "index.html").write_text(render_challenge_page(res, ch, links))
results.append(res)
(out / "index.html").write_text(render_index(results, sources, {"challenge_repo": challenge_repo_url}))
(out / "player.js").write_text(player_js())
return results
def player_js() -> str:
"""The Web Audio player shipped with every site (also copied by solution repositories' site builders)."""
from importlib import resources
return resources.files("challenge_tools").joinpath("player.js").read_text()