"""Compute all scores and write the results site."""
from __future__ import annotations
import datetime as dt
import html
import json
import shutil
from pathlib import Path
import numpy as np
from .loading import load_challenges, load_submissions
from .metrics import compute_metric, metric_names
from .reference import collect_order, select_reference
from .wav import read_wav, resolve_output
def score_challenge(challenge: dict, submissions: list[dict], out_audio: Path | None = None) -> dict:
"""Load every output for one challenge, compute pairwise errors, choose the reference, and return a results dict."""
spec = challenge["output"]
names = metric_names(challenge)
primary = challenge["metrics"]["primary"]
signals: dict[str, np.ndarray] = {}
rows: dict[str, dict] = {}
for sub in submissions:
for sol in sub["solutions"]:
key = f"{sub['submission_id']}/{sol['solution_id']}"
p = resolve_output(sol["output"], sub["_path"].parent)
signals[key] = read_wav(p, spec)
audio_rel = None
if out_audio is not None:
dest = out_audio / sub["submission_id"] / f"{sol['solution_id']}.wav"
dest.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(p, dest)
audio_rel = f"audio/{sub['submission_id']}/{sol['solution_id']}.wav"
rows[key] = {
"key": key,
"submission_id": sub["submission_id"],
"solution_id": sol["solution_id"],
"submitter": sub["submitter"]["name"],
"date": sub.get("date"),
"method": sol.get("method"),
"parameters": sol.get("parameters"),
"timing_seconds": sol.get("timing_seconds"),
"hardware": sol.get("hardware"),
"code_url": sol.get("code_url") or sub.get("code_url"),
"viz_url": sol.get("viz_url") or sub.get("viz_url"),
"more_accurate_than": [f"{sub['submission_id']}/{w}" for w in sol.get("more_accurate_than", [])],
"notes": sol.get("notes"),
"audio": audio_rel,
"peak": float(np.max(np.abs(signals[key]))),
}
keys = list(signals)
pairwise = {name: {a: {} for a in keys} for name in names}
for name in names:
params = challenge["metrics"][name]
for a in keys:
for r in keys:
pairwise[name][a][r] = 0.0 if a == r else compute_metric(name, signals[a], signals[r], params)
info = collect_order(submissions)
explicit = challenge.get("reference_solution")
if explicit is not None and explicit not in keys:
raise KeyError(f"reference_solution {explicit!r} is not among the solutions")
selection = select_reference(info, pairwise[primary], explicit)
ref = selection["reference"]
for key, row in rows.items():
row["errors"] = {name: pairwise[name][key][ref] for name in names} if ref else {}
row["is_reference"] = key == ref
# Check every asserted pair against the chosen reference, for the report.
violations = []
if ref:
for worse, better, _sid in info.pairs:
for name in names:
if not pairwise[name][better][ref] < pairwise[name][worse][ref]:
violations.append({"metric": name, "worse": worse, "better": better})
ordered = sorted(rows.values(), key=lambda r: (r["errors"].get(primary, float("inf")), r["key"]))
return {
"challenge_id": challenge["challenge_id"],
"title": challenge.get("title"),
"generated": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
"metrics": names,
"primary_metric": primary,
"reference": ref,
"reference_selection": selection,
"order_violations_vs_reference": violations,
"solutions": ordered,
"pairwise": pairwise,
"submissions": [
{k: v for k, v in sub.items() if not k.startswith("_") and k != "solutions"} for sub in submissions
],
}
# ---------------------------------------------------------------- HTML
_CSS = """
:root { --fg: #1b1b1b; --bg: #ffffff; --muted: #666; --line: #ddd; --accent: #2a6db0; --ref: #e8f4e8; }
@media (prefers-color-scheme: dark) { :root { --fg: #e6e6e6; --bg: #151515; --muted: #999; --line: #333; --accent: #7ab3e8; --ref: #1f3320; } }
body { font-family: system-ui, sans-serif; color: var(--fg); background: var(--bg); margin: 0; padding: 16px; line-height: 1.45; }
main { max-width: 1200px; margin: 0 auto; }
a { color: var(--accent); }
table { border-collapse: collapse; width: 100%; font-size: 0.92em; }
th, td { border-bottom: 1px solid var(--line); padding: 6px 8px; text-align: left; vertical-align: top; }
th { position: sticky; top: 0; background: var(--bg); }
td.num { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; }
tr.ref { background: var(--ref); }
.muted { color: var(--muted); }
.wrap { overflow-x: auto; }
audio { width: 180px; height: 28px; }
details { margin: 1em 0; }
code { font-size: 0.9em; }
"""
def _fmt(x, digits=4):
if x is None:
return ""
if isinstance(x, float):
if x == 0:
return "0"
return f"{x:.{digits}g}"
return html.escape(str(x))
def _link(url, text):
if not url:
return ""
return f'{html.escape(text)}'
def render_challenge_page(res: dict, challenge: dict, links: dict) -> str:
names = res["metrics"]
primary = res["primary_metric"]
head = "".join(
f"
{html.escape(n)}{' (primary)' if n == primary else ''}
" for n in names
)
body = []
for r in res["solutions"]:
errs = "".join(f'
{_fmt(r["errors"].get(n))}
' for n in names)
params = html.escape(json.dumps(r["parameters"])) if r["parameters"] else ""
mat = ", ".join(html.escape(k.split("/", 1)[1]) for k in r["more_accurate_than"])
audio = f'' if r["audio"] else ""
linkbits = " ".join(x for x in [_link(r["code_url"], "code"), _link(r["viz_url"], "viz"), _link(r["audio"], "wav")] if x)
body.append(
f'
{html.escape(res.get("title") or "")}. {"Challenge specification. " if spec_link else ""}results.json. Generated {html.escape(res["generated"])}.
Errors are relative to the reference solution {html.escape(res["reference"] or "none")}, chosen by rule {html.escape(sel["rule"])}. Rows are sorted by the primary metric.
Submission
Solution
Method
{head}
Time (s)
Hardware
Claimed more accurate than
Output
{"".join(body)}
Reference selection
Candidates are the maximal elements of each submission's asserted order. The score is the number of asserted pairs from other submissions whose order is preserved by the primary error to this candidate. Ties are broken by the mean primary error to the other candidates.
Candidate
Pairs preserved
Mean error to other candidates
Violated pairs
{cand_rows}
Order check against the chosen reference
{viol_html}
Pairwise errors (row vs column as reference){"".join(pair_tables)}
"""
def render_index(results: list[dict], links: dict) -> str:
items = "".join(
f'
{html.escape(r["challenge_id"])}: {html.escape(r.get("title") or "")} '
f'({len(r["solutions"])} solutions, reference {html.escape(r["reference"] or "none")})
'
for r in results
)
repo = links.get("challenge_repo")
sols = links.get("solutions_repo")
return f"""
Challenge results