1"""Compute all scores and write the comparison site."""
2from __future__ import annotations
4import datetime as dt
5import html
6import json
7import shutil
8from pathlib import Path
10import numpy as np
12from .loading import Source, load_challenges, load_submissions, public_fields
13from .metrics import compute_metric, metric_names
14from .reference import collect_order, select_reference
15from .wav import read_wav, resolve_output
18def score_challenge(challenge: dict, submissions: list[dict], out_audio: Path | None = None) -> dict:
19 """Load every output for one challenge, compute pairwise errors, choose the reference, and return a results dict."""
20 spec = challenge["output"]
21 names = metric_names(challenge)
22 primary = challenge["metrics"]["primary"]
24 signals: dict[str, np.ndarray] = {}
25 rows: dict[str, dict] = {}
26 for sub in submissions:
27 src: Source = sub["_source"]
28 for sol in sub["solutions"]:
29 key = f"{sub['_key']}/{sol['solution_id']}"
30 p = resolve_output(sol["output"], sub["_path"].parent)
31 signals[key] = read_wav(p, spec)
32 audio_rel = None
33 if out_audio is not None:
34 dest = out_audio / sub["_key"] / f"{sol['solution_id']}.wav"
35 dest.parent.mkdir(parents=True, exist_ok=True)
36 shutil.copyfile(p, dest)
37 audio_rel = f"audio/{sub['_key']}/{sol['solution_id']}.wav"
38 fields_url = None
39 if "fields" in sol:
40 fpath = sol["fields"]["path"]
41 fields_url = fpath if fpath.startswith("http") else src.blob_url(sub["_path"].parent / fpath)
42 rows[key] = {
43 "key": key,
44 "source": src.label,
45 "source_url": src.url,
46 "submission_id": sub["submission_id"],
47 "submission_url": src.blob_url(sub["_path"]),
48 "solution_id": sol["solution_id"],
49 "submitter": sub["submitter"]["name"],
50 "date": sub.get("date"),
51 "method": sol.get("method"),
52 "parameters": sol.get("parameters"),
53 "timing_seconds": sol.get("timing_seconds"),
54 "hardware": sol.get("hardware"),
55 "code_url": sol.get("code_url") or sub.get("code_url"),
56 "viz_url": sol.get("viz_url") or sub.get("viz_url"),
57 "fields_url": fields_url,
58 "more_accurate_than": [f"{sub['_key']}/{w}" for w in sol.get("more_accurate_than", [])],
59 "notes": sol.get("notes"),
60 "audio": audio_rel,
61 "peak": float(np.max(np.abs(signals[key]))),
62 }
64 keys = list(signals)
65 pairwise = {name: {a: {} for a in keys} for name in names}
66 for name in names:
67 params = challenge["metrics"][name]
68 for a in keys:
69 for r in keys:
70 pairwise[name][a][r] = 0.0 if a == r else compute_metric(name, signals[a], signals[r], params)
72 info = collect_order(submissions)
73 explicit = challenge.get("reference_solution")
74 if explicit is not None and explicit not in keys:
75 raise KeyError(f"reference_solution {explicit!r} is not among the solutions")
76 selection = select_reference(info, pairwise[primary], explicit)
77 ref = selection["reference"]
78 for key, row in rows.items():
79 row["errors"] = {name: pairwise[name][key][ref] for name in names} if ref else {}
80 row["is_reference"] = key == ref
82 violations = []
83 if ref:
84 for worse, better, _sid in info.pairs:
85 for name in names:
86 if not pairwise[name][better][ref] < pairwise[name][worse][ref]:
87 violations.append({"metric": name, "worse": worse, "better": better})
89 ordered = sorted(rows.values(), key=lambda r: (r["errors"].get(primary, float("inf")), r["key"]))
90 return {
91 "challenge_id": challenge["challenge_id"],
92 "title": challenge.get("title"),
93 "generated": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
94 "metrics": names,
95 "primary_metric": primary,
96 "reference": ref,
97 "reference_selection": selection,
98 "order_violations_vs_reference": violations,
99 "solutions": ordered,
100 "pairwise": pairwise,
101 "submissions": [
102 {**{k: v for k, v in public_fields(sub).items() if k != "solutions"}, "key": sub["_key"], "source": sub["_source"].label, "source_url": sub["_source"].url}
103 for sub in submissions
104 ],
105 }
108# ---------------------------------------------------------------- HTML
110_CSS = """
111:root { --fg: #1b1b1b; --bg: #ffffff; --muted: #666; --line: #ddd; --accent: #2a6db0; --ref: #e8f4e8; }
112@media (prefers-color-scheme: dark) { :root { --fg: #e6e6e6; --bg: #151515; --muted: #999; --line: #333; --accent: #7ab3e8; --ref: #1f3320; } }
113body { font-family: system-ui, sans-serif; color: var(--fg); background: var(--bg); margin: 0; padding: 16px; line-height: 1.45; }
114main { max-width: 1300px; margin: 0 auto; }
115a { color: var(--accent); }
116table { border-collapse: collapse; width: 100%; font-size: 0.92em; }
117th, td { border-bottom: 1px solid var(--line); padding: 6px 8px; text-align: left; vertical-align: top; }
118th { position: sticky; top: 0; background: var(--bg); }
119td.num { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; }
120tr.ref { background: var(--ref); }
121.muted { color: var(--muted); }
122.wrap { overflow-x: auto; }
123audio { width: 180px; height: 28px; }
124details { margin: 1em 0; }
125code { font-size: 0.9em; }
126"""
129def _fmt(x, digits=4):
130 if x is None:
131 return ""
132 if isinstance(x, float):
133 if x == 0:
134 return "0"
135 return f"{x:.{digits}g}"
136 return html.escape(str(x))
139def _link(url, text):
140 if not url:
141 return ""
142 return f'<a href="{html.escape(url)}">{html.escape(text)}</a>'
145def render_challenge_page(res: dict, challenge: dict, links: dict) -> str:
146 names = res["metrics"]
147 primary = res["primary_metric"]
148 head = "".join(f"<th>{html.escape(n)}{' (primary)' if n == primary else ''}</th>" for n in names)
149 body = []
150 for r in res["solutions"]:
151 errs = "".join(f'<td class="num">{_fmt(r["errors"].get(n))}</td>' for n in names)
152 params = html.escape(json.dumps(r["parameters"])) if r["parameters"] else ""
153 mat = ", ".join(html.escape(k.rsplit("/", 1)[1]) for k in r["more_accurate_than"])
154 audio = f'<audio controls preload="none" src="{html.escape(r["audio"])}"></audio>' if r["audio"] else ""
155 linkbits = " ".join(x for x in [
156 _link(r["audio"], "wav"), _link(r["fields_url"], "fields"), _link(r["code_url"], "code"), _link(r["viz_url"], "viz"),
157 ] if x)
158 sub_cell = _link(r["submission_url"], r["submission_id"]) if r["submission_url"] else html.escape(r["submission_id"])
159 src_cell = _link(r["source_url"], r["source"]) if r["source"] else ""
160 body.append(
161 f'<tr class="{"ref" if r["is_reference"] else ""}">'
162 f'<td>{sub_cell}<br><span class="muted">{src_cell}{" · " if src_cell else ""}{html.escape(r["submitter"])}</span></td>'
163 f'<td>{html.escape(r["solution_id"])}{" <b>(reference)</b>" if r["is_reference"] else ""}</td>'
164 f'<td>{html.escape(r["method"] or "")}<br><span class="muted"><code>{params}</code></span></td>'
165 f"{errs}"
166 f'<td class="num">{_fmt(r["timing_seconds"], 3)}</td>'
167 f'<td>{html.escape(r["hardware"] or "")}</td>'
168 f'<td class="muted">{mat}</td>'
169 f"<td>{audio}<br>{linkbits}</td>"
170 "</tr>"
171 )
172 sel = res["reference_selection"]
173 cand_rows = "".join(
174 f"<tr><td>{html.escape(c['candidate'])}</td><td class='num'>{c['pairs_preserved']}/{c['pairs_total']}</td>"
175 f"<td class='num'>{_fmt(c['convergence_gap'])}</td><td class='num'>{_fmt(c['mean_error_to_other_candidates'])}</td>"
176 f"<td>{html.escape(', '.join(f'{w} vs {b}' for w, b in c['violated_pairs']))}</td></tr>"
177 for c in sel.get("candidates", [])
178 )
179 viol = res["order_violations_vs_reference"]
180 viol_html = (
181 "<p>All asserted accuracy orderings are preserved by both metrics under the chosen reference.</p>"
182 if not viol
183 else "<ul>" + "".join(
184 f"<li>{html.escape(v['metric'])}: {html.escape(v['better'])} asserted more accurate than {html.escape(v['worse'])}, but its error is not smaller</li>"
185 for v in viol
186 ) + "</ul>"
187 )
188 pair_tables = []
189 keys = [r["key"] for r in res["solutions"]]
190 for n in names:
191 hdr = "".join(f"<th>{html.escape(k)}</th>" for k in keys)
192 rws = "".join(
193 f"<tr><th>{html.escape(a)}</th>" + "".join(f"<td class='num'>{_fmt(res['pairwise'][n][a][b], 3)}</td>" for b in keys) + "</tr>"
194 for a in keys
195 )
196 pair_tables.append(f"<h4>{html.escape(n)}</h4><div class='wrap'><table><tr><th></th>{hdr}</tr>{rws}</table></div>")
197 spec_link = links.get("spec")
198 return f"""<!doctype html>
199<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
200<title>{html.escape(res["challenge_id"])} results</title><style>{_CSS}</style></head>
201<body><main>
202<p><a href="../">All challenges</a></p>
203<h1>{html.escape(res["challenge_id"])}</h1>
204<p>{html.escape(res.get("title") or "")}. {"<a href='" + html.escape(spec_link) + "'>Challenge specification</a>. " if spec_link else ""}<a href="results.json">results.json</a>. Generated {html.escape(res["generated"])}.</p>
205<p>Errors are relative to the reference solution <b>{html.escape(res["reference"] or "none")}</b>, chosen by rule <i>{html.escape(sel["rule"])}</i>. Rows are sorted by the primary metric. Submissions are named as <i>repository/submission</i>.</p>
206<div class="wrap"><table>
207<tr><th>Submission</th><th>Solution</th><th>Method</th>{head}<th>Time (s)</th><th>Hardware</th><th>Claimed more accurate than</th><th>Output</th></tr>
208{"".join(body)}
209</table></div>
210<details><summary>Reference selection</summary>
211<p>Candidates are the maximal elements of each submission's asserted order. The score is the number of asserted pairs from other submissions whose order is preserved by the primary error to this candidate. Ties are broken first by the convergence gap (the primary error between the candidate and the nearest solution of its own submission asserted to be less accurate; smaller means a tighter convergence claim), then by the mean primary error to the other candidates.</p>
212<div class="wrap"><table><tr><th>Candidate</th><th>Pairs preserved</th><th>Convergence gap</th><th>Mean error to other candidates</th><th>Violated pairs</th></tr>{cand_rows}</table></div>
213<h4>Order check against the chosen reference</h4>{viol_html}
214</details>
215<details><summary>Pairwise errors (row vs column as reference)</summary>{"".join(pair_tables)}</details>
216</main></body></html>
217"""
220def render_index(results: list[dict], sources: list[Source], links: dict) -> str:
221 items = "".join(
222 f'<li><a href="{html.escape(r["challenge_id"])}/">{html.escape(r["challenge_id"])}</a>: {html.escape(r.get("title") or "")} '
223 f'<span class="muted">({len(r["solutions"])} solutions, reference {html.escape(r["reference"] or "none")})</span></li>'
224 for r in results
225 )
226 srcs = "".join(
227 f"<li>{_link(s.url, s.repo or s.label or str(s.dir)) if s.url else html.escape(str(s.dir))}"
228 f"{' <span class=muted>(' + html.escape(s.branch) + ')</span>' if s.branch else ''}"
229 f"{': ' + html.escape(s.description) if s.description else ''}</li>"
230 for s in sources
231 )
232 repo = links.get("challenge_repo")
233 return f"""<!doctype html>
234<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
235<title>Challenge results</title><style>{_CSS}</style></head>
236<body><main>
237<h1>Challenge results</h1>
238<p>{"Specifications and registry: <a href='" + html.escape(repo) + "'>" + html.escape(repo) + "</a>." if repo else ""}</p>
239<h2>Challenges</h2><ul>{items}</ul>
240<h2>Registered solution repositories</h2><ul>{srcs}</ul>
241</main></body></html>
242"""
245def build(challenges_dir: Path, sources: list[Source], out: Path, challenge_repo_url: str | None = None) -> list[dict]:
246 challenges = load_challenges(challenges_dir)
247 submissions = load_submissions(sources)
248 out.mkdir(parents=True, exist_ok=True)
249 results = []
250 for cid, ch in challenges.items():
251 subs = [s for s in submissions if s["challenge_id"] == cid]
252 cdir = out / cid
253 cdir.mkdir(parents=True, exist_ok=True)
254 res = score_challenge(ch, subs, cdir / "audio") if subs else {
255 "challenge_id": cid, "title": ch.get("title"), "generated": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
256 "metrics": metric_names(ch), "primary_metric": ch["metrics"]["primary"], "reference": None,
257 "reference_selection": {"rule": "none", "candidates": []}, "order_violations_vs_reference": [], "solutions": [], "pairwise": {}, "submissions": [],
258 }
259 with open(cdir / "results.json", "w") as f:
260 json.dump(res, f, indent=1)
261 links = {"spec": f"{challenge_repo_url}/blob/main/challenges/{cid}/README.md" if challenge_repo_url else None}
262 (cdir / "index.html").write_text(render_challenge_page(res, ch, links))
263 results.append(res)
264 (out / "index.html").write_text(render_index(results, sources, {"challenge_repo": challenge_repo_url}))
265 return results