1"""Compute all scores and write the comparison site."""
2from __future__ import annotations
4import datetime as dt
5import html
6import json
7import shutil
8from pathlib import Path
10import numpy as np
12from .loading import Source, load_challenges, load_submissions, public_fields
13from .metrics import compute_metric, metric_names
14from .reference import collect_order, select_reference
15from .wav import read_wav, resolve_output
18def score_challenge(challenge: dict, submissions: list[dict], out_audio: Path | None = None) -> dict:
19 """Load every output for one challenge, compute pairwise errors, choose the reference, and return a results dict."""
20 spec = challenge["output"]
21 names = metric_names(challenge)
22 primary = challenge["metrics"]["primary"]
24 signals: dict[str, np.ndarray] = {}
25 rows: dict[str, dict] = {}
26 for sub in submissions:
27 src: Source = sub["_source"]
28 for sol in sub["solutions"]:
29 key = f"{sub['_key']}/{sol['solution_id']}"
30 p = resolve_output(sol["output"], sub["_path"].parent)
31 signals[key] = read_wav(p, spec)
32 audio_rel = None
33 if out_audio is not None:
34 dest = out_audio / sub["_key"] / f"{sol['solution_id']}.wav"
35 dest.parent.mkdir(parents=True, exist_ok=True)
36 shutil.copyfile(p, dest)
37 audio_rel = f"audio/{sub['_key']}/{sol['solution_id']}.wav"
38 fields_url = None
39 if "fields" in sol:
40 fpath = sol["fields"]["path"]
41 fields_url = fpath if fpath.startswith("http") else src.blob_url(sub["_path"].parent / fpath)
42 rows[key] = {
43 "key": key,
44 "source": src.label,
45 "source_url": src.url,
46 "submission_id": sub["submission_id"],
47 "submission_url": src.blob_url(sub["_path"]),
48 "solution_id": sol["solution_id"],
49 "submitter": sub["submitter"]["name"],
50 "date": sub.get("date"),
51 "method": sol.get("method"),
52 "parameters": sol.get("parameters"),
53 "timing_seconds": sol.get("timing_seconds"),
54 "hardware": sol.get("hardware"),
55 "code_url": sol.get("code_url") or sub.get("code_url"),
56 "viz_url": sol.get("viz_url") or sub.get("viz_url"),
57 "fields_url": fields_url,
58 "more_accurate_than": [f"{sub['_key']}/{w}" for w in sol.get("more_accurate_than", [])],
59 "notes": sol.get("notes"),
60 "audio": audio_rel,
61 "peak": float(np.max(np.abs(signals[key]))),
62 }
64 keys = list(signals)
65 pairwise = {name: {a: {} for a in keys} for name in names}
66 for name in names:
67 params = challenge["metrics"][name]
68 for a in keys:
69 for r in keys:
70 pairwise[name][a][r] = 0.0 if a == r else compute_metric(name, signals[a], signals[r], params)
72 info = collect_order(submissions)
73 explicit = challenge.get("reference_solution")
74 if explicit is not None and explicit not in keys:
75 raise KeyError(f"reference_solution {explicit!r} is not among the solutions")
76 selection = select_reference(info, pairwise[primary], explicit)
77 ref = selection["reference"]
78 for key, row in rows.items():
79 row["errors"] = {name: pairwise[name][key][ref] for name in names} if ref else {}
80 row["is_reference"] = key == ref
82 violations = []
83 if ref:
84 for worse, better, _sid in info.pairs:
85 for name in names:
86 if not pairwise[name][better][ref] < pairwise[name][worse][ref]:
87 violations.append({"metric": name, "worse": worse, "better": better})
89 ordered = sorted(rows.values(), key=lambda r: (r["errors"].get(primary, float("inf")), r["key"]))
90 return {
91 "challenge_id": challenge["challenge_id"],
92 "title": challenge.get("title"),
93 "generated": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
94 "metrics": names,
95 "primary_metric": primary,
96 "reference": ref,
97 "reference_selection": selection,
98 "order_violations_vs_reference": violations,
99 "solutions": ordered,
100 "pairwise": pairwise,
101 "submissions": [
102 {**{k: v for k, v in public_fields(sub).items() if k != "solutions"}, "key": sub["_key"], "source": sub["_source"].label, "source_url": sub["_source"].url}
103 for sub in submissions
104 ],
105 }
108# ---------------------------------------------------------------- HTML
110_CSS = """
111:root { --fg: #1b1b1b; --bg: #ffffff; --muted: #666; --line: #ddd; --accent: #2a6db0; --ref: #e8f4e8; }
112@media (prefers-color-scheme: dark) { :root { --fg: #e6e6e6; --bg: #151515; --muted: #999; --line: #333; --accent: #7ab3e8; --ref: #1f3320; } }
113body { font-family: system-ui, sans-serif; color: var(--fg); background: var(--bg); margin: 0; padding: 16px; line-height: 1.45; }
114main { max-width: 1300px; margin: 0 auto; }
115a { color: var(--accent); }
116table { border-collapse: collapse; width: 100%; font-size: 0.92em; }
117th, td { border-bottom: 1px solid var(--line); padding: 6px 8px; text-align: left; vertical-align: top; }
118th { position: sticky; top: 0; background: var(--bg); }
119td.num { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; }
120tr.ref { background: var(--ref); }
121.muted { color: var(--muted); }
122.wrap { overflow-x: auto; }
123details { margin: 1em 0; }
124code { font-size: 0.9em; }
125"""
128def _fmt(x, digits=4):
129 if x is None:
130 return ""
131 if isinstance(x, float):
132 if x == 0:
133 return "0"
134 return f"{x:.{digits}g}"
135 return html.escape(str(x))
138def _link(url, text):
139 if not url:
140 return ""
141 return f'<a href="{html.escape(url)}">{html.escape(text)}</a>'
144def render_challenge_page(res: dict, challenge: dict, links: dict) -> str:
145 names = res["metrics"]
146 primary = res["primary_metric"]
147 head = "".join(f"<th>{html.escape(n)}{' (primary)' if n == primary else ''}</th>" for n in names)
148 body = []
149 for r in res["solutions"]:
150 errs = "".join(f'<td class="num">{_fmt(r["errors"].get(n))}</td>' for n in names)
151 params = html.escape(json.dumps(r["parameters"])) if r["parameters"] else ""
152 mat = ", ".join(html.escape(k.rsplit("/", 1)[1]) for k in r["more_accurate_than"])
153 audio = f'<div class="cplayer" data-src="{html.escape(r["audio"])}"></div>' if r["audio"] else ""
154 linkbits = " ".join(x for x in [
155 _link(r["audio"], "wav"), _link(r["fields_url"], "fields"), _link(r["code_url"], "code"), _link(r["viz_url"], "viz"),
156 ] if x)
157 sub_cell = _link(r["submission_url"], r["submission_id"]) if r["submission_url"] else html.escape(r["submission_id"])
158 src_cell = _link(r["source_url"], r["source"]) if r["source"] else ""
159 body.append(
160 f'<tr class="{"ref" if r["is_reference"] else ""}">'
161 f'<td>{sub_cell}<br><span class="muted">{src_cell}{" · " if src_cell else ""}{html.escape(r["submitter"])}</span></td>'
162 f'<td>{html.escape(r["solution_id"])}{" <b>(reference)</b>" if r["is_reference"] else ""}</td>'
163 f'<td>{html.escape(r["method"] or "")}<br><span class="muted"><code>{params}</code></span></td>'
164 f"{errs}"
165 f'<td class="num">{_fmt(r["timing_seconds"], 3)}</td>'
166 f'<td>{html.escape(r["hardware"] or "")}</td>'
167 f'<td class="muted">{mat}</td>'
168 f"<td>{audio}<br>{linkbits}</td>"
169 "</tr>"
170 )
171 sel = res["reference_selection"]
172 cand_rows = "".join(
173 f"<tr><td>{html.escape(c['candidate'])}</td><td class='num'>{c['pairs_preserved']}/{c['pairs_total']}</td>"
174 f"<td class='num'>{_fmt(c['convergence_gap'])}</td><td class='num'>{_fmt(c['mean_error_to_other_candidates'])}</td>"
175 f"<td>{html.escape(', '.join(f'{w} vs {b}' for w, b in c['violated_pairs']))}</td></tr>"
176 for c in sel.get("candidates", [])
177 )
178 viol = res["order_violations_vs_reference"]
179 viol_html = (
180 "<p>All asserted accuracy orderings are preserved by both metrics under the chosen reference.</p>"
181 if not viol
182 else "<ul>" + "".join(
183 f"<li>{html.escape(v['metric'])}: {html.escape(v['better'])} asserted more accurate than {html.escape(v['worse'])}, but its error is not smaller</li>"
184 for v in viol
185 ) + "</ul>"
186 )
187 pair_tables = []
188 keys = [r["key"] for r in res["solutions"]]
189 for n in names:
190 hdr = "".join(f"<th>{html.escape(k)}</th>" for k in keys)
191 rws = "".join(
192 f"<tr><th>{html.escape(a)}</th>" + "".join(f"<td class='num'>{_fmt(res['pairwise'][n][a][b], 3)}</td>" for b in keys) + "</tr>"
193 for a in keys
194 )
195 pair_tables.append(f"<h4>{html.escape(n)}</h4><div class='wrap'><table><tr><th></th>{hdr}</tr>{rws}</table></div>")
196 spec_link = links.get("spec")
197 playback_gain = float((challenge.get("playback") or {}).get("gain_db", 20))
198 return f"""<!doctype html>
199<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
200<title>{html.escape(res["challenge_id"])} results</title><style>{_CSS}</style></head>
201<body><main>
202<p><a href="../">All challenges</a></p>
203<h1>{html.escape(res["challenge_id"])}</h1>
204<p>{html.escape(res.get("title") or "")}. {"<a href='" + html.escape(spec_link) + "'>Challenge specification</a>. " if spec_link else ""}<a href="results.json">results.json</a>. Generated {html.escape(res["generated"])}.</p>
205<p>Errors are relative to the reference solution <b>{html.escape(res["reference"] or "none")}</b>, chosen by rule <i>{html.escape(sel["rule"])}</i>. Rows are sorted by the primary metric. Submissions are named as <i>repository/submission</i>.</p>
206<div id="playback"></div>
207<div class="wrap"><table>
208<tr><th>Submission</th><th>Solution</th><th>Method</th>{head}<th>Time (s)</th><th>Hardware</th><th>Claimed more accurate than</th><th>Output</th></tr>
209{"".join(body)}
210</table></div>
211<details><summary>Reference selection</summary>
212<p>Candidates are the maximal elements of each submission's asserted order. The score is the number of asserted pairs from other submissions whose order is preserved by the primary error to this candidate. Ties are broken first by the convergence gap (the primary error between the candidate and the nearest solution of its own submission asserted to be less accurate; smaller means a tighter convergence claim), then by the mean primary error to the other candidates.</p>
213<div class="wrap"><table><tr><th>Candidate</th><th>Pairs preserved</th><th>Convergence gap</th><th>Mean error to other candidates</th><th>Violated pairs</th></tr>{cand_rows}</table></div>
214<h4>Order check against the chosen reference</h4>{viol_html}
215</details>
216<details><summary>Pairwise errors (row vs column as reference)</summary>{"".join(pair_tables)}</details>
217</main>
218<script src="../player.js?v={player_version()}"></script>
219<script>ChallengePlayer.controls(document.getElementById("playback"), {{gainDb: {playback_gain}, limiter: true}}); ChallengePlayer.mount();</script>
220</body></html>
221"""
224def render_index(results: list[dict], sources: list[Source], links: dict) -> str:
225 items = "".join(
226 f'<li><a href="{html.escape(r["challenge_id"])}/">{html.escape(r["challenge_id"])}</a>: {html.escape(r.get("title") or "")} '
227 f'<span class="muted">({len(r["solutions"])} solutions, reference {html.escape(r["reference"] or "none")})</span></li>'
228 for r in results
229 )
230 srcs = "".join(
231 f"<li>{_link(s.url, s.repo or s.label or str(s.dir)) if s.url else html.escape(str(s.dir))}"
232 f"{' <span class=muted>(' + html.escape(s.branch) + ')</span>' if s.branch else ''}"
233 f"{': ' + html.escape(s.description) if s.description else ''}</li>"
234 for s in sources
235 )
236 repo = links.get("challenge_repo")
237 return f"""<!doctype html>
238<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
239<title>Challenge results</title><style>{_CSS}</style></head>
240<body><main>
241<h1>Challenge results</h1>
242<p>{"Specifications and registry: <a href='" + html.escape(repo) + "'>" + html.escape(repo) + "</a>." if repo else ""}</p>
243<h2>Challenges</h2><ul>{items}</ul>
244<h2>Registered solution repositories</h2><ul>{srcs}</ul>
245</main></body></html>
246"""
249def build(challenges_dir: Path, sources: list[Source], out: Path, challenge_repo_url: str | None = None) -> list[dict]:
250 challenges = load_challenges(challenges_dir)
251 submissions = load_submissions(sources)
252 out.mkdir(parents=True, exist_ok=True)
253 results = []
254 for cid, ch in challenges.items():
255 subs = [s for s in submissions if s["challenge_id"] == cid]
256 cdir = out / cid
257 cdir.mkdir(parents=True, exist_ok=True)
258 res = score_challenge(ch, subs, cdir / "audio") if subs else {
259 "challenge_id": cid, "title": ch.get("title"), "generated": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
260 "metrics": metric_names(ch), "primary_metric": ch["metrics"]["primary"], "reference": None,
261 "reference_selection": {"rule": "none", "candidates": []}, "order_violations_vs_reference": [], "solutions": [], "pairwise": {}, "submissions": [],
262 }
263 with open(cdir / "results.json", "w") as f:
264 json.dump(res, f, indent=1)
265 links = {"spec": f"{challenge_repo_url}/blob/main/challenges/{cid}/README.md" if challenge_repo_url else None}
266 (cdir / "index.html").write_text(render_challenge_page(res, ch, links))
267 results.append(res)
268 (out / "index.html").write_text(render_index(results, sources, {"challenge_repo": challenge_repo_url}))
269 (out / "player.js").write_text(player_js())
270 return results
273def player_version() -> str:
274 """Short hash of the player script, used as a cache-busting query on the script tag."""
275 import hashlib
277 return hashlib.sha1(player_js().encode()).hexdigest()[:10]
280def player_js() -> str:
281 """The Web Audio player shipped with every site (also copied by solution repositories' site builders)."""
282 from importlib import resources
284 return resources.files("challenge_tools").joinpath("player.js").read_text()