1"""Compute all scores and write the comparison site."""
2from __future__ import annotations
4import datetime as dt
5import html
6import json
7import shutil
8from pathlib import Path
10import numpy as np
12from .loading import Source, load_challenges, load_submissions, public_fields
13from .metrics import compute_metric, metric_names
14from .reference import collect_order, select_reference
15from .wav import read_wav, resolve_output
18def score_challenge(challenge: dict, submissions: list[dict], out_audio: Path | None = None) -> dict:
19 """Load every output for one challenge, compute pairwise errors, choose the reference, and return a results dict."""
20 spec = challenge["output"]
21 names = metric_names(challenge)
22 primary = challenge["metrics"]["primary"]
24 signals: dict[str, np.ndarray] = {}
25 rows: dict[str, dict] = {}
26 for sub in submissions:
27 src: Source = sub["_source"]
28 for sol in sub["solutions"]:
29 key = f"{sub['_key']}/{sol['solution_id']}"
30 p = resolve_output(sol["output"], sub["_path"].parent)
31 signals[key] = read_wav(p, spec)
32 audio_rel = None
33 if out_audio is not None:
34 dest = out_audio / sub["_key"] / f"{sol['solution_id']}.wav"
35 dest.parent.mkdir(parents=True, exist_ok=True)
36 shutil.copyfile(p, dest)
37 audio_rel = f"audio/{sub['_key']}/{sol['solution_id']}.wav"
38 fields_url = None
39 if "fields" in sol:
40 fpath = sol["fields"]["path"]
41 fields_url = fpath if fpath.startswith("http") else src.blob_url(sub["_path"].parent / fpath)
42 rows[key] = {
43 "key": key,
44 "source": src.label,
45 "source_url": src.url,
46 "submission_id": sub["submission_id"],
47 "submission_url": src.blob_url(sub["_path"]),
48 "solution_id": sol["solution_id"],
49 "submitter": sub["submitter"]["name"],
50 "date": sub.get("date"),
51 "method": sol.get("method"),
52 "parameters": sol.get("parameters"),
53 "timing_seconds": sol.get("timing_seconds"),
54 "hardware": sol.get("hardware"),
55 "code_url": sol.get("code_url") or sub.get("code_url"),
56 "viz_url": sol.get("viz_url") or sub.get("viz_url"),
57 "fields_url": fields_url,
58 "more_accurate_than": [f"{sub['_key']}/{w}" for w in sol.get("more_accurate_than", [])],
59 "notes": sol.get("notes"),
60 "audio": audio_rel,
61 "peak": float(np.max(np.abs(signals[key]))),
62 }
64 keys = list(signals)
65 pairwise = {name: {a: {} for a in keys} for name in names}
66 for name in names:
67 params = challenge["metrics"][name]
68 for a in keys:
69 for r in keys:
70 pairwise[name][a][r] = 0.0 if a == r else compute_metric(name, signals[a], signals[r], params)
72 info = collect_order(submissions)
73 explicit = challenge.get("reference_solution")
74 if explicit is not None and explicit not in keys:
75 raise KeyError(f"reference_solution {explicit!r} is not among the solutions")
76 selection = select_reference(info, pairwise[primary], explicit)
77 ref = selection["reference"]
78 for key, row in rows.items():
79 row["errors"] = {name: pairwise[name][key][ref] for name in names} if ref else {}
80 row["is_reference"] = key == ref
82 violations = []
83 if ref:
84 for worse, better, _sid in info.pairs:
85 for name in names:
86 if not pairwise[name][better][ref] < pairwise[name][worse][ref]:
87 violations.append({"metric": name, "worse": worse, "better": better})
89 ordered = sorted(rows.values(), key=lambda r: (r["errors"].get(primary, float("inf")), r["key"]))
90 return {
91 "challenge_id": challenge["challenge_id"],
92 "title": challenge.get("title"),
93 "generated": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
94 "metrics": names,
95 "primary_metric": primary,
96 "reference": ref,
97 "reference_selection": selection,
98 "order_violations_vs_reference": violations,
99 "solutions": ordered,
100 "pairwise": pairwise,
101 "submissions": [
102 {**{k: v for k, v in public_fields(sub).items() if k != "solutions"}, "key": sub["_key"], "source": sub["_source"].label, "source_url": sub["_source"].url}
103 for sub in submissions
104 ],
105 }
108# ---------------------------------------------------------------- HTML
110_CSS = """
111:root { --fg: #1b1b1b; --bg: #ffffff; --muted: #666; --line: #ddd; --accent: #2a6db0; --ref: #e8f4e8; }
112@media (prefers-color-scheme: dark) { :root { --fg: #e6e6e6; --bg: #151515; --muted: #999; --line: #333; --accent: #7ab3e8; --ref: #1f3320; } }
113body { font-family: system-ui, sans-serif; color: var(--fg); background: var(--bg); margin: 0; padding: 16px; line-height: 1.45; }
114main { max-width: 1300px; margin: 0 auto; }
115a { color: var(--accent); }
116table { border-collapse: collapse; width: 100%; font-size: 0.92em; }
117th, td { border-bottom: 1px solid var(--line); padding: 6px 8px; text-align: left; vertical-align: top; }
118th { position: sticky; top: 0; background: var(--bg); }
119td.num { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; }
120tr.ref { background: var(--ref); }
121.muted { color: var(--muted); }
122.wrap { overflow-x: auto; }
123details { margin: 1em 0; }
124code { font-size: 0.9em; }
125"""
128def _fmt(x, digits=4):
129 if x is None:
130 return ""
131 if isinstance(x, float):
132 if x == 0:
133 return "0"
134 return f"{x:.{digits}g}"
135 return html.escape(str(x))
138def _link(url, text):
139 if not url:
140 return ""
141 return f'<a href="{html.escape(url)}">{html.escape(text)}</a>'
144def render_challenge_page(res: dict, challenge: dict, links: dict, scene: bool = False) -> str:
145 names = res["metrics"]
146 primary = res["primary_metric"]
147 head = "".join(f"<th>{html.escape(n)}{' (primary)' if n == primary else ''}</th>" for n in names)
148 body = []
149 for r in res["solutions"]:
150 errs = "".join(f'<td class="num">{_fmt(r["errors"].get(n))}</td>' for n in names)
151 params = html.escape(json.dumps(r["parameters"])) if r["parameters"] else ""
152 mat = ", ".join(html.escape(k.rsplit("/", 1)[1]) for k in r["more_accurate_than"])
153 audio = f'<div class="cplayer" data-src="{html.escape(r["audio"])}"></div>' if r["audio"] else ""
154 linkbits = " ".join(x for x in [
155 _link(r["audio"], "wav"), _link(r["fields_url"], "fields"), _link(r["code_url"], "code"), _link(r["viz_url"], "viz"),
156 ] if x)
157 sub_cell = _link(r["submission_url"], r["submission_id"]) if r["submission_url"] else html.escape(r["submission_id"])
158 src_cell = _link(r["source_url"], r["source"]) if r["source"] else ""
159 body.append(
160 f'<tr class="{"ref" if r["is_reference"] else ""}">'
161 f'<td>{sub_cell}<br><span class="muted">{src_cell}{" · " if src_cell else ""}{html.escape(r["submitter"])}</span></td>'
162 f'<td>{html.escape(r["solution_id"])}{" <b>(reference)</b>" if r["is_reference"] else ""}</td>'
163 f'<td>{html.escape(r["method"] or "")}<br><span class="muted"><code>{params}</code></span></td>'
164 f"{errs}"
165 f'<td class="num">{_fmt(r["timing_seconds"], 3)}</td>'
166 f'<td>{html.escape(r["hardware"] or "")}</td>'
167 f'<td class="muted">{mat}</td>'
168 f"<td>{audio}<br>{linkbits}</td>"
169 "</tr>"
170 )
171 sel = res["reference_selection"]
172 cand_rows = "".join(
173 f"<tr><td>{html.escape(c['candidate'])}</td><td class='num'>{c['pairs_preserved']}/{c['pairs_total']}</td>"
174 f"<td class='num'>{_fmt(c['convergence_gap'])}</td><td class='num'>{_fmt(c['mean_error_to_other_candidates'])}</td>"
175 f"<td>{html.escape(', '.join(f'{w} vs {b}' for w, b in c['violated_pairs']))}</td></tr>"
176 for c in sel.get("candidates", [])
177 )
178 viol = res["order_violations_vs_reference"]
179 viol_html = (
180 "<p>All asserted accuracy orderings are preserved by both metrics under the chosen reference.</p>"
181 if not viol
182 else "<ul>" + "".join(
183 f"<li>{html.escape(v['metric'])}: {html.escape(v['better'])} asserted more accurate than {html.escape(v['worse'])}, but its error is not smaller</li>"
184 for v in viol
185 ) + "</ul>"
186 )
187 pair_tables = []
188 keys = [r["key"] for r in res["solutions"]]
189 for n in names:
190 hdr = "".join(f"<th>{html.escape(k)}</th>" for k in keys)
191 rws = "".join(
192 f"<tr><th>{html.escape(a)}</th>" + "".join(f"<td class='num'>{_fmt(res['pairwise'][n][a][b], 3)}</td>" for b in keys) + "</tr>"
193 for a in keys
194 )
195 pair_tables.append(f"<h4>{html.escape(n)}</h4><div class='wrap'><table><tr><th></th>{hdr}</tr>{rws}</table></div>")
196 spec_link = links.get("spec")
197 playback_gain = float((challenge.get("playback") or {}).get("gain_db", 20))
198 return f"""<!doctype html>
199<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
200<title>{html.escape(res["challenge_id"])} results</title><style>{_CSS}</style></head>
201<body><main>
202<p><a href="../">All challenges</a></p>
203<h1>{html.escape(res["challenge_id"])}</h1>
204<p>{html.escape(res.get("title") or "")}. {"<a href='" + html.escape(spec_link) + "'>Challenge specification</a>. " if spec_link else ""}<a href="results.json">results.json</a>. Generated {html.escape(res["generated"])}.</p>
205<p>Errors are relative to the reference solution <b>{html.escape(res["reference"] or "none")}</b>, chosen by rule <i>{html.escape(sel["rule"])}</i>. Rows are sorted by the primary metric. Submissions are named as <i>repository/submission</i>.</p>
206{"<h3 id='geometry'>Geometry</h3><div class='cscene' data-src='scene.json'></div>" if scene else ""}
207<div id="playback"></div>
208<div class="wrap"><table>
209<tr><th>Submission</th><th>Solution</th><th>Method</th>{head}<th>Time (s)</th><th>Hardware</th><th>Claimed more accurate than</th><th>Output</th></tr>
210{"".join(body)}
211</table></div>
212<details><summary>Reference selection</summary>
213<p>Candidates are the maximal elements of each submission's asserted order. The score is the number of asserted pairs from other submissions whose order is preserved by the primary error to this candidate. Ties are broken first by the convergence gap (the primary error between the candidate and the nearest solution of its own submission asserted to be less accurate; smaller means a tighter convergence claim), then by the mean primary error to the other candidates.</p>
214<div class="wrap"><table><tr><th>Candidate</th><th>Pairs preserved</th><th>Convergence gap</th><th>Mean error to other candidates</th><th>Violated pairs</th></tr>{cand_rows}</table></div>
215<h4>Order check against the chosen reference</h4>{viol_html}
216</details>
217<details><summary>Pairwise errors (row vs column as reference)</summary>{"".join(pair_tables)}</details>
218</main>
219<script src="../player.js?v={player_version()}"></script>
220<script>ChallengePlayer.controls(document.getElementById("playback"), {{gainDb: {playback_gain}, limiter: true}}); ChallengePlayer.mount();</script>
221{"<script src='../viewer.js?v=" + asset_version("viewer.js") + "'></script><script>SceneViewer.mount();</script>" if scene else ""}
222</body></html>
223"""
226def render_index(results: list[dict], sources: list[Source], links: dict) -> str:
227 items = "".join(
228 f'<li><a href="{html.escape(r["challenge_id"])}/">{html.escape(r["challenge_id"])}</a>: {html.escape(r.get("title") or "")} '
229 f'<span class="muted">({len(r["solutions"])} solutions, reference {html.escape(r["reference"] or "none")})</span>'
230 f'{" <a href=\"" + html.escape(r["challenge_id"]) + "/#geometry\">3D model</a>" if r.get("scene") else ""}</li>'
231 for r in results
232 )
233 srcs = "".join(
234 f"<li>{_link(s.url, s.repo or s.label or str(s.dir)) if s.url else html.escape(str(s.dir))}"
235 f"{' <span class=muted>(' + html.escape(s.branch) + ')</span>' if s.branch else ''}"
236 f"{': ' + html.escape(s.description) if s.description else ''}</li>"
237 for s in sources
238 )
239 repo = links.get("challenge_repo")
240 return f"""<!doctype html>
241<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
242<title>Challenge results</title><style>{_CSS}</style></head>
243<body><main>
244<h1>Challenge results</h1>
245<p>{"Specifications and registry: <a href='" + html.escape(repo) + "'>" + html.escape(repo) + "</a>." if repo else ""}</p>
246<h2>Challenges</h2><ul>{items}</ul>
247<h2>Registered solution repositories</h2><ul>{srcs}</ul>
248</main></body></html>
249"""
252def build(challenges_dir: Path, sources: list[Source], out: Path, challenge_repo_url: str | None = None) -> list[dict]:
253 challenges = load_challenges(challenges_dir)
254 submissions = load_submissions(sources)
255 out.mkdir(parents=True, exist_ok=True)
256 results = []
257 for cid, ch in challenges.items():
258 subs = [s for s in submissions if s["challenge_id"] == cid]
259 cdir = out / cid
260 cdir.mkdir(parents=True, exist_ok=True)
261 res = score_challenge(ch, subs, cdir / "audio") if subs else {
262 "challenge_id": cid, "title": ch.get("title"), "generated": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
263 "metrics": metric_names(ch), "primary_metric": ch["metrics"]["primary"], "reference": None,
264 "reference_selection": {"rule": "none", "candidates": []}, "order_violations_vs_reference": [], "solutions": [], "pairwise": {}, "submissions": [],
265 }
266 with open(cdir / "results.json", "w") as f:
267 json.dump(res, f, indent=1)
268 links = {"spec": f"{challenge_repo_url}/blob/main/challenges/{cid}/README.md" if challenge_repo_url else None}
269 scene = challenges_dir / cid / "scene.json"
270 if scene.exists():
271 shutil.copyfile(scene, cdir / "scene.json")
272 res["scene"] = "scene.json"
273 (cdir / "index.html").write_text(render_challenge_page(res, ch, links, scene=scene.exists()))
274 results.append(res)
275 (out / "index.html").write_text(render_index(results, sources, {"challenge_repo": challenge_repo_url}))
276 (out / "player.js").write_text(player_js())
277 (out / "viewer.js").write_text(asset_text("viewer.js"))
278 return results
281def asset_text(name: str) -> str:
282 """Contents of a static asset shipped with the package (player.js, viewer.js)."""
283 return (Path(__file__).parent / name).read_text()
286def asset_version(name: str) -> str:
287 """Short hash of an asset, used as a cache-busting query on its script tag."""
288 import hashlib
290 return hashlib.sha1(asset_text(name).encode()).hexdigest()[:10]
293def player_version() -> str:
294 return asset_version("player.js")
297def player_js() -> str:
298 return asset_text("player.js")
301def viewer_js() -> str:
302 return asset_text("viewer.js")