03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 1"""Compute all scores and write the comparison site."""
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 2from __future__ import annotations
4import datetime as dt
5import html
6import json
7import shutil
8from pathlib import Path
10import numpy as np
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 12from .loading import Source, load_challenges, load_submissions, public_fields
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 13from .metrics import compute_metric, metric_names
14from .reference import collect_order, select_reference
15from .wav import read_wav, resolve_output
18def score_challenge(challenge: dict, submissions: list[dict], out_audio: Path | None = None) -> dict:
19 """Load every output for one challenge, compute pairwise errors, choose the reference, and return a results dict."""
20 spec = challenge["output"]
21 names = metric_names(challenge)
22 primary = challenge["metrics"]["primary"]
24 signals: dict[str, np.ndarray] = {}
25 rows: dict[str, dict] = {}
26 for sub in submissions:
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 27 src: Source = sub["_source"]
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 28 for sol in sub["solutions"]:
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 29 key = f"{sub['_key']}/{sol['solution_id']}"
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 30 p = resolve_output(sol["output"], sub["_path"].parent)
31 signals[key] = read_wav(p, spec)
32 audio_rel = None
33 if out_audio is not None:
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 34 dest = out_audio / sub["_key"] / f"{sol['solution_id']}.wav"
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 35 dest.parent.mkdir(parents=True, exist_ok=True)
36 shutil.copyfile(p, dest)
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 37 audio_rel = f"audio/{sub['_key']}/{sol['solution_id']}.wav"
38 fields_url = None
39 if "fields" in sol:
40 fpath = sol["fields"]["path"]
41 fields_url = fpath if fpath.startswith("http") else src.blob_url(sub["_path"].parent / fpath)
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 42 rows[key] = {
43 "key": key,
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 44 "source": src.label,
45 "source_url": src.url,
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 46 "submission_id": sub["submission_id"],
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 47 "submission_url": src.blob_url(sub["_path"]),
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 48 "solution_id": sol["solution_id"],
49 "submitter": sub["submitter"]["name"],
50 "date": sub.get("date"),
51 "method": sol.get("method"),
52 "parameters": sol.get("parameters"),
53 "timing_seconds": sol.get("timing_seconds"),
54 "hardware": sol.get("hardware"),
55 "code_url": sol.get("code_url") or sub.get("code_url"),
56 "viz_url": sol.get("viz_url") or sub.get("viz_url"),
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 57 "fields_url": fields_url,
58 "more_accurate_than": [f"{sub['_key']}/{w}" for w in sol.get("more_accurate_than", [])],
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 59 "notes": sol.get("notes"),
60 "audio": audio_rel,
61 "peak": float(np.max(np.abs(signals[key]))),
62 }
64 keys = list(signals)
65 pairwise = {name: {a: {} for a in keys} for name in names}
66 for name in names:
67 params = challenge["metrics"][name]
68 for a in keys:
69 for r in keys:
70 pairwise[name][a][r] = 0.0 if a == r else compute_metric(name, signals[a], signals[r], params)
72 info = collect_order(submissions)
73 explicit = challenge.get("reference_solution")
74 if explicit is not None and explicit not in keys:
75 raise KeyError(f"reference_solution {explicit!r} is not among the solutions")
76 selection = select_reference(info, pairwise[primary], explicit)
77 ref = selection["reference"]
78 for key, row in rows.items():
79 row["errors"] = {name: pairwise[name][key][ref] for name in names} if ref else {}
80 row["is_reference"] = key == ref
82 violations = []
83 if ref:
84 for worse, better, _sid in info.pairs:
85 for name in names:
86 if not pairwise[name][better][ref] < pairwise[name][worse][ref]:
87 violations.append({"metric": name, "worse": worse, "better": better})
89 ordered = sorted(rows.values(), key=lambda r: (r["errors"].get(primary, float("inf")), r["key"]))
90 return {
91 "challenge_id": challenge["challenge_id"],
92 "title": challenge.get("title"),
93 "generated": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
94 "metrics": names,
95 "primary_metric": primary,
96 "reference": ref,
97 "reference_selection": selection,
98 "order_violations_vs_reference": violations,
99 "solutions": ordered,
100 "pairwise": pairwise,
101 "submissions": [
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 102 {**{k: v for k, v in public_fields(sub).items() if k != "solutions"}, "key": sub["_key"], "source": sub["_source"].label, "source_url": sub["_source"].url}
103 for sub in submissions
105 }
108# ---------------------------------------------------------------- HTML
110_CSS = """
111:root { --fg: #1b1b1b; --bg: #ffffff; --muted: #666; --line: #ddd; --accent: #2a6db0; --ref: #e8f4e8; }
112@media (prefers-color-scheme: dark) { :root { --fg: #e6e6e6; --bg: #151515; --muted: #999; --line: #333; --accent: #7ab3e8; --ref: #1f3320; } }
113body { font-family: system-ui, sans-serif; color: var(--fg); background: var(--bg); margin: 0; padding: 16px; line-height: 1.45; }
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 114main { max-width: 1300px; margin: 0 auto; }
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 115a { color: var(--accent); }
116table { border-collapse: collapse; width: 100%; font-size: 0.92em; }
117th, td { border-bottom: 1px solid var(--line); padding: 6px 8px; text-align: left; vertical-align: top; }
118th { position: sticky; top: 0; background: var(--bg); }
119td.num { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; }
120tr.ref { background: var(--ref); }
121.muted { color: var(--muted); }
122.wrap { overflow-x: auto; }
123audio { width: 180px; height: 28px; }
124details { margin: 1em 0; }
125code { font-size: 0.9em; }
126"""
129def _fmt(x, digits=4):
130 if x is None:
131 return ""
132 if isinstance(x, float):
133 if x == 0:
134 return "0"
135 return f"{x:.{digits}g}"
136 return html.escape(str(x))
139def _link(url, text):
140 if not url:
141 return ""
142 return f'<a href="{html.escape(url)}">{html.escape(text)}</a>'
145def render_challenge_page(res: dict, challenge: dict, links: dict) -> str:
146 names = res["metrics"]
147 primary = res["primary_metric"]
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 148 head = "".join(f"<th>{html.escape(n)}{' (primary)' if n == primary else ''}</th>" for n in names)
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 149 body = []
150 for r in res["solutions"]:
151 errs = "".join(f'<td class="num">{_fmt(r["errors"].get(n))}</td>' for n in names)
152 params = html.escape(json.dumps(r["parameters"])) if r["parameters"] else ""
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 153 mat = ", ".join(html.escape(k.rsplit("/", 1)[1]) for k in r["more_accurate_than"])
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 154 audio = f'<audio controls preload="none" src="{html.escape(r["audio"])}"></audio>' if r["audio"] else ""
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 155 linkbits = " ".join(x for x in [
156 _link(r["audio"], "wav"), _link(r["fields_url"], "fields"), _link(r["code_url"], "code"), _link(r["viz_url"], "viz"),
157 ] if x)
158 sub_cell = _link(r["submission_url"], r["submission_id"]) if r["submission_url"] else html.escape(r["submission_id"])
159 src_cell = _link(r["source_url"], r["source"]) if r["source"] else ""
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 160 body.append(
161 f'<tr class="{"ref" if r["is_reference"] else ""}">'
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 162 f'<td>{sub_cell}<br><span class="muted">{src_cell}{" · " if src_cell else ""}{html.escape(r["submitter"])}</span></td>'
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 163 f'<td>{html.escape(r["solution_id"])}{" <b>(reference)</b>" if r["is_reference"] else ""}</td>'
164 f'<td>{html.escape(r["method"] or "")}<br><span class="muted"><code>{params}</code></span></td>'
165 f"{errs}"
166 f'<td class="num">{_fmt(r["timing_seconds"], 3)}</td>'
167 f'<td>{html.escape(r["hardware"] or "")}</td>'
168 f'<td class="muted">{mat}</td>'
169 f"<td>{audio}<br>{linkbits}</td>"
170 "</tr>"
171 )
172 sel = res["reference_selection"]
173 cand_rows = "".join(
174 f"<tr><td>{html.escape(c['candidate'])}</td><td class='num'>{c['pairs_preserved']}/{c['pairs_total']}</td>"
c183eeaBreak reference ties by convergence gap; document the rule's limitationsJeremy Magland 175 f"<td class='num'>{_fmt(c['convergence_gap'])}</td><td class='num'>{_fmt(c['mean_error_to_other_candidates'])}</td>"
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 176 f"<td>{html.escape(', '.join(f'{w} vs {b}' for w, b in c['violated_pairs']))}</td></tr>"
177 for c in sel.get("candidates", [])
178 )
179 viol = res["order_violations_vs_reference"]
180 viol_html = (
181 "<p>All asserted accuracy orderings are preserved by both metrics under the chosen reference.</p>"
182 if not viol
183 else "<ul>" + "".join(
184 f"<li>{html.escape(v['metric'])}: {html.escape(v['better'])} asserted more accurate than {html.escape(v['worse'])}, but its error is not smaller</li>"
185 for v in viol
186 ) + "</ul>"
187 )
188 pair_tables = []
189 keys = [r["key"] for r in res["solutions"]]
190 for n in names:
191 hdr = "".join(f"<th>{html.escape(k)}</th>" for k in keys)
192 rws = "".join(
193 f"<tr><th>{html.escape(a)}</th>" + "".join(f"<td class='num'>{_fmt(res['pairwise'][n][a][b], 3)}</td>" for b in keys) + "</tr>"
194 for a in keys
195 )
196 pair_tables.append(f"<h4>{html.escape(n)}</h4><div class='wrap'><table><tr><th></th>{hdr}</tr>{rws}</table></div>")
197 spec_link = links.get("spec")
198 return f"""<!doctype html>
199<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
200<title>{html.escape(res["challenge_id"])} results</title><style>{_CSS}</style></head>
201<body><main>
202<p><a href="../">All challenges</a></p>
203<h1>{html.escape(res["challenge_id"])}</h1>
204<p>{html.escape(res.get("title") or "")}. {"<a href='" + html.escape(spec_link) + "'>Challenge specification</a>. " if spec_link else ""}<a href="results.json">results.json</a>. Generated {html.escape(res["generated"])}.</p>
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 205<p>Errors are relative to the reference solution <b>{html.escape(res["reference"] or "none")}</b>, chosen by rule <i>{html.escape(sel["rule"])}</i>. Rows are sorted by the primary metric. Submissions are named as <i>repository/submission</i>.</p>
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 206<div class="wrap"><table>
207<tr><th>Submission</th><th>Solution</th><th>Method</th>{head}<th>Time (s)</th><th>Hardware</th><th>Claimed more accurate than</th><th>Output</th></tr>
208{"".join(body)}
209</table></div>
210<details><summary>Reference selection</summary>
c183eeaBreak reference ties by convergence gap; document the rule's limitationsJeremy Magland 211<p>Candidates are the maximal elements of each submission's asserted order. The score is the number of asserted pairs from other submissions whose order is preserved by the primary error to this candidate. Ties are broken first by the convergence gap (the primary error between the candidate and the nearest solution of its own submission asserted to be less accurate; smaller means a tighter convergence claim), then by the mean primary error to the other candidates.</p>
212<div class="wrap"><table><tr><th>Candidate</th><th>Pairs preserved</th><th>Convergence gap</th><th>Mean error to other candidates</th><th>Violated pairs</th></tr>{cand_rows}</table></div>
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 213<h4>Order check against the chosen reference</h4>{viol_html}
214</details>
215<details><summary>Pairwise errors (row vs column as reference)</summary>{"".join(pair_tables)}</details>
216</main></body></html>
217"""
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 220def render_index(results: list[dict], sources: list[Source], links: dict) -> str:
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 221 items = "".join(
222 f'<li><a href="{html.escape(r["challenge_id"])}/">{html.escape(r["challenge_id"])}</a>: {html.escape(r.get("title") or "")} '
223 f'<span class="muted">({len(r["solutions"])} solutions, reference {html.escape(r["reference"] or "none")})</span></li>'
224 for r in results
225 )
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 226 srcs = "".join(
227 f"<li>{_link(s.url, s.repo or s.label or str(s.dir)) if s.url else html.escape(str(s.dir))}"
228 f"{' <span class=muted>(' + html.escape(s.branch) + ')</span>' if s.branch else ''}"
229 f"{': ' + html.escape(s.description) if s.description else ''}</li>"
230 for s in sources
231 )
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 232 repo = links.get("challenge_repo")
233 return f"""<!doctype html>
234<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
235<title>Challenge results</title><style>{_CSS}</style></head>
236<body><main>
237<h1>Challenge results</h1>
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 238<p>{"Specifications and registry: <a href='" + html.escape(repo) + "'>" + html.escape(repo) + "</a>." if repo else ""}</p>
239<h2>Challenges</h2><ul>{items}</ul>
240<h2>Registered solution repositories</h2><ul>{srcs}</ul>
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 241</main></body></html>
242"""
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 245def build(challenges_dir: Path, sources: list[Source], out: Path, challenge_repo_url: str | None = None) -> list[dict]:
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 246 challenges = load_challenges(challenges_dir)
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 247 submissions = load_submissions(sources)
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 248 out.mkdir(parents=True, exist_ok=True)
249 results = []
250 for cid, ch in challenges.items():
251 subs = [s for s in submissions if s["challenge_id"] == cid]
252 cdir = out / cid
253 cdir.mkdir(parents=True, exist_ok=True)
254 res = score_challenge(ch, subs, cdir / "audio") if subs else {
255 "challenge_id": cid, "title": ch.get("title"), "generated": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
256 "metrics": metric_names(ch), "primary_metric": ch["metrics"]["primary"], "reference": None,
257 "reference_selection": {"rule": "none", "candidates": []}, "order_violations_vs_reference": [], "solutions": [], "pairwise": {}, "submissions": [],
258 }
259 with open(cdir / "results.json", "w") as f:
260 json.dump(res, f, indent=1)
261 links = {"spec": f"{challenge_repo_url}/blob/main/challenges/{cid}/README.md" if challenge_repo_url else None}
262 (cdir / "index.html").write_text(render_challenge_page(res, ch, links))
263 results.append(res)
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 264 (out / "index.html").write_text(render_index(results, sources, {"challenge_repo": challenge_repo_url}))
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 265 return results