ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 1"""Compute all scores and write the results site."""
2from __future__ import annotations
4import datetime as dt
5import html
6import json
7import shutil
8from pathlib import Path
10import numpy as np
12from .loading import load_challenges, load_submissions
13from .metrics import compute_metric, metric_names
14from .reference import collect_order, select_reference
15from .wav import read_wav, resolve_output
18def score_challenge(challenge: dict, submissions: list[dict], out_audio: Path | None = None) -> dict:
19 """Load every output for one challenge, compute pairwise errors, choose the reference, and return a results dict."""
20 spec = challenge["output"]
21 names = metric_names(challenge)
22 primary = challenge["metrics"]["primary"]
24 signals: dict[str, np.ndarray] = {}
25 rows: dict[str, dict] = {}
26 for sub in submissions:
27 for sol in sub["solutions"]:
28 key = f"{sub['submission_id']}/{sol['solution_id']}"
29 p = resolve_output(sol["output"], sub["_path"].parent)
30 signals[key] = read_wav(p, spec)
31 audio_rel = None
32 if out_audio is not None:
33 dest = out_audio / sub["submission_id"] / f"{sol['solution_id']}.wav"
34 dest.parent.mkdir(parents=True, exist_ok=True)
35 shutil.copyfile(p, dest)
36 audio_rel = f"audio/{sub['submission_id']}/{sol['solution_id']}.wav"
37 rows[key] = {
38 "key": key,
39 "submission_id": sub["submission_id"],
40 "solution_id": sol["solution_id"],
41 "submitter": sub["submitter"]["name"],
42 "date": sub.get("date"),
43 "method": sol.get("method"),
44 "parameters": sol.get("parameters"),
45 "timing_seconds": sol.get("timing_seconds"),
46 "hardware": sol.get("hardware"),
47 "code_url": sol.get("code_url") or sub.get("code_url"),
48 "viz_url": sol.get("viz_url") or sub.get("viz_url"),
49 "more_accurate_than": [f"{sub['submission_id']}/{w}" for w in sol.get("more_accurate_than", [])],
50 "notes": sol.get("notes"),
51 "audio": audio_rel,
52 "peak": float(np.max(np.abs(signals[key]))),
53 }
55 keys = list(signals)
56 pairwise = {name: {a: {} for a in keys} for name in names}
57 for name in names:
58 params = challenge["metrics"][name]
59 for a in keys:
60 for r in keys:
61 pairwise[name][a][r] = 0.0 if a == r else compute_metric(name, signals[a], signals[r], params)
63 info = collect_order(submissions)
64 explicit = challenge.get("reference_solution")
65 if explicit is not None and explicit not in keys:
66 raise KeyError(f"reference_solution {explicit!r} is not among the solutions")
67 selection = select_reference(info, pairwise[primary], explicit)
68 ref = selection["reference"]
69 for key, row in rows.items():
70 row["errors"] = {name: pairwise[name][key][ref] for name in names} if ref else {}
71 row["is_reference"] = key == ref
73 # Check every asserted pair against the chosen reference, for the report.
74 violations = []
75 if ref:
76 for worse, better, _sid in info.pairs:
77 for name in names:
78 if not pairwise[name][better][ref] < pairwise[name][worse][ref]:
79 violations.append({"metric": name, "worse": worse, "better": better})
81 ordered = sorted(rows.values(), key=lambda r: (r["errors"].get(primary, float("inf")), r["key"]))
82 return {
83 "challenge_id": challenge["challenge_id"],
84 "title": challenge.get("title"),
85 "generated": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
86 "metrics": names,
87 "primary_metric": primary,
88 "reference": ref,
89 "reference_selection": selection,
90 "order_violations_vs_reference": violations,
91 "solutions": ordered,
92 "pairwise": pairwise,
93 "submissions": [
94 {k: v for k, v in sub.items() if not k.startswith("_") and k != "solutions"} for sub in submissions
95 ],
96 }
99# ---------------------------------------------------------------- HTML
101_CSS = """
102:root { --fg: #1b1b1b; --bg: #ffffff; --muted: #666; --line: #ddd; --accent: #2a6db0; --ref: #e8f4e8; }
103@media (prefers-color-scheme: dark) { :root { --fg: #e6e6e6; --bg: #151515; --muted: #999; --line: #333; --accent: #7ab3e8; --ref: #1f3320; } }
104body { font-family: system-ui, sans-serif; color: var(--fg); background: var(--bg); margin: 0; padding: 16px; line-height: 1.45; }
105main { max-width: 1200px; margin: 0 auto; }
106a { color: var(--accent); }
107table { border-collapse: collapse; width: 100%; font-size: 0.92em; }
108th, td { border-bottom: 1px solid var(--line); padding: 6px 8px; text-align: left; vertical-align: top; }
109th { position: sticky; top: 0; background: var(--bg); }
110td.num { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; }
111tr.ref { background: var(--ref); }
112.muted { color: var(--muted); }
113.wrap { overflow-x: auto; }
114audio { width: 180px; height: 28px; }
115details { margin: 1em 0; }
116code { font-size: 0.9em; }
117"""
120def _fmt(x, digits=4):
121 if x is None:
122 return ""
123 if isinstance(x, float):
124 if x == 0:
125 return "0"
126 return f"{x:.{digits}g}"
127 return html.escape(str(x))
130def _link(url, text):
131 if not url:
132 return ""
133 return f'<a href="{html.escape(url)}">{html.escape(text)}</a>'
136def render_challenge_page(res: dict, challenge: dict, links: dict) -> str:
137 names = res["metrics"]
138 primary = res["primary_metric"]
139 head = "".join(
140 f"<th>{html.escape(n)}{' (primary)' if n == primary else ''}</th>" for n in names
141 )
142 body = []
143 for r in res["solutions"]:
144 errs = "".join(f'<td class="num">{_fmt(r["errors"].get(n))}</td>' for n in names)
145 params = html.escape(json.dumps(r["parameters"])) if r["parameters"] else ""
146 mat = ", ".join(html.escape(k.split("/", 1)[1]) for k in r["more_accurate_than"])
147 audio = f'<audio controls preload="none" src="{html.escape(r["audio"])}"></audio>' if r["audio"] else ""
148 linkbits = " ".join(x for x in [_link(r["code_url"], "code"), _link(r["viz_url"], "viz"), _link(r["audio"], "wav")] if x)
149 body.append(
150 f'<tr class="{"ref" if r["is_reference"] else ""}">'
151 f'<td>{html.escape(r["submission_id"])}<br><span class="muted">{html.escape(r["submitter"])}</span></td>'
152 f'<td>{html.escape(r["solution_id"])}{" <b>(reference)</b>" if r["is_reference"] else ""}</td>'
153 f'<td>{html.escape(r["method"] or "")}<br><span class="muted"><code>{params}</code></span></td>'
154 f"{errs}"
155 f'<td class="num">{_fmt(r["timing_seconds"], 3)}</td>'
156 f'<td>{html.escape(r["hardware"] or "")}</td>'
157 f'<td class="muted">{mat}</td>'
158 f"<td>{audio}<br>{linkbits}</td>"
159 "</tr>"
160 )
161 sel = res["reference_selection"]
162 cand_rows = "".join(
163 f"<tr><td>{html.escape(c['candidate'])}</td><td class='num'>{c['pairs_preserved']}/{c['pairs_total']}</td>"
c183eeaBreak reference ties by convergence gap; document the rule's limitationsJeremy Magland 164 f"<td class='num'>{_fmt(c['convergence_gap'])}</td><td class='num'>{_fmt(c['mean_error_to_other_candidates'])}</td>"
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 165 f"<td>{html.escape(', '.join(f'{w} vs {b}' for w, b in c['violated_pairs']))}</td></tr>"
166 for c in sel.get("candidates", [])
167 )
168 viol = res["order_violations_vs_reference"]
169 viol_html = (
170 "<p>All asserted accuracy orderings are preserved by both metrics under the chosen reference.</p>"
171 if not viol
172 else "<ul>" + "".join(
173 f"<li>{html.escape(v['metric'])}: {html.escape(v['better'])} asserted more accurate than {html.escape(v['worse'])}, but its error is not smaller</li>"
174 for v in viol
175 ) + "</ul>"
176 )
177 pair_tables = []
178 keys = [r["key"] for r in res["solutions"]]
179 for n in names:
180 hdr = "".join(f"<th>{html.escape(k)}</th>" for k in keys)
181 rws = "".join(
182 f"<tr><th>{html.escape(a)}</th>" + "".join(f"<td class='num'>{_fmt(res['pairwise'][n][a][b], 3)}</td>" for b in keys) + "</tr>"
183 for a in keys
184 )
185 pair_tables.append(f"<h4>{html.escape(n)}</h4><div class='wrap'><table><tr><th></th>{hdr}</tr>{rws}</table></div>")
186 spec_link = links.get("spec")
187 return f"""<!doctype html>
188<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
189<title>{html.escape(res["challenge_id"])} results</title><style>{_CSS}</style></head>
190<body><main>
191<p><a href="../">All challenges</a></p>
192<h1>{html.escape(res["challenge_id"])}</h1>
193<p>{html.escape(res.get("title") or "")}. {"<a href='" + html.escape(spec_link) + "'>Challenge specification</a>. " if spec_link else ""}<a href="results.json">results.json</a>. Generated {html.escape(res["generated"])}.</p>
194<p>Errors are relative to the reference solution <b>{html.escape(res["reference"] or "none")}</b>, chosen by rule <i>{html.escape(sel["rule"])}</i>. Rows are sorted by the primary metric.</p>
195<div class="wrap"><table>
196<tr><th>Submission</th><th>Solution</th><th>Method</th>{head}<th>Time (s)</th><th>Hardware</th><th>Claimed more accurate than</th><th>Output</th></tr>
197{"".join(body)}
198</table></div>
199<details><summary>Reference selection</summary>
c183eeaBreak reference ties by convergence gap; document the rule's limitationsJeremy Magland 200<p>Candidates are the maximal elements of each submission's asserted order. The score is the number of asserted pairs from other submissions whose order is preserved by the primary error to this candidate. Ties are broken first by the convergence gap (the primary error between the candidate and the nearest solution of its own submission asserted to be less accurate; smaller means a tighter convergence claim), then by the mean primary error to the other candidates.</p>
201<div class="wrap"><table><tr><th>Candidate</th><th>Pairs preserved</th><th>Convergence gap</th><th>Mean error to other candidates</th><th>Violated pairs</th></tr>{cand_rows}</table></div>
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 202<h4>Order check against the chosen reference</h4>{viol_html}
203</details>
204<details><summary>Pairwise errors (row vs column as reference)</summary>{"".join(pair_tables)}</details>
205</main></body></html>
206"""
209def render_index(results: list[dict], links: dict) -> str:
210 items = "".join(
211 f'<li><a href="{html.escape(r["challenge_id"])}/">{html.escape(r["challenge_id"])}</a>: {html.escape(r.get("title") or "")} '
212 f'<span class="muted">({len(r["solutions"])} solutions, reference {html.escape(r["reference"] or "none")})</span></li>'
213 for r in results
214 )
215 repo = links.get("challenge_repo")
216 sols = links.get("solutions_repo")
217 return f"""<!doctype html>
218<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
219<title>Challenge results</title><style>{_CSS}</style></head>
220<body><main>
221<h1>Challenge results</h1>
222<p>{"Specifications: <a href='" + html.escape(repo) + "'>" + html.escape(repo) + "</a>. " if repo else ""}{"Solutions: <a href='" + html.escape(sols) + "'>" + html.escape(sols) + "</a>." if sols else ""}</p>
223<ul>{items}</ul>
224</main></body></html>
225"""
228def build(challenges_dir: Path, solutions_dir: Path, out: Path, challenge_repo_url: str | None = None, solutions_repo_url: str | None = None) -> list[dict]:
229 challenges = load_challenges(challenges_dir)
230 submissions = load_submissions(solutions_dir)
231 out.mkdir(parents=True, exist_ok=True)
232 results = []
233 for cid, ch in challenges.items():
234 subs = [s for s in submissions if s["challenge_id"] == cid]
235 cdir = out / cid
236 cdir.mkdir(parents=True, exist_ok=True)
237 res = score_challenge(ch, subs, cdir / "audio") if subs else {
238 "challenge_id": cid, "title": ch.get("title"), "generated": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
239 "metrics": metric_names(ch), "primary_metric": ch["metrics"]["primary"], "reference": None,
240 "reference_selection": {"rule": "none", "candidates": []}, "order_violations_vs_reference": [], "solutions": [], "pairwise": {}, "submissions": [],
241 }
242 with open(cdir / "results.json", "w") as f:
243 json.dump(res, f, indent=1)
244 links = {"spec": f"{challenge_repo_url}/blob/main/challenges/{cid}/README.md" if challenge_repo_url else None}
245 (cdir / "index.html").write_text(render_challenge_page(res, ch, links))
246 results.append(res)
247 (out / "index.html").write_text(render_index(results, {"challenge_repo": challenge_repo_url, "solutions_repo": solutions_repo_url}))
248 return results