Shared challenge tools: schema, validation, metrics, reference selection, comparison site; moved out of the string repository
16 changed files+1271−0
.gitignoreadded+8−0View file
@@ -0,0 +1,8 @@
1+__pycache__/
2+*.egg-info/
3+build/
4+dist/
5+.pytest_cache/
6+.challenge-cache/
7+clones/
8+site/
.mochi/workflows/test.ymladded+14−0View file
@@ -0,0 +1,14 @@
1+name: Test
2+on:
3+ push:
4+ workflow_dispatch:
5+jobs:
6+ test:
7+ runs-on: ubuntu-latest
8+ steps:
9+ - uses: actions/checkout@v4
10+ - uses: actions/setup-python@v5
11+ with:
12+ python-version: '3.12'
13+ - run: pip install . pytest
14+ - run: pytest -q tests
README.mdadded+47−0View file
@@ -0,0 +1,47 @@
1+# challenge-tools
2+
3+Shared tooling for the instrument simulation challenges: the submission schema, validation of submissions, the error measures, the reference-selection rule, and the generator of the static comparison site. Every challenge repository (for example [challenges/string](https://vault1.magland.org/challenges/string) and [challenges/plate](https://vault1.magland.org/challenges/plate)) installs this package in its workflow and runs it against the solution repositories listed in its registry. Solution repositories run only the validator.
4+
5+## Install
6+
7+```
8+pip install git+https://vault1.magland.org/challenges/challenge-tools
9+```
10+
11+## Commands
12+
13+```
14+challenge-tools fetch --registry registry/solution-repos.json --clones clones
15+challenge-tools validate --challenges challenges --registry registry/solution-repos.json --clones clones
16+challenge-tools build --challenges challenges --registry registry/solution-repos.json --clones clones --out site
17+challenge-tools validate --challenges <spec-repo>/challenges --solutions solutions # a solution repository checking itself
18+```
19+
20+`fetch` clones or updates every registered repository (shallow, one branch, anonymously; `git lfs pull` is run when git-lfs is available). `validate` checks each submission JSON against [schema.json](schema.json), the wav files against the challenge's output specification, and any field exports against the challenge's recommended grid. `build` validates, computes every pairwise error, chooses the reference, and writes the site.
21+
22+## What a challenge must provide
23+
24+A directory `challenges/<challenge-id>/` with a `README.md` (the specification) and a `challenge.json` with at least:
25+
26+```json
27+{
28+ "challenge_id": "...",
29+ "output": {"sample_rate_hz": 48000, "num_samples": 96000, "channels": 1, "sample_format": "float32", "signal_scale": 1.0},
30+ "metrics": {"primary": "waveform_l2", "waveform_l2": {}, "spectrogram_l2": {"n_fft": 2048, "hop": 512}},
31+ "fields": {"format": "hdf5", "axes": ["t", "x"], "shape": [4001, 201]},
32+ "reference_solution": null
33+}
34+```
35+
36+`fields` is optional and describes the grid of the optional field export: datasets named after `axes` plus a dataset `u` of the given shape. `reference_solution` overrides the automatic reference with a `repo/submission/solution` key.
37+
38+## Scoring and reference selection
39+
40+The error measures live in `tools/challenge_tools/metrics.py` and the reference rule in `tools/challenge_tools/reference.py`. The challenge READMEs are the specification; this code is intended to follow them exactly. The rule: within a submission, `more_accurate_than` asserts a partial order; candidates for reference are the maximal elements of each submission's order; a candidate is scored by the fraction of asserted pairs from other submissions whose order it reproduces under the primary error; ties are broken by the candidate's convergence gap (error to its nearest asserted-worse sibling) and then by mean error to the other candidates.
41+
42+## Development
43+
44+```
45+pip install -e . pytest
46+pytest tests
47+```
pyproject.tomladded+20−0View file
@@ -0,0 +1,20 @@
1+[build-system]
2+requires = ["setuptools>=68"]
3+build-backend = "setuptools.build_meta"
4+
5+[project]
6+name = "challenge-tools"
7+version = "0.1.0"
8+description = "Schema, validation, scoring, and comparison-site generation for the instrument simulation challenges"
9+requires-python = ">=3.10"
10+dependencies = ["numpy", "scipy", "jsonschema", "h5py"]
11+
12+[project.scripts]
13+challenge-tools = "challenge_tools.cli:main"
14+
15+[tool.setuptools]
16+package-dir = {"" = "tools"}
17+packages = ["challenge_tools"]
18+
19+[tool.setuptools.package-data]
20+challenge_tools = ["*.json"]
schema.jsonadded+143−0View file
@@ -0,0 +1,143 @@
1+{
2+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3+ "$id": "https://vault1.magland.org/challenges/string/schema/solution.schema.json",
4+ "title": "Challenge solution submission",
5+ "description": "One submission file: a set of solutions to one challenge from one submitter, with an optional accuracy partial order among them.",
6+ "type": "object",
7+ "required": [
8+ "schema_version",
9+ "submission_id",
10+ "challenge_id",
11+ "submitter",
12+ "solutions"
13+ ],
14+ "additionalProperties": false,
15+ "properties": {
16+ "schema_version": {
17+ "const": 1
18+ },
19+ "submission_id": {
20+ "type": "string",
21+ "pattern": "^[a-z0-9][a-z0-9-]*$",
22+ "description": "Identifier of this submission. Must equal the file name without .json."
23+ },
24+ "challenge_id": {
25+ "type": "string",
26+ "pattern": "^[a-z0-9][a-z0-9-]*$",
27+ "description": "The challenge this submission solves. Must equal the name of the directory containing the file."
28+ },
29+ "submitter": {
30+ "type": "object",
31+ "required": [
32+ "name"
33+ ],
34+ "additionalProperties": false,
35+ "properties": {
36+ "name": {
37+ "type": "string"
38+ },
39+ "email": {
40+ "type": "string"
41+ },
42+ "url": {
43+ "type": "string",
44+ "format": "uri"
45+ }
46+ }
47+ },
48+ "date": {
49+ "type": "string",
50+ "format": "date"
51+ },
52+ "description": {
53+ "type": "string"
54+ },
55+ "code_url": {
56+ "type": "string",
57+ "format": "uri",
58+ "description": "Source code for the method, applying to every solution unless overridden."
59+ },
60+ "viz_url": {
61+ "type": "string",
62+ "format": "uri",
63+ "description": "A web application visualizing the solutions."
64+ },
65+ "solutions": {
66+ "type": "array",
67+ "minItems": 1,
68+ "items": {
69+ "type": "object",
70+ "required": [
71+ "solution_id",
72+ "output"
73+ ],
74+ "additionalProperties": false,
75+ "properties": {
76+ "solution_id": {
77+ "type": "string",
78+ "pattern": "^[a-z0-9][a-z0-9-]*$",
79+ "description": "Unique within the submission."
80+ },
81+ "output": {
82+ "type": "string",
83+ "description": "Path to the wav file relative to the directory containing this JSON file, or an http(s) URL."
84+ },
85+ "method": {
86+ "type": "string",
87+ "description": "Short name of the numerical method."
88+ },
89+ "parameters": {
90+ "type": "object",
91+ "description": "Discretization parameters, free form."
92+ },
93+ "timing_seconds": {
94+ "type": "number",
95+ "minimum": 0,
96+ "description": "Self-reported wall-clock time to produce the output, excluding file I/O."
97+ },
98+ "hardware": {
99+ "type": "string"
100+ },
101+ "code_url": {
102+ "type": "string",
103+ "format": "uri"
104+ },
105+ "viz_url": {
106+ "type": "string",
107+ "format": "uri"
108+ },
109+ "more_accurate_than": {
110+ "type": "array",
111+ "items": {
112+ "type": "string"
113+ },
114+ "description": "solution_ids in this same file that the submitter asserts are less accurate than this one."
115+ },
116+ "notes": {
117+ "type": "string"
118+ },
119+ "fields": {
120+ "type": "object",
121+ "required": [
122+ "path"
123+ ],
124+ "additionalProperties": false,
125+ "description": "Optional export of the solution field on the grid recommended by the challenge, for visualization. Not scored.",
126+ "properties": {
127+ "path": {
128+ "type": "string",
129+ "description": "Path relative to the directory containing this JSON file, or an http(s) URL."
130+ },
131+ "format": {
132+ "const": "hdf5"
133+ },
134+ "description": {
135+ "type": "string"
136+ }
137+ }
138+ }
139+ }
140+ }
141+ }
142+ }
143+}
tests/test_tools.pyadded+99−0View file
@@ -0,0 +1,99 @@
1+import numpy as np
2+
3+from challenge_tools.metrics import spectrogram, spectrogram_l2, waveform_l2
4+from challenge_tools.reference import OrderInfo, collect_order, maximal_elements, select_reference
5+
6+
7+def test_waveform_l2():
8+ r = np.array([1.0, 2.0, 2.0])
9+ assert waveform_l2(r, r) == 0.0
10+ assert abs(waveform_l2(r * 1.1, r) - 0.1) < 1e-12
11+
12+
13+def test_spectrogram_shape():
14+ x = np.zeros(96000)
15+ s = spectrogram(x, 2048, 512)
16+ assert s.shape == ((96000 - 2048) // 512 + 1, 1025)
17+
18+
19+def test_spectrogram_phase_insensitive():
20+ t = np.arange(96000) / 48000
21+ a = np.sin(2 * np.pi * 440 * t)
22+ b = np.sin(2 * np.pi * 440 * t + 1.0)
23+ assert waveform_l2(a, b) > 0.5
24+ assert spectrogram_l2(a, b, 2048, 512) < 1e-2
25+
26+
27+def test_reference_selection_prefers_order_preserving_candidate():
28+ subs = [
29+ {"submission_id": "x", "solutions": [
30+ {"solution_id": "a", "more_accurate_than": []},
31+ {"solution_id": "b", "more_accurate_than": ["a"]},
32+ ]},
33+ {"submission_id": "y", "solutions": [
34+ {"solution_id": "c", "more_accurate_than": []},
35+ {"solution_id": "d", "more_accurate_than": ["c"]},
36+ ]},
37+ ]
38+ info = collect_order(subs)
39+ assert set(maximal_elements(info)) == {"x/b", "y/d"}
40+ # Truth: x/b is exact; y/d is wrong. Errors to x/b respect y's chain, errors to y/d do not respect x's chain.
41+ keys = ["x/a", "x/b", "y/c", "y/d"]
42+ truth = {"x/a": 1.0, "x/b": 0.0, "y/c": 3.0, "y/d": 2.5}
43+ err = {a: {b: abs(truth[a] - truth[b]) for b in keys} for a in keys}
44+ # under y/d: err(x/b)=2.5 > err(x/a)=1.5, so x's pair is violated
45+ rep = select_reference(info, err)
46+ assert rep["reference"] == "x/b"
47+ assert rep["candidates"][0]["score"] == 1.0
48+
49+
50+def test_reference_selection_medoid_fallback():
51+ info = OrderInfo(pairs=[], submission_of={"p/a": "p", "q/b": "q", "r/c": "r"})
52+ keys = list(info.submission_of)
53+ pos = {"p/a": 0.0, "q/b": 1.0, "r/c": 5.0}
54+ err = {a: {b: abs(pos[a] - pos[b]) for b in keys} for a in keys}
55+ rep = select_reference(info, err)
56+ assert rep["reference"] == "q/b"
57+
58+
59+def test_reference_selection_tie_broken_by_convergence_gap():
60+ subs = [
61+ {"submission_id": "x", "solutions": [
62+ {"solution_id": "a", "more_accurate_than": []},
63+ {"solution_id": "b", "more_accurate_than": ["a"]},
64+ ]},
65+ {"submission_id": "y", "solutions": [
66+ {"solution_id": "c", "more_accurate_than": []},
67+ {"solution_id": "d", "more_accurate_than": ["c"]},
68+ ]},
69+ ]
70+ info = collect_order(subs)
71+ keys = ["x/a", "x/b", "y/c", "y/d"]
72+ # both chains monotone toward 0; x/b is much closer to its sibling than y/d is to its own
73+ truth = {"x/a": 0.03, "x/b": 0.0001, "y/c": 1.0, "y/d": 0.01}
74+ err = {a: {b: abs(truth[a] - truth[b]) for b in keys} for a in keys}
75+ rep = select_reference(info, err)
76+ assert [c["score"] for c in rep["candidates"]] == [1.0, 1.0]
77+ assert rep["reference"] == "x/b"
78+
79+
80+def test_fields_check_axes(tmp_path):
81+ import h5py
82+ import numpy as np
83+
84+ from challenge_tools.validate import check_fields_file, is_lfs_pointer
85+
86+ p = tmp_path / "f.h5"
87+ with h5py.File(p, "w") as f:
88+ f.create_dataset("x", data=np.linspace(0, 1, 5))
89+ f.create_dataset("y", data=np.linspace(0, 1, 3))
90+ f.create_dataset("t", data=np.linspace(0, 1, 7))
91+ f.create_dataset("u", data=np.zeros((7, 3, 5), dtype=np.float32))
92+ ch = {"fields": {"axes": ["t", "y", "x"], "shape": [7, 3, 5]}}
93+ assert check_fields_file(p, ch) == []
94+ ch_bad = {"fields": {"axes": ["t", "y", "x"], "shape": [7, 3, 6]}}
95+ assert any("recommends" in m for m in check_fields_file(p, ch_bad))
96+ ptr = tmp_path / "ptr.h5"
97+ ptr.write_text("version https://git-lfs.github.com/spec/v1\noid sha256:abc\nsize 1\n")
98+ assert is_lfs_pointer(ptr)
99+ assert check_fields_file(ptr, ch) == []
tools/challenge_tools/__init__.pyadded+1−0View file
@@ -0,0 +1 @@
1+"""Tools for the instrument simulation challenges: validation, metrics, reference selection, results site."""
tools/challenge_tools/__main__.pyadded+3−0View file
@@ -0,0 +1,3 @@
1+from .cli import main
2+
3+main()
tools/challenge_tools/build.pyadded+265−0View file
@@ -0,0 +1,265 @@
1+"""Compute all scores and write the comparison site."""
2+from __future__ import annotations
3+
4+import datetime as dt
5+import html
6+import json
7+import shutil
8+from pathlib import Path
9+
10+import numpy as np
11+
12+from .loading import Source, load_challenges, load_submissions, public_fields
13+from .metrics import compute_metric, metric_names
14+from .reference import collect_order, select_reference
15+from .wav import read_wav, resolve_output
16+
17+
18+def score_challenge(challenge: dict, submissions: list[dict], out_audio: Path | None = None) -> dict:
19+ """Load every output for one challenge, compute pairwise errors, choose the reference, and return a results dict."""
20+ spec = challenge["output"]
21+ names = metric_names(challenge)
22+ primary = challenge["metrics"]["primary"]
23+
24+ signals: dict[str, np.ndarray] = {}
25+ rows: dict[str, dict] = {}
26+ for sub in submissions:
27+ src: Source = sub["_source"]
28+ for sol in sub["solutions"]:
29+ key = f"{sub['_key']}/{sol['solution_id']}"
30+ p = resolve_output(sol["output"], sub["_path"].parent)
31+ signals[key] = read_wav(p, spec)
32+ audio_rel = None
33+ if out_audio is not None:
34+ dest = out_audio / sub["_key"] / f"{sol['solution_id']}.wav"
35+ dest.parent.mkdir(parents=True, exist_ok=True)
36+ shutil.copyfile(p, dest)
37+ audio_rel = f"audio/{sub['_key']}/{sol['solution_id']}.wav"
38+ fields_url = None
39+ if "fields" in sol:
40+ fpath = sol["fields"]["path"]
41+ fields_url = fpath if fpath.startswith("http") else src.blob_url(sub["_path"].parent / fpath)
42+ rows[key] = {
43+ "key": key,
44+ "source": src.label,
45+ "source_url": src.url,
46+ "submission_id": sub["submission_id"],
47+ "submission_url": src.blob_url(sub["_path"]),
48+ "solution_id": sol["solution_id"],
49+ "submitter": sub["submitter"]["name"],
50+ "date": sub.get("date"),
51+ "method": sol.get("method"),
52+ "parameters": sol.get("parameters"),
53+ "timing_seconds": sol.get("timing_seconds"),
54+ "hardware": sol.get("hardware"),
55+ "code_url": sol.get("code_url") or sub.get("code_url"),
56+ "viz_url": sol.get("viz_url") or sub.get("viz_url"),
57+ "fields_url": fields_url,
58+ "more_accurate_than": [f"{sub['_key']}/{w}" for w in sol.get("more_accurate_than", [])],
59+ "notes": sol.get("notes"),
60+ "audio": audio_rel,
61+ "peak": float(np.max(np.abs(signals[key]))),
62+ }
63+
64+ keys = list(signals)
65+ pairwise = {name: {a: {} for a in keys} for name in names}
66+ for name in names:
67+ params = challenge["metrics"][name]
68+ for a in keys:
69+ for r in keys:
70+ pairwise[name][a][r] = 0.0 if a == r else compute_metric(name, signals[a], signals[r], params)
71+
72+ info = collect_order(submissions)
73+ explicit = challenge.get("reference_solution")
74+ if explicit is not None and explicit not in keys:
75+ raise KeyError(f"reference_solution {explicit!r} is not among the solutions")
76+ selection = select_reference(info, pairwise[primary], explicit)
77+ ref = selection["reference"]
78+ for key, row in rows.items():
79+ row["errors"] = {name: pairwise[name][key][ref] for name in names} if ref else {}
80+ row["is_reference"] = key == ref
81+
82+ violations = []
83+ if ref:
84+ for worse, better, _sid in info.pairs:
85+ for name in names:
86+ if not pairwise[name][better][ref] < pairwise[name][worse][ref]:
87+ violations.append({"metric": name, "worse": worse, "better": better})
88+
89+ ordered = sorted(rows.values(), key=lambda r: (r["errors"].get(primary, float("inf")), r["key"]))
90+ return {
91+ "challenge_id": challenge["challenge_id"],
92+ "title": challenge.get("title"),
93+ "generated": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
94+ "metrics": names,
95+ "primary_metric": primary,
96+ "reference": ref,
97+ "reference_selection": selection,
98+ "order_violations_vs_reference": violations,
99+ "solutions": ordered,
100+ "pairwise": pairwise,
101+ "submissions": [
102+ {**{k: v for k, v in public_fields(sub).items() if k != "solutions"}, "key": sub["_key"], "source": sub["_source"].label, "source_url": sub["_source"].url}
103+ for sub in submissions
104+ ],
105+ }
106+
107+
108+# ---------------------------------------------------------------- HTML
109+
110+_CSS = """
111+:root { --fg: #1b1b1b; --bg: #ffffff; --muted: #666; --line: #ddd; --accent: #2a6db0; --ref: #e8f4e8; }
112+@media (prefers-color-scheme: dark) { :root { --fg: #e6e6e6; --bg: #151515; --muted: #999; --line: #333; --accent: #7ab3e8; --ref: #1f3320; } }
113+body { font-family: system-ui, sans-serif; color: var(--fg); background: var(--bg); margin: 0; padding: 16px; line-height: 1.45; }
114+main { max-width: 1300px; margin: 0 auto; }
115+a { color: var(--accent); }
116+table { border-collapse: collapse; width: 100%; font-size: 0.92em; }
117+th, td { border-bottom: 1px solid var(--line); padding: 6px 8px; text-align: left; vertical-align: top; }
118+th { position: sticky; top: 0; background: var(--bg); }
119+td.num { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; }
120+tr.ref { background: var(--ref); }
121+.muted { color: var(--muted); }
122+.wrap { overflow-x: auto; }
123+audio { width: 180px; height: 28px; }
124+details { margin: 1em 0; }
125+code { font-size: 0.9em; }
126+"""
127+
128+
129+def _fmt(x, digits=4):
130+ if x is None:
131+ return ""
132+ if isinstance(x, float):
133+ if x == 0:
134+ return "0"
135+ return f"{x:.{digits}g}"
136+ return html.escape(str(x))
137+
138+
139+def _link(url, text):
140+ if not url:
141+ return ""
142+ return f'<a href="{html.escape(url)}">{html.escape(text)}</a>'
143+
144+
145+def render_challenge_page(res: dict, challenge: dict, links: dict) -> str:
146+ names = res["metrics"]
147+ primary = res["primary_metric"]
148+ head = "".join(f"<th>{html.escape(n)}{' (primary)' if n == primary else ''}</th>" for n in names)
149+ body = []
150+ for r in res["solutions"]:
151+ errs = "".join(f'<td class="num">{_fmt(r["errors"].get(n))}</td>' for n in names)
152+ params = html.escape(json.dumps(r["parameters"])) if r["parameters"] else ""
153+ mat = ", ".join(html.escape(k.rsplit("/", 1)[1]) for k in r["more_accurate_than"])
154+ audio = f'<audio controls preload="none" src="{html.escape(r["audio"])}"></audio>' if r["audio"] else ""
155+ linkbits = " ".join(x for x in [
156+ _link(r["audio"], "wav"), _link(r["fields_url"], "fields"), _link(r["code_url"], "code"), _link(r["viz_url"], "viz"),
157+ ] if x)
158+ sub_cell = _link(r["submission_url"], r["submission_id"]) if r["submission_url"] else html.escape(r["submission_id"])
159+ src_cell = _link(r["source_url"], r["source"]) if r["source"] else ""
160+ body.append(
161+ f'<tr class="{"ref" if r["is_reference"] else ""}">'
162+ f'<td>{sub_cell}<br><span class="muted">{src_cell}{" · " if src_cell else ""}{html.escape(r["submitter"])}</span></td>'
163+ f'<td>{html.escape(r["solution_id"])}{" <b>(reference)</b>" if r["is_reference"] else ""}</td>'
164+ f'<td>{html.escape(r["method"] or "")}<br><span class="muted"><code>{params}</code></span></td>'
165+ f"{errs}"
166+ f'<td class="num">{_fmt(r["timing_seconds"], 3)}</td>'
167+ f'<td>{html.escape(r["hardware"] or "")}</td>'
168+ f'<td class="muted">{mat}</td>'
169+ f"<td>{audio}<br>{linkbits}</td>"
170+ "</tr>"
171+ )
172+ sel = res["reference_selection"]
173+ cand_rows = "".join(
174+ f"<tr><td>{html.escape(c['candidate'])}</td><td class='num'>{c['pairs_preserved']}/{c['pairs_total']}</td>"
175+ f"<td class='num'>{_fmt(c['convergence_gap'])}</td><td class='num'>{_fmt(c['mean_error_to_other_candidates'])}</td>"
176+ f"<td>{html.escape(', '.join(f'{w} vs {b}' for w, b in c['violated_pairs']))}</td></tr>"
177+ for c in sel.get("candidates", [])
178+ )
179+ viol = res["order_violations_vs_reference"]
180+ viol_html = (
181+ "<p>All asserted accuracy orderings are preserved by both metrics under the chosen reference.</p>"
182+ if not viol
183+ else "<ul>" + "".join(
184+ f"<li>{html.escape(v['metric'])}: {html.escape(v['better'])} asserted more accurate than {html.escape(v['worse'])}, but its error is not smaller</li>"
185+ for v in viol
186+ ) + "</ul>"
187+ )
188+ pair_tables = []
189+ keys = [r["key"] for r in res["solutions"]]
190+ for n in names:
191+ hdr = "".join(f"<th>{html.escape(k)}</th>" for k in keys)
192+ rws = "".join(
193+ f"<tr><th>{html.escape(a)}</th>" + "".join(f"<td class='num'>{_fmt(res['pairwise'][n][a][b], 3)}</td>" for b in keys) + "</tr>"
194+ for a in keys
195+ )
196+ pair_tables.append(f"<h4>{html.escape(n)}</h4><div class='wrap'><table><tr><th></th>{hdr}</tr>{rws}</table></div>")
197+ spec_link = links.get("spec")
198+ return f"""<!doctype html>
199+<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
200+<title>{html.escape(res["challenge_id"])} results</title><style>{_CSS}</style></head>
201+<body><main>
202+<p><a href="../">All challenges</a></p>
203+<h1>{html.escape(res["challenge_id"])}</h1>
204+<p>{html.escape(res.get("title") or "")}. {"<a href='" + html.escape(spec_link) + "'>Challenge specification</a>. " if spec_link else ""}<a href="results.json">results.json</a>. Generated {html.escape(res["generated"])}.</p>
205+<p>Errors are relative to the reference solution <b>{html.escape(res["reference"] or "none")}</b>, chosen by rule <i>{html.escape(sel["rule"])}</i>. Rows are sorted by the primary metric. Submissions are named as <i>repository/submission</i>.</p>
206+<div class="wrap"><table>
207+<tr><th>Submission</th><th>Solution</th><th>Method</th>{head}<th>Time (s)</th><th>Hardware</th><th>Claimed more accurate than</th><th>Output</th></tr>
208+{"".join(body)}
209+</table></div>
210+<details><summary>Reference selection</summary>
211+<p>Candidates are the maximal elements of each submission's asserted order. The score is the number of asserted pairs from other submissions whose order is preserved by the primary error to this candidate. Ties are broken first by the convergence gap (the primary error between the candidate and the nearest solution of its own submission asserted to be less accurate; smaller means a tighter convergence claim), then by the mean primary error to the other candidates.</p>
212+<div class="wrap"><table><tr><th>Candidate</th><th>Pairs preserved</th><th>Convergence gap</th><th>Mean error to other candidates</th><th>Violated pairs</th></tr>{cand_rows}</table></div>
213+<h4>Order check against the chosen reference</h4>{viol_html}
214+</details>
215+<details><summary>Pairwise errors (row vs column as reference)</summary>{"".join(pair_tables)}</details>
216+</main></body></html>
217+"""
218+
219+
220+def render_index(results: list[dict], sources: list[Source], links: dict) -> str:
221+ items = "".join(
222+ f'<li><a href="{html.escape(r["challenge_id"])}/">{html.escape(r["challenge_id"])}</a>: {html.escape(r.get("title") or "")} '
223+ f'<span class="muted">({len(r["solutions"])} solutions, reference {html.escape(r["reference"] or "none")})</span></li>'
224+ for r in results
225+ )
226+ srcs = "".join(
227+ f"<li>{_link(s.url, s.repo or s.label or str(s.dir)) if s.url else html.escape(str(s.dir))}"
228+ f"{' <span class=muted>(' + html.escape(s.branch) + ')</span>' if s.branch else ''}"
229+ f"{': ' + html.escape(s.description) if s.description else ''}</li>"
230+ for s in sources
231+ )
232+ repo = links.get("challenge_repo")
233+ return f"""<!doctype html>
234+<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
235+<title>Challenge results</title><style>{_CSS}</style></head>
236+<body><main>
237+<h1>Challenge results</h1>
238+<p>{"Specifications and registry: <a href='" + html.escape(repo) + "'>" + html.escape(repo) + "</a>." if repo else ""}</p>
239+<h2>Challenges</h2><ul>{items}</ul>
240+<h2>Registered solution repositories</h2><ul>{srcs}</ul>
241+</main></body></html>
242+"""
243+
244+
245+def build(challenges_dir: Path, sources: list[Source], out: Path, challenge_repo_url: str | None = None) -> list[dict]:
246+ challenges = load_challenges(challenges_dir)
247+ submissions = load_submissions(sources)
248+ out.mkdir(parents=True, exist_ok=True)
249+ results = []
250+ for cid, ch in challenges.items():
251+ subs = [s for s in submissions if s["challenge_id"] == cid]
252+ cdir = out / cid
253+ cdir.mkdir(parents=True, exist_ok=True)
254+ res = score_challenge(ch, subs, cdir / "audio") if subs else {
255+ "challenge_id": cid, "title": ch.get("title"), "generated": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
256+ "metrics": metric_names(ch), "primary_metric": ch["metrics"]["primary"], "reference": None,
257+ "reference_selection": {"rule": "none", "candidates": []}, "order_violations_vs_reference": [], "solutions": [], "pairwise": {}, "submissions": [],
258+ }
259+ with open(cdir / "results.json", "w") as f:
260+ json.dump(res, f, indent=1)
261+ links = {"spec": f"{challenge_repo_url}/blob/main/challenges/{cid}/README.md" if challenge_repo_url else None}
262+ (cdir / "index.html").write_text(render_challenge_page(res, ch, links))
263+ results.append(res)
264+ (out / "index.html").write_text(render_index(results, sources, {"challenge_repo": challenge_repo_url}))
265+ return results
tools/challenge_tools/cli.pyadded+83−0View file
@@ -0,0 +1,83 @@
1+from __future__ import annotations
2+
3+import argparse
4+import sys
5+from pathlib import Path
6+
7+from .loading import Source, fetch_registry, local_source, registry_sources
8+
9+DEFAULT_VAULT = "https://vault1.magland.org"
10+
11+
12+def _add_source_args(p: argparse.ArgumentParser) -> None:
13+ g = p.add_argument_group("solution sources (one of)")
14+ g.add_argument("--solutions", type=Path, help="a local directory containing <challenge-id>/<submission-id>.json")
15+ g.add_argument("--registry", type=Path, help="registry JSON listing solution repositories")
16+ g.add_argument("--clones", type=Path, default=Path("clones"), help="where 'fetch' put the registered repositories")
17+ g.add_argument("--vault-url", default=DEFAULT_VAULT)
18+
19+
20+def _sources(args) -> list[Source]:
21+ if args.solutions is not None and args.registry is not None:
22+ raise SystemExit("give either --solutions or --registry, not both")
23+ if args.solutions is not None:
24+ return [local_source(args.solutions)]
25+ if args.registry is not None:
26+ return registry_sources(args.registry, args.clones, args.vault_url)
27+ raise SystemExit("one of --solutions or --registry is required")
28+
29+
30+def main(argv=None):
31+ ap = argparse.ArgumentParser(prog="challenge-tools", description="Validate submissions and build the comparison site.")
32+ sp = ap.add_subparsers(dest="cmd", required=True)
33+
34+ f = sp.add_parser("fetch", help="clone or update the registered solution repositories")
35+ f.add_argument("--registry", required=True, type=Path)
36+ f.add_argument("--clones", type=Path, default=Path("clones"))
37+ f.add_argument("--vault-url", default=DEFAULT_VAULT)
38+
39+ v = sp.add_parser("validate", help="check submission files and their outputs")
40+ v.add_argument("--challenges", required=True, type=Path, help="directory containing <challenge-id>/challenge.json")
41+ _add_source_args(v)
42+ v.add_argument("--no-outputs", action="store_true", help="skip reading the wav and fields files")
43+
44+ b = sp.add_parser("build", help="compute scores and write the static comparison site")
45+ b.add_argument("--challenges", required=True, type=Path)
46+ _add_source_args(b)
47+ b.add_argument("--out", required=True, type=Path)
48+ b.add_argument("--challenge-repo-url", default=None)
49+
50+ args = ap.parse_args(argv)
51+ if args.cmd == "fetch":
52+ fetch_registry(args.registry, args.clones, args.vault_url)
53+ return 0
54+ if args.cmd == "validate":
55+ from .validate import validate
56+
57+ problems = validate(args.challenges, _sources(args), check_outputs=not args.no_outputs)
58+ for p in problems:
59+ print("PROBLEM:", p, file=sys.stderr)
60+ print(f"{len(problems)} problem(s)")
61+ return 1 if problems else 0
62+ if args.cmd == "build":
63+ from .build import build
64+ from .validate import validate
65+
66+ sources = _sources(args)
67+ problems = validate(args.challenges, sources)
68+ if problems:
69+ for p in problems:
70+ print("PROBLEM:", p, file=sys.stderr)
71+ return 1
72+ results = build(args.challenges, sources, args.out, args.challenge_repo_url)
73+ for r in results:
74+ print(f"{r['challenge_id']}: {len(r['solutions'])} solutions, reference = {r['reference']}")
75+ for s in r["solutions"]:
76+ errs = " ".join(f"{k}={v:.3e}" for k, v in s["errors"].items())
77+ print(f" {s['key']:50s} {errs} t={s['timing_seconds']}")
78+ return 0
79+ return 2
80+
81+
82+if __name__ == "__main__":
83+ sys.exit(main())
tools/challenge_tools/loading.pyadded+128−0View file
@@ -0,0 +1,128 @@
1+"""Loading challenge definitions, solution sources, and submission files."""
2+from __future__ import annotations
3+
4+import json
5+import subprocess
6+from dataclasses import dataclass
7+from pathlib import Path
8+
9+
10+@dataclass
11+class Source:
12+ """One place solutions come from: a registered repository clone, or a local directory."""
13+
14+ label: str | None # short name used to namespace submission ids; None for a single local directory
15+ dir: Path # directory holding <challenge-id>/<submission-id>.json
16+ repo: str | None = None # "<collection>/<repo>"
17+ url: str | None = None # web URL of the repository
18+ branch: str | None = None
19+ root: Path | None = None # clone root, to express files as repository-relative paths
20+ description: str | None = None
21+
22+ def blob_url(self, path: Path) -> str | None:
23+ if self.url is None or self.root is None:
24+ return None
25+ try:
26+ rel = Path(path).resolve().relative_to(self.root.resolve())
27+ except ValueError:
28+ return None
29+ return f"{self.url}/blob/{self.branch or 'main'}/{rel.as_posix()}"
30+
31+
32+def load_challenges(challenges_dir: Path) -> dict[str, dict]:
33+ out = {}
34+ for p in sorted(Path(challenges_dir).glob("*/challenge.json")):
35+ with open(p) as f:
36+ ch = json.load(f)
37+ ch["_dir"] = p.parent
38+ out[ch["challenge_id"]] = ch
39+ return out
40+
41+
42+def read_registry(registry_path: Path) -> list[dict]:
43+ with open(registry_path) as f:
44+ reg = json.load(f)
45+ repos = reg["repos"]
46+ for r in repos:
47+ r.setdefault("branch", "main")
48+ r.setdefault("path", "solutions")
49+ r["label"] = r.get("label") or r["repo"].split("/")[-1]
50+ labels = [r["label"] for r in repos]
51+ if len(set(labels)) != len(labels):
52+ raise ValueError("registry labels are not unique; give colliding entries an explicit 'label'")
53+ return repos
54+
55+
56+def clone_dir(clones: Path, entry: dict) -> Path:
57+ return Path(clones) / entry["label"]
58+
59+
60+def fetch_registry(registry_path: Path, clones: Path, vault_url: str) -> list[Source]:
61+ """Clone or update every registered repository (shallow, one branch). Anonymous, so public repositories only."""
62+ sources = []
63+ clones.mkdir(parents=True, exist_ok=True)
64+ for entry in read_registry(registry_path):
65+ dest = clone_dir(clones, entry)
66+ remote = f"{vault_url.rstrip('/')}/{entry['repo']}"
67+ if (dest / ".git").exists():
68+ subprocess.run(["git", "-C", str(dest), "fetch", "--depth", "1", "origin", entry["branch"]], check=True)
69+ subprocess.run(["git", "-C", str(dest), "checkout", "-q", "--force", "FETCH_HEAD"], check=True)
70+ else:
71+ subprocess.run(["git", "clone", "-q", "--depth", "1", "--branch", entry["branch"], remote, str(dest)], check=True)
72+ # bring in LFS objects when git-lfs is available; otherwise pointer files stay and are tolerated downstream
73+ if subprocess.run(["git", "lfs", "version"], capture_output=True).returncode == 0:
74+ lfs = subprocess.run(["git", "-C", str(dest), "lfs", "pull"], capture_output=True, text=True)
75+ if lfs.returncode != 0:
76+ print(f"warning: git lfs pull failed for {entry['repo']}: {lfs.stderr.strip()[:200]}")
77+ sha = subprocess.run(["git", "-C", str(dest), "rev-parse", "HEAD"], check=True, capture_output=True, text=True).stdout.strip()
78+ print(f"{entry['repo']}@{entry['branch']} -> {dest} ({sha[:10]})")
79+ sources.append(source_from_entry(entry, clones, vault_url))
80+ return sources
81+
82+
83+def source_from_entry(entry: dict, clones: Path, vault_url: str) -> Source:
84+ root = clone_dir(clones, entry)
85+ return Source(
86+ label=entry["label"],
87+ dir=root / entry["path"],
88+ repo=entry["repo"],
89+ url=f"{vault_url.rstrip('/')}/{entry['repo']}",
90+ branch=entry["branch"],
91+ root=root,
92+ description=entry.get("description"),
93+ )
94+
95+
96+def registry_sources(registry_path: Path, clones: Path, vault_url: str) -> list[Source]:
97+ return [source_from_entry(e, clones, vault_url) for e in read_registry(registry_path)]
98+
99+
100+def local_source(solutions_dir: Path) -> Source:
101+ return Source(label=None, dir=Path(solutions_dir))
102+
103+
104+def submission_key(sub: dict) -> str:
105+ label = sub.get("_source").label if sub.get("_source") else None
106+ return f"{label}/{sub['submission_id']}" if label else sub["submission_id"]
107+
108+
109+def load_submissions(sources: list[Source]) -> list[dict]:
110+ """Every submission JSON under <source.dir>/<challenge-id>/*.json, with _source, _path, _dir_challenge, _key set."""
111+ out = []
112+ for src in sources:
113+ if not src.dir.exists():
114+ print(f"warning: solutions directory {src.dir} does not exist")
115+ continue
116+ for p in sorted(src.dir.glob("*/*.json")):
117+ with open(p) as f:
118+ sub = json.load(f)
119+ sub["_source"] = src
120+ sub["_path"] = p
121+ sub["_dir_challenge"] = p.parent.name
122+ sub["_key"] = submission_key(sub)
123+ out.append(sub)
124+ return out
125+
126+
127+def public_fields(sub: dict) -> dict:
128+ return {k: v for k, v in sub.items() if not k.startswith("_")}
tools/challenge_tools/metrics.pyadded+36−0View file
@@ -0,0 +1,36 @@
1+"""Error measures between a solution and a reference, as defined in each challenge README."""
2+from __future__ import annotations
3+
4+import numpy as np
5+
6+
7+def waveform_l2(a: np.ndarray, r: np.ndarray) -> float:
8+ return float(np.linalg.norm(a - r) / np.linalg.norm(r))
9+
10+
11+def spectrogram(x: np.ndarray, n_fft: int, hop: int) -> np.ndarray:
12+ """Magnitude STFT with a periodic Hann window, no padding. Shape (frames, n_fft//2 + 1)."""
13+ j = np.arange(n_fft)
14+ g = 0.5 * (1.0 - np.cos(2.0 * np.pi * j / n_fft))
15+ m = (len(x) - n_fft) // hop + 1
16+ idx = j[None, :] + hop * np.arange(m)[:, None]
17+ frames = x[idx] * g[None, :]
18+ return np.abs(np.fft.rfft(frames, axis=1))
19+
20+
21+def spectrogram_l2(a: np.ndarray, r: np.ndarray, n_fft: int, hop: int) -> float:
22+ sa = spectrogram(a, n_fft, hop)
23+ sr = spectrogram(r, n_fft, hop)
24+ return float(np.linalg.norm(sa - sr) / np.linalg.norm(sr))
25+
26+
27+def compute_metric(name: str, a: np.ndarray, r: np.ndarray, params: dict) -> float:
28+ if name == "waveform_l2":
29+ return waveform_l2(a, r)
30+ if name == "spectrogram_l2":
31+ return spectrogram_l2(a, r, int(params["n_fft"]), int(params["hop"]))
32+ raise KeyError(f"unknown metric {name}")
33+
34+
35+def metric_names(challenge: dict) -> list[str]:
36+ return [k for k in challenge["metrics"] if k != "primary"]
tools/challenge_tools/reference.pyadded+87−0View file
@@ -0,0 +1,87 @@
1+"""Choosing the reference solution from the submitted solutions.
2+
3+Each submission may assert a partial order among its own solutions through
4+``more_accurate_than``. Only orderings within a submission are trusted. A candidate
5+reference R is scored by the fraction of asserted pairs (worse, better) from *other*
6+submissions for which err(better, R) < err(worse, R) under the primary metric.
7+Candidates are the maximal elements of each submission's order. Ties are broken by the
8+smallest mean primary error to the other candidates; with no asserted pairs at all this
9+tie-break is the whole rule, so the reference is then the medoid of the candidates.
10+"""
11+from __future__ import annotations
12+
13+from dataclasses import dataclass, field
14+
15+
16+@dataclass
17+class OrderInfo:
18+ # (worse, better, submission_key) triples over global solution keys "[source/]submission/solution"
19+ pairs: list[tuple[str, str, str]] = field(default_factory=list)
20+ # global key -> submission id
21+ submission_of: dict[str, str] = field(default_factory=dict)
22+
23+
24+def collect_order(submissions: list[dict]) -> OrderInfo:
25+ info = OrderInfo()
26+ for sub in submissions:
27+ sid = sub.get("_key", sub["submission_id"])
28+ for sol in sub["solutions"]:
29+ key = f"{sid}/{sol['solution_id']}"
30+ info.submission_of[key] = sid
31+ for worse in sol.get("more_accurate_than", []):
32+ info.pairs.append((f"{sid}/{worse}", key, sid))
33+ return info
34+
35+
36+def maximal_elements(info: OrderInfo) -> list[str]:
37+ dominated = {worse for worse, _better, _sid in info.pairs}
38+ return [k for k in info.submission_of if k not in dominated]
39+
40+
41+def select_reference(
42+ info: OrderInfo,
43+ err: dict[str, dict[str, float]],
44+ explicit: str | None = None,
45+) -> dict:
46+ """Return a report dict with keys: reference, candidates (list of dicts), rule."""
47+ if explicit is not None:
48+ return {"reference": explicit, "rule": "explicit", "candidates": []}
49+
50+ candidates = maximal_elements(info)
51+ rows = []
52+ for cand in candidates:
53+ external = [p for p in info.pairs if p[2] != info.submission_of[cand]]
54+ preserved = 0
55+ violated = []
56+ for worse, better, _sid in external:
57+ if err[better][cand] < err[worse][cand]:
58+ preserved += 1
59+ else:
60+ violated.append([worse, better])
61+ others = [c for c in candidates if c != cand]
62+ mean_err = sum(err[c][cand] for c in others) / len(others) if others else 0.0
63+ siblings = [worse for worse, better, _sid in info.pairs if better == cand]
64+ gap = min((err[w][cand] for w in siblings), default=None)
65+ rows.append(
66+ {
67+ "candidate": cand,
68+ "pairs_total": len(external),
69+ "pairs_preserved": preserved,
70+ "score": (preserved / len(external)) if external else None,
71+ "convergence_gap": gap,
72+ "mean_error_to_other_candidates": mean_err,
73+ "violated_pairs": violated,
74+ }
75+ )
76+
77+ def sort_key(row):
78+ score = row["score"] if row["score"] is not None else -1.0
79+ gap = row["convergence_gap"] if row["convergence_gap"] is not None else float("inf")
80+ return (-score, gap, row["mean_error_to_other_candidates"])
81+
82+ rows.sort(key=sort_key)
83+ return {
84+ "reference": rows[0]["candidate"] if rows else None,
85+ "rule": "partial-order-agreement",
86+ "candidates": rows,
87+ }
tools/challenge_tools/solution.schema.jsonadded+143−0View file
@@ -0,0 +1,143 @@
1+{
2+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3+ "$id": "https://vault1.magland.org/challenges/string/schema/solution.schema.json",
4+ "title": "Challenge solution submission",
5+ "description": "One submission file: a set of solutions to one challenge from one submitter, with an optional accuracy partial order among them.",
6+ "type": "object",
7+ "required": [
8+ "schema_version",
9+ "submission_id",
10+ "challenge_id",
11+ "submitter",
12+ "solutions"
13+ ],
14+ "additionalProperties": false,
15+ "properties": {
16+ "schema_version": {
17+ "const": 1
18+ },
19+ "submission_id": {
20+ "type": "string",
21+ "pattern": "^[a-z0-9][a-z0-9-]*$",
22+ "description": "Identifier of this submission. Must equal the file name without .json."
23+ },
24+ "challenge_id": {
25+ "type": "string",
26+ "pattern": "^[a-z0-9][a-z0-9-]*$",
27+ "description": "The challenge this submission solves. Must equal the name of the directory containing the file."
28+ },
29+ "submitter": {
30+ "type": "object",
31+ "required": [
32+ "name"
33+ ],
34+ "additionalProperties": false,
35+ "properties": {
36+ "name": {
37+ "type": "string"
38+ },
39+ "email": {
40+ "type": "string"
41+ },
42+ "url": {
43+ "type": "string",
44+ "format": "uri"
45+ }
46+ }
47+ },
48+ "date": {
49+ "type": "string",
50+ "format": "date"
51+ },
52+ "description": {
53+ "type": "string"
54+ },
55+ "code_url": {
56+ "type": "string",
57+ "format": "uri",
58+ "description": "Source code for the method, applying to every solution unless overridden."
59+ },
60+ "viz_url": {
61+ "type": "string",
62+ "format": "uri",
63+ "description": "A web application visualizing the solutions."
64+ },
65+ "solutions": {
66+ "type": "array",
67+ "minItems": 1,
68+ "items": {
69+ "type": "object",
70+ "required": [
71+ "solution_id",
72+ "output"
73+ ],
74+ "additionalProperties": false,
75+ "properties": {
76+ "solution_id": {
77+ "type": "string",
78+ "pattern": "^[a-z0-9][a-z0-9-]*$",
79+ "description": "Unique within the submission."
80+ },
81+ "output": {
82+ "type": "string",
83+ "description": "Path to the wav file relative to the directory containing this JSON file, or an http(s) URL."
84+ },
85+ "method": {
86+ "type": "string",
87+ "description": "Short name of the numerical method."
88+ },
89+ "parameters": {
90+ "type": "object",
91+ "description": "Discretization parameters, free form."
92+ },
93+ "timing_seconds": {
94+ "type": "number",
95+ "minimum": 0,
96+ "description": "Self-reported wall-clock time to produce the output, excluding file I/O."
97+ },
98+ "hardware": {
99+ "type": "string"
100+ },
101+ "code_url": {
102+ "type": "string",
103+ "format": "uri"
104+ },
105+ "viz_url": {
106+ "type": "string",
107+ "format": "uri"
108+ },
109+ "more_accurate_than": {
110+ "type": "array",
111+ "items": {
112+ "type": "string"
113+ },
114+ "description": "solution_ids in this same file that the submitter asserts are less accurate than this one."
115+ },
116+ "notes": {
117+ "type": "string"
118+ },
119+ "fields": {
120+ "type": "object",
121+ "required": [
122+ "path"
123+ ],
124+ "additionalProperties": false,
125+ "description": "Optional export of the solution field on the grid recommended by the challenge, for visualization. Not scored.",
126+ "properties": {
127+ "path": {
128+ "type": "string",
129+ "description": "Path relative to the directory containing this JSON file, or an http(s) URL."
130+ },
131+ "format": {
132+ "const": "hdf5"
133+ },
134+ "description": {
135+ "type": "string"
136+ }
137+ }
138+ }
139+ }
140+ }
141+ }
142+ }
143+}
tools/challenge_tools/validate.pyadded+142−0View file
@@ -0,0 +1,142 @@
1+"""Validation of submission files against the schema and the challenge output spec."""
2+from __future__ import annotations
3+
4+import json
5+import sys
6+from importlib import resources
7+from pathlib import Path
8+
9+import jsonschema
10+
11+from .loading import Source, load_challenges, load_submissions, public_fields
12+from .wav import read_wav, resolve_output
13+
14+
15+def _schema() -> dict:
16+ with resources.files("challenge_tools").joinpath("solution.schema.json").open() as f:
17+ return json.load(f)
18+
19+
20+def _acyclic(edges: list[tuple[str, str]], nodes: list[str]) -> bool:
21+ adj = {n: [] for n in nodes}
22+ for a, b in edges:
23+ adj[a].append(b)
24+ state = {n: 0 for n in nodes}
25+
26+ def visit(n):
27+ state[n] = 1
28+ for m in adj[n]:
29+ if state[m] == 1 or (state[m] == 0 and not visit(m)):
30+ return False
31+ state[n] = 2
32+ return True
33+
34+ return all(state[n] == 2 or visit(n) for n in nodes)
35+
36+
37+LFS_POINTER_PREFIX = b"version https://git-lfs.github.com/spec/v1"
38+
39+
40+def is_lfs_pointer(path: Path) -> bool:
41+ try:
42+ with open(path, "rb") as f:
43+ return f.read(len(LFS_POINTER_PREFIX)) == LFS_POINTER_PREFIX
44+ except OSError:
45+ return False
46+
47+
48+def check_fields_file(path: Path, challenge: dict) -> list[str]:
49+ """Open an optional HDF5 field export and check its datasets against the challenge's grid.
50+
51+ A git LFS pointer file (the object was not fetched) is reported on stderr and otherwise accepted."""
52+ import h5py
53+
54+ if is_lfs_pointer(path):
55+ print(f"note: {path} is a git LFS pointer whose object was not fetched; fields not checked", file=sys.stderr)
56+ return []
57+ problems = []
58+ spec = challenge.get("fields") or {}
59+ with h5py.File(path, "r") as f:
60+ if "u" not in f:
61+ return ["fields file lacks dataset 'u'"]
62+ axes = spec.get("axes")
63+ if axes is None:
64+ axes = ["t", "y", "x"] if "y" in f else ["t", "x"]
65+ for name in axes:
66+ if name not in f:
67+ problems.append(f"fields file lacks axis dataset {name!r}")
68+ if problems:
69+ return problems
70+ expected = tuple(int(f[name].shape[0]) for name in axes)
71+ if tuple(f["u"].shape) != expected:
72+ problems.append(f"fields dataset u has shape {tuple(f['u'].shape)}, expected {expected} from axes {axes}")
73+ if spec.get("shape") and tuple(f["u"].shape) != tuple(spec["shape"]):
74+ problems.append(f"fields grid is {tuple(f['u'].shape)}, challenge recommends {tuple(spec['shape'])}")
75+ return problems
76+
77+
78+def validate(challenges_dir: Path, sources: list[Source], check_outputs: bool = True) -> list[str]:
79+ """Return a list of problems (empty means valid)."""
80+ problems: list[str] = []
81+ challenges = load_challenges(challenges_dir)
82+ if not challenges:
83+ problems.append(f"no challenge.json found under {challenges_dir}")
84+ schema = _schema()
85+ validator = jsonschema.Draft202012Validator(schema, format_checker=jsonschema.FormatChecker())
86+ seen = set()
87+ for sub in load_submissions(sources):
88+ path: Path = sub["_path"]
89+ tag = str(path)
90+ errs = sorted(validator.iter_errors(public_fields(sub)), key=str)
91+ for e in errs:
92+ loc = "/".join(str(x) for x in e.absolute_path)
93+ problems.append(f"{tag}: schema: {loc}: {e.message}")
94+ if errs:
95+ continue
96+ if sub["submission_id"] != path.stem:
97+ problems.append(f"{tag}: submission_id {sub['submission_id']!r} does not match file name")
98+ if sub["challenge_id"] != sub["_dir_challenge"]:
99+ problems.append(f"{tag}: challenge_id {sub['challenge_id']!r} does not match directory {sub['_dir_challenge']!r}")
100+ if sub["challenge_id"] not in challenges:
101+ problems.append(f"{tag}: unknown challenge_id {sub['challenge_id']!r}")
102+ continue
103+ key = (sub["challenge_id"], sub["_key"])
104+ if key in seen:
105+ problems.append(f"{tag}: duplicate submission_id for this challenge")
106+ seen.add(key)
107+ ids = [s["solution_id"] for s in sub["solutions"]]
108+ if len(set(ids)) != len(ids):
109+ problems.append(f"{tag}: duplicate solution_id")
110+ edges = []
111+ for s in sub["solutions"]:
112+ for worse in s.get("more_accurate_than", []):
113+ if worse not in ids:
114+ problems.append(f"{tag}: {s['solution_id']}: more_accurate_than names unknown solution {worse!r}")
115+ elif worse == s["solution_id"]:
116+ problems.append(f"{tag}: {s['solution_id']}: more_accurate_than names itself")
117+ else:
118+ edges.append((worse, s["solution_id"]))
119+ if not _acyclic(edges, ids):
120+ problems.append(f"{tag}: more_accurate_than contains a cycle")
121+ if check_outputs:
122+ challenge = challenges[sub["challenge_id"]]
123+ for s in sub["solutions"]:
124+ try:
125+ p = resolve_output(s["output"], path.parent)
126+ if not p.exists():
127+ problems.append(f"{tag}: {s['solution_id']}: output not found: {s['output']}")
128+ continue
129+ read_wav(p, challenge["output"])
130+ except Exception as e: # noqa: BLE001
131+ problems.append(f"{tag}: {s['solution_id']}: {e}")
132+ if "fields" in s:
133+ try:
134+ fp = resolve_output(s["fields"]["path"], path.parent)
135+ if not fp.exists():
136+ problems.append(f"{tag}: {s['solution_id']}: fields file not found: {s['fields']['path']}")
137+ else:
138+ for pr in check_fields_file(fp, challenge):
139+ problems.append(f"{tag}: {s['solution_id']}: {pr}")
140+ except Exception as e: # noqa: BLE001
141+ problems.append(f"{tag}: {s['solution_id']}: fields: {e}")
142+ return problems
tools/challenge_tools/wav.pyadded+52−0View file
@@ -0,0 +1,52 @@
1+"""Reading and checking wav deliverables."""
2+from __future__ import annotations
3+
4+import hashlib
5+import os
6+import urllib.request
7+from pathlib import Path
8+
9+import numpy as np
10+from scipy.io import wavfile
11+
12+
13+def resolve_output(output: str, json_dir: Path, cache_dir: Path | None = None) -> Path:
14+ """Return a local path for a solution output, downloading URLs into cache_dir."""
15+ if output.startswith("http://") or output.startswith("https://"):
16+ cache_dir = cache_dir or Path(os.environ.get("CHALLENGE_CACHE", ".challenge-cache"))
17+ cache_dir.mkdir(parents=True, exist_ok=True)
18+ name = hashlib.sha256(output.encode()).hexdigest()[:16] + ".wav"
19+ dest = cache_dir / name
20+ if not dest.exists():
21+ urllib.request.urlretrieve(output, dest)
22+ return dest
23+ p = (json_dir / output).resolve()
24+ return p
25+
26+
27+def read_wav(path: Path, spec: dict) -> np.ndarray:
28+ """Read a wav and check it against the challenge's output spec. Returns float64 samples (mono)."""
29+ rate, data = wavfile.read(path)
30+ problems = []
31+ if rate != spec["sample_rate_hz"]:
32+ problems.append(f"sample rate {rate} != {spec['sample_rate_hz']}")
33+ if spec.get("sample_format") == "float32" and data.dtype != np.float32:
34+ problems.append(f"sample format {data.dtype} != float32")
35+ channels = 1 if data.ndim == 1 else data.shape[1]
36+ if channels != spec["channels"]:
37+ problems.append(f"channels {channels} != {spec['channels']}")
38+ n = data.shape[0]
39+ if n != spec["num_samples"]:
40+ problems.append(f"num samples {n} != {spec['num_samples']}")
41+ if not np.all(np.isfinite(data)):
42+ problems.append("non-finite samples")
43+ if problems:
44+ raise ValueError(f"{path}: " + "; ".join(problems))
45+ if data.ndim > 1:
46+ data = data[:, 0]
47+ return data.astype(np.float64)
48+
49+
50+def write_wav(path: Path, samples: np.ndarray, sample_rate: int) -> None:
51+ path.parent.mkdir(parents=True, exist_ok=True)
52+ wavfile.write(path, sample_rate, np.asarray(samples, dtype=np.float32))