dc14526Shared challenge tools: schema, validation, metrics, reference selection, comparison site; moved out of the string repositoryJeremy Magland 1import numpy as np
3from challenge_tools.metrics import spectrogram, spectrogram_l2, waveform_l2
4from challenge_tools.reference import OrderInfo, collect_order, maximal_elements, select_reference
7def test_waveform_l2():
8 r = np.array([1.0, 2.0, 2.0])
9 assert waveform_l2(r, r) == 0.0
10 assert abs(waveform_l2(r * 1.1, r) - 0.1) < 1e-12
13def test_spectrogram_shape():
14 x = np.zeros(96000)
15 s = spectrogram(x, 2048, 512)
16 assert s.shape == ((96000 - 2048) // 512 + 1, 1025)
19def test_spectrogram_phase_insensitive():
20 t = np.arange(96000) / 48000
21 a = np.sin(2 * np.pi * 440 * t)
22 b = np.sin(2 * np.pi * 440 * t + 1.0)
23 assert waveform_l2(a, b) > 0.5
24 assert spectrogram_l2(a, b, 2048, 512) < 1e-2
27def test_reference_selection_prefers_order_preserving_candidate():
28 subs = [
29 {"submission_id": "x", "solutions": [
30 {"solution_id": "a", "more_accurate_than": []},
31 {"solution_id": "b", "more_accurate_than": ["a"]},
32 ]},
33 {"submission_id": "y", "solutions": [
34 {"solution_id": "c", "more_accurate_than": []},
35 {"solution_id": "d", "more_accurate_than": ["c"]},
36 ]},
37 ]
38 info = collect_order(subs)
39 assert set(maximal_elements(info)) == {"x/b", "y/d"}
40 # Truth: x/b is exact; y/d is wrong. Errors to x/b respect y's chain, errors to y/d do not respect x's chain.
41 keys = ["x/a", "x/b", "y/c", "y/d"]
42 truth = {"x/a": 1.0, "x/b": 0.0, "y/c": 3.0, "y/d": 2.5}
43 err = {a: {b: abs(truth[a] - truth[b]) for b in keys} for a in keys}
44 # under y/d: err(x/b)=2.5 > err(x/a)=1.5, so x's pair is violated
45 rep = select_reference(info, err)
46 assert rep["reference"] == "x/b"
47 assert rep["candidates"][0]["score"] == 1.0
50def test_reference_selection_medoid_fallback():
51 info = OrderInfo(pairs=[], submission_of={"p/a": "p", "q/b": "q", "r/c": "r"})
52 keys = list(info.submission_of)
53 pos = {"p/a": 0.0, "q/b": 1.0, "r/c": 5.0}
54 err = {a: {b: abs(pos[a] - pos[b]) for b in keys} for a in keys}
55 rep = select_reference(info, err)
56 assert rep["reference"] == "q/b"
59def test_reference_selection_tie_broken_by_convergence_gap():
60 subs = [
61 {"submission_id": "x", "solutions": [
62 {"solution_id": "a", "more_accurate_than": []},
63 {"solution_id": "b", "more_accurate_than": ["a"]},
64 ]},
65 {"submission_id": "y", "solutions": [
66 {"solution_id": "c", "more_accurate_than": []},
67 {"solution_id": "d", "more_accurate_than": ["c"]},
68 ]},
69 ]
70 info = collect_order(subs)
71 keys = ["x/a", "x/b", "y/c", "y/d"]
72 # both chains monotone toward 0; x/b is much closer to its sibling than y/d is to its own
73 truth = {"x/a": 0.03, "x/b": 0.0001, "y/c": 1.0, "y/d": 0.01}
74 err = {a: {b: abs(truth[a] - truth[b]) for b in keys} for a in keys}
75 rep = select_reference(info, err)
76 assert [c["score"] for c in rep["candidates"]] == [1.0, 1.0]
77 assert rep["reference"] == "x/b"
80def test_fields_check_axes(tmp_path):
81 import h5py
82 import numpy as np
84 from challenge_tools.validate import check_fields_file, is_lfs_pointer
86 p = tmp_path / "f.h5"
87 with h5py.File(p, "w") as f:
88 f.create_dataset("x", data=np.linspace(0, 1, 5))
89 f.create_dataset("y", data=np.linspace(0, 1, 3))
90 f.create_dataset("t", data=np.linspace(0, 1, 7))
91 f.create_dataset("u", data=np.zeros((7, 3, 5), dtype=np.float32))
92 ch = {"fields": {"axes": ["t", "y", "x"], "shape": [7, 3, 5]}}
93 assert check_fields_file(p, ch) == []
94 ch_bad = {"fields": {"axes": ["t", "y", "x"], "shape": [7, 3, 6]}}
95 assert any("recommends" in m for m in check_fields_file(p, ch_bad))
96 ptr = tmp_path / "ptr.h5"
97 ptr.write_text("version https://git-lfs.github.com/spec/v1\noid sha256:abc\nsize 1\n")
98 assert is_lfs_pointer(ptr)
99 assert check_fields_file(ptr, ch) == []
f99e1d0Interactive 3D geometry view: viewer.js (raw WebGL) renders a challenge's scene.json on its results pageJeremy Magland 100
102def test_build_embeds_scene(tmp_path):
103 import json
104 from challenge_tools.build import build
106 cdir = tmp_path / "challenges" / "demo-v1"
107 cdir.mkdir(parents=True)
108 (cdir / "README.md").write_text("# demo")
109 json.dump({"challenge_id": "demo-v1", "output": {"sample_rate_hz": 48000, "num_samples": 10, "channels": 1, "sample_format": "float32", "signal_scale": 1.0},
110 "metrics": {"primary": "waveform_l2", "waveform_l2": {}}, "reference_solution": None}, open(cdir / "challenge.json", "w"))
111 json.dump({"surfaces": [{"vertices": [[0, 0, 0], [1, 0, 0], [0, 1, 0]], "faces": [[0, 1, 2]]}]}, open(cdir / "scene.json", "w"))
112 out = tmp_path / "site"
113 build(tmp_path / "challenges", [], out)
114 assert (out / "demo-v1" / "scene.json").exists()
115 page = (out / "demo-v1" / "index.html").read_text()
116 assert "scene.json" in page and "viewer.js" in page
117 assert "3D model" in (out / "index.html").read_text()
118 assert "SceneViewer" in (out / "viewer.js").read_text()