dc14526Shared challenge tools: schema, validation, metrics, reference selection, comparison site; moved out of the string repositoryJeremy Magland 1"""Validation of submission files against the schema and the challenge output spec."""
2from __future__ import annotations
4import json
5import sys
6from importlib import resources
7from pathlib import Path
9import jsonschema
11from .loading import Source, load_challenges, load_submissions, public_fields
12from .wav import read_wav, resolve_output
15def _schema() -> dict:
16 with resources.files("challenge_tools").joinpath("solution.schema.json").open() as f:
17 return json.load(f)
20def _acyclic(edges: list[tuple[str, str]], nodes: list[str]) -> bool:
21 adj = {n: [] for n in nodes}
22 for a, b in edges:
23 adj[a].append(b)
24 state = {n: 0 for n in nodes}
26 def visit(n):
27 state[n] = 1
28 for m in adj[n]:
29 if state[m] == 1 or (state[m] == 0 and not visit(m)):
30 return False
31 state[n] = 2
32 return True
34 return all(state[n] == 2 or visit(n) for n in nodes)
37LFS_POINTER_PREFIX = b"version https://git-lfs.github.com/spec/v1"
40def is_lfs_pointer(path: Path) -> bool:
41 try:
42 with open(path, "rb") as f:
43 return f.read(len(LFS_POINTER_PREFIX)) == LFS_POINTER_PREFIX
44 except OSError:
45 return False
48def check_fields_file(path: Path, challenge: dict) -> list[str]:
49 """Open an optional HDF5 field export and check its datasets against the challenge's grid.
51 A git LFS pointer file (the object was not fetched) is reported on stderr and otherwise accepted."""
52 import h5py
54 if is_lfs_pointer(path):
55 print(f"note: {path} is a git LFS pointer whose object was not fetched; fields not checked", file=sys.stderr)
56 return []
57 problems = []
58 spec = challenge.get("fields") or {}
59 with h5py.File(path, "r") as f:
60 if "u" not in f:
61 return ["fields file lacks dataset 'u'"]
62 axes = spec.get("axes")
63 if axes is None:
64 axes = ["t", "y", "x"] if "y" in f else ["t", "x"]
65 for name in axes:
66 if name not in f:
67 problems.append(f"fields file lacks axis dataset {name!r}")
68 if problems:
69 return problems
70 expected = tuple(int(f[name].shape[0]) for name in axes)
71 if tuple(f["u"].shape) != expected:
72 problems.append(f"fields dataset u has shape {tuple(f['u'].shape)}, expected {expected} from axes {axes}")
73 if spec.get("shape") and tuple(f["u"].shape) != tuple(spec["shape"]):
74 problems.append(f"fields grid is {tuple(f['u'].shape)}, challenge recommends {tuple(spec['shape'])}")
75 return problems
78def validate(challenges_dir: Path, sources: list[Source], check_outputs: bool = True) -> list[str]:
79 """Return a list of problems (empty means valid)."""
80 problems: list[str] = []
81 challenges = load_challenges(challenges_dir)
82 if not challenges:
83 problems.append(f"no challenge.json found under {challenges_dir}")
84 schema = _schema()
85 validator = jsonschema.Draft202012Validator(schema, format_checker=jsonschema.FormatChecker())
86 seen = set()
87 for sub in load_submissions(sources):
88 path: Path = sub["_path"]
89 tag = str(path)
90 errs = sorted(validator.iter_errors(public_fields(sub)), key=str)
91 for e in errs:
92 loc = "/".join(str(x) for x in e.absolute_path)
93 problems.append(f"{tag}: schema: {loc}: {e.message}")
94 if errs:
95 continue
96 if sub["submission_id"] != path.stem:
97 problems.append(f"{tag}: submission_id {sub['submission_id']!r} does not match file name")
98 if sub["challenge_id"] != sub["_dir_challenge"]:
99 problems.append(f"{tag}: challenge_id {sub['challenge_id']!r} does not match directory {sub['_dir_challenge']!r}")
100 if sub["challenge_id"] not in challenges:
101 problems.append(f"{tag}: unknown challenge_id {sub['challenge_id']!r}")
102 continue
103 key = (sub["challenge_id"], sub["_key"])
104 if key in seen:
105 problems.append(f"{tag}: duplicate submission_id for this challenge")
106 seen.add(key)
107 ids = [s["solution_id"] for s in sub["solutions"]]
108 if len(set(ids)) != len(ids):
109 problems.append(f"{tag}: duplicate solution_id")
110 edges = []
111 for s in sub["solutions"]:
112 for worse in s.get("more_accurate_than", []):
113 if worse not in ids:
114 problems.append(f"{tag}: {s['solution_id']}: more_accurate_than names unknown solution {worse!r}")
115 elif worse == s["solution_id"]:
116 problems.append(f"{tag}: {s['solution_id']}: more_accurate_than names itself")
117 else:
118 edges.append((worse, s["solution_id"]))
119 if not _acyclic(edges, ids):
120 problems.append(f"{tag}: more_accurate_than contains a cycle")
121 if check_outputs:
122 challenge = challenges[sub["challenge_id"]]
123 for s in sub["solutions"]:
124 try:
125 p = resolve_output(s["output"], path.parent)
126 if not p.exists():
127 problems.append(f"{tag}: {s['solution_id']}: output not found: {s['output']}")
128 continue
129 read_wav(p, challenge["output"])
130 except Exception as e: # noqa: BLE001
131 problems.append(f"{tag}: {s['solution_id']}: {e}")
132 if "fields" in s:
133 try:
134 fp = resolve_output(s["fields"]["path"], path.parent)
135 if not fp.exists():
136 problems.append(f"{tag}: {s['solution_id']}: fields file not found: {s['fields']['path']}")
137 else:
138 for pr in check_fields_file(fp, challenge):
139 problems.append(f"{tag}: {s['solution_id']}: {pr}")
140 except Exception as e: # noqa: BLE001
141 problems.append(f"{tag}: {s['solution_id']}: fields: {e}")
142 return problems