"""Validation of submission files against the schema and the challenge output spec.""" from __future__ import annotations import json from importlib import resources from pathlib import Path import jsonschema from .loading import Source, load_challenges, load_submissions, public_fields from .wav import read_wav, resolve_output def _schema() -> dict: with resources.files("challenge_tools").joinpath("solution.schema.json").open() as f: return json.load(f) def _acyclic(edges: list[tuple[str, str]], nodes: list[str]) -> bool: adj = {n: [] for n in nodes} for a, b in edges: adj[a].append(b) state = {n: 0 for n in nodes} def visit(n): state[n] = 1 for m in adj[n]: if state[m] == 1 or (state[m] == 0 and not visit(m)): return False state[n] = 2 return True return all(state[n] == 2 or visit(n) for n in nodes) def check_fields_file(path: Path, challenge: dict) -> list[str]: """Open an optional HDF5 field export and check the datasets against the challenge's grid.""" import h5py problems = [] spec = challenge.get("fields") with h5py.File(path, "r") as f: for name in ("x", "t", "u"): if name not in f: problems.append(f"fields file lacks dataset {name!r}") if problems: return problems nx, nt = f["x"].shape[0], f["t"].shape[0] if f["u"].shape != (nt, nx): problems.append(f"fields dataset u has shape {f['u'].shape}, expected {(nt, nx)}") if spec: if nx != spec["nx"] or nt != spec["nt"]: problems.append(f"fields grid is {nx} x {nt}, challenge recommends nx={spec['nx']} nt={spec['nt']}") return problems def validate(challenges_dir: Path, sources: list[Source], check_outputs: bool = True) -> list[str]: """Return a list of problems (empty means valid).""" problems: list[str] = [] challenges = load_challenges(challenges_dir) if not challenges: problems.append(f"no challenge.json found under {challenges_dir}") schema = _schema() validator = jsonschema.Draft202012Validator(schema, format_checker=jsonschema.FormatChecker()) seen = set() for sub in load_submissions(sources): path: Path = sub["_path"] tag = str(path) errs = sorted(validator.iter_errors(public_fields(sub)), key=str) for e in errs: loc = "/".join(str(x) for x in e.absolute_path) problems.append(f"{tag}: schema: {loc}: {e.message}") if errs: continue if sub["submission_id"] != path.stem: problems.append(f"{tag}: submission_id {sub['submission_id']!r} does not match file name") if sub["challenge_id"] != sub["_dir_challenge"]: problems.append(f"{tag}: challenge_id {sub['challenge_id']!r} does not match directory {sub['_dir_challenge']!r}") if sub["challenge_id"] not in challenges: problems.append(f"{tag}: unknown challenge_id {sub['challenge_id']!r}") continue key = (sub["challenge_id"], sub["_key"]) if key in seen: problems.append(f"{tag}: duplicate submission_id for this challenge") seen.add(key) ids = [s["solution_id"] for s in sub["solutions"]] if len(set(ids)) != len(ids): problems.append(f"{tag}: duplicate solution_id") edges = [] for s in sub["solutions"]: for worse in s.get("more_accurate_than", []): if worse not in ids: problems.append(f"{tag}: {s['solution_id']}: more_accurate_than names unknown solution {worse!r}") elif worse == s["solution_id"]: problems.append(f"{tag}: {s['solution_id']}: more_accurate_than names itself") else: edges.append((worse, s["solution_id"])) if not _acyclic(edges, ids): problems.append(f"{tag}: more_accurate_than contains a cycle") if check_outputs: challenge = challenges[sub["challenge_id"]] for s in sub["solutions"]: try: p = resolve_output(s["output"], path.parent) if not p.exists(): problems.append(f"{tag}: {s['solution_id']}: output not found: {s['output']}") continue read_wav(p, challenge["output"]) except Exception as e: # noqa: BLE001 problems.append(f"{tag}: {s['solution_id']}: {e}") if "fields" in s: try: fp = resolve_output(s["fields"]["path"], path.parent) if not fp.exists(): problems.append(f"{tag}: {s['solution_id']}: fields file not found: {s['fields']['path']}") else: for pr in check_fields_file(fp, challenge): problems.append(f"{tag}: {s['solution_id']}: {pr}") except Exception as e: # noqa: BLE001 problems.append(f"{tag}: {s['solution_id']}: fields: {e}") return problems