ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 1from __future__ import annotations
3import argparse
4import sys
5from pathlib import Path
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 7from .loading import Source, fetch_registry, local_source, registry_sources
9DEFAULT_VAULT = "https://vault1.magland.org"
12def _add_source_args(p: argparse.ArgumentParser) -> None:
13 g = p.add_argument_group("solution sources (one of)")
14 g.add_argument("--solutions", type=Path, help="a local directory containing <challenge-id>/<submission-id>.json")
15 g.add_argument("--registry", type=Path, help="registry JSON listing solution repositories")
16 g.add_argument("--clones", type=Path, default=Path("clones"), help="where 'fetch' put the registered repositories")
17 g.add_argument("--vault-url", default=DEFAULT_VAULT)
20def _sources(args) -> list[Source]:
21 if args.solutions is not None and args.registry is not None:
22 raise SystemExit("give either --solutions or --registry, not both")
23 if args.solutions is not None:
24 return [local_source(args.solutions)]
25 if args.registry is not None:
26 return registry_sources(args.registry, args.clones, args.vault_url)
27 raise SystemExit("one of --solutions or --registry is required")
30def main(argv=None):
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 31 ap = argparse.ArgumentParser(prog="challenge-tools", description="Validate submissions and build the comparison site.")
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 32 sp = ap.add_subparsers(dest="cmd", required=True)
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 34 f = sp.add_parser("fetch", help="clone or update the registered solution repositories")
35 f.add_argument("--registry", required=True, type=Path)
36 f.add_argument("--clones", type=Path, default=Path("clones"))
37 f.add_argument("--vault-url", default=DEFAULT_VAULT)
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 39 v = sp.add_parser("validate", help="check submission files and their outputs")
40 v.add_argument("--challenges", required=True, type=Path, help="directory containing <challenge-id>/challenge.json")
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 41 _add_source_args(v)
42 v.add_argument("--no-outputs", action="store_true", help="skip reading the wav and fields files")
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 44 b = sp.add_parser("build", help="compute scores and write the static comparison site")
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 45 b.add_argument("--challenges", required=True, type=Path)
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 46 _add_source_args(b)
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 47 b.add_argument("--out", required=True, type=Path)
48 b.add_argument("--challenge-repo-url", default=None)
50 args = ap.parse_args(argv)
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 51 if args.cmd == "fetch":
52 fetch_registry(args.registry, args.clones, args.vault_url)
53 return 0
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 54 if args.cmd == "validate":
55 from .validate import validate
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 57 problems = validate(args.challenges, _sources(args), check_outputs=not args.no_outputs)
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 58 for p in problems:
59 print("PROBLEM:", p, file=sys.stderr)
60 print(f"{len(problems)} problem(s)")
61 return 1 if problems else 0
62 if args.cmd == "build":
63 from .build import build
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 64 from .validate import validate
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 66 sources = _sources(args)
67 problems = validate(args.challenges, sources)
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 68 if problems:
69 for p in problems:
70 print("PROBLEM:", p, file=sys.stderr)
71 return 1
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 72 results = build(args.challenges, sources, args.out, args.challenge_repo_url)
ddfc839Add string-pluck-v1 challenge, solution schema, and scoring toolsJeremy Magland 73 for r in results:
74 print(f"{r['challenge_id']}: {len(r['solutions'])} solutions, reference = {r['reference']}")
75 for s in r["solutions"]:
76 errs = " ".join(f"{k}={v:.3e}" for k, v in s["errors"].items())
03370d1Register solution repositories and build the comparison site here; optional HDF5 field export in the schema and specJeremy Magland 77 print(f" {s['key']:50s} {errs} t={s['timing_seconds']}")
79 return 2
82if __name__ == "__main__":
83 sys.exit(main())