1from __future__ import annotations
3import argparse
4import sys
5from pathlib import Path
8def main(argv=None):
9 ap = argparse.ArgumentParser(prog="challenge-tools", description="Validate submissions and build the results site.")
10 sp = ap.add_subparsers(dest="cmd", required=True)
12 v = sp.add_parser("validate", help="check submission files and their outputs")
13 v.add_argument("--challenges", required=True, type=Path, help="directory containing <challenge-id>/challenge.json")
14 v.add_argument("--solutions", required=True, type=Path, help="directory containing <challenge-id>/<submission-id>.json")
15 v.add_argument("--no-outputs", action="store_true", help="skip reading the wav files")
17 b = sp.add_parser("build", help="compute scores and write the static results site")
18 b.add_argument("--challenges", required=True, type=Path)
19 b.add_argument("--solutions", required=True, type=Path)
20 b.add_argument("--out", required=True, type=Path)
21 b.add_argument("--challenge-repo-url", default=None)
22 b.add_argument("--solutions-repo-url", default=None)
24 args = ap.parse_args(argv)
25 if args.cmd == "validate":
26 from .validate import validate
28 problems = validate(args.challenges, args.solutions, check_outputs=not args.no_outputs)
29 for p in problems:
30 print("PROBLEM:", p, file=sys.stderr)
31 print(f"{len(problems)} problem(s)")
32 return 1 if problems else 0
33 if args.cmd == "build":
34 from .validate import validate
35 from .build import build
37 problems = validate(args.challenges, args.solutions)
38 if problems:
39 for p in problems:
40 print("PROBLEM:", p, file=sys.stderr)
41 return 1
42 results = build(args.challenges, args.solutions, args.out, args.challenge_repo_url, args.solutions_repo_url)
43 for r in results:
44 print(f"{r['challenge_id']}: {len(r['solutions'])} solutions, reference = {r['reference']}")
45 for s in r["solutions"]:
46 errs = " ".join(f"{k}={v:.3e}" for k, v in s["errors"].items())
47 print(f" {s['key']:40s} {errs} t={s['timing_seconds']}")
48 return 0
49 return 2
52if __name__ == "__main__":
53 sys.exit(main())