from __future__ import annotations import argparse import sys from pathlib import Path def main(argv=None): ap = argparse.ArgumentParser(prog="challenge-tools", description="Validate submissions and build the results site.") sp = ap.add_subparsers(dest="cmd", required=True) v = sp.add_parser("validate", help="check submission files and their outputs") v.add_argument("--challenges", required=True, type=Path, help="directory containing /challenge.json") v.add_argument("--solutions", required=True, type=Path, help="directory containing /.json") v.add_argument("--no-outputs", action="store_true", help="skip reading the wav files") b = sp.add_parser("build", help="compute scores and write the static results site") b.add_argument("--challenges", required=True, type=Path) b.add_argument("--solutions", required=True, type=Path) b.add_argument("--out", required=True, type=Path) b.add_argument("--challenge-repo-url", default=None) b.add_argument("--solutions-repo-url", default=None) args = ap.parse_args(argv) if args.cmd == "validate": from .validate import validate problems = validate(args.challenges, args.solutions, check_outputs=not args.no_outputs) for p in problems: print("PROBLEM:", p, file=sys.stderr) print(f"{len(problems)} problem(s)") return 1 if problems else 0 if args.cmd == "build": from .validate import validate from .build import build problems = validate(args.challenges, args.solutions) if problems: for p in problems: print("PROBLEM:", p, file=sys.stderr) return 1 results = build(args.challenges, args.solutions, args.out, args.challenge_repo_url, args.solutions_repo_url) for r in results: print(f"{r['challenge_id']}: {len(r['solutions'])} solutions, reference = {r['reference']}") for s in r["solutions"]: errs = " ".join(f"{k}={v:.3e}" for k, v in s["errors"].items()) print(f" {s['key']:40s} {errs} t={s['timing_seconds']}") return 0 return 2 if __name__ == "__main__": sys.exit(main())