1"""Put the results of run.py side by side.
3 python compare.py [results_dir]
5Prints a table of all runs found in results_dir (default results/) and, for
6every problem and case run with both backends, the speedup and the relative
7difference of the surface fields. The table is also written to results_dir/comparison.md.
8"""
10import glob
11import json
12import os
13import sys
15import numpy as np
18def fmt(x, spec):
19 return "" if x is None else format(x, spec)
22def main():
23 rdir = sys.argv[1] if len(sys.argv) > 1 else os.path.join(os.path.dirname(os.path.abspath(__file__)), "results")
24 runs = {}
25 for f in sorted(glob.glob(os.path.join(rdir, "*", "summary.json"))):
26 s = json.load(open(f))
27 s["dir"] = os.path.dirname(f)
28 runs[(s.get("problem", "room"), s["case"], s["backend"])] = s
29 if not runs:
30 print(f"no results in {rdir}")
31 return
33 lines = ["| problem | case | k | points | backend | machine | setup (s) | solve (s) | setup + solve (s) | "
34 "iterations | self-test error | energy mismatch |",
35 "|---|---|---|---|---|---|---|---|---|---|---|---|"]
36 order = {"small": 0, "medium": 1, "large": 2}
37 sort_key = lambda pc: (pc[0] != "room", pc[0], order.get(pc[1], 9))
38 for (problem, case, backend), s in sorted(runs.items(), key=lambda kv: (sort_key(kv[0][:2]), kv[0][2])):
39 m = s["machine"]
40 where = m.get("gpu", "").split(",")[0] if backend == "gpu" else f"{m.get('cpu', '?')}, {m.get('cpu_count')} threads"
41 lines.append(f"| {problem} | {case} | {s['k']:.4g} | {s['npts']:,} | {backend} | {where} | {s['setup_s']:.2f} | "
42 f"{s['solve_s']:.2f} | {s['setup_plus_solve_s']:.2f} | {s['niter']} | "
43 f"{fmt(s.get('selftest_err'), '.1e')} | {s['energy_mismatch']:.1e} |")
45 lines += ["", "| problem | case | speedup (setup + solve) | surface field difference |", "|---|---|---|---|"]
46 for problem, case in sorted({pc[:2] for pc in runs}, key=sort_key):
47 if (problem, case, "gpu") in runs and (problem, case, "fmm3dbie") in runs:
48 g, c = runs[(problem, case, "gpu")], runs[(problem, case, "fmm3dbie")]
49 ug = np.load(os.path.join(g["dir"], "field.npz"))["u_wall"]
50 uc = np.load(os.path.join(c["dir"], "field.npz"))["u_wall"]
51 diff = np.linalg.norm(ug - uc) / np.linalg.norm(uc) if ug.shape == uc.shape else float("nan")
52 lines.append(f"| {problem} | {case} | {c['setup_plus_solve_s'] / g['setup_plus_solve_s']:.0f}x | {diff:.1e} |")
53 text = "\n".join(lines)
54 print(text)
55 with open(os.path.join(rdir, "comparison.md"), "w") as f:
56 f.write(text + "\n")
59if __name__ == "__main__":
60 main()