1/**
2 * Benchmark orchestration: for each run, synthesize one dataset (run r
3 * uses seed + r, so the whole benchmark is reproducible from the config),
4 * hand the same dataset to every enabled algorithm through the worker,
5 * and score each result against the ground truth. Algorithms run
6 * sequentially on one worker so timings are comparable and a runaway
7 * script can be cancelled by killing the worker.
8 */
9import { buildTraining } from './training.ts';
10import { score } from './score.ts';
11import { synthesize } from './synthesize.ts';
12import type { AlgorithmDef, RunResult, SnippetLibrary, SynthConfig } from './types.ts';
13import { FS } from './types.ts';
14import type { AlgoRequest, AlgoResponse } from './algo.worker.ts';
16export interface BenchmarkProgress {
17 runIndex: number;
18 runCount: number;
19 algorithmName: string;
20 done: number;
21 total: number;
22}
24export interface BenchmarkOutput {
25 runs: { seed: number; x: Float32Array; truth: { t: number; typeIndex: number; gain: number }[] }[];
26 results: RunResult[];
27}
29export class BenchmarkRunner {
30 private worker: Worker | null = null;
31 private nextId = 1;
32 cancelled = false;
34 cancel(): void {
35 this.cancelled = true;
36 this.worker?.terminate();
37 this.worker = null;
38 }
40 private exec(req: Omit<AlgoRequest, 'id'>): Promise<AlgoResponse> {
41 this.worker ??= new Worker(new URL('./algo.worker.ts', import.meta.url), { type: 'module' });
42 const worker = this.worker;
43 const id = this.nextId++;
44 return new Promise((resolve, reject) => {
45 const onMsg = (e: MessageEvent<AlgoResponse>) => {
46 if (e.data.id !== id) return;
47 worker.removeEventListener('message', onMsg);
48 resolve(e.data);
49 };
50 worker.addEventListener('message', onMsg);
51 worker.addEventListener('error', (e) => reject(new Error(e.message)), { once: true });
52 worker.postMessage({ id, ...req });
53 });
54 }
56 async run(
57 lib: SnippetLibrary,
58 cfg: SynthConfig,
59 runCount: number,
60 toleranceMs: number,
61 algorithms: AlgorithmDef[],
62 onProgress: (p: BenchmarkProgress) => void,
63 ): Promise<BenchmarkOutput> {
64 this.cancelled = false;
65 const runs: BenchmarkOutput['runs'] = [];
66 const results: RunResult[] = [];
67 const total = runCount * algorithms.length;
68 let done = 0;
69 for (let r = 0; r < runCount && !this.cancelled; r++) {
70 const ds = synthesize(lib, cfg, r);
71 runs.push({ seed: cfg.seed + r, x: ds.x, truth: ds.truth });
72 const training = buildTraining(lib, ds.trainIndices);
73 for (const algo of algorithms) {
74 if (this.cancelled) break;
75 onProgress({ runIndex: r, runCount, algorithmName: algo.name, done, total });
76 const resp = await this.exec({ source: algo.source, x: ds.x, fs: FS, training });
77 done++;
78 if (resp.ok && resp.detections) {
79 results.push({
80 runIndex: r,
81 seed: cfg.seed + r,
82 algorithmId: algo.id,
83 score: score(ds.truth, resp.detections, lib.names.length, toleranceMs / 1000),
84 elapsedMs: resp.elapsedMs,
85 logs: resp.logs,
86 detections: resp.detections,
87 });
88 } else {
89 results.push({
90 runIndex: r,
91 seed: cfg.seed + r,
92 algorithmId: algo.id,
93 score: score(ds.truth, [], lib.names.length, toleranceMs / 1000),
94 elapsedMs: resp.elapsedMs,
95 error: resp.error ?? 'unknown error',
96 logs: resp.logs,
97 detections: [],
98 });
99 }
100 }
101 }
102 this.worker?.terminate();
103 this.worker = null;
104 return { runs, results };
105 }
106}