21abc12audio-spike-sorting: benchmark framework for audio event detection and classificationJeremy Magland 1/**
2 * Benchmark orchestration: for each run, synthesize one dataset (run r
3 * uses seed + r, so the whole benchmark is reproducible from the config),
4 * hand the same dataset to every enabled algorithm through the worker,
5 * and score each result against the ground truth. Algorithms run
6 * sequentially on one worker so timings are comparable and a runaway
7 * script can be cancelled by killing the worker.
8 */
9import { buildTraining } from './training.ts';
10import { score } from './score.ts';
11import { synthesize } from './synthesize.ts';
12import type { AlgorithmDef, RunResult, SnippetLibrary, SynthConfig } from './types.ts';
13import { FS } from './types.ts';
14import type { AlgoRequest, AlgoResponse } from './algo.worker.ts';
16export interface BenchmarkProgress {
17 runIndex: number;
18 runCount: number;
19 algorithmName: string;
20 done: number;
21 total: number;
22}
24export interface BenchmarkOutput {
16328b9Download runs as a .zip for offline development: WAVs, ground truth, training snippets, algorithm sources, and a numbl evaluation harness that reproduces the browser scoresJeremy Magland 25 runs: {
26 seed: number;
27 x: Float32Array;
28 truth: { t: number; typeIndex: number; gain: number }[];
29 /** Per type: the training snippets this run handed to every
30 * algorithm, concatenated, so an export can reproduce the run. */
31 training: { data: Float32Array; count: number }[];
32 }[];
21abc12audio-spike-sorting: benchmark framework for audio event detection and classificationJeremy Magland 33 results: RunResult[];
34}
36export class BenchmarkRunner {
37 private worker: Worker | null = null;
38 private nextId = 1;
39 cancelled = false;
41 cancel(): void {
42 this.cancelled = true;
43 this.worker?.terminate();
44 this.worker = null;
45 }
47 private exec(req: Omit<AlgoRequest, 'id'>): Promise<AlgoResponse> {
48 this.worker ??= new Worker(new URL('./algo.worker.ts', import.meta.url), { type: 'module' });
49 const worker = this.worker;
50 const id = this.nextId++;
51 return new Promise((resolve, reject) => {
52 const onMsg = (e: MessageEvent<AlgoResponse>) => {
53 if (e.data.id !== id) return;
54 worker.removeEventListener('message', onMsg);
55 resolve(e.data);
56 };
57 worker.addEventListener('message', onMsg);
58 worker.addEventListener('error', (e) => reject(new Error(e.message)), { once: true });
59 worker.postMessage({ id, ...req });
60 });
61 }
63 async run(
64 lib: SnippetLibrary,
65 cfg: SynthConfig,
66 runCount: number,
67 toleranceMs: number,
68 algorithms: AlgorithmDef[],
69 onProgress: (p: BenchmarkProgress) => void,
70 ): Promise<BenchmarkOutput> {
71 this.cancelled = false;
72 const runs: BenchmarkOutput['runs'] = [];
73 const results: RunResult[] = [];
74 const total = runCount * algorithms.length;
75 let done = 0;
76 for (let r = 0; r < runCount && !this.cancelled; r++) {
77 const ds = synthesize(lib, cfg, r);
78 const training = buildTraining(lib, ds.trainIndices);
16328b9Download runs as a .zip for offline development: WAVs, ground truth, training snippets, algorithm sources, and a numbl evaluation harness that reproduces the browser scoresJeremy Magland 79 runs.push({
80 seed: cfg.seed + r,
81 x: ds.x,
82 truth: ds.truth,
83 training: training.snips.map((s) => ({ data: Float32Array.from(s.data), count: s.count })),
84 });
21abc12audio-spike-sorting: benchmark framework for audio event detection and classificationJeremy Magland 85 for (const algo of algorithms) {
86 if (this.cancelled) break;
87 onProgress({ runIndex: r, runCount, algorithmName: algo.name, done, total });
88 const resp = await this.exec({ source: algo.source, x: ds.x, fs: FS, training });
89 done++;
90 if (resp.ok && resp.detections) {
91 results.push({
92 runIndex: r,
93 seed: cfg.seed + r,
94 algorithmId: algo.id,
95 score: score(ds.truth, resp.detections, lib.names.length, toleranceMs / 1000),
96 elapsedMs: resp.elapsedMs,
97 logs: resp.logs,
98 detections: resp.detections,
99 });
100 } else {
101 results.push({
102 runIndex: r,
103 seed: cfg.seed + r,
104 algorithmId: algo.id,
105 score: score(ds.truth, [], lib.names.length, toleranceMs / 1000),
106 elapsedMs: resp.elapsedMs,
107 error: resp.error ?? 'unknown error',
108 logs: resp.logs,
109 detections: [],
110 });
111 }
112 }
113 }
114 this.worker?.terminate();
115 this.worker = null;
116 return { runs, results };
117 }
118}