21abc12audio-spike-sorting: benchmark framework for audio event detection and classificationJeremy Magland 1/**
2 * End-to-end pipeline test without a microphone: three synthetic event
3 * types (a low ringing tone, a high short tone, a noise burst) stand in
4 * for recorded taps. Builds a snippet library, synthesizes datasets across
5 * runs, executes each baseline algorithm through numbl, and prints scores.
6 * The matched filter and the kNN sorter should sit far above the naive
7 * floor; the run fails loudly if they do not.
8 */
16328b9Download runs as a .zip for offline development: WAVs, ground truth, training snippets, algorithm sources, and a numbl evaluation harness that reproduces the browser scoresJeremy Magland 9import { FS, type SynthConfig } from '../src/bench/types.ts';
21abc12audio-spike-sorting: benchmark framework for audio event detection and classificationJeremy Magland 10import { synthesize } from '../src/bench/synthesize.ts';
11import { score } from '../src/bench/score.ts';
12import { buildTraining } from '../src/bench/training.ts';
13import { runAlgorithm } from '../src/bench/execute.ts';
16328b9Download runs as a .zip for offline development: WAVs, ground truth, training snippets, algorithm sources, and a numbl evaluation harness that reproduces the browser scoresJeremy Magland 14import { makeSyntheticLibrary } from './synthetic-lib.ts';
21abc12audio-spike-sorting: benchmark framework for audio event detection and classificationJeremy Magland 15import matchedFilterSrc from '../src/algorithms/template_matched_filter.m?raw';
16import onsetKnnSrc from '../src/algorithms/onset_knn.m?raw';
32dec4fAdd silence_gap_knn as a built-in; stop tracking the tmp/ offline workspaceJeremy Magland 17import silenceGapKnnSrc from '../src/algorithms/silence_gap_knn.m?raw';
7facb2fAdd overlap_pursuit as a built-in: anchored matching pursuit over merged silence blocks (boundary onsets exact, coverage-forced interior events, duplicate pursuit with a same-type veto); 0.958 mean accuracy on the 3-type 20%-overlap export where the prior best scored 0.730Jeremy Magland 18import overlapPursuitSrc from '../src/algorithms/overlap_pursuit.m?raw';
21abc12audio-spike-sorting: benchmark framework for audio event detection and classificationJeremy Magland 19import naiveSrc from '../src/algorithms/naive_threshold.m?raw';
16328b9Download runs as a .zip for offline development: WAVs, ground truth, training snippets, algorithm sources, and a numbl evaluation harness that reproduces the browser scoresJeremy Magland 21const lib = makeSyntheticLibrary();
21abc12audio-spike-sorting: benchmark framework for audio event detection and classificationJeremy Magland 22
23const cfg: SynthConfig = {
24 durationS: 30,
25 eventsPerType: 30,
26 gainMinDb: -12,
27 overlap: 0.2,
df3f613Add a coincidence knob: a fraction of overlapping events land within a small window of their host's onset, the hardest case for detection and one the scorer's greedy matching handles with a documented caveatJeremy Magland 28 coincidence: 0,
29 coincidenceMs: 5,
21abc12audio-spike-sorting: benchmark framework for audio event detection and classificationJeremy Magland 30 trainFraction: 0.3,
31 seed: 1,
32};
34const algos = [
35 { name: 'template_matched_filter', source: matchedFilterSrc },
36 { name: 'onset_knn', source: onsetKnnSrc },
32dec4fAdd silence_gap_knn as a built-in; stop tracking the tmp/ offline workspaceJeremy Magland 37 { name: 'silence_gap_knn', source: silenceGapKnnSrc },
7facb2fAdd overlap_pursuit as a built-in: anchored matching pursuit over merged silence blocks (boundary onsets exact, coverage-forced interior events, duplicate pursuit with a same-type veto); 0.958 mean accuracy on the 3-type 20%-overlap export where the prior best scored 0.730Jeremy Magland 38 { name: 'overlap_pursuit', source: overlapPursuitSrc },
21abc12audio-spike-sorting: benchmark framework for audio event detection and classificationJeremy Magland 39 { name: 'naive_threshold', source: naiveSrc },
40];
42const NRUNS = 2;
43const TOL = 0.02;
44const acc: Record<string, number[]> = {};
45for (let r = 0; r < NRUNS; r++) {
46 const ds = synthesize(lib, cfg, r);
47 const training = buildTraining(lib, ds.trainIndices);
48 console.log(`run ${r}: ${ds.truth.length} ground-truth events, ${ds.x.length} samples`);
49 for (const a of algos) {
50 const t0 = Date.now();
51 try {
52 const out = runAlgorithm(a.source, ds.x, FS, training);
53 const s = score(ds.truth, out.detections, lib.names.length, TOL);
54 (acc[a.name] ??= []).push(s.accuracy);
55 console.log(
56 ` ${a.name.padEnd(26)} acc=${s.accuracy.toFixed(3)} correct=${s.correct}` +
57 ` miss=${s.missed} fp=${s.falsePositives} misclass=${s.misclassified}` +
58 ` (${Date.now() - t0} ms)`,
59 );
60 } catch (err) {
61 console.log(` ${a.name.padEnd(26)} ERROR: ${(err as Error).message}`);
62 }
63 }
64}
66const mean = (v: number[]) => v.reduce((a, b) => a + b, 0) / Math.max(v.length, 1);
67console.log('\nmean accuracy over runs:');
68for (const a of algos) console.log(` ${a.name.padEnd(26)} ${mean(acc[a.name] ?? []).toFixed(3)}`);
df3f613Add a coincidence knob: a fraction of overlapping events land within a small window of their host's onset, the hardest case for detection and one the scorer's greedy matching handles with a documented caveatJeremy Magland 70// The coincidence knob must actually place events near-simultaneously:
71// with every overlapper aimed at its host's onset, most consecutive
72// ground-truth pairs closer than a snippet length should be inside the
73// coincidence window, where they are outside it when the knob is off.
74function nearPairs(coincidence: number, windowMs: number): { close: number; overlapping: number } {
75 const ds = synthesize(lib, { ...cfg, overlap: 0.5, coincidence, coincidenceMs: windowMs }, 0);
76 let close = 0, overlapping = 0;
77 for (let i = 1; i < ds.truth.length; i++) {
78 const dtMs = (ds.truth[i].t - ds.truth[i - 1].t) * 1000;
79 if (dtMs < 200) overlapping++;
80 if (dtMs <= windowMs) close++;
81 }
82 return { close, overlapping };
83}
84const off = nearPairs(0, 5);
85const on = nearPairs(1, 5);
86console.log(`\ncoincidence off: ${off.close}/${off.overlapping} overlapping pairs within 5 ms`);
87console.log(`coincidence on: ${on.close}/${on.overlapping} overlapping pairs within 5 ms`);
88if (!(on.close > 0.6 * on.overlapping)) {
89 console.error('FAIL: coincidence knob did not place events near-simultaneously');
90 process.exit(1);
91}
92if (!(off.close < 0.35 * off.overlapping)) {
93 console.error('FAIL: events cluster within the coincidence window with the knob off');
94 process.exit(1);
95}
21abc12audio-spike-sorting: benchmark framework for audio event detection and classificationJeremy Magland 97const mf = mean(acc['template_matched_filter'] ?? []);
98const knn = mean(acc['onset_knn'] ?? []);
99if (!(mf > 0.7)) { console.error('FAIL: matched filter accuracy too low'); process.exit(1); }
100if (!(knn > 0.5)) { console.error('FAIL: onset_knn accuracy too low'); process.exit(1); }
101console.log('PASS');