16328b9Download runs as a .zip for offline development: WAVs, ground truth, training snippets, algorithm sources, and a numbl evaluation harness that reproduces the browser scoresJeremy Magland 1/**
2 * Round-trip test of the dataset download: build a benchmark in-process,
3 * write the offline archive's files to a scratch directory (unzipped;
4 * the zip wrapper is byte bookkeeping tested separately), run the shipped
5 * evaluate.m through the real numbl CLI, and require the per-run
6 * accuracies it prints to match the in-process scores exactly. Float32
7 * WAVs round-trip the samples bit-exactly, so any disagreement is a bug
8 * in the harness, not noise.
9 */
10import { execFileSync } from 'node:child_process';
11import { mkdirSync, writeFileSync, rmSync } from 'node:fs';
12import { dirname, join } from 'node:path';
13import { FS, type SynthConfig } from '../src/bench/types.ts';
14import { synthesize } from '../src/bench/synthesize.ts';
15import { buildTraining } from '../src/bench/training.ts';
16import { runAlgorithm } from '../src/bench/execute.ts';
17import { score } from '../src/bench/score.ts';
18import { buildOfflineFiles, type OfflineExportInput } from '../src/export/offline.ts';
19import { zipStore } from '../src/export/zip.ts';
20import { makeSyntheticLibrary } from './synthetic-lib.ts';
21import matchedFilterSrc from '../src/algorithms/template_matched_filter.m?raw';
22import onsetKnnSrc from '../src/algorithms/onset_knn.m?raw';
32dec4fAdd silence_gap_knn as a built-in; stop tracking the tmp/ offline workspaceJeremy Magland 23import silenceGapKnnSrc from '../src/algorithms/silence_gap_knn.m?raw';
7facb2fAdd overlap_pursuit as a built-in: anchored matching pursuit over merged silence blocks (boundary onsets exact, coverage-forced interior events, duplicate pursuit with a same-type veto); 0.958 mean accuracy on the 3-type 20%-overlap export where the prior best scored 0.730Jeremy Magland 24import overlapPursuitSrc from '../src/algorithms/overlap_pursuit.m?raw';
16328b9Download runs as a .zip for offline development: WAVs, ground truth, training snippets, algorithm sources, and a numbl evaluation harness that reproduces the browser scoresJeremy Magland 25import naiveSrc from '../src/algorithms/naive_threshold.m?raw';
27const lib = makeSyntheticLibrary();
28const cfg: SynthConfig = {
29 durationS: 20, eventsPerType: 15, gainMinDb: -12,
df3f613Add a coincidence knob: a fraction of overlapping events land within a small window of their host's onset, the hardest case for detection and one the scorer's greedy matching handles with a documented caveatJeremy Magland 30 overlap: 0.2, coincidence: 0, coincidenceMs: 5, trainFraction: 0.3, seed: 1,
16328b9Download runs as a .zip for offline development: WAVs, ground truth, training snippets, algorithm sources, and a numbl evaluation harness that reproduces the browser scoresJeremy Magland 31};
32const NRUNS = 2;
33const TOL_MS = 20;
34const algos = [
35 { name: 'template_matched_filter', source: matchedFilterSrc },
36 { name: 'onset_knn', source: onsetKnnSrc },
32dec4fAdd silence_gap_knn as a built-in; stop tracking the tmp/ offline workspaceJeremy Magland 37 { name: 'silence_gap_knn', source: silenceGapKnnSrc },
7facb2fAdd overlap_pursuit as a built-in: anchored matching pursuit over merged silence blocks (boundary onsets exact, coverage-forced interior events, duplicate pursuit with a same-type veto); 0.958 mean accuracy on the 3-type 20%-overlap export where the prior best scored 0.730Jeremy Magland 38 { name: 'overlap_pursuit', source: overlapPursuitSrc },
16328b9Download runs as a .zip for offline development: WAVs, ground truth, training snippets, algorithm sources, and a numbl evaluation harness that reproduces the browser scoresJeremy Magland 39 { name: 'naive_threshold', source: naiveSrc },
40];
42// in-process benchmark, collecting expected per-run accuracies
43const runs: OfflineExportInput['runs'] = [];
44const expected = new Map<string, string[]>();
45for (let r = 0; r < NRUNS; r++) {
46 const ds = synthesize(lib, cfg, r);
47 const training = buildTraining(lib, ds.trainIndices);
48 runs.push({
49 x: ds.x,
50 truth: ds.truth,
51 training: training.snips.map((s) => ({ data: Float32Array.from(s.data), count: s.count })),
52 });
53 for (const a of algos) {
54 const out = runAlgorithm(a.source, ds.x, FS, training);
55 const s = score(ds.truth, out.detections, lib.names.length, TOL_MS / 1000);
56 (expected.get(a.name) ?? expected.set(a.name, []).get(a.name)!).push(s.accuracy.toFixed(3));
57 }
58}
60// write the archive's files to a scratch directory
61const files = buildOfflineFiles({
62 names: lib.names,
63 fs: FS,
64 pre: lib.pre,
65 post: lib.post,
66 toleranceMs: TOL_MS,
67 config: cfg,
68 algorithms: algos.map((a) => ({ source: a.source })),
69 runs,
70});
71const zip = zipStore(files);
72console.log(`archive: ${files.length} files, ${(zip.length / 1e6).toFixed(1)} MB stored`);
74const dir = join(process.env.TMPDIR ?? '/tmp', `asstest-offline-${process.pid}`);
75rmSync(dir, { recursive: true, force: true });
76for (const f of files) {
77 mkdirSync(join(dir, dirname(f.name)), { recursive: true });
78 writeFileSync(join(dir, f.name), f.data);
79}
81// run the shipped harness through the numbl CLI
82console.log('running: npx numbl run evaluate.m');
83const stdout = execFileSync('npx', ['numbl', 'run', 'evaluate.m'], {
84 cwd: dir, encoding: 'utf8', timeout: 600000,
85});
86console.log(stdout);
88// compare per-run accuracies, algorithm by algorithm
89let failures = 0;
90for (const a of algos) {
91 const block = stdout.split(`=== ${a.name} ===`)[1]?.split('===')[0] ?? '';
92 const got = [...block.matchAll(/run\s+\d+: accuracy (\d\.\d{3})/g)].map((m) => m[1]);
93 const want = expected.get(a.name)!;
94 const ok = got.length === want.length && got.every((g, i) => g === want[i]);
95 console.log(`${a.name.padEnd(26)} browser=[${want.join(', ')}] offline=[${got.join(', ')}] ${ok ? 'MATCH' : 'MISMATCH'}`);
96 if (!ok) failures++;
97}
98rmSync(dir, { recursive: true, force: true });
99if (failures) { console.error('OFFLINE FAIL'); process.exit(1); }
100console.log('OFFLINE PASS');