16328b9Download runs as a .zip for offline development: WAVs, ground truth, training snippets, algorithm sources, and a numbl evaluation harness that reproduces the browser scoresJeremy Magland 1/**
2 * Round-trip test of the dataset download: build a benchmark in-process,
3 * write the offline archive's files to a scratch directory (unzipped;
4 * the zip wrapper is byte bookkeeping tested separately), run the shipped
5 * evaluate.m through the real numbl CLI, and require the per-run
6 * accuracies it prints to match the in-process scores exactly. Float32
7 * WAVs round-trip the samples bit-exactly, so any disagreement is a bug
8 * in the harness, not noise.
9 */
10import { execFileSync } from 'node:child_process';
11import { mkdirSync, writeFileSync, rmSync } from 'node:fs';
12import { dirname, join } from 'node:path';
13import { FS, type SynthConfig } from '../src/bench/types.ts';
14import { synthesize } from '../src/bench/synthesize.ts';
15import { buildTraining } from '../src/bench/training.ts';
16import { runAlgorithm } from '../src/bench/execute.ts';
17import { score } from '../src/bench/score.ts';
18import { buildOfflineFiles, type OfflineExportInput } from '../src/export/offline.ts';
19import { zipStore } from '../src/export/zip.ts';
20import { makeSyntheticLibrary } from './synthetic-lib.ts';
21import matchedFilterSrc from '../src/algorithms/template_matched_filter.m?raw';
22import onsetKnnSrc from '../src/algorithms/onset_knn.m?raw';
32dec4fAdd silence_gap_knn as a built-in; stop tracking the tmp/ offline workspaceJeremy Magland 23import silenceGapKnnSrc from '../src/algorithms/silence_gap_knn.m?raw';
16328b9Download runs as a .zip for offline development: WAVs, ground truth, training snippets, algorithm sources, and a numbl evaluation harness that reproduces the browser scoresJeremy Magland 24import naiveSrc from '../src/algorithms/naive_threshold.m?raw';
26const lib = makeSyntheticLibrary();
27const cfg: SynthConfig = {
28 durationS: 20, eventsPerType: 15, gainMinDb: -12,
df3f613Add a coincidence knob: a fraction of overlapping events land within a small window of their host's onset, the hardest case for detection and one the scorer's greedy matching handles with a documented caveatJeremy Magland 29 overlap: 0.2, coincidence: 0, coincidenceMs: 5, trainFraction: 0.3, seed: 1,
16328b9Download runs as a .zip for offline development: WAVs, ground truth, training snippets, algorithm sources, and a numbl evaluation harness that reproduces the browser scoresJeremy Magland 30};
31const NRUNS = 2;
32const TOL_MS = 20;
33const algos = [
34 { name: 'template_matched_filter', source: matchedFilterSrc },
35 { name: 'onset_knn', source: onsetKnnSrc },
32dec4fAdd silence_gap_knn as a built-in; stop tracking the tmp/ offline workspaceJeremy Magland 36 { name: 'silence_gap_knn', source: silenceGapKnnSrc },
16328b9Download runs as a .zip for offline development: WAVs, ground truth, training snippets, algorithm sources, and a numbl evaluation harness that reproduces the browser scoresJeremy Magland 37 { name: 'naive_threshold', source: naiveSrc },
38];
40// in-process benchmark, collecting expected per-run accuracies
41const runs: OfflineExportInput['runs'] = [];
42const expected = new Map<string, string[]>();
43for (let r = 0; r < NRUNS; r++) {
44 const ds = synthesize(lib, cfg, r);
45 const training = buildTraining(lib, ds.trainIndices);
46 runs.push({
47 x: ds.x,
48 truth: ds.truth,
49 training: training.snips.map((s) => ({ data: Float32Array.from(s.data), count: s.count })),
50 });
51 for (const a of algos) {
52 const out = runAlgorithm(a.source, ds.x, FS, training);
53 const s = score(ds.truth, out.detections, lib.names.length, TOL_MS / 1000);
54 (expected.get(a.name) ?? expected.set(a.name, []).get(a.name)!).push(s.accuracy.toFixed(3));
55 }
56}
58// write the archive's files to a scratch directory
59const files = buildOfflineFiles({
60 names: lib.names,
61 fs: FS,
62 pre: lib.pre,
63 post: lib.post,
64 toleranceMs: TOL_MS,
65 config: cfg,
66 algorithms: algos.map((a) => ({ source: a.source })),
67 runs,
68});
69const zip = zipStore(files);
70console.log(`archive: ${files.length} files, ${(zip.length / 1e6).toFixed(1)} MB stored`);
72const dir = join(process.env.TMPDIR ?? '/tmp', `asstest-offline-${process.pid}`);
73rmSync(dir, { recursive: true, force: true });
74for (const f of files) {
75 mkdirSync(join(dir, dirname(f.name)), { recursive: true });
76 writeFileSync(join(dir, f.name), f.data);
77}
79// run the shipped harness through the numbl CLI
80console.log('running: npx numbl run evaluate.m');
81const stdout = execFileSync('npx', ['numbl', 'run', 'evaluate.m'], {
82 cwd: dir, encoding: 'utf8', timeout: 600000,
83});
84console.log(stdout);
86// compare per-run accuracies, algorithm by algorithm
87let failures = 0;
88for (const a of algos) {
89 const block = stdout.split(`=== ${a.name} ===`)[1]?.split('===')[0] ?? '';
90 const got = [...block.matchAll(/run\s+\d+: accuracy (\d\.\d{3})/g)].map((m) => m[1]);
91 const want = expected.get(a.name)!;
92 const ok = got.length === want.length && got.every((g, i) => g === want[i]);
93 console.log(`${a.name.padEnd(26)} browser=[${want.join(', ')}] offline=[${got.join(', ')}] ${ok ? 'MATCH' : 'MISMATCH'}`);
94 if (!ok) failures++;
95}
96rmSync(dir, { recursive: true, force: true });
97if (failures) { console.error('OFFLINE FAIL'); process.exit(1); }
98console.log('OFFLINE PASS');