1% evaluate.m Run every listed algorithm on every run in runs/ and score
2% it against the ground truth, printing the same numbers the browser
3% benchmark reports.
4%
5% Usage:
6% npx numbl run evaluate.m
7%
8% To evaluate your own algorithm, add a file algorithms/<name>.m defining
9% function [times, labels] = <name>(x, fs, training)
10% and add '<name>' to the list below. See README.md for the interface.
12algorithms = {{{ALGORITHM_LIST}}};
14addpath('algorithms');
15addpath('tools');
17info = dataset_info();
18nTypes = numel(info.names);
19tol = info.tolerance_ms / 1000;
21for a = 1:numel(algorithms)
22 name = algorithms{a};
23 fprintf('\n=== %s ===\n', name);
24 accs = zeros(info.n_runs, 1);
25 totals = zeros(1, 4);
26 per_type = zeros(nTypes, 5);
27 for r = 1:info.n_runs
28 run_dir = sprintf('runs/run_%02d', r);
29 [x, fs] = load_wav(sprintf('%s/recording.wav', run_dir));
30 gt = load_csv(sprintf('%s/ground_truth.csv', run_dir), 3);
31 training = struct();
32 training.names = info.names;
33 training.pre = info.pre;
34 snips = cell(1, nTypes);
35 for k = 1:nTypes
36 sk = load_wav(sprintf('%s/training_type_%02d.wav', run_dir, k));
37 snips{k} = reshape(sk, info.snip_len, []);
38 end
39 training.snips = snips;
40 [times, labels] = feval(name, x, fs, training);
41 sc = score_detections(gt(:, 1), gt(:, 2), times, labels, tol, nTypes);
42 accs(r) = sc.accuracy;
43 totals = totals + [sc.correct, sc.misclassified, sc.missed, sc.false_positives];
44 per_type = per_type + sc.per_type;
45 fprintf('run %2d: accuracy %.3f (correct %d, misclassified %d, missed %d, false pos %d)\n', ...
46 r, sc.accuracy, sc.correct, sc.misclassified, sc.missed, sc.false_positives);
47 end
48 fprintf('mean accuracy %.3f +/- %.3f | correct %d, misclassified %d, missed %d, false pos %d\n', ...
49 mean(accs), std(accs), totals(1), totals(2), totals(3), totals(4));
50 fprintf('per event type (all runs):\n');
51 for k = 1:nTypes
52 d = sum(per_type(k, 1:4));
53 if d > 0
54 ak = per_type(k, 1) / d;
55 else
56 ak = 1;
57 end
58 fprintf(' %s: acc %.3f correct %d misclassified %d missed %d false pos %d truth %d\n', ...
59 info.names{k}, ak, per_type(k, 1), per_type(k, 2), per_type(k, 3), per_type(k, 4), per_type(k, 5));
60 end
61end