% evaluate.m  Run every listed algorithm on every run in runs/ and score
% it against the ground truth, printing the same numbers the browser
% benchmark reports.
%
% Usage:
%   npx numbl run evaluate.m
%
% To evaluate your own algorithm, add a file algorithms/<name>.m defining
%   function [times, labels] = <name>(x, fs, training)
% and add '<name>' to the list below. See README.md for the interface.

algorithms = {{{ALGORITHM_LIST}}};

addpath('algorithms');
addpath('tools');

info = dataset_info();
nTypes = numel(info.names);
tol = info.tolerance_ms / 1000;

for a = 1:numel(algorithms)
  name = algorithms{a};
  fprintf('\n=== %s ===\n', name);
  accs = zeros(info.n_runs, 1);
  totals = zeros(1, 4);
  per_type = zeros(nTypes, 5);
  for r = 1:info.n_runs
    run_dir = sprintf('runs/run_%02d', r);
    [x, fs] = load_wav(sprintf('%s/recording.wav', run_dir));
    gt = load_csv(sprintf('%s/ground_truth.csv', run_dir), 3);
    training = struct();
    training.names = info.names;
    training.pre = info.pre;
    snips = cell(1, nTypes);
    for k = 1:nTypes
      sk = load_wav(sprintf('%s/training_type_%02d.wav', run_dir, k));
      snips{k} = reshape(sk, info.snip_len, []);
    end
    training.snips = snips;
    [times, labels] = feval(name, x, fs, training);
    sc = score_detections(gt(:, 1), gt(:, 2), times, labels, tol, nTypes);
    accs(r) = sc.accuracy;
    totals = totals + [sc.correct, sc.misclassified, sc.missed, sc.false_positives];
    per_type = per_type + sc.per_type;
    fprintf('run %2d: accuracy %.3f  (correct %d, misclassified %d, missed %d, false pos %d)\n', ...
      r, sc.accuracy, sc.correct, sc.misclassified, sc.missed, sc.false_positives);
  end
  fprintf('mean accuracy %.3f +/- %.3f  |  correct %d, misclassified %d, missed %d, false pos %d\n', ...
    mean(accs), std(accs), totals(1), totals(2), totals(3), totals(4));
  fprintf('per event type (all runs):\n');
  for k = 1:nTypes
    d = sum(per_type(k, 1:4));
    if d > 0
      ak = per_type(k, 1) / d;
    else
      ak = 1;
    end
    fprintf('  %s: acc %.3f  correct %d  misclassified %d  missed %d  false pos %d  truth %d\n', ...
      info.names{k}, ak, per_type(k, 1), per_type(k, 2), per_type(k, 3), per_type(k, 4), per_type(k, 5));
  end
end