#!/bin/bash
# Retrains on the current dataset every INTERVAL seconds while clips accumulate.
#   ml/retrain-loop.sh DATASET_DIR [interval s] [epochs] [tag [extra train.py arguments, e.g. --logf --arch spec1d]]
# The tag names the run directories and the report and history files (report-TAG.html).
set -u
DATA=$1
INTERVAL=${2:-3600}
EPOCHS=${3:-60}
TAG=${4:-}
ML="$DATA/ml"
mkdir -p "$ML"
while true; do
  RUN="$ML/run-$(date +%Y%m%d-%H%M%S)${TAG:+-$TAG}"
  echo "=== $(date) training into $RUN"
  python3 "$(dirname "$0")/train.py" --data "$DATA" --out "$RUN" --epochs "$EPOCHS" ${TAG:+--tag "$TAG"} "${@:5}" 2>&1 | tail -12
  sleep "$INTERVAL"
done