2# Retrains on the current dataset every INTERVAL seconds while clips accumulate.
3# ml/retrain-loop.sh DATASET_DIR [interval s] [epochs] [tag [extra train.py arguments, e.g. --logf --arch spec1d]]
4# The tag names the run directories and the report and history files (report-TAG.html).
5set -u
6DATA=$1
7INTERVAL=${2:-3600}
8EPOCHS=${3:-60}
9TAG=${4:-}
10ML="$DATA/ml"
11mkdir -p "$ML"
12while true; do
13 RUN="$ML/run-$(date +%Y%m%d-%H%M%S)${TAG:+-$TAG}"
14 echo "=== $(date) training into $RUN"
15 python3 "$(dirname "$0")/train.py" --data "$DATA" --out "$RUN" --epochs "$EPOCHS" ${TAG:+--tag "$TAG"} "${@:5}" 2>&1 | tail -12
16 sleep "$INTERVAL"
17done