#!/bin/bash
# Pulls the clips and the index of a dataset rendered on another machine into a
# local directory, repeatedly, so that they can be listened to while the run
# progresses. The index is snapshotted first and every clip missing locally is
# fetched, so the local index never names a clip that is not there yet.
#   scripts/pull-dataset.sh [remote host] [remote dir] [local dir] [interval s]
set -u
REMOTE=${1:-office}
RDIR=${2:-steelsim31-dataset/v1}
LDIR=${3:-dataset}
INTERVAL=${4:-120}
mkdir -p "$LDIR/clips"
cp "$(dirname "$0")/gallery.html" "$LDIR/gallery.html"
while true; do
  if ssh -x -o BatchMode=yes "$REMOTE" "cat $RDIR/index.jsonl 2>/dev/null" > "$LDIR/index.jsonl.tmp"; then
    ssh -x -o BatchMode=yes "$REMOTE" "ls $RDIR/clips" | sort > "$LDIR/.remote-clips"
    ls "$LDIR/clips" | sort > "$LDIR/.local-clips"
    comm -13 "$LDIR/.local-clips" "$LDIR/.remote-clips" > "$LDIR/.missing"
    if [ -s "$LDIR/.missing" ]; then
      ssh -x -o BatchMode=yes "$REMOTE" "cd $RDIR/clips && tar cf - $(tr '\n' ' ' < "$LDIR/.missing")" | tar xf - -C "$LDIR/clips"
    fi
    mv "$LDIR/index.jsonl.tmp" "$LDIR/index.jsonl"
    # The training report and history, if a model is being trained there.
    ssh -x -o BatchMode=yes "$REMOTE" "cd $RDIR && tar cf - ml/report*.html ml/history*.jsonl 2>/dev/null" | tar xf - -C "$LDIR" 2>/dev/null
    echo "$(date +%H:%M:%S) $(wc -l < "$LDIR/index.jsonl") clips in index, $(ls "$LDIR/clips" | wc -l) wav files local"
  fi
  sleep "$INTERVAL"
done