2# Pulls the clips and the index of a dataset rendered on another machine into a
3# local directory, repeatedly, so that they can be listened to while the run
4# progresses. The index is snapshotted first and every clip missing locally is
5# fetched, so the local index never names a clip that is not there yet.
6# scripts/pull-dataset.sh [remote host] [remote dir] [local dir] [interval s]
7set -u
8REMOTE=${1:-office}
9RDIR=${2:-steelsim31-dataset/v1}
10LDIR=${3:-dataset}
11INTERVAL=${4:-120}
12mkdir -p "$LDIR/clips"
13cp "$(dirname "$0")/gallery.html" "$LDIR/gallery.html"
14while true; do
15 if ssh -x -o BatchMode=yes "$REMOTE" "cat $RDIR/index.jsonl 2>/dev/null" > "$LDIR/index.jsonl.tmp"; then
16 ssh -x -o BatchMode=yes "$REMOTE" "ls $RDIR/clips" | sort > "$LDIR/.remote-clips"
17 ls "$LDIR/clips" | sort > "$LDIR/.local-clips"
18 comm -13 "$LDIR/.local-clips" "$LDIR/.remote-clips" > "$LDIR/.missing"
19 if [ -s "$LDIR/.missing" ]; then
20 ssh -x -o BatchMode=yes "$REMOTE" "cd $RDIR/clips && tar cf - $(tr '\n' ' ' < "$LDIR/.missing")" | tar xf - -C "$LDIR/clips"
21 fi
22 mv "$LDIR/index.jsonl.tmp" "$LDIR/index.jsonl"
23 # The training report and history, if a model is being trained there.
24 ssh -x -o BatchMode=yes "$REMOTE" "cd $RDIR && tar cf - ml/report*.html ml/history*.jsonl 2>/dev/null" | tar xf - -C "$LDIR" 2>/dev/null
25 echo "$(date +%H:%M:%S) $(wc -l < "$LDIR/index.jsonl") clips in index, $(ls "$LDIR/clips" | wc -l) wav files local"
26 fi
27 sleep "$INTERVAL"
28done