2# Build the GitHub Pages site: download the first STEP chunk of the ABC
3# dataset, extract the first N models, gzip each file, and write an index.
4#
5# Env:
6# CHUNK_ARCHIVE path to an already-downloaded abc_0000_step_v00.7z
7# (skips the ~1.6 GB download)
8# WORK working directory (default: ./build)
9# N number of models to host (default: 1000)
10set -euo pipefail
12REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd)
13WORK=${WORK:-"$REPO_ROOT/build"}
14N=${N:-1000}
16mkdir -p "$WORK"
17cd "$WORK"
19# The name recorded in index.json always refers to the canonical source chunk.
20CHUNK_NAME=abc_0000_step_v00.7z
22# Mirror holding a repack of just the first 1000 model directories of the
23# canonical chunk (byte-identical files). Used because archive.nyu.edu
24# rate-limits by IP (mod_qos) and often redirects CI runners to an HTML
25# restrictions page instead of serving the file.
26MIRROR_URL=https://github.com/concept-collection/abc-step-1000/releases/download/data-v00/abc_0000_step_v00_first1000.7z
28is_7z() { [ "$(head -c 2 "$1" 2> /dev/null)" = "7z" ]; }
30if [ -z "${CHUNK_ARCHIVE:-}" ]; then
31 wget -q https://deep-geometry.github.io/abc-dataset/data/step_v00.txt
32 CHUNK_URL=$(sed '1q;d' step_v00.txt | awk '{print $1}')
33 for attempt in 1 2 3; do
34 echo "Downloading $CHUNK_NAME from $CHUNK_URL (attempt $attempt) ..."
35 wget -q --no-check-certificate "$CHUNK_URL" -O chunk.7z || true
36 is_7z chunk.7z && break
37 echo "Response is not a 7z archive (rate-limited?); retrying in 30 s"
38 sleep 30
39 done
40 if ! is_7z chunk.7z; then
41 echo "Falling back to mirror: $MIRROR_URL"
42 wget -q "$MIRROR_URL" -O chunk.7z
43 is_7z chunk.7z
44 fi
45 CHUNK_ARCHIVE="$WORK/chunk.7z"
46fi
48# Extract only the first N model directories; the full chunk holds 10000
49# models (~15 GB uncompressed), far more than we need or than CI disk allows.
50echo "Extracting first $N model directories from $CHUNK_NAME ..."
51seq -f '%08g/*' 0 $((N - 1)) > include.txt
52rm -rf extracted
537z x -y "$CHUNK_ARCHIVE" -i@include.txt -oextracted > /dev/null
55rm -rf site
56mkdir -p site/step
57find extracted -name '*.step' | sort | head -n "$N" > files.txt
58echo "Compressing $(wc -l < files.txt) STEP files ..."
59xargs -a files.txt -P "$(nproc)" -I{} \
60 sh -c 'gzip -9 -c "$1" > "site/step/$(basename "$1").gz"' _ {}
62python3 "$REPO_ROOT/scripts/make_index.py" files.txt site "$CHUNK_NAME"
63cp "$REPO_ROOT/site/index.html" site/
65echo "Done. Site size:"
66du -sh site