1/**
2 * Filling the shared cache from the command line, so that a machine with a GPU
3 * and nothing to do can contribute without a browser window open on it.
4 *
5 * The walk, the runs and the uploads are the page's (src/cache/fillWalk.ts,
6 * src/cache/runSpec.ts); what is here is the shell around them — Dawn instead
7 * of a browser's WebGPU, a key from the environment instead of localStorage,
8 * and lines of text instead of a status bar.
9 */
10import { tmpdir } from 'node:os';
11import { requestShtDevice, describeAdapter } from '../sht/sht.ts';
12import { mModelByKey } from '../mgpu/registry.ts';
13import { formatFailure } from '../mgpu/errors.ts';
14import {
15 fmtChoice,
16 GEOMETRY_CHOICES,
17 MODEL_CHOICES,
18 T_END_CHOICE,
19 type DiscreteChoice,
20} from '../cache/options.ts';
21import { autoOrder, specForTarget, type AutoTarget } from '../cache/autoWalk.ts';
22import { headCached, lookupFor, verifyApiKey } from '../cache/client.ts';
23import { SolverSession } from '../cache/solver.ts';
24import { fillWalk } from '../cache/fillWalk.ts';
25import { setScratchDir } from '../cache/h5file.ts';
26import { stepsFor } from '../cache/spec.ts';
27import type { RunSummary } from '../cache/runSpec.ts';
28import { installWebGpu, errMsg, isSoftwareAdapter, NO_ADAPTER_HINT } from './webgpu.ts';
29import { KEY_ENV, keyPath, maskKey, promptSecret, resolveKey, saveKey } from './key.ts';
31const HELP = `turing-surface-cache — fill the shared cache of Turing patterns
33Usage
34 fill [options] work through the parameter space, contributing what is
35 missing, until stopped (ctrl-C)
36 fill --dry-run [N] show the first N targets and whether they are cached
37 login save an upload key for later runs
38 --help
40Options
41 --key <key> upload key; otherwise $${KEY_ENV}, otherwise the saved key
42 --limit <n> stop after n solutions have been computed
43 --model <key> only targets of one model (schnakenberg, brusselator,
44 allencahn)
45 --tend <list> replace the end-time list, e.g. --tend 5,10 (for testing:
46 a short run hashes to its own honest cache entry)
48An upload key is required: the walk exists to contribute. Solutions are read
49by everyone and written only by key holders.
51This is build ${__BUILD_ID__}. npx keys its install directory on the whole
52URL it was given, so a newer build comes from the command the page offers,
53whose URL carries the build it belongs to.`;
55interface Options {
56 command: 'fill' | 'login' | 'help';
57 key?: string;
58 limit: number;
59 model?: string;
60 dryRun: number;
61}
63function parseArgs(argv: string[]): Options {
64 const opts: Options = { command: 'fill', limit: Infinity, dryRun: 0 };
65 const rest = [...argv];
66 if (rest[0] === 'fill' || rest[0] === 'login') opts.command = rest.shift() as 'fill' | 'login';
67 /** A count option whose value may be left off (--dry-run, --dry-run 40). */
68 const count = (fallback: number): number => {
69 const next = rest[0];
70 if (next && /^\d+$/.test(next)) return Number(rest.shift());
71 return fallback;
72 };
73 while (rest.length) {
74 const arg = rest.shift()!;
75 if (arg === '--help' || arg === '-h') opts.command = 'help';
76 else if (arg === '--key') opts.key = rest.shift();
77 else if (arg === '--limit') opts.limit = Number(rest.shift());
78 else if (arg === '--model') opts.model = rest.shift();
79 else if (arg === '--dry-run') opts.dryRun = count(20);
80 else if (arg === '--tend') setEndTimes(rest.shift());
81 else throw new Error(`unknown option ${arg}`);
82 }
83 if (opts.model && !mModelByKey(opts.model)) throw new Error(`unknown model ${opts.model}`);
84 if (!(opts.limit > 0)) throw new Error('--limit wants a positive number');
85 return opts;
86}
88/** The page's ?tend hook, spelled as an option (src/main.ts). */
89function setEndTimes(list: string | undefined): void {
90 const values = (list ?? '')
91 .split(',')
92 .map(Number)
93 .filter((v) => Number.isFinite(v) && v > 0);
94 if (!values.length) throw new Error('--tend wants a comma-separated list of end times');
95 T_END_CHOICE.values = values;
96 T_END_CHOICE.value = values[0];
97}
99// ---------------------------------------------------------------- output
100const tty = process.stdout.isTTY === true;
101/** Written in place on a terminal, and only every so often when piped, so a
102 * log file does not fill with progress. */
103const LOG_EVERY_MS = 30_000;
104let liveLine = false;
106function say(line = ''): void {
107 if (liveLine) {
108 process.stdout.write('\n');
109 liveLine = false;
110 }
111 process.stdout.write(`${line}\n`);
112}
114/** One line that keeps being rewritten while a run advances. */
115function live(line: string): void {
116 if (!tty) return;
117 process.stdout.write(`\r${line.padEnd(78).slice(0, 78)}`);
118 liveLine = true;
119}
121const plural = (n: number, word: string): string => `${n} ${word}${n === 1 ? '' : 's'}`;
123function duration(seconds: number): string {
124 if (!Number.isFinite(seconds)) return '?';
125 if (seconds < 90) return `${seconds.toFixed(0)}s`;
126 const m = Math.floor(seconds / 60);
127 return m < 90 ? `${m}m${String(Math.round(seconds - 60 * m)).padStart(2, '0')}s` : `${(m / 60).toFixed(1)}h`;
128}
130/** Parameters in the order the app lists them, not the order they were built. */
131const paramList = (params: Record<string, number>, choices: DiscreteChoice[]): string =>
132 choices
133 .filter((c) => c.key in params)
134 .map((c) => `${c.key}=${fmtChoice(params[c.key])}`)
135 .join(' ');
137/** What a target is, in one line. */
138function describe(target: AutoTarget): string {
139 const geomChoices = GEOMETRY_CHOICES[target.geometry] ?? [];
140 const geom = geomChoices.length
141 ? `${target.geometry} ${paramList(target.geometryParams, geomChoices)}`
142 : target.geometry;
143 return (
144 `${target.model} ${paramList(target.params, MODEL_CHOICES[target.model])} · ${geom} · ` +
145 `${plural(target.distance, 'knob')} from the defaults`
146 );
147}
149const doneLine = (run: RunSummary): string =>
150 `computed in ${duration(run.seconds)}` +
151 (run.warmFrom !== null ? ` (resumed from cached t = ${fmtChoice(run.warmFrom)})` : '');
153// ---------------------------------------------------------------- commands
154async function login(): Promise<void> {
155 const key = await promptSecret('upload API key: ');
156 if (!key) throw new Error('nothing entered');
157 let ok: boolean;
158 try {
159 ok = await verifyApiKey(key);
160 } catch (e) {
161 say(`could not reach the upload service to check the key (${errMsg(e)}) — saving anyway.`);
162 ok = true;
163 }
164 if (!ok) throw new Error('that key is not allowed to upload — nothing saved');
165 say(`key saved to ${await saveKey(key)}`);
166}
168async function dryRun(opts: Options, targets: AutoTarget[]): Promise<void> {
169 const shown = targets.slice(0, opts.dryRun);
170 say(`the first ${plural(shown.length, 'target')} of ${targets.length.toLocaleString()}, ` +
171 `nearest the defaults first:`);
172 say();
173 let cached = 0;
174 // A handful at a time: a HEAD apiece, and the answers are wanted in order.
175 const width = String(shown.length).length;
176 for (let i = 0; i < shown.length; i += 8) {
177 const batch = shown.slice(i, i + 8);
178 const present = await Promise.all(
179 batch.map(async (t) => (await headCached(await lookupFor(specForTarget(t)))) === true),
180 );
181 present.forEach((isThere, k) => {
182 if (isThere) cached++;
183 say(
184 ` [${String(i + k + 1).padStart(width)}] ${isThere ? 'cached ' : 'missing'} ` +
185 describe(batch[k]),
186 );
187 });
188 }
189 say();
190 say(`${cached} of ${shown.length} already cached; the walk would compute the other ` +
191 `${shown.length - cached}.`);
192}
194async function fill(opts: Options, targets: AutoTarget[], apiKey: string): Promise<void> {
195 const runtime = await installWebGpu();
196 const device = await requestShtDevice().catch((e: unknown) => {
197 throw new Error(`${errMsg(e)}\n${NO_ADAPTER_HINT}`);
198 });
199 const adapter = await describeAdapter(device);
200 say(`build ${__BUILD_ID__} · ${runtime} · ${adapter}`);
201 say(`uploads enabled (key ${maskKey(apiKey)})`);
202 if (isSoftwareAdapter(adapter)) {
203 say('');
204 say(`WARNING: ${adapter} is a software rasterizer, not a GPU. Runs here are`);
205 say(' perhaps a thousand times slower than on hardware — fast enough to look');
206 say(' like it is working, slow enough to be worth nothing. Check that the');
207 say(' machine has a GPU and its driver, or stop now.');
208 }
209 say('');
210 say(`${targets.length.toLocaleString()} targets, nearest the defaults first; ` +
211 `ctrl-C stops after the current run.`);
212 say('');
214 const solver = new SolverSession(device, 1, {
215 onCompiling: (m) => say(` compiling ${m.label}…`),
216 });
218 let stopping = false;
219 process.on('SIGINT', () => {
220 if (stopping) process.exit(130);
221 stopping = true;
222 say('');
223 say('stopping after this run — ctrl-C again to give up on it.');
224 });
226 let index = 0;
227 let computed = 0;
228 let uploads = 0;
229 let slowNoted = false;
230 let lastLog = 0;
231 const counts = await fillWalk({
232 targets,
233 solver,
234 adapter,
235 runtime,
236 apiKey: () => apiKey,
237 beforeTarget: (target) => {
238 index++;
239 return specForTarget(target);
240 },
241 events: {
242 onTarget: (target) => say(`[${index}] ${describe(target)}`),
243 onCached: () => say(' already cached'),
244 onComputing: (_target, spec) =>
245 say(` computing to t = ${fmtChoice(spec.tEnd)} ` +
246 `(${stepsFor(spec).toLocaleString()} steps)`),
247 onPhase: (phase) => {
248 if (phase.kind === 'warm-search') say(' looking for a shorter cached run…');
249 else if (phase.kind === 'seeding') say(' seeding…');
250 else if (phase.kind === 'uploading') {
251 say(` ${doneLine(phase.run)} — uploading ${plural(phase.started, 'file')}…`);
252 }
253 },
254 onProgress: (p) => {
255 const eta = p.rate > 0 ? (p.totalSteps - p.steps) / p.rate : Infinity;
256 const line =
257 ` t = ${p.t.toFixed(1)} / ${fmtChoice(p.tEnd)} ` +
258 `${(100 * p.fraction).toFixed(0)}% ${p.rate.toFixed(0)} steps/s ` +
259 `eta ${duration(eta)}` +
260 (p.uploadsStarted ? ` uploaded ${p.uploadsDone}/${p.uploadsStarted}` : '');
261 live(line);
262 if (!tty && performance.now() - lastLog > LOG_EVERY_MS) {
263 lastLog = performance.now();
264 say(line);
265 }
266 // A rate this low means the run is on a software rasterizer, or on a
267 // GPU so busy it may as well be. Said once, not every chunk.
268 if (!slowNoted && p.rate > 0 && p.rate < 20 && p.steps > 500) {
269 slowNoted = true;
270 say(` NOTE: ${p.rate.toFixed(1)} steps/s is far below what a GPU does ` +
271 `(${adapter}).`);
272 }
273 },
274 onUploaded: () => uploads++,
275 onOutcome: (_target, _spec, outcome) => {
276 if (outcome.kind === 'done') {
277 computed++;
278 const times = [...outcome.uploaded].sort((a, b) => a - b).map(fmtChoice).join(', ');
279 say(` ${doneLine(outcome)} — ` +
280 (outcome.uploaded.length
281 ? `uploaded ${plural(outcome.uploaded.length, 'solution')} (t = ${times})`
282 : 'nothing uploaded'));
283 for (const err of outcome.uploadErrors) say(` upload failed: ${err}`);
284 } else if (outcome.kind === 'diverged') {
285 say(` diverged at t = ${outcome.t.toFixed(2)} — discarded, nothing uploaded`);
286 } else if (outcome.kind === 'stopped') {
287 say(` stopped at t = ${outcome.t.toFixed(2)}`);
288 }
289 if (computed >= opts.limit) stopping = true;
290 },
291 onFailure: (_target, spec, e) => {
292 const model = mModelByKey(spec.model);
293 say(` failed: ${formatFailure(e, model?.source ?? '')}`);
294 },
295 walkStopped: () => stopping,
296 stopRequested: () => stopping,
297 },
298 });
300 say('');
301 say(`stopped — computed ${counts.computed}, skipped ${counts.skipped} already cached` +
302 (counts.failed ? `, ${counts.failed} failed` : '') +
303 `; ${plural(uploads, 'file')} uploaded.`);
304 solver.destroy();
305 device.destroy();
306}
308// ---------------------------------------------------------------- main
309async function main(): Promise<void> {
310 // h5wasm's node build writes through to the real filesystem, so its scratch
311 // files need a real directory to live in (src/cache/h5file.ts).
312 setScratchDir(tmpdir());
313 const opts = parseArgs(process.argv.slice(2));
314 if (opts.command === 'help') {
315 say(HELP);
316 return;
317 }
318 if (opts.command === 'login') {
319 await login();
320 return;
321 }
322 const targets = autoOrder().filter((t) => !opts.model || t.model === opts.model);
323 say('turing-surface-cache — the shared cache of Turing patterns on curved surfaces');
324 if (opts.dryRun) {
325 await dryRun(opts, targets);
326 return;
327 }
328 const apiKey = await resolveKey(opts.key);
329 if (!apiKey) {
330 throw new Error(
331 'no upload key. The walk contributes solutions, so it needs one:\n' +
332 ` ${KEY_ENV}=… npx <this command>\n` +
333 `or save one for later runs with \`login\` (kept in ${keyPath()}).`,
334 );
335 }
336 await fill(opts, targets, apiKey);
337}
339main().catch((e: unknown) => {
340 say('');
341 say(errMsg(e));
342 process.exitCode = 1;
343});