2 * Filling the shared cache from the command line, so that a machine with a GPU
3 * and nothing to do can contribute without a browser window open on it.
4 *
5 * The walk, the runs and the uploads are the page's (src/cache/fillWalk.ts,
6 * src/cache/runSpec.ts); what is here is the shell around them — Dawn instead
7 * of a browser's WebGPU, a key from the environment instead of localStorage,
8 * and lines of text instead of a status bar.
9 */
b0546a9Fix the command line's file writing, and how it gets updatedJeremy Magland 10import { tmpdir } from 'node:os';
c2317d0Fill the cache from the command line, without a browserJeremy Magland 11import { requestShtDevice, describeAdapter } from '../sht/sht.ts';
12import { mModelByKey } from '../mgpu/registry.ts';
13import { formatFailure } from '../mgpu/errors.ts';
14import {
15 fmtChoice,
16 GEOMETRY_CHOICES,
17 MODEL_CHOICES,
18 T_END_CHOICE,
19 type DiscreteChoice,
20} from '../cache/options.ts';
21import { autoOrder, specForTarget, type AutoTarget } from '../cache/autoWalk.ts';
e53205fShow one parameter across its range, not only at a pointJeremy Magland 22import { readSweep, specsForSweep, sweepChoice } from '../cache/selection.ts';
c2317d0Fill the cache from the command line, without a browserJeremy Magland 23import { headCached, lookupFor, verifyApiKey } from '../cache/client.ts';
e53205fShow one parameter across its range, not only at a pointJeremy Magland 24import type { CacheSpec } from '../cache/spec.ts';
c2317d0Fill the cache from the command line, without a browserJeremy Magland 25import { SolverSession } from '../cache/solver.ts';
26import { fillWalk } from '../cache/fillWalk.ts';
b0546a9Fix the command line's file writing, and how it gets updatedJeremy Magland 27import { setScratchDir } from '../cache/h5file.ts';
c2317d0Fill the cache from the command line, without a browserJeremy Magland 28import { stepsFor } from '../cache/spec.ts';
29import type { RunSummary } from '../cache/runSpec.ts';
30import { installWebGpu, errMsg, isSoftwareAdapter, NO_ADAPTER_HINT } from './webgpu.ts';
31import { KEY_ENV, keyPath, maskKey, promptSecret, resolveKey, saveKey } from './key.ts';
33const HELP = `turing-surface-cache — fill the shared cache of Turing patterns
35Usage
36 fill [options] work through the parameter space, contributing what is
37 missing, until stopped (ctrl-C)
e53205fShow one parameter across its range, not only at a pointJeremy Magland 38 sweep '<url>' fill one parameter sweep: the argument is the sweep
39 page's URL, whose fragment says which parameter runs
40 over its values and what everything else is fixed to
c2317d0Fill the cache from the command line, without a browserJeremy Magland 41 fill --dry-run [N] show the first N targets and whether they are cached
e53205fShow one parameter across its range, not only at a pointJeremy Magland 42 (works for sweep too)
c2317d0Fill the cache from the command line, without a browserJeremy Magland 43 login save an upload key for later runs
44 --help
46Options
47 --key <key> upload key; otherwise $${KEY_ENV}, otherwise the saved key
48 --limit <n> stop after n solutions have been computed
49 --model <key> only targets of one model (schnakenberg, brusselator,
e53205fShow one parameter across its range, not only at a pointJeremy Magland 50 allencahn); the full walk only — a sweep names its model
c2317d0Fill the cache from the command line, without a browserJeremy Magland 51 --tend <list> replace the end-time list, e.g. --tend 5,10 (for testing:
52 a short run hashes to its own honest cache entry)
54An upload key is required: the walk exists to contribute. Solutions are read
b0546a9Fix the command line's file writing, and how it gets updatedJeremy Magland 55by everyone and written only by key holders.
57This is build ${__BUILD_ID__}. npx keys its install directory on the whole
58URL it was given, so a newer build comes from the command the page offers,
59whose URL carries the build it belongs to.`;
61interface Options {
e53205fShow one parameter across its range, not only at a pointJeremy Magland 62 command: 'fill' | 'sweep' | 'login' | 'help';
64 limit: number;
65 model?: string;
66 dryRun: number;
e53205fShow one parameter across its range, not only at a pointJeremy Magland 67 /** The sweep page's URL (the `sweep` command's argument). */
68 sweepUrl?: string;
69 /** --tend was given, so a sweep URL's own ?tend must not override it. */
70 tendGiven: boolean;
73function parseArgs(argv: string[]): Options {
e53205fShow one parameter across its range, not only at a pointJeremy Magland 74 const opts: Options = { command: 'fill', limit: Infinity, dryRun: 0, tendGiven: false };
c2317d0Fill the cache from the command line, without a browserJeremy Magland 75 const rest = [...argv];
e53205fShow one parameter across its range, not only at a pointJeremy Magland 76 if (rest[0] === 'fill' || rest[0] === 'login' || rest[0] === 'sweep') {
77 opts.command = rest.shift() as Options['command'];
78 }
c2317d0Fill the cache from the command line, without a browserJeremy Magland 79 /** A count option whose value may be left off (--dry-run, --dry-run 40). */
80 const count = (fallback: number): number => {
81 const next = rest[0];
82 if (next && /^\d+$/.test(next)) return Number(rest.shift());
83 return fallback;
84 };
85 while (rest.length) {
86 const arg = rest.shift()!;
87 if (arg === '--help' || arg === '-h') opts.command = 'help';
88 else if (arg === '--key') opts.key = rest.shift();
89 else if (arg === '--limit') opts.limit = Number(rest.shift());
90 else if (arg === '--model') opts.model = rest.shift();
91 else if (arg === '--dry-run') opts.dryRun = count(20);
e53205fShow one parameter across its range, not only at a pointJeremy Magland 92 else if (arg === '--tend') {
93 setEndTimes(rest.shift());
94 opts.tendGiven = true;
95 }
96 else if (opts.command === 'sweep' && !opts.sweepUrl && !arg.startsWith('-')) {
97 opts.sweepUrl = arg;
98 } else throw new Error(`unknown option ${arg}`);
100 if (opts.model && !mModelByKey(opts.model)) throw new Error(`unknown model ${opts.model}`);
e53205fShow one parameter across its range, not only at a pointJeremy Magland 101 if (opts.model && opts.command === 'sweep') {
102 throw new Error('--model applies to the full walk only; a sweep link names its model');
103 }
104 if (opts.command === 'sweep' && !opts.sweepUrl) {
105 throw new Error("sweep wants the sweep page's URL (copy the command from that page)");
106 }
c2317d0Fill the cache from the command line, without a browserJeremy Magland 107 if (!(opts.limit > 0)) throw new Error('--limit wants a positive number');
108 return opts;
109}
111/** The page's ?tend hook, spelled as an option (src/main.ts). */
112function setEndTimes(list: string | undefined): void {
113 const values = (list ?? '')
114 .split(',')
115 .map(Number)
116 .filter((v) => Number.isFinite(v) && v > 0);
117 if (!values.length) throw new Error('--tend wants a comma-separated list of end times');
118 T_END_CHOICE.values = values;
119 T_END_CHOICE.value = values[0];
120}
122// ---------------------------------------------------------------- output
123const tty = process.stdout.isTTY === true;
124/** Written in place on a terminal, and only every so often when piped, so a
125 * log file does not fill with progress. */
126const LOG_EVERY_MS = 30_000;
127let liveLine = false;
129function say(line = ''): void {
130 if (liveLine) {
131 process.stdout.write('\n');
132 liveLine = false;
133 }
134 process.stdout.write(`${line}\n`);
135}
137/** One line that keeps being rewritten while a run advances. */
138function live(line: string): void {
139 if (!tty) return;
140 process.stdout.write(`\r${line.padEnd(78).slice(0, 78)}`);
141 liveLine = true;
142}
144const plural = (n: number, word: string): string => `${n} ${word}${n === 1 ? '' : 's'}`;
146function duration(seconds: number): string {
147 if (!Number.isFinite(seconds)) return '?';
148 if (seconds < 90) return `${seconds.toFixed(0)}s`;
149 const m = Math.floor(seconds / 60);
150 return m < 90 ? `${m}m${String(Math.round(seconds - 60 * m)).padStart(2, '0')}s` : `${(m / 60).toFixed(1)}h`;
151}
153/** Parameters in the order the app lists them, not the order they were built. */
154const paramList = (params: Record<string, number>, choices: DiscreteChoice[]): string =>
155 choices
156 .filter((c) => c.key in params)
157 .map((c) => `${c.key}=${fmtChoice(params[c.key])}`)
158 .join(' ');
160/** What a target is, in one line. */
161function describe(target: AutoTarget): string {
162 const geomChoices = GEOMETRY_CHOICES[target.geometry] ?? [];
163 const geom = geomChoices.length
164 ? `${target.geometry} ${paramList(target.geometryParams, geomChoices)}`
165 : target.geometry;
166 return (
167 `${target.model} ${paramList(target.params, MODEL_CHOICES[target.model])} · ${geom} · ` +
168 `${plural(target.distance, 'knob')} from the defaults`
169 );
170}
172const doneLine = (run: RunSummary): string =>
173 `computed in ${duration(run.seconds)}` +
174 (run.warmFrom !== null ? ` (resumed from cached t = ${fmtChoice(run.warmFrom)})` : '');
e53205fShow one parameter across its range, not only at a pointJeremy Magland 176// ---------------------------------------------------------------- plans
177/**
178 * What to work through: the full walk, or one sweep. The run loop, the
179 * progress lines and the outcome reporting are identical either way; a plan
180 * is only which targets exist, which spec each names, and how to say so.
181 */
182interface Plan {
183 targets: AutoTarget[];
184 specFor(target: AutoTarget): CacheSpec;
185 label(target: AutoTarget): string;
186 /** Printed once, under the header. */
187 intro: string[];
188}
190function autoPlan(opts: Options): Plan {
191 const targets = autoOrder().filter((t) => !opts.model || t.model === opts.model);
192 return {
193 targets,
194 specFor: specForTarget,
195 label: describe,
196 intro: [`${targets.length.toLocaleString()} targets, nearest the defaults first.`],
197 };
198}
200/**
201 * A sweep, read from the sweep page's own URL: one serialization
202 * (src/cache/selection.ts) shared with the page, so the copied command and
203 * the page it came from always mean the same solutions. Only the fragment is
204 * consulted — plus the page's ?tend test hook, honored the way the pages
205 * honor it, so a command copied from a test page still names what that page
206 * showed.
207 */
208function sweepPlan(url: string, tendGiven: boolean): Plan {
209 const query = url.match(/\?([^#]*)/)?.[1];
210 const tend = query ? new URLSearchParams(query).get('tend') : null;
211 if (tend && !tendGiven) setEndTimes(tend);
212 const fragment = url.includes('#') ? url.slice(url.indexOf('#') + 1) : url;
213 const sweep = readSweep(new URLSearchParams(fragment));
214 if (!sweep) {
215 throw new Error(
216 'that is not a sweep link: its fragment must carry the selection and ' +
217 'sweep=<param>. Copy the command from the sweep page.',
218 );
219 }
220 const { sel, key, values } = sweep;
221 const choice = sweepChoice(sweep);
222 const specs = specsForSweep(sweep);
223 const specByValue = new Map(specs.map(({ value, spec }) => [value, spec]));
224 const fixed = paramList(
225 Object.fromEntries(Object.entries(sel.params).filter(([k]) => k !== key)),
226 MODEL_CHOICES[sel.model],
227 );
228 const geomChoices = GEOMETRY_CHOICES[sel.geometry] ?? [];
229 const geom = geomChoices.length
230 ? `${sel.geometry} ${paramList(sel.geometryParams, geomChoices)}`
231 : sel.geometry;
232 return {
233 targets: specs.map(
234 ({ spec }): AutoTarget => ({
235 model: spec.model,
236 params: { ...spec.params },
237 geometry: spec.geometry,
238 geometryParams: { ...spec.geometryParams },
239 distance: 0,
240 }),
241 ),
242 specFor: (target) => specByValue.get(target.params[key])!,
243 label: (target) => `${key} = ${fmtChoice(target.params[key])}`,
244 intro: [
245 `sweep: ${sel.model}, ${key} over ${plural(values.length, 'value')} ` +
246 `(${values.map(fmtChoice).join(', ')})` +
247 (values.length === choice.values.length &&
248 values.every((v, i) => v === choice.values[i])
249 ? ''
250 : ' — a custom list, so the auto-fill walk will not have filled it'),
251 `fixed: ${fixed} · ${geom} · seed ${sel.seed} · t = ${fmtChoice(sel.tEnd)}`,
252 ],
253 };
254}
c2317d0Fill the cache from the command line, without a browserJeremy Magland 256// ---------------------------------------------------------------- commands
257async function login(): Promise<void> {
258 const key = await promptSecret('upload API key: ');
259 if (!key) throw new Error('nothing entered');
260 let ok: boolean;
261 try {
262 ok = await verifyApiKey(key);
263 } catch (e) {
264 say(`could not reach the upload service to check the key (${errMsg(e)}) — saving anyway.`);
265 ok = true;
266 }
267 if (!ok) throw new Error('that key is not allowed to upload — nothing saved');
268 say(`key saved to ${await saveKey(key)}`);
269}
e53205fShow one parameter across its range, not only at a pointJeremy Magland 271async function dryRun(opts: Options, plan: Plan): Promise<void> {
272 const shown = plan.targets.slice(0, opts.dryRun);
273 for (const line of plan.intro) say(line);
274 say(`the first ${plural(shown.length, 'target')} of ${plan.targets.length.toLocaleString()}:`);
276 let cached = 0;
277 // A handful at a time: a HEAD apiece, and the answers are wanted in order.
278 const width = String(shown.length).length;
279 for (let i = 0; i < shown.length; i += 8) {
280 const batch = shown.slice(i, i + 8);
281 const present = await Promise.all(
e53205fShow one parameter across its range, not only at a pointJeremy Magland 282 batch.map(async (t) => (await headCached(await lookupFor(plan.specFor(t)))) === true),
284 present.forEach((isThere, k) => {
285 if (isThere) cached++;
286 say(
287 ` [${String(i + k + 1).padStart(width)}] ${isThere ? 'cached ' : 'missing'} ` +
e53205fShow one parameter across its range, not only at a pointJeremy Magland 288 plan.label(batch[k]),
290 });
291 }
292 say();
293 say(`${cached} of ${shown.length} already cached; the walk would compute the other ` +
294 `${shown.length - cached}.`);
295}
e53205fShow one parameter across its range, not only at a pointJeremy Magland 297async function fill(opts: Options, plan: Plan, apiKey: string): Promise<void> {
c2317d0Fill the cache from the command line, without a browserJeremy Magland 298 const runtime = await installWebGpu();
299 const device = await requestShtDevice().catch((e: unknown) => {
300 throw new Error(`${errMsg(e)}\n${NO_ADAPTER_HINT}`);
301 });
302 const adapter = await describeAdapter(device);
b0546a9Fix the command line's file writing, and how it gets updatedJeremy Magland 303 say(`build ${__BUILD_ID__} · ${runtime} · ${adapter}`);
c2317d0Fill the cache from the command line, without a browserJeremy Magland 304 say(`uploads enabled (key ${maskKey(apiKey)})`);
305 if (isSoftwareAdapter(adapter)) {
306 say('');
307 say(`WARNING: ${adapter} is a software rasterizer, not a GPU. Runs here are`);
308 say(' perhaps a thousand times slower than on hardware — fast enough to look');
309 say(' like it is working, slow enough to be worth nothing. Check that the');
310 say(' machine has a GPU and its driver, or stop now.');
311 }
312 say('');
e53205fShow one parameter across its range, not only at a pointJeremy Magland 313 for (const line of plan.intro) say(line);
314 say('ctrl-C stops after the current run.');
317 const solver = new SolverSession(device, 1, {
318 onCompiling: (m) => say(` compiling ${m.label}…`),
319 });
321 let stopping = false;
322 process.on('SIGINT', () => {
323 if (stopping) process.exit(130);
324 stopping = true;
325 say('');
326 say('stopping after this run — ctrl-C again to give up on it.');
327 });
329 let index = 0;
330 let computed = 0;
331 let uploads = 0;
332 let slowNoted = false;
333 let lastLog = 0;
334 const counts = await fillWalk({
e53205fShow one parameter across its range, not only at a pointJeremy Magland 335 targets: plan.targets,
337 adapter,
c2317d0Fill the cache from the command line, without a browserJeremy Magland 339 apiKey: () => apiKey,
340 beforeTarget: (target) => {
341 index++;
e53205fShow one parameter across its range, not only at a pointJeremy Magland 342 return plan.specFor(target);
344 events: {
e53205fShow one parameter across its range, not only at a pointJeremy Magland 345 onTarget: (target) => say(`[${index}] ${plan.label(target)}`),
c2317d0Fill the cache from the command line, without a browserJeremy Magland 346 onCached: () => say(' already cached'),
347 onComputing: (_target, spec) =>
348 say(` computing to t = ${fmtChoice(spec.tEnd)} ` +
349 `(${stepsFor(spec).toLocaleString()} steps)`),
350 onPhase: (phase) => {
351 if (phase.kind === 'warm-search') say(' looking for a shorter cached run…');
352 else if (phase.kind === 'seeding') say(' seeding…');
353 else if (phase.kind === 'uploading') {
354 say(` ${doneLine(phase.run)} — uploading ${plural(phase.started, 'file')}…`);
355 }
356 },
357 onProgress: (p) => {
358 const eta = p.rate > 0 ? (p.totalSteps - p.steps) / p.rate : Infinity;
359 const line =
360 ` t = ${p.t.toFixed(1)} / ${fmtChoice(p.tEnd)} ` +
361 `${(100 * p.fraction).toFixed(0)}% ${p.rate.toFixed(0)} steps/s ` +
362 `eta ${duration(eta)}` +
363 (p.uploadsStarted ? ` uploaded ${p.uploadsDone}/${p.uploadsStarted}` : '');
364 live(line);
365 if (!tty && performance.now() - lastLog > LOG_EVERY_MS) {
366 lastLog = performance.now();
367 say(line);
368 }
369 // A rate this low means the run is on a software rasterizer, or on a
370 // GPU so busy it may as well be. Said once, not every chunk.
371 if (!slowNoted && p.rate > 0 && p.rate < 20 && p.steps > 500) {
372 slowNoted = true;
373 say(` NOTE: ${p.rate.toFixed(1)} steps/s is far below what a GPU does ` +
374 `(${adapter}).`);
375 }
376 },
377 onUploaded: () => uploads++,
378 onOutcome: (_target, _spec, outcome) => {
379 if (outcome.kind === 'done') {
380 computed++;
381 const times = [...outcome.uploaded].sort((a, b) => a - b).map(fmtChoice).join(', ');
382 say(` ${doneLine(outcome)} — ` +
383 (outcome.uploaded.length
384 ? `uploaded ${plural(outcome.uploaded.length, 'solution')} (t = ${times})`
385 : 'nothing uploaded'));
386 for (const err of outcome.uploadErrors) say(` upload failed: ${err}`);
387 } else if (outcome.kind === 'diverged') {
388 say(` diverged at t = ${outcome.t.toFixed(2)} — discarded, nothing uploaded`);
389 } else if (outcome.kind === 'stopped') {
390 say(` stopped at t = ${outcome.t.toFixed(2)}`);
391 }
392 if (computed >= opts.limit) stopping = true;
393 },
394 onFailure: (_target, spec, e) => {
395 const model = mModelByKey(spec.model);
396 say(` failed: ${formatFailure(e, model?.source ?? '')}`);
397 },
398 walkStopped: () => stopping,
399 stopRequested: () => stopping,
400 },
401 });
403 say('');
404 say(`stopped — computed ${counts.computed}, skipped ${counts.skipped} already cached` +
405 (counts.failed ? `, ${counts.failed} failed` : '') +
406 `; ${plural(uploads, 'file')} uploaded.`);
407 solver.destroy();
408 device.destroy();
409}
411// ---------------------------------------------------------------- main
412async function main(): Promise<void> {
b0546a9Fix the command line's file writing, and how it gets updatedJeremy Magland 413 // h5wasm's node build writes through to the real filesystem, so its scratch
414 // files need a real directory to live in (src/cache/h5file.ts).
415 setScratchDir(tmpdir());
c2317d0Fill the cache from the command line, without a browserJeremy Magland 416 const opts = parseArgs(process.argv.slice(2));
417 if (opts.command === 'help') {
418 say(HELP);
419 return;
420 }
421 if (opts.command === 'login') {
422 await login();
423 return;
424 }
426 opts.command === 'sweep' ? sweepPlan(opts.sweepUrl!, opts.tendGiven) : autoPlan(opts);
c2317d0Fill the cache from the command line, without a browserJeremy Magland 427 say('turing-surface-cache — the shared cache of Turing patterns on curved surfaces');
428 if (opts.dryRun) {
e53205fShow one parameter across its range, not only at a pointJeremy Magland 429 await dryRun(opts, plan);
431 }
432 const apiKey = await resolveKey(opts.key);
433 if (!apiKey) {
434 throw new Error(
435 'no upload key. The walk contributes solutions, so it needs one:\n' +
436 ` ${KEY_ENV}=… npx <this command>\n` +
437 `or save one for later runs with \`login\` (kept in ${keyPath()}).`,
438 );
439 }
e53205fShow one parameter across its range, not only at a pointJeremy Magland 440 await fill(opts, plan, apiKey);
443main().catch((e: unknown) => {
444 say('');
445 say(errMsg(e));
446 process.exitCode = 1;
447});