1/**
2 * Filling the shared cache from the command line, so that a machine with a GPU
3 * and nothing to do can contribute without a browser window open on it.
4 *
5 * The walk, the runs and the uploads are the page's (src/cache/fillWalk.ts,
6 * src/cache/runSpec.ts); what is here is the shell around them — Dawn instead
7 * of a browser's WebGPU, a key from the environment instead of localStorage,
8 * and lines of text instead of a status bar.
9 */
10import { tmpdir } from 'node:os';
11import { requestShtDevice, describeAdapter } from '../sht/sht.ts';
12import { mModelByKey } from '../mgpu/registry.ts';
13import { formatFailure } from '../mgpu/errors.ts';
14import {
15 fmtChoice,
16 GEOMETRY_CHOICES,
17 MODEL_CHOICES,
18 T_END_CHOICE,
19 type DiscreteChoice,
20} from '../cache/options.ts';
21import { autoOrder, specForTarget, type AutoTarget } from '../cache/autoWalk.ts';
22import { readSweep, specsForSweep, sweepChoice } from '../cache/selection.ts';
23import { headCached, lookupFor, verifyApiKey } from '../cache/client.ts';
24import type { CacheSpec } from '../cache/spec.ts';
25import { SolverSession } from '../cache/solver.ts';
26import { fillWalk } from '../cache/fillWalk.ts';
27import { setScratchDir } from '../cache/h5file.ts';
28import { stepsFor } from '../cache/spec.ts';
29import type { RunSummary } from '../cache/runSpec.ts';
30import { installWebGpu, errMsg, isSoftwareAdapter, NO_ADAPTER_HINT } from './webgpu.ts';
31import { KEY_ENV, keyPath, maskKey, promptSecret, resolveKey, saveKey } from './key.ts';
33const HELP = `turing-surface-cache — fill the shared cache of Turing patterns
35Usage
36 fill [options] work through the parameter space, contributing what is
37 missing, until stopped (ctrl-C)
38 sweep '<url>' fill one parameter sweep: the argument is the sweep
39 page's URL, whose fragment says which parameter runs
40 over its values and what everything else is fixed to
41 fill --dry-run [N] show the first N targets and whether they are cached
42 (works for sweep too)
43 login save an upload key for later runs
44 --help
46Options
47 --key <key> upload key; otherwise $${KEY_ENV}, otherwise the saved key
48 --limit <n> stop after n solutions have been computed
49 --model <key> only targets of one model (schnakenberg, brusselator,
50 allencahn); the full walk only — a sweep names its model
51 --tend <list> replace the end-time list, e.g. --tend 5,10 (for testing:
52 a short run hashes to its own honest cache entry)
54An upload key is required: the walk exists to contribute. Solutions are read
55by everyone and written only by key holders.
57This is build ${__BUILD_ID__}. npx keys its install directory on the whole
58URL it was given, so a newer build comes from the command the page offers,
59whose URL carries the build it belongs to.`;
61interface Options {
62 command: 'fill' | 'sweep' | 'login' | 'help';
63 key?: string;
64 limit: number;
65 model?: string;
66 dryRun: number;
67 /** The sweep page's URL (the `sweep` command's argument). */
68 sweepUrl?: string;
69 /** --tend was given, so a sweep URL's own ?tend must not override it. */
70 tendGiven: boolean;
71}
73function parseArgs(argv: string[]): Options {
74 const opts: Options = { command: 'fill', limit: Infinity, dryRun: 0, tendGiven: false };
75 const rest = [...argv];
76 if (rest[0] === 'fill' || rest[0] === 'login' || rest[0] === 'sweep') {
77 opts.command = rest.shift() as Options['command'];
78 }
79 /** A count option whose value may be left off (--dry-run, --dry-run 40). */
80 const count = (fallback: number): number => {
81 const next = rest[0];
82 if (next && /^\d+$/.test(next)) return Number(rest.shift());
83 return fallback;
84 };
85 while (rest.length) {
86 const arg = rest.shift()!;
87 if (arg === '--help' || arg === '-h') opts.command = 'help';
88 else if (arg === '--key') opts.key = rest.shift();
89 else if (arg === '--limit') opts.limit = Number(rest.shift());
90 else if (arg === '--model') opts.model = rest.shift();
91 else if (arg === '--dry-run') opts.dryRun = count(20);
92 else if (arg === '--tend') {
93 setEndTimes(rest.shift());
94 opts.tendGiven = true;
95 }
96 else if (opts.command === 'sweep' && !opts.sweepUrl && !arg.startsWith('-')) {
97 opts.sweepUrl = arg;
98 } else throw new Error(`unknown option ${arg}`);
99 }
100 if (opts.model && !mModelByKey(opts.model)) throw new Error(`unknown model ${opts.model}`);
101 if (opts.model && opts.command === 'sweep') {
102 throw new Error('--model applies to the full walk only; a sweep link names its model');
103 }
104 if (opts.command === 'sweep' && !opts.sweepUrl) {
105 throw new Error("sweep wants the sweep page's URL (copy the command from that page)");
106 }
107 if (!(opts.limit > 0)) throw new Error('--limit wants a positive number');
108 return opts;
109}
111/** The page's ?tend hook, spelled as an option (src/main.ts). */
112function setEndTimes(list: string | undefined): void {
113 const values = (list ?? '')
114 .split(',')
115 .map(Number)
116 .filter((v) => Number.isFinite(v) && v > 0);
117 if (!values.length) throw new Error('--tend wants a comma-separated list of end times');
118 T_END_CHOICE.values = values;
119 T_END_CHOICE.value = values[0];
120}
122// ---------------------------------------------------------------- output
123const tty = process.stdout.isTTY === true;
124/** Written in place on a terminal, and only every so often when piped, so a
125 * log file does not fill with progress. */
126const LOG_EVERY_MS = 30_000;
127let liveLine = false;
129function say(line = ''): void {
130 if (liveLine) {
131 process.stdout.write('\n');
132 liveLine = false;
133 }
134 process.stdout.write(`${line}\n`);
135}
137/** One line that keeps being rewritten while a run advances. */
138function live(line: string): void {
139 if (!tty) return;
140 process.stdout.write(`\r${line.padEnd(78).slice(0, 78)}`);
141 liveLine = true;
142}
144const plural = (n: number, word: string): string => `${n} ${word}${n === 1 ? '' : 's'}`;
146function duration(seconds: number): string {
147 if (!Number.isFinite(seconds)) return '?';
148 if (seconds < 90) return `${seconds.toFixed(0)}s`;
149 const m = Math.floor(seconds / 60);
150 return m < 90 ? `${m}m${String(Math.round(seconds - 60 * m)).padStart(2, '0')}s` : `${(m / 60).toFixed(1)}h`;
151}
153/** Parameters in the order the app lists them, not the order they were built. */
154const paramList = (params: Record<string, number>, choices: DiscreteChoice[]): string =>
155 choices
156 .filter((c) => c.key in params)
157 .map((c) => `${c.key}=${fmtChoice(params[c.key])}`)
158 .join(' ');
160/** What a target is, in one line. */
161function describe(target: AutoTarget): string {
162 const geomChoices = GEOMETRY_CHOICES[target.geometry] ?? [];
163 const geom = geomChoices.length
164 ? `${target.geometry} ${paramList(target.geometryParams, geomChoices)}`
165 : target.geometry;
166 return (
167 `${target.model} ${paramList(target.params, MODEL_CHOICES[target.model])} · ${geom} · ` +
168 `${plural(target.distance, 'knob')} from the defaults`
169 );
170}
172const doneLine = (run: RunSummary): string =>
173 `computed in ${duration(run.seconds)}` +
174 (run.warmFrom !== null ? ` (resumed from cached t = ${fmtChoice(run.warmFrom)})` : '');
176// ---------------------------------------------------------------- plans
177/**
178 * What to work through: the full walk, or one sweep. The run loop, the
179 * progress lines and the outcome reporting are identical either way; a plan
180 * is only which targets exist, which spec each names, and how to say so.
181 */
182interface Plan {
183 targets: AutoTarget[];
184 specFor(target: AutoTarget): CacheSpec;
185 label(target: AutoTarget): string;
186 /** Printed once, under the header. */
187 intro: string[];
188}
190function autoPlan(opts: Options): Plan {
191 const targets = autoOrder().filter((t) => !opts.model || t.model === opts.model);
192 return {
193 targets,
194 specFor: specForTarget,
195 label: describe,
196 intro: [`${targets.length.toLocaleString()} targets, nearest the defaults first.`],
197 };
198}
200/**
201 * A sweep, read from the sweep page's own URL: one serialization
202 * (src/cache/selection.ts) shared with the page, so the copied command and
203 * the page it came from always mean the same solutions. Only the fragment is
204 * consulted — plus the page's ?tend test hook, honored the way the pages
205 * honor it, so a command copied from a test page still names what that page
206 * showed.
207 */
208function sweepPlan(url: string, tendGiven: boolean): Plan {
209 const query = url.match(/\?([^#]*)/)?.[1];
210 const tend = query ? new URLSearchParams(query).get('tend') : null;
211 if (tend && !tendGiven) setEndTimes(tend);
212 const fragment = url.includes('#') ? url.slice(url.indexOf('#') + 1) : url;
213 const sweep = readSweep(new URLSearchParams(fragment));
214 if (!sweep) {
215 throw new Error(
216 'that is not a sweep link: its fragment must carry the selection and ' +
217 'sweep=<param>. Copy the command from the sweep page.',
218 );
219 }
220 const { sel, key, values } = sweep;
221 const choice = sweepChoice(sweep);
222 const specs = specsForSweep(sweep);
223 const specByValue = new Map(specs.map(({ value, spec }) => [value, spec]));
224 const fixed = paramList(
225 Object.fromEntries(Object.entries(sel.params).filter(([k]) => k !== key)),
226 MODEL_CHOICES[sel.model],
227 );
228 const geomChoices = GEOMETRY_CHOICES[sel.geometry] ?? [];
229 const geom = geomChoices.length
230 ? `${sel.geometry} ${paramList(sel.geometryParams, geomChoices)}`
231 : sel.geometry;
232 return {
233 targets: specs.map(
234 ({ spec }): AutoTarget => ({
235 model: spec.model,
236 params: { ...spec.params },
237 geometry: spec.geometry,
238 geometryParams: { ...spec.geometryParams },
239 distance: 0,
240 }),
241 ),
242 specFor: (target) => specByValue.get(target.params[key])!,
243 label: (target) => `${key} = ${fmtChoice(target.params[key])}`,
244 intro: [
245 `sweep: ${sel.model}, ${key} over ${plural(values.length, 'value')} ` +
246 `(${values.map(fmtChoice).join(', ')})` +
247 (values.length === choice.values.length &&
248 values.every((v, i) => v === choice.values[i])
249 ? ''
250 : ' — a custom list, so the auto-fill walk will not have filled it'),
251 `fixed: ${fixed} · ${geom} · seed ${sel.seed} · t = ${fmtChoice(sel.tEnd)}`,
252 ],
253 };
254}
256// ---------------------------------------------------------------- commands
257async function login(): Promise<void> {
258 const key = await promptSecret('upload API key: ');
259 if (!key) throw new Error('nothing entered');
260 let ok: boolean;
261 try {
262 ok = await verifyApiKey(key);
263 } catch (e) {
264 say(`could not reach the upload service to check the key (${errMsg(e)}) — saving anyway.`);
265 ok = true;
266 }
267 if (!ok) throw new Error('that key is not allowed to upload — nothing saved');
268 say(`key saved to ${await saveKey(key)}`);
269}
271async function dryRun(opts: Options, plan: Plan): Promise<void> {
272 const shown = plan.targets.slice(0, opts.dryRun);
273 for (const line of plan.intro) say(line);
274 say(`the first ${plural(shown.length, 'target')} of ${plan.targets.length.toLocaleString()}:`);
275 say();
276 let cached = 0;
277 // A handful at a time: a HEAD apiece, and the answers are wanted in order.
278 const width = String(shown.length).length;
279 for (let i = 0; i < shown.length; i += 8) {
280 const batch = shown.slice(i, i + 8);
281 const present = await Promise.all(
282 batch.map(async (t) => (await headCached(await lookupFor(plan.specFor(t)))) === true),
283 );
284 present.forEach((isThere, k) => {
285 if (isThere) cached++;
286 say(
287 ` [${String(i + k + 1).padStart(width)}] ${isThere ? 'cached ' : 'missing'} ` +
288 plan.label(batch[k]),
289 );
290 });
291 }
292 say();
293 say(`${cached} of ${shown.length} already cached; the walk would compute the other ` +
294 `${shown.length - cached}.`);
295}
297async function fill(opts: Options, plan: Plan, apiKey: string): Promise<void> {
298 const runtime = await installWebGpu();
299 const device = await requestShtDevice().catch((e: unknown) => {
300 throw new Error(`${errMsg(e)}\n${NO_ADAPTER_HINT}`);
301 });
302 const adapter = await describeAdapter(device);
303 say(`build ${__BUILD_ID__} · ${runtime} · ${adapter}`);
304 say(`uploads enabled (key ${maskKey(apiKey)})`);
305 if (isSoftwareAdapter(adapter)) {
306 say('');
307 say(`WARNING: ${adapter} is a software rasterizer, not a GPU. Runs here are`);
308 say(' perhaps a thousand times slower than on hardware — fast enough to look');
309 say(' like it is working, slow enough to be worth nothing. Check that the');
310 say(' machine has a GPU and its driver, or stop now.');
311 }
312 say('');
313 for (const line of plan.intro) say(line);
314 say('ctrl-C stops after the current run.');
315 say('');
317 const solver = new SolverSession(device, 1, {
318 onCompiling: (m) => say(` compiling ${m.label}…`),
319 });
321 let stopping = false;
322 process.on('SIGINT', () => {
323 if (stopping) process.exit(130);
324 stopping = true;
325 say('');
326 say('stopping after this run — ctrl-C again to give up on it.');
327 });
329 let index = 0;
330 let computed = 0;
331 let uploads = 0;
332 let slowNoted = false;
333 let lastLog = 0;
334 const counts = await fillWalk({
335 targets: plan.targets,
336 solver,
337 adapter,
338 runtime,
339 apiKey: () => apiKey,
340 beforeTarget: (target) => {
341 index++;
342 return plan.specFor(target);
343 },
344 events: {
345 onTarget: (target) => say(`[${index}] ${plan.label(target)}`),
346 onCached: () => say(' already cached'),
347 onComputing: (_target, spec) =>
348 say(` computing to t = ${fmtChoice(spec.tEnd)} ` +
349 `(${stepsFor(spec).toLocaleString()} steps)`),
350 onPhase: (phase) => {
351 if (phase.kind === 'warm-search') say(' looking for a shorter cached run…');
352 else if (phase.kind === 'seeding') say(' seeding…');
353 else if (phase.kind === 'uploading') {
354 say(` ${doneLine(phase.run)} — uploading ${plural(phase.started, 'file')}…`);
355 }
356 },
357 onProgress: (p) => {
358 const eta = p.rate > 0 ? (p.totalSteps - p.steps) / p.rate : Infinity;
359 const line =
360 ` t = ${p.t.toFixed(1)} / ${fmtChoice(p.tEnd)} ` +
361 `${(100 * p.fraction).toFixed(0)}% ${p.rate.toFixed(0)} steps/s ` +
362 `eta ${duration(eta)}` +
363 (p.uploadsStarted ? ` uploaded ${p.uploadsDone}/${p.uploadsStarted}` : '');
364 live(line);
365 if (!tty && performance.now() - lastLog > LOG_EVERY_MS) {
366 lastLog = performance.now();
367 say(line);
368 }
369 // A rate this low means the run is on a software rasterizer, or on a
370 // GPU so busy it may as well be. Said once, not every chunk.
371 if (!slowNoted && p.rate > 0 && p.rate < 20 && p.steps > 500) {
372 slowNoted = true;
373 say(` NOTE: ${p.rate.toFixed(1)} steps/s is far below what a GPU does ` +
374 `(${adapter}).`);
375 }
376 },
377 onUploaded: () => uploads++,
378 onOutcome: (_target, _spec, outcome) => {
379 if (outcome.kind === 'done') {
380 computed++;
381 const times = [...outcome.uploaded].sort((a, b) => a - b).map(fmtChoice).join(', ');
382 say(` ${doneLine(outcome)} — ` +
383 (outcome.uploaded.length
384 ? `uploaded ${plural(outcome.uploaded.length, 'solution')} (t = ${times})`
385 : 'nothing uploaded'));
386 for (const err of outcome.uploadErrors) say(` upload failed: ${err}`);
387 } else if (outcome.kind === 'diverged') {
388 say(` diverged at t = ${outcome.t.toFixed(2)} — discarded, nothing uploaded`);
389 } else if (outcome.kind === 'stopped') {
390 say(` stopped at t = ${outcome.t.toFixed(2)}`);
391 }
392 if (computed >= opts.limit) stopping = true;
393 },
394 onFailure: (_target, spec, e) => {
395 const model = mModelByKey(spec.model);
396 say(` failed: ${formatFailure(e, model?.source ?? '')}`);
397 },
398 walkStopped: () => stopping,
399 stopRequested: () => stopping,
400 },
401 });
403 say('');
404 say(`stopped — computed ${counts.computed}, skipped ${counts.skipped} already cached` +
405 (counts.failed ? `, ${counts.failed} failed` : '') +
406 `; ${plural(uploads, 'file')} uploaded.`);
407 solver.destroy();
408 device.destroy();
409}
411// ---------------------------------------------------------------- main
412async function main(): Promise<void> {
413 // h5wasm's node build writes through to the real filesystem, so its scratch
414 // files need a real directory to live in (src/cache/h5file.ts).
415 setScratchDir(tmpdir());
416 const opts = parseArgs(process.argv.slice(2));
417 if (opts.command === 'help') {
418 say(HELP);
419 return;
420 }
421 if (opts.command === 'login') {
422 await login();
423 return;
424 }
425 const plan =
426 opts.command === 'sweep' ? sweepPlan(opts.sweepUrl!, opts.tendGiven) : autoPlan(opts);
427 say('turing-surface-cache — the shared cache of Turing patterns on curved surfaces');
428 if (opts.dryRun) {
429 await dryRun(opts, plan);
430 return;
431 }
432 const apiKey = await resolveKey(opts.key);
433 if (!apiKey) {
434 throw new Error(
435 'no upload key. The walk contributes solutions, so it needs one:\n' +
436 ` ${KEY_ENV}=… npx <this command>\n` +
437 `or save one for later runs with \`login\` (kept in ${keyPath()}).`,
438 );
439 }
440 await fill(opts, plan, apiKey);
441}
443main().catch((e: unknown) => {
444 say('');
445 say(errMsg(e));
446 process.exitCode = 1;
447});