2 * Filling the shared cache from the command line, so that a machine with a GPU
3 * and nothing to do can contribute without a browser window open on it.
4 *
5 * The walk, the runs and the uploads are the page's (src/cache/fillWalk.ts,
6 * src/cache/runSpec.ts); what is here is the shell around them — Dawn instead
7 * of a browser's WebGPU, a key from the environment instead of localStorage,
8 * and lines of text instead of a status bar.
9 */
b0546a9Fix the command line's file writing, and how it gets updatedJeremy Magland 10import { tmpdir } from 'node:os';
c2317d0Fill the cache from the command line, without a browserJeremy Magland 11import { requestShtDevice, describeAdapter } from '../sht/sht.ts';
12import { mModelByKey } from '../mgpu/registry.ts';
13import { formatFailure } from '../mgpu/errors.ts';
14import {
15 fmtChoice,
16 GEOMETRY_CHOICES,
17 MODEL_CHOICES,
18 T_END_CHOICE,
19 type DiscreteChoice,
20} from '../cache/options.ts';
21import { autoOrder, specForTarget, type AutoTarget } from '../cache/autoWalk.ts';
22import { headCached, lookupFor, verifyApiKey } from '../cache/client.ts';
23import { SolverSession } from '../cache/solver.ts';
24import { fillWalk } from '../cache/fillWalk.ts';
b0546a9Fix the command line's file writing, and how it gets updatedJeremy Magland 25import { setScratchDir } from '../cache/h5file.ts';
c2317d0Fill the cache from the command line, without a browserJeremy Magland 26import { stepsFor } from '../cache/spec.ts';
27import type { RunSummary } from '../cache/runSpec.ts';
28import { installWebGpu, errMsg, isSoftwareAdapter, NO_ADAPTER_HINT } from './webgpu.ts';
29import { KEY_ENV, keyPath, maskKey, promptSecret, resolveKey, saveKey } from './key.ts';
31const HELP = `turing-surface-cache — fill the shared cache of Turing patterns
33Usage
34 fill [options] work through the parameter space, contributing what is
35 missing, until stopped (ctrl-C)
36 fill --dry-run [N] show the first N targets and whether they are cached
37 login save an upload key for later runs
38 --help
40Options
41 --key <key> upload key; otherwise $${KEY_ENV}, otherwise the saved key
42 --limit <n> stop after n solutions have been computed
43 --model <key> only targets of one model (schnakenberg, brusselator,
44 allencahn)
45 --tend <list> replace the end-time list, e.g. --tend 5,10 (for testing:
46 a short run hashes to its own honest cache entry)
48An upload key is required: the walk exists to contribute. Solutions are read
b0546a9Fix the command line's file writing, and how it gets updatedJeremy Magland 49by everyone and written only by key holders.
51This is build ${__BUILD_ID__}. npx keys its install directory on the whole
52URL it was given, so a newer build comes from the command the page offers,
53whose URL carries the build it belongs to.`;
55interface Options {
56 command: 'fill' | 'login' | 'help';
57 key?: string;
58 limit: number;
59 model?: string;
60 dryRun: number;
61}
63function parseArgs(argv: string[]): Options {
64 const opts: Options = { command: 'fill', limit: Infinity, dryRun: 0 };
65 const rest = [...argv];
66 if (rest[0] === 'fill' || rest[0] === 'login') opts.command = rest.shift() as 'fill' | 'login';
67 /** A count option whose value may be left off (--dry-run, --dry-run 40). */
68 const count = (fallback: number): number => {
69 const next = rest[0];
70 if (next && /^\d+$/.test(next)) return Number(rest.shift());
71 return fallback;
72 };
73 while (rest.length) {
74 const arg = rest.shift()!;
75 if (arg === '--help' || arg === '-h') opts.command = 'help';
76 else if (arg === '--key') opts.key = rest.shift();
77 else if (arg === '--limit') opts.limit = Number(rest.shift());
78 else if (arg === '--model') opts.model = rest.shift();
79 else if (arg === '--dry-run') opts.dryRun = count(20);
80 else if (arg === '--tend') setEndTimes(rest.shift());
81 else throw new Error(`unknown option ${arg}`);
82 }
83 if (opts.model && !mModelByKey(opts.model)) throw new Error(`unknown model ${opts.model}`);
84 if (!(opts.limit > 0)) throw new Error('--limit wants a positive number');
85 return opts;
86}
88/** The page's ?tend hook, spelled as an option (src/main.ts). */
89function setEndTimes(list: string | undefined): void {
90 const values = (list ?? '')
91 .split(',')
92 .map(Number)
93 .filter((v) => Number.isFinite(v) && v > 0);
94 if (!values.length) throw new Error('--tend wants a comma-separated list of end times');
95 T_END_CHOICE.values = values;
96 T_END_CHOICE.value = values[0];
97}
99// ---------------------------------------------------------------- output
100const tty = process.stdout.isTTY === true;
101/** Written in place on a terminal, and only every so often when piped, so a
102 * log file does not fill with progress. */
103const LOG_EVERY_MS = 30_000;
104let liveLine = false;
106function say(line = ''): void {
107 if (liveLine) {
108 process.stdout.write('\n');
109 liveLine = false;
110 }
111 process.stdout.write(`${line}\n`);
112}
114/** One line that keeps being rewritten while a run advances. */
115function live(line: string): void {
116 if (!tty) return;
117 process.stdout.write(`\r${line.padEnd(78).slice(0, 78)}`);
118 liveLine = true;
119}
121const plural = (n: number, word: string): string => `${n} ${word}${n === 1 ? '' : 's'}`;
123function duration(seconds: number): string {
124 if (!Number.isFinite(seconds)) return '?';
125 if (seconds < 90) return `${seconds.toFixed(0)}s`;
126 const m = Math.floor(seconds / 60);
127 return m < 90 ? `${m}m${String(Math.round(seconds - 60 * m)).padStart(2, '0')}s` : `${(m / 60).toFixed(1)}h`;
128}
130/** Parameters in the order the app lists them, not the order they were built. */
131const paramList = (params: Record<string, number>, choices: DiscreteChoice[]): string =>
132 choices
133 .filter((c) => c.key in params)
134 .map((c) => `${c.key}=${fmtChoice(params[c.key])}`)
135 .join(' ');
137/** What a target is, in one line. */
138function describe(target: AutoTarget): string {
139 const geomChoices = GEOMETRY_CHOICES[target.geometry] ?? [];
140 const geom = geomChoices.length
141 ? `${target.geometry} ${paramList(target.geometryParams, geomChoices)}`
142 : target.geometry;
143 return (
144 `${target.model} ${paramList(target.params, MODEL_CHOICES[target.model])} · ${geom} · ` +
145 `${plural(target.distance, 'knob')} from the defaults`
146 );
147}
149const doneLine = (run: RunSummary): string =>
150 `computed in ${duration(run.seconds)}` +
151 (run.warmFrom !== null ? ` (resumed from cached t = ${fmtChoice(run.warmFrom)})` : '');
153// ---------------------------------------------------------------- commands
154async function login(): Promise<void> {
155 const key = await promptSecret('upload API key: ');
156 if (!key) throw new Error('nothing entered');
157 let ok: boolean;
158 try {
159 ok = await verifyApiKey(key);
160 } catch (e) {
161 say(`could not reach the upload service to check the key (${errMsg(e)}) — saving anyway.`);
162 ok = true;
163 }
164 if (!ok) throw new Error('that key is not allowed to upload — nothing saved');
165 say(`key saved to ${await saveKey(key)}`);
166}
168async function dryRun(opts: Options, targets: AutoTarget[]): Promise<void> {
169 const shown = targets.slice(0, opts.dryRun);
170 say(`the first ${plural(shown.length, 'target')} of ${targets.length.toLocaleString()}, ` +
171 `nearest the defaults first:`);
172 say();
173 let cached = 0;
174 // A handful at a time: a HEAD apiece, and the answers are wanted in order.
175 const width = String(shown.length).length;
176 for (let i = 0; i < shown.length; i += 8) {
177 const batch = shown.slice(i, i + 8);
178 const present = await Promise.all(
179 batch.map(async (t) => (await headCached(await lookupFor(specForTarget(t)))) === true),
180 );
181 present.forEach((isThere, k) => {
182 if (isThere) cached++;
183 say(
184 ` [${String(i + k + 1).padStart(width)}] ${isThere ? 'cached ' : 'missing'} ` +
185 describe(batch[k]),
186 );
187 });
188 }
189 say();
190 say(`${cached} of ${shown.length} already cached; the walk would compute the other ` +
191 `${shown.length - cached}.`);
192}
194async function fill(opts: Options, targets: AutoTarget[], apiKey: string): Promise<void> {
195 const runtime = await installWebGpu();
196 const device = await requestShtDevice().catch((e: unknown) => {
197 throw new Error(`${errMsg(e)}\n${NO_ADAPTER_HINT}`);
198 });
199 const adapter = await describeAdapter(device);
b0546a9Fix the command line's file writing, and how it gets updatedJeremy Magland 200 say(`build ${__BUILD_ID__} · ${runtime} · ${adapter}`);
c2317d0Fill the cache from the command line, without a browserJeremy Magland 201 say(`uploads enabled (key ${maskKey(apiKey)})`);
202 if (isSoftwareAdapter(adapter)) {
203 say('');
204 say(`WARNING: ${adapter} is a software rasterizer, not a GPU. Runs here are`);
205 say(' perhaps a thousand times slower than on hardware — fast enough to look');
206 say(' like it is working, slow enough to be worth nothing. Check that the');
207 say(' machine has a GPU and its driver, or stop now.');
208 }
209 say('');
210 say(`${targets.length.toLocaleString()} targets, nearest the defaults first; ` +
211 `ctrl-C stops after the current run.`);
212 say('');
214 const solver = new SolverSession(device, 1, {
215 onCompiling: (m) => say(` compiling ${m.label}…`),
216 });
218 let stopping = false;
219 process.on('SIGINT', () => {
220 if (stopping) process.exit(130);
221 stopping = true;
222 say('');
223 say('stopping after this run — ctrl-C again to give up on it.');
224 });
226 let index = 0;
227 let computed = 0;
228 let uploads = 0;
229 let slowNoted = false;
230 let lastLog = 0;
231 const counts = await fillWalk({
232 targets,
233 solver,
234 adapter,
c2317d0Fill the cache from the command line, without a browserJeremy Magland 236 apiKey: () => apiKey,
237 beforeTarget: (target) => {
238 index++;
239 return specForTarget(target);
240 },
241 events: {
242 onTarget: (target) => say(`[${index}] ${describe(target)}`),
243 onCached: () => say(' already cached'),
244 onComputing: (_target, spec) =>
245 say(` computing to t = ${fmtChoice(spec.tEnd)} ` +
246 `(${stepsFor(spec).toLocaleString()} steps)`),
247 onPhase: (phase) => {
248 if (phase.kind === 'warm-search') say(' looking for a shorter cached run…');
249 else if (phase.kind === 'seeding') say(' seeding…');
250 else if (phase.kind === 'uploading') {
251 say(` ${doneLine(phase.run)} — uploading ${plural(phase.started, 'file')}…`);
252 }
253 },
254 onProgress: (p) => {
255 const eta = p.rate > 0 ? (p.totalSteps - p.steps) / p.rate : Infinity;
256 const line =
257 ` t = ${p.t.toFixed(1)} / ${fmtChoice(p.tEnd)} ` +
258 `${(100 * p.fraction).toFixed(0)}% ${p.rate.toFixed(0)} steps/s ` +
259 `eta ${duration(eta)}` +
260 (p.uploadsStarted ? ` uploaded ${p.uploadsDone}/${p.uploadsStarted}` : '');
261 live(line);
262 if (!tty && performance.now() - lastLog > LOG_EVERY_MS) {
263 lastLog = performance.now();
264 say(line);
265 }
266 // A rate this low means the run is on a software rasterizer, or on a
267 // GPU so busy it may as well be. Said once, not every chunk.
268 if (!slowNoted && p.rate > 0 && p.rate < 20 && p.steps > 500) {
269 slowNoted = true;
270 say(` NOTE: ${p.rate.toFixed(1)} steps/s is far below what a GPU does ` +
271 `(${adapter}).`);
272 }
273 },
274 onUploaded: () => uploads++,
275 onOutcome: (_target, _spec, outcome) => {
276 if (outcome.kind === 'done') {
277 computed++;
278 const times = [...outcome.uploaded].sort((a, b) => a - b).map(fmtChoice).join(', ');
279 say(` ${doneLine(outcome)} — ` +
280 (outcome.uploaded.length
281 ? `uploaded ${plural(outcome.uploaded.length, 'solution')} (t = ${times})`
282 : 'nothing uploaded'));
283 for (const err of outcome.uploadErrors) say(` upload failed: ${err}`);
284 } else if (outcome.kind === 'diverged') {
285 say(` diverged at t = ${outcome.t.toFixed(2)} — discarded, nothing uploaded`);
286 } else if (outcome.kind === 'stopped') {
287 say(` stopped at t = ${outcome.t.toFixed(2)}`);
288 }
289 if (computed >= opts.limit) stopping = true;
290 },
291 onFailure: (_target, spec, e) => {
292 const model = mModelByKey(spec.model);
293 say(` failed: ${formatFailure(e, model?.source ?? '')}`);
294 },
295 walkStopped: () => stopping,
296 stopRequested: () => stopping,
297 },
298 });
300 say('');
301 say(`stopped — computed ${counts.computed}, skipped ${counts.skipped} already cached` +
302 (counts.failed ? `, ${counts.failed} failed` : '') +
303 `; ${plural(uploads, 'file')} uploaded.`);
304 solver.destroy();
305 device.destroy();
306}
308// ---------------------------------------------------------------- main
309async function main(): Promise<void> {
b0546a9Fix the command line's file writing, and how it gets updatedJeremy Magland 310 // h5wasm's node build writes through to the real filesystem, so its scratch
311 // files need a real directory to live in (src/cache/h5file.ts).
312 setScratchDir(tmpdir());
c2317d0Fill the cache from the command line, without a browserJeremy Magland 313 const opts = parseArgs(process.argv.slice(2));
314 if (opts.command === 'help') {
315 say(HELP);
316 return;
317 }
318 if (opts.command === 'login') {
319 await login();
320 return;
321 }
322 const targets = autoOrder().filter((t) => !opts.model || t.model === opts.model);
323 say('turing-surface-cache — the shared cache of Turing patterns on curved surfaces');
324 if (opts.dryRun) {
325 await dryRun(opts, targets);
326 return;
327 }
328 const apiKey = await resolveKey(opts.key);
329 if (!apiKey) {
330 throw new Error(
331 'no upload key. The walk contributes solutions, so it needs one:\n' +
332 ` ${KEY_ENV}=… npx <this command>\n` +
333 `or save one for later runs with \`login\` (kept in ${keyPath()}).`,
334 );
335 }
336 await fill(opts, targets, apiKey);
337}
339main().catch((e: unknown) => {
340 say('');
341 say(errMsg(e));
342 process.exitCode = 1;
343});