2 * Filling the shared cache from the command line, so that a machine with a GPU
3 * and nothing to do can contribute without a browser window open on it.
4 *
5 * The walk, the runs and the uploads are the page's (src/cache/fillWalk.ts,
6 * src/cache/runSpec.ts); what is here is the shell around them — Dawn instead
7 * of a browser's WebGPU, a key from the environment instead of localStorage,
8 * and lines of text instead of a status bar.
9 */
10import { requestShtDevice, describeAdapter } from '../sht/sht.ts';
11import { mModelByKey } from '../mgpu/registry.ts';
12import { formatFailure } from '../mgpu/errors.ts';
13import {
14 fmtChoice,
15 GEOMETRY_CHOICES,
16 MODEL_CHOICES,
17 T_END_CHOICE,
18 type DiscreteChoice,
19} from '../cache/options.ts';
20import { autoOrder, specForTarget, type AutoTarget } from '../cache/autoWalk.ts';
21import { headCached, lookupFor, verifyApiKey } from '../cache/client.ts';
22import { SolverSession } from '../cache/solver.ts';
23import { fillWalk } from '../cache/fillWalk.ts';
24import { stepsFor } from '../cache/spec.ts';
25import type { RunSummary } from '../cache/runSpec.ts';
26import { installWebGpu, errMsg, isSoftwareAdapter, NO_ADAPTER_HINT } from './webgpu.ts';
27import { KEY_ENV, keyPath, maskKey, promptSecret, resolveKey, saveKey } from './key.ts';
29const HELP = `turing-surface-cache — fill the shared cache of Turing patterns
31Usage
32 fill [options] work through the parameter space, contributing what is
33 missing, until stopped (ctrl-C)
34 fill --dry-run [N] show the first N targets and whether they are cached
35 login save an upload key for later runs
36 --help
38Options
39 --key <key> upload key; otherwise $${KEY_ENV}, otherwise the saved key
40 --limit <n> stop after n solutions have been computed
41 --model <key> only targets of one model (schnakenberg, brusselator,
42 allencahn)
43 --tend <list> replace the end-time list, e.g. --tend 5,10 (for testing:
44 a short run hashes to its own honest cache entry)
46An upload key is required: the walk exists to contribute. Solutions are read
47by everyone and written only by key holders.`;
49interface Options {
50 command: 'fill' | 'login' | 'help';
51 key?: string;
52 limit: number;
53 model?: string;
54 dryRun: number;
55}
57function parseArgs(argv: string[]): Options {
58 const opts: Options = { command: 'fill', limit: Infinity, dryRun: 0 };
59 const rest = [...argv];
60 if (rest[0] === 'fill' || rest[0] === 'login') opts.command = rest.shift() as 'fill' | 'login';
61 /** A count option whose value may be left off (--dry-run, --dry-run 40). */
62 const count = (fallback: number): number => {
63 const next = rest[0];
64 if (next && /^\d+$/.test(next)) return Number(rest.shift());
65 return fallback;
66 };
67 while (rest.length) {
68 const arg = rest.shift()!;
69 if (arg === '--help' || arg === '-h') opts.command = 'help';
70 else if (arg === '--key') opts.key = rest.shift();
71 else if (arg === '--limit') opts.limit = Number(rest.shift());
72 else if (arg === '--model') opts.model = rest.shift();
73 else if (arg === '--dry-run') opts.dryRun = count(20);
74 else if (arg === '--tend') setEndTimes(rest.shift());
75 else throw new Error(`unknown option ${arg}`);
76 }
77 if (opts.model && !mModelByKey(opts.model)) throw new Error(`unknown model ${opts.model}`);
78 if (!(opts.limit > 0)) throw new Error('--limit wants a positive number');
79 return opts;
80}
82/** The page's ?tend hook, spelled as an option (src/main.ts). */
83function setEndTimes(list: string | undefined): void {
84 const values = (list ?? '')
85 .split(',')
86 .map(Number)
87 .filter((v) => Number.isFinite(v) && v > 0);
88 if (!values.length) throw new Error('--tend wants a comma-separated list of end times');
89 T_END_CHOICE.values = values;
90 T_END_CHOICE.value = values[0];
91}
93// ---------------------------------------------------------------- output
94const tty = process.stdout.isTTY === true;
95/** Written in place on a terminal, and only every so often when piped, so a
96 * log file does not fill with progress. */
97const LOG_EVERY_MS = 30_000;
98let liveLine = false;
100function say(line = ''): void {
101 if (liveLine) {
102 process.stdout.write('\n');
103 liveLine = false;
104 }
105 process.stdout.write(`${line}\n`);
106}
108/** One line that keeps being rewritten while a run advances. */
109function live(line: string): void {
110 if (!tty) return;
111 process.stdout.write(`\r${line.padEnd(78).slice(0, 78)}`);
112 liveLine = true;
113}
115const plural = (n: number, word: string): string => `${n} ${word}${n === 1 ? '' : 's'}`;
117function duration(seconds: number): string {
118 if (!Number.isFinite(seconds)) return '?';
119 if (seconds < 90) return `${seconds.toFixed(0)}s`;
120 const m = Math.floor(seconds / 60);
121 return m < 90 ? `${m}m${String(Math.round(seconds - 60 * m)).padStart(2, '0')}s` : `${(m / 60).toFixed(1)}h`;
122}
124/** Parameters in the order the app lists them, not the order they were built. */
125const paramList = (params: Record<string, number>, choices: DiscreteChoice[]): string =>
126 choices
127 .filter((c) => c.key in params)
128 .map((c) => `${c.key}=${fmtChoice(params[c.key])}`)
129 .join(' ');
131/** What a target is, in one line. */
132function describe(target: AutoTarget): string {
133 const geomChoices = GEOMETRY_CHOICES[target.geometry] ?? [];
134 const geom = geomChoices.length
135 ? `${target.geometry} ${paramList(target.geometryParams, geomChoices)}`
136 : target.geometry;
137 return (
138 `${target.model} ${paramList(target.params, MODEL_CHOICES[target.model])} · ${geom} · ` +
139 `${plural(target.distance, 'knob')} from the defaults`
140 );
141}
143const doneLine = (run: RunSummary): string =>
144 `computed in ${duration(run.seconds)}` +
145 (run.warmFrom !== null ? ` (resumed from cached t = ${fmtChoice(run.warmFrom)})` : '');
147// ---------------------------------------------------------------- commands
148async function login(): Promise<void> {
149 const key = await promptSecret('upload API key: ');
150 if (!key) throw new Error('nothing entered');
151 let ok: boolean;
152 try {
153 ok = await verifyApiKey(key);
154 } catch (e) {
155 say(`could not reach the upload service to check the key (${errMsg(e)}) — saving anyway.`);
156 ok = true;
157 }
158 if (!ok) throw new Error('that key is not allowed to upload — nothing saved');
159 say(`key saved to ${await saveKey(key)}`);
160}
162async function dryRun(opts: Options, targets: AutoTarget[]): Promise<void> {
163 const shown = targets.slice(0, opts.dryRun);
164 say(`the first ${plural(shown.length, 'target')} of ${targets.length.toLocaleString()}, ` +
165 `nearest the defaults first:`);
166 say();
167 let cached = 0;
168 // A handful at a time: a HEAD apiece, and the answers are wanted in order.
169 const width = String(shown.length).length;
170 for (let i = 0; i < shown.length; i += 8) {
171 const batch = shown.slice(i, i + 8);
172 const present = await Promise.all(
173 batch.map(async (t) => (await headCached(await lookupFor(specForTarget(t)))) === true),
174 );
175 present.forEach((isThere, k) => {
176 if (isThere) cached++;
177 say(
178 ` [${String(i + k + 1).padStart(width)}] ${isThere ? 'cached ' : 'missing'} ` +
179 describe(batch[k]),
180 );
181 });
182 }
183 say();
184 say(`${cached} of ${shown.length} already cached; the walk would compute the other ` +
185 `${shown.length - cached}.`);
186}
188async function fill(opts: Options, targets: AutoTarget[], apiKey: string): Promise<void> {
189 const runtime = await installWebGpu();
190 const device = await requestShtDevice().catch((e: unknown) => {
191 throw new Error(`${errMsg(e)}\n${NO_ADAPTER_HINT}`);
192 });
193 const adapter = await describeAdapter(device);
194 say(`${runtime} · ${adapter}`);
195 say(`uploads enabled (key ${maskKey(apiKey)})`);
196 if (isSoftwareAdapter(adapter)) {
197 say('');
198 say(`WARNING: ${adapter} is a software rasterizer, not a GPU. Runs here are`);
199 say(' perhaps a thousand times slower than on hardware — fast enough to look');
200 say(' like it is working, slow enough to be worth nothing. Check that the');
201 say(' machine has a GPU and its driver, or stop now.');
202 }
203 say('');
204 say(`${targets.length.toLocaleString()} targets, nearest the defaults first; ` +
205 `ctrl-C stops after the current run.`);
206 say('');
208 const solver = new SolverSession(device, 1, {
209 onCompiling: (m) => say(` compiling ${m.label}…`),
210 });
212 let stopping = false;
213 process.on('SIGINT', () => {
214 if (stopping) process.exit(130);
215 stopping = true;
216 say('');
217 say('stopping after this run — ctrl-C again to give up on it.');
218 });
220 let index = 0;
221 let computed = 0;
222 let uploads = 0;
223 let slowNoted = false;
224 let lastLog = 0;
225 const counts = await fillWalk({
226 targets,
227 solver,
228 adapter,
229 apiKey: () => apiKey,
230 beforeTarget: (target) => {
231 index++;
232 return specForTarget(target);
233 },
234 events: {
235 onTarget: (target) => say(`[${index}] ${describe(target)}`),
236 onCached: () => say(' already cached'),
237 onComputing: (_target, spec) =>
238 say(` computing to t = ${fmtChoice(spec.tEnd)} ` +
239 `(${stepsFor(spec).toLocaleString()} steps)`),
240 onPhase: (phase) => {
241 if (phase.kind === 'warm-search') say(' looking for a shorter cached run…');
242 else if (phase.kind === 'seeding') say(' seeding…');
243 else if (phase.kind === 'uploading') {
244 say(` ${doneLine(phase.run)} — uploading ${plural(phase.started, 'file')}…`);
245 }
246 },
247 onProgress: (p) => {
248 const eta = p.rate > 0 ? (p.totalSteps - p.steps) / p.rate : Infinity;
249 const line =
250 ` t = ${p.t.toFixed(1)} / ${fmtChoice(p.tEnd)} ` +
251 `${(100 * p.fraction).toFixed(0)}% ${p.rate.toFixed(0)} steps/s ` +
252 `eta ${duration(eta)}` +
253 (p.uploadsStarted ? ` uploaded ${p.uploadsDone}/${p.uploadsStarted}` : '');
254 live(line);
255 if (!tty && performance.now() - lastLog > LOG_EVERY_MS) {
256 lastLog = performance.now();
257 say(line);
258 }
259 // A rate this low means the run is on a software rasterizer, or on a
260 // GPU so busy it may as well be. Said once, not every chunk.
261 if (!slowNoted && p.rate > 0 && p.rate < 20 && p.steps > 500) {
262 slowNoted = true;
263 say(` NOTE: ${p.rate.toFixed(1)} steps/s is far below what a GPU does ` +
264 `(${adapter}).`);
265 }
266 },
267 onUploaded: () => uploads++,
268 onOutcome: (_target, _spec, outcome) => {
269 if (outcome.kind === 'done') {
270 computed++;
271 const times = [...outcome.uploaded].sort((a, b) => a - b).map(fmtChoice).join(', ');
272 say(` ${doneLine(outcome)} — ` +
273 (outcome.uploaded.length
274 ? `uploaded ${plural(outcome.uploaded.length, 'solution')} (t = ${times})`
275 : 'nothing uploaded'));
276 for (const err of outcome.uploadErrors) say(` upload failed: ${err}`);
277 } else if (outcome.kind === 'diverged') {
278 say(` diverged at t = ${outcome.t.toFixed(2)} — discarded, nothing uploaded`);
279 } else if (outcome.kind === 'stopped') {
280 say(` stopped at t = ${outcome.t.toFixed(2)}`);
281 }
282 if (computed >= opts.limit) stopping = true;
283 },
284 onFailure: (_target, spec, e) => {
285 const model = mModelByKey(spec.model);
286 say(` failed: ${formatFailure(e, model?.source ?? '')}`);
287 },
288 walkStopped: () => stopping,
289 stopRequested: () => stopping,
290 },
291 });
293 say('');
294 say(`stopped — computed ${counts.computed}, skipped ${counts.skipped} already cached` +
295 (counts.failed ? `, ${counts.failed} failed` : '') +
296 `; ${plural(uploads, 'file')} uploaded.`);
297 solver.destroy();
298 device.destroy();
299}
301// ---------------------------------------------------------------- main
302async function main(): Promise<void> {
303 const opts = parseArgs(process.argv.slice(2));
304 if (opts.command === 'help') {
305 say(HELP);
306 return;
307 }
308 if (opts.command === 'login') {
309 await login();
310 return;
311 }
312 const targets = autoOrder().filter((t) => !opts.model || t.model === opts.model);
313 say('turing-surface-cache — the shared cache of Turing patterns on curved surfaces');
314 if (opts.dryRun) {
315 await dryRun(opts, targets);
316 return;
317 }
318 const apiKey = await resolveKey(opts.key);
319 if (!apiKey) {
320 throw new Error(
321 'no upload key. The walk contributes solutions, so it needs one:\n' +
322 ` ${KEY_ENV}=… npx <this command>\n` +
323 `or save one for later runs with \`login\` (kept in ${keyPath()}).`,
324 );
325 }
326 await fill(opts, targets, apiKey);
327}
329main().catch((e: unknown) => {
330 say('');
331 say(errMsg(e));
332 process.exitCode = 1;
333});