Blame scripts/bench.ts at cde22ea37ca3b1ab2cd016bd80054ad1db1e8727

concept-collection / turing-sphere-2

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago1/**

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago2 * Command-line benchmark: run exactly what the browser runs — the same .m

3 * models, lowered by numbl and compiled to the same WGSL kernels, over the same

4 * transforms — on desktop WebGPU (Google Dawn, via the optional `webgpu`

5 * package), and report ms/step. The app prints the matching command under its

6 * stats line; copy it and run it here for an apples-to-apples comparison.

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago7 *

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago8 * npm run bench -- --preset schnak-spots --lmax 63 --steps 2000 --seed 1 \

9 * --a 0.1 --b 0.9 --D1 0.0004 --D2 0.008 --dt 0.05

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago10 *

11 * The only thing missing here is the rendering: this is the solver alone.

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago12 *

13 * Two numbers are reported, because they answer different questions:

14 * - throughput: a batch of steps submitted together, awaited once. This is how

15 * the app runs, and what keeping the state in GPU buffers is for.

16 * - latency: one step per submit, each awaited. Comparable to a design that

17 * reads back every step, and the only way to get a per-step distribution.

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago18 */

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago19import { requestShtDevice, describeAdapter } from '../src/sht/sht.ts';

20import { ModelSession } from '../src/mgpu/session.ts';

21import { presets } from '../src/mgpu/registry.ts';

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago22import {

23 parseArgs,

24 modelForSpec,

25 resolvePreset,

26 formatCommand,

27 BENCH_COMMAND,

28 DEFAULT_LMAX,

29 DEFAULT_SEED,

30 DEFAULT_STEPS,

31 DEFAULT_WARMUP,

32 type RunSpec,

33} from '../src/bench/runSpec.ts';

0f3abbd Separate solver time from frame time, and add a cross-environment checkJeremy Magland 20 days ago34import { digestOf, formatDigest } from '../src/mgpu/digest.ts';

61e12f1 Write the solver in MATLAB and compile it to WebGPUJeremy Magland 20 days ago35import { installWebGpu, errMsg, NO_ADAPTER_HINT } from './nodeWebGpu.ts';

0f3abbd Separate solver time from frame time, and add a cross-environment checkJeremy Magland 20 days ago36import { writeFileSync } from 'node:fs';

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago37

38const USAGE = `usage: ${BENCH_COMMAND} [options]

40 --preset <key> ${presets.map((p) => p.key).join(' | ')}

41 (default ${presets[0].key})

42 --lmax <n> spherical harmonic truncation (default ${DEFAULT_LMAX})

43 --steps <n> timed steps (default ${DEFAULT_STEPS})

44 --warmup <n> untimed steps first (default ${DEFAULT_WARMUP})

45 --seed <n> initial-noise seed (default ${DEFAULT_SEED})

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago46 --batch <n> steps per submit for the throughput number (default 16)

0f3abbd Separate solver time from frame time, and add a cross-environment checkJeremy Magland 20 days ago47 --digest after timing, re-run exactly --steps steps from the seed and

48 print a digest of the final state

49 --dump-state <f> like --digest, and write the state to <f> as JSON, for

50 scripts/compare-env.mjs to compare against a browser run

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago51 --<param> <v> any parameter of the preset's model, e.g. --dt 0.05

52 --json machine-readable output

53 --help

55The browser app shows the command for whatever it is currently simulating;

56copy it from under the stats line to compare the same run here.`;

58function fail(msg: string, code = 1): never {

59 console.error(`bench: ${msg}`);

60 process.exit(code);

61}

63// ---------------------------------------------------------------- arguments

64const argv = process.argv.slice(2);

65if (argv.includes('--help') || argv.includes('-h')) {

66 console.log(USAGE);

67 process.exit(0);

68}

69const wantJson = argv.includes('--json');

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago70let batch = 16;

0f3abbd Separate solver time from frame time, and add a cross-environment checkJeremy Magland 20 days ago71let dumpState: string | null = null;

72let wantDigest = false;

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago73const rest: string[] = [];

74for (let i = 0; i < argv.length; i++) {

0f3abbd Separate solver time from frame time, and add a cross-environment checkJeremy Magland 20 days ago75 const a = argv[i];

76 if (a === '--json') continue;

77 if (a === '--digest') {

78 wantDigest = true;

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago79 continue;

80 }

0f3abbd Separate solver time from frame time, and add a cross-environment checkJeremy Magland 20 days ago81 const valued = (name: string): string | null => {

82 if (a === `--${name}`) return argv[++i];

83 if (a.startsWith(`--${name}=`)) return a.slice(name.length + 3);

84 return null;

85 };

86 const b = valued('batch');

87 if (b !== null) {

88 batch = Number(b);

89 continue;

90 }

91 const d = valued('dump-state');

92 if (d !== null) {

93 dumpState = d;

94 wantDigest = true;

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago95 continue;

96 }

0f3abbd Separate solver time from frame time, and add a cross-environment checkJeremy Magland 20 days ago97 rest.push(a);

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago98}

99if (!Number.isInteger(batch) || batch < 1) fail(`--batch must be an integer >= 1`, 2);

100

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago101let spec: RunSpec;

102try {

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago103 spec = parseArgs(rest);

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago104} catch (e) {

105 fail(`${errMsg(e)}\n\n${USAGE}`, 2);

106}

107

108// ---------------------------------------------------------------- statistics

109interface Timing {

110 meanMs: number;

111 medianMs: number;

112 p05Ms: number;

113 p95Ms: number;

114 minMs: number;

115 totalMs: number;

116 stepsPerSec: number;

117}

118

119function timing(samples: Float64Array): Timing {

120 const sorted = Float64Array.from(samples).sort();

121 const q = (p: number): number =>

122 sorted[Math.min(sorted.length - 1, Math.floor(p * sorted.length))];

123 let total = 0;

124 for (const v of samples) total += v;

125 const mean = total / samples.length;

126 return {

127 meanMs: mean,

128 medianMs: q(0.5),

129 p05Ms: q(0.05),

130 p95Ms: q(0.95),

131 minMs: sorted[0],

132 totalMs: total,

133 stepsPerSec: 1000 / mean,

134 };

135}

136

137function fieldRange(v: ArrayLike<number>): { min: number; max: number } {

138 let min = Infinity;

139 let max = -Infinity;

140 for (let i = 0; i < v.length; i++) {

141 if (v[i] < min) min = v[i];

142 if (v[i] > max) max = v[i];

143 }

144 return { min, max };

145}

146

147// ---------------------------------------------------------------- run

148const model = modelForSpec(spec);

149const { preset } = resolvePreset(spec.preset);

150

151let device: GPUDevice | null = null;

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago152let session: ModelSession | null = null;

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago153

154try {

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago155 const runtime = await installWebGpu();

156 device = await requestShtDevice().catch((e: unknown) => {

157 throw new Error(`${errMsg(e)}\n${NO_ADAPTER_HINT}`);

158 });

159 const adapter = await describeAdapter(device);

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago160

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago161 session = await ModelSession.create({

162 device,

163 model,

164 params: spec.params,

165 lmax: spec.lmax,

166 });

167 session.seed(spec.seed);

168

169 const plan = session.describe();

170 const kernels = plan.step.filter((l) => l.startsWith('kernel')).length;

171 const cfg = session.cfg;

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago172

173 if (!wantJson) {

174 console.log(`turing-sphere bench — solver only, no rendering\n`);

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago175 console.log(` preset ${preset.label} (models/${model.key}.m: ${model.species.join(', ')})`);

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago176 console.log(

177 ` params ${model.params.map((p) => `${p.key}=${spec.params[p.key]}`).join(' ')}`,

178 );

179 console.log(

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago180 ` grid lmax ${cfg.lmax} · ${cfg.nlat}×${cfg.nphi} · nlm ${session.sht.nlm.toLocaleString()}`,

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago181 );

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago182 console.log(` compiled ${plan.step.length} GPU ops/step (${kernels} generated kernels)`);

0f3abbd Separate solver time from frame time, and add a cross-environment checkJeremy Magland 20 days ago183 console.log(` fourier ${session.sht.fourierMode.toUpperCase()} stage`);

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago184 console.log(` backend WebGPU fp32${adapter ? ` — ${adapter}` : ''}\n ${runtime}`);

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago185 console.log(` run ${spec.warmup} warmup + ${spec.steps} timed steps, seed ${spec.seed}\n`);

186 }

187

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago188 const done = (): Promise<undefined> => device!.queue.onSubmittedWorkDone();

189

190 session.step(spec.warmup);

191 await done();

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago192

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago193 // --- throughput: batches submitted together, awaited once each ---

194 const batches = Math.max(1, Math.ceil(spec.steps / batch));

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago195 const progress = !wantJson && process.stderr.isTTY;

196 let lastReport = performance.now();

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago197 const tp0 = performance.now();

198 let stepsRun = 0;

199 for (let b = 0; b < batches; b++) {

200 const n = Math.min(batch, spec.steps - stepsRun);

201 session.step(n);

202 await done();

203 stepsRun += n;

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago204 if (progress && performance.now() - lastReport > 1000) {

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago205 const so_far = (performance.now() - tp0) / stepsRun;

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago206 process.stderr.write(

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago207 `\r\x1b[K ${stepsRun}/${spec.steps} steps · ${so_far.toFixed(2)} ms/step`,

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago208 );

209 lastReport = performance.now();

210 }

211 }

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago212 const throughputMs = (performance.now() - tp0) / stepsRun;

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago213 if (progress) process.stderr.write('\r\x1b[K');

214

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago215 // --- latency: one step per submit, for the distribution ---

216 const latencySteps = Math.min(spec.steps, 200);

217 const samples = new Float64Array(latencySteps);

218 for (let s = 0; s < latencySteps; s++) {

219 const t0 = performance.now();

220 session.step(1);

221 await done();

222 samples[s] = performance.now() - t0;

223 }

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago224 const t = timing(samples);

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago225

226 const field = await session.read(model.species[0]);

227 const range = fieldRange(field);

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago228 let finite = true;

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago229 for (const v of field) if (!Number.isFinite(v)) finite = false;

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago230

0f3abbd Separate solver time from frame time, and add a cross-environment checkJeremy Magland 20 days ago231 // A reproducible state to compare across machines: exactly `--steps` steps

232 // from the seed, separate from the timed runs above (which step a different

233 // number of times to measure throughput and latency).

234 let digest = null;

235 let state: Float32Array | null = null;

236 if (wantDigest) {

237 session.seed(spec.seed);

238 session.step(spec.steps);

239 await done();

240 state = await session.read(model.state[0]);

241 digest = digestOf(state, session.sht.fourierMode, adapter);

242 }

243

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago244 if (wantJson) {

245 console.log(

246 JSON.stringify(

247 {

248 command: formatCommand(spec),

249 spec,

250 model: model.key,

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago251 backend: { adapter, runtime },

252 grid: { lmax: cfg.lmax, nlat: cfg.nlat, nphi: cfg.nphi, nlm: session.sht.nlm },

253 compiled: { opsPerStep: plan.step.length, kernels },

0f3abbd Separate solver time from frame time, and add a cross-environment checkJeremy Magland 20 days ago254 digest,

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago255 throughput: { batch, msPerStep: throughputMs, stepsPerSec: 1000 / throughputMs },

256 latency: t,

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago257 state: {

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago258 t: session.t,

259 steps: session.steps,

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago260 species: model.species[0],

261 min: range.min,

262 max: range.max,

263 contrast: range.max - range.min,

264 finite,

265 },

266 },

267 null,

268 2,

269 ),

270 );

271 } else {

272 console.log(

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago273 ` ${throughputMs.toFixed(2)} ms/step ${(1000 / throughputMs).toFixed(1)} steps/s ` +

274 `${(spec.params.dt * (1000 / throughputMs)).toFixed(2)} model time/s` +

275 ` (batches of ${batch})`,

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago276 );

277 console.log(

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago278 ` one step per submit: ${t.meanMs.toFixed(2)} ms mean · median ${t.medianMs.toFixed(2)} · ` +

279 `p05 ${t.p05Ms.toFixed(2)} · p95 ${t.p95Ms.toFixed(2)} · min ${t.minMs.toFixed(2)}`,

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago280 );

281 console.log(

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago282 ` after ${session.steps} steps: t = ${session.t.toFixed(2)}, ` +

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago283 `${model.species[0]} ∈ [${range.min.toFixed(4)}, ${range.max.toFixed(4)}] ` +

284 `(contrast ${(range.max - range.min).toFixed(4)})${finite ? '' : ' — NOT FINITE'}`,

285 );

0f3abbd Separate solver time from frame time, and add a cross-environment checkJeremy Magland 20 days ago286 if (digest) {

287 console.log(`\n state after ${spec.steps} steps from seed ${spec.seed}:`);

288 console.log(` ${formatDigest(digest)}`);

289 }

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago290 console.log(

0f3abbd Separate solver time from frame time, and add a cross-environment checkJeremy Magland 20 days ago291 `\n The app's stats line reports the same solver number (batched steps,\n` +

292 ` nothing read back) plus a separate ms/frame that carries the readback\n` +

293 ` and the rendering. Compare solver with solver.`,

294 );

295 }

296

297 if (dumpState && state && digest) {

298 writeFileSync(

299 dumpState,

300 JSON.stringify({ command: formatCommand(spec), spec, digest, state: [...state] }),

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago301 );

0f3abbd Separate solver time from frame time, and add a cross-environment checkJeremy Magland 20 days ago302 if (!wantJson) console.log(`\n wrote ${dumpState}`);

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago303 }

304

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago305 session.destroy();

306 device.destroy();

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago307 process.exit(finite ? 0 : 1);

308} catch (e) {

35d91fa Delete the TypeScript solver; the .m models are the only implementationJeremy Magland 20 days ago309 session?.destroy();

15a77e2 Add a desktop WebGPU benchmark and show its command in the appJeremy Magland 21 days ago310 device?.destroy();

311 fail(errMsg(e));

312}