Do not assume a linked library leaves stdout alone
SHTns writes to stdout unconditionally — the GPU it found, the FFT layout
it chose, which VkFFT it linked — none of it gated by shtns_verbose().
So shtbench_gpu's --json output does not start at byte 0, and
compare-native.mjs reported a working CUDA run as "did not print JSON".
Find the object instead: every producer prints `{` alone on a line and
`}` alone on the last.
Two other things that made that failure hard to read:
- a failed run reported the FIRST six lines of its output, which for the
WebGPU side is Dawn's two start-up warnings and nothing useful. Report
the tail instead, drop the known noise, and print the command to re-run
by hand.
- ratios were labelled "x webgpu" even when webgpu was the side that
failed and the baseline had silently become something else. Name the
baseline that was actually used, and say so when it is not webgpu.
1 changed file+66−12
scripts/compare-native.mjsmodified+66−12View file
@@ -96,6 +96,50 @@ const tmp = (tag) => join(tmpdir(), `turing-sphere-native-${tag}-${process.pid}.
9696 const cleanup = [];
9797
9898 // ------------------------------------------------------------------- runners
99+/**
100+ * Pull our JSON object out of stdout.
101+ *
102+ * A linked library shares the process's stdout, and SHTns writes to it
103+ * unconditionally — it announces the GPU it found, which FFT layout it chose and
104+ * which VkFFT it linked, none of it gated by shtns_verbose(). So the object may
105+ * not start at byte 0. Every producer here prints it with `{` alone on a line and
106+ * `}` alone on the last, which is enough to find it.
107+ */
108+function extractJson(out) {
109+ try {
110+ return JSON.parse(out);
111+ } catch {
112+ /* there is something else on stdout; find where the object starts */
113+ }
114+ const start = out.search(/^\{$/m);
115+ const ends = [...out.matchAll(/^\}$/gm)];
116+ if (start < 0 || !ends.length) return null;
117+ const last = ends[ends.length - 1];
118+ try {
119+ return JSON.parse(out.slice(start, last.index + 1));
120+ } catch {
121+ return null;
122+ }
123+}
124+
125+/** Dawn reports two of these on every start-up; they are not the failure. */
126+const NOISE = /^Warning: max(Dynamic|Compute|Storage)/;
127+
128+/** The tail of a failed run's output, which is where the actual error is. */
129+function failureDetail(r, cmd, args) {
130+ const lines = `${r.stderr ?? ''}\n${r.stdout ?? ''}`
131+ .split('\n')
132+ .map((s) => s.trimEnd())
133+ .filter((s) => s && !NOISE.test(s));
134+ const tail = lines.slice(-14).map((l) => ` ${l}`);
135+ return [
136+ `exit ${r.status}`,
137+ ...tail,
138+ ` re-run it alone to see everything:`,
139+ ` ${cmd} ${args.join(' ')}`,
140+ ].join('\n');
141+}
142+
99143 /** Run one side and parse its --json output. `ok: false` with a reason if it is
100144 * not available here — a missing binary, no adapter, no CUDA. */
101145 function run(label, cmd, args, statePath) {
@@ -108,15 +152,14 @@ function run(label, cmd, args, statePath) {
108152 maxBuffer: 256 * 1024 * 1024,
109153 });
110154 if (r.error) return { label, ok: false, why: r.error.message };
111- if (r.status !== 0) {
112- const detail = (r.stderr || r.stdout || '').trim().split('\n').slice(0, 6).join('\n ');
113- return { label, ok: false, why: detail || `exit ${r.status}` };
114- }
115- let json;
116- try {
117- json = JSON.parse(r.stdout);
118- } catch {
119- return { label, ok: false, why: `did not print JSON:\n ${r.stdout.slice(0, 300)}` };
155+ if (r.status !== 0) return { label, ok: false, why: failureDetail(r, cmd, full) };
156+ const json = extractJson(r.stdout);
157+ if (!json) {
158+ return {
159+ label,
160+ ok: false,
161+ why: `printed no JSON object:\n${failureDetail(r, cmd, full)}`,
162+ };
120163 }
121164 let state = null;
122165 if (statePath && existsSync(statePath)) {
@@ -245,8 +288,12 @@ if (gridProblems.length) {
245288 }
246289
247290 // -------------------------------------------------------------------- report
291+// Ratios are against the WGSL run, which is the point of the comparison. If that
292+// is the side that failed, fall back to whatever did run and say so, rather than
293+// printing "1.00x webgpu" for a run webgpu had no part in.
248294 const rate = (r) => r.json.throughput.msPerStep;
249-const base = rate(good.find((r) => r.label === 'webgpu') ?? good[0]);
295+const baseRun = good.find((r) => r.label === 'webgpu') ?? good[0];
296+const base = rate(baseRun);
250297
251298 if (wantJson) {
252299 console.log(
@@ -263,6 +310,7 @@ if (wantJson) {
263310 threads: Number(threads),
264311 },
265312 grid: gridOf(ref),
313+ baseline: baseRun.label,
266314 runs: results.map((r) =>
267315 r.ok
268316 ? {
@@ -270,7 +318,7 @@ if (wantJson) {
270318 msPerStep: rate(r),
271319 stepsPerSec: r.json.throughput.stepsPerSec,
272320 encodeMsPerStep: r.json.throughput.encodeMsPerStep,
273- ratioToWebgpu: rate(r) / base,
321+ ratioToBaseline: rate(r) / base,
274322 precision: r.json.backend.precision,
275323 adapter: r.json.backend.adapter,
276324 library: r.json.backend.library,
@@ -301,7 +349,7 @@ if (wantJson) {
301349 console.log(
302350 ` ${r.label.padEnd(11)} ${ms.toFixed(3)} ${unit} ` +
303351 `${r.json.throughput.stepsPerSec.toFixed(0)}/s ` +
304- `${r.label === 'webgpu' ? '(baseline)' : `${ratio.toFixed(2)}x webgpu`} ` +
352+ `${r === baseRun ? '(baseline)' : `${ratio.toFixed(2)}x ${baseRun.label}`} ` +
305353 `${r.json.backend.precision}`,
306354 );
307355 console.log(
@@ -310,6 +358,12 @@ if (wantJson) {
310358 ? ` · CPU-side launching ${r.json.throughput.encodeMsPerStep.toFixed(3)} ms/step`
311359 : ''),
312360 );
361+ if (r === baseRun && baseRun.label !== 'webgpu') {
362+ console.log(
363+ ` ${''.padEnd(11)} webgpu did not run, so this is the baseline instead — which is` +
364+ `\n ${''.padEnd(11)} not the comparison you wanted. Fix that side first.`,
365+ );
366+ }
313367 }
314368
315369 // Same caution compare-perf.mjs takes: a ratio between two different devices