Connect work-precision curves by resolution, not time; strengthen the timing protocol
8 changed files+69−37
docs/problems/laplace-dirichlet-2d.mdmodified+22−8View file
@@ -70,14 +70,28 @@ test it.
7070 ## Timing protocol
7171
7272 One run is a full call of the solver, including its own discretization,
73-assembly, solve, and evaluation. The harness performs one untimed warmup
74-run (which absorbs numbl's JIT compilation and is recorded separately as
75-the cold time), then N timed runs (N = 3 unless stated), and reports the
76-median as the solve time. All timing is tic/toc inside the MATLAB
77-session, so browser and node runs measure the same thing. Times from
78-different machines are not comparable; every result records its
79-environment, and comparisons across environments are the reader's
80-responsibility.
73+assembly, solve, and evaluation. The harness performs two untimed warmup
74+runs (the first of which is recorded separately as the cold time), then N
75+timed runs (N = 5 unless stated), and reports the **fastest** of them as
76+the solve time. The minimum is used rather than the mean or median
77+because everything that interferes with a measurement (scheduling, other
78+load, residual JIT compilation) only ever adds time, so the fastest run
79+is the least contaminated estimate of the solver's own cost; every
80+individual timing is recorded in the result file regardless. Where a
81+sweep shares one process across resolutions, as the MATLAB runner does,
82+warmup runs precede the sweep as well, so that the first resolution does
83+not absorb the session's one-time costs. All timing is tic/toc inside the
84+solver's own runtime, so browser, node and MATLAB runs measure the same
85+thing. Times from different machines are not comparable; every result
86+records its environment, and comparisons across environments are the
87+reader's responsibility.
88+
89+Note that solve time need not increase with the resolution parameter n. A
90+solver whose cost is dominated by a per-target near-field computation can
91+get *slower* as n decreases, since coarser panels put more targets in the
92+near field. Work-precision curves are therefore drawn as parametric
93+curves in n, and may double back in time; a point that is both slower and
94+less accurate than another point on the same curve is simply dominated.
8195
8296 ## Solver interface
8397
src/app/components/WorkPrecisionChart.tsxmodified+10−2View file
@@ -205,14 +205,22 @@ export function WorkPrecisionChart({ curves }: { curves: ChartCurve[] }) {
205205 </text>
206206 {/* curves */}
207207 {nonEmpty.map((c) => {
208- const pts = [...c.points].sort((a, b) => a.solveSeconds - b.solveSeconds);
208+ // Connected in order of the solver's own resolution parameter,
209+ // which is what traces the curve. Time need not increase with n
210+ // (a solver can get both slower and less accurate as n falls),
211+ // so the curve may double back; ordering by time instead would
212+ // produce a meaningless zigzag.
213+ const pts = [...c.points].sort((a, b) => a.n - b.n);
209214 const path = pts
210215 .map(
211216 (p, i) =>
212217 `${i === 0 ? "M" : "L"}${sx(p.solveSeconds).toFixed(1)},${sy(Math.max(p.relMax, 1e-17)).toFixed(1)}`
213218 )
214219 .join("");
215- const last = pts[pts.length - 1];
220+ // Label at the rightmost point, which need not be the last one.
221+ const last = pts.reduce((a, b) =>
222+ b.solveSeconds > a.solveSeconds ? b : a
223+ );
216224 return (
217225 <g key={c.key}>
218226 <path
src/app/pages/AboutPage.tsxmodified+7−3View file
@@ -31,9 +31,13 @@ export function AboutPage() {
3131 error against compute time, traced out as the solver's resolution
3232 parameter varies. Errors are measured at a fixed set of evaluation
3333 points defined per instance, relative to the reference solution.
34- Timing is one untimed warmup run (which absorbs JIT compilation),
35- then the median of repeated timed runs; a run includes the solver's
36- own discretization, assembly, solve, and evaluation.
34+ Timing is two untimed warmup runs followed by several timed runs, of
35+ which the fastest is reported: interference only ever adds time, so
36+ the fastest run is the least contaminated estimate of the solver's
37+ own cost. A run includes the solver's own discretization, assembly,
38+ solve, and evaluation. Curves are traced in order of the solver's
39+ resolution parameter, and may double back in time, since a solver's
40+ cost need not increase with resolution.
3741 </p>
3842
3943 <h2>Results and provenance</h2>
src/cli/main.tsmodified+2−2View file
@@ -119,7 +119,7 @@ function listCommand() {
119119 }
120120
121121 async function runCommand(flags: Record<string, string>) {
122- const repeats = flags.repeats ? parseInt(flags.repeats, 10) : 3;
122+ const repeats = flags.repeats ? parseInt(flags.repeats, 10) : 5;
123123 const maxN = flags["max-n"] ? parseInt(flags["max-n"], 10) : undefined;
124124 const outDir = resolve(flags.out ?? "fastandaccurate-results-out");
125125 const instances = flags.instance
@@ -259,7 +259,7 @@ async function main() {
259259 " --solver-file <f.m> A custom solver file (requires --solver-id)",
260260 " --solver-id <name> Identifier for the custom solver",
261261 " --solver-version <v> Version string for the custom solver",
262- " --repeats <N> Timed repeats per point (default 3)",
262+ " --repeats <N> Timed repeats per point (default 5)",
263263 " --max-n <N> Restrict the sweep to n <= N",
264264 " --label <text> Free-text machine label recorded in results",
265265 " --out <dir> Output directory (default fastandaccurate-results-out)",
src/cli/matlabRun.tsmodified+9−7View file
@@ -88,15 +88,22 @@ export function runMatlabSweep(opts: MatlabSweepOptions): MatlabSweepResult {
8888 `nrep = ${repeats};`,
8989 `prob = build_problem(${instance.a}, ${instance.k}, ${instance.d}, 0);`,
9090 "results = cell(numel(ns), 1);",
91+ "% Session-level warmup: the whole sweep shares one MATLAB process, so",
92+ "% without this the first resolution absorbs all of the one-time cost",
93+ "% (loading the solver's dependencies, quadrature tables, JIT).",
94+ "for w = 1:2",
95+ " solver(prob, ns(max(1, floor(numel(ns)/2))));",
96+ "end",
9197 "for i = 1:numel(ns)",
9298 " n = ns(i);",
9399 " tic; out = solver(prob, n); cold = toc;",
100+ " out = solver(prob, n);",
94101 " times = zeros(nrep, 1);",
95102 " for r = 1:nrep",
96103 " tic; out = solver(prob, n); times(r) = toc;",
97104 " end",
98105 " results{i} = struct('n', n, 'cold', cold, 'times', times, 'ueval', out.uEval);",
99- " fprintf('point n=%d done (%.3fs)\\n', n, median(times));",
106+ " fprintf('point n=%d done (%.3fs)\\n', n, min(times));",
100107 "end",
101108 "payload = struct('matlabVersion', version, 'results', {results});",
102109 "fid = fopen('out_results.json', 'w');",
@@ -127,15 +134,10 @@ export function runMatlabSweep(opts: MatlabSweepOptions): MatlabSweepResult {
127134 : [payload.results];
128135 const points = entries.map((e) => {
129136 const times = asArray(e.times);
130- const sorted = [...times].sort((x, y) => x - y);
131- const median =
132- sorted.length % 2 === 1
133- ? sorted[(sorted.length - 1) / 2]
134- : (sorted[sorted.length / 2 - 1] + sorted[sorted.length / 2]) / 2;
135137 const { relMax, relL2 } = evalErrors(instance, e.ueval);
136138 return {
137139 n: e.n,
138- solveSeconds: median,
140+ solveSeconds: Math.min(...times),
139141 solveSecondsAll: times,
140142 coldSeconds: e.cold,
141143 relMax,
src/harness/resultSchema.tsmodified+5−1View file
@@ -30,6 +30,7 @@ export interface ResultEnvironment {
3030
3131 export interface ResultPoint {
3232 n: number;
33+ /** The fastest of the timed runs. */
3334 solveSeconds: number;
3435 solveSecondsAll: number[];
3536 coldSeconds: number;
@@ -57,6 +58,8 @@ export interface ResultFile {
5758 protocol: {
5859 warmupRuns: number;
5960 timedRuns: number;
61+ /** Which statistic of the timed runs is reported as solveSeconds. */
62+ statistic: "min";
6063 timer: string;
6164 };
6265 createdUtc: string;
@@ -105,8 +108,9 @@ export async function buildResultFile(opts: {
105108 solver: opts.solver,
106109 environment: opts.environment,
107110 protocol: {
108- warmupRuns: 1,
111+ warmupRuns: 2,
109112 timedRuns: opts.repeats,
113+ statistic: "min",
110114 timer: opts.timer ?? "numbl tic/toc",
111115 },
112116 createdUtc: new Date().toISOString(),
src/harness/runner.tsmodified+13−13View file
@@ -1,9 +1,12 @@
11 // Runs one solver at one resolution on one instance and measures it.
2-// The timing protocol (see docs/problems/laplace-dirichlet-2d.md): one
3-// untimed warmup run absorbs JIT compilation, then `repeats` timed runs
4-// whose median is the reported solve time. The warmup itself is timed
5-// and reported as the cold time. All timing is MATLAB tic/toc inside the
6-// numbl session, so browser and node measure the same thing.
2+// The timing protocol (see docs/problems/laplace-dirichlet-2d.md): two
3+// untimed warmup runs absorb JIT compilation, then `repeats` timed runs
4+// of which the fastest is reported. The minimum is the least contaminated
5+// estimator of the solver's own cost, since everything that interferes
6+// (scheduling, residual compilation, other load) only ever adds time; all
7+// the individual timings are recorded too. The first warmup is timed and
8+// reported separately as the cold time. All timing is MATLAB tic/toc
9+// inside the numbl session, so browser and node measure the same thing.
710
811 import { runNumblScript } from "./numblRun";
912 import type { Laplace2dInstance } from "../problems/laplace2d/spec";
@@ -21,7 +24,7 @@ export interface MatlabSources {
2124 export interface RunPointRequest {
2225 instance: Laplace2dInstance;
2326 n: number;
24- /** Timed repeats after the warmup (default 3). */
27+ /** Timed repeats after the warmups (default 5). */
2528 repeats?: number;
2629 /** Also evaluate the solution on the visualization grid. */
2730 wantGrid?: boolean;
@@ -30,6 +33,7 @@ export interface RunPointRequest {
3033
3134 export interface RunPoint {
3235 n: number;
36+ /** The fastest of the timed runs. */
3337 solveSeconds: number;
3438 solveSecondsAll: number[];
3539 coldSeconds: number;
@@ -46,12 +50,13 @@ function numLiteral(x: number): string {
4650
4751 export function runPoint(req: RunPointRequest): RunPoint {
4852 const { instance, n } = req;
49- const repeats = req.repeats ?? 3;
53+ const repeats = req.repeats ?? 5;
5054 const wantGrid = req.wantGrid ?? false;
5155 const main = [
5256 "% generated by the fastandaccurate harness",
5357 `prob = build_problem(${numLiteral(instance.a)}, ${numLiteral(instance.k)}, ${numLiteral(instance.d)}, ${wantGrid ? 1 : 0});`,
5458 `tic; out = solver(prob, ${n}); res_cold = toc;`,
59+ `out = solver(prob, ${n});`,
5560 `res_times = zeros(${repeats}, 1);`,
5661 `for irep = 1:${repeats}`,
5762 ` tic; out = solver(prob, ${n}); res_times(irep) = toc;`,
@@ -70,15 +75,10 @@ export function runPoint(req: RunPointRequest): RunPoint {
7075 ["res_cold", "res_times", "res_ueval", "res_ugrid"]
7176 );
7277 const times = Array.from(vars.res_times);
73- const sorted = [...times].sort((x, y) => x - y);
74- const median =
75- sorted.length % 2 === 1
76- ? sorted[(sorted.length - 1) / 2]
77- : (sorted[sorted.length / 2 - 1] + sorted[sorted.length / 2]) / 2;
7878 const { relMax, relL2 } = evalErrors(instance, vars.res_ueval);
7979 return {
8080 n,
81- solveSeconds: median,
81+ solveSeconds: Math.min(...times),
8282 solveSecondsAll: times,
8383 coldSeconds: vars.res_cold[0],
8484 relMax,
src/harness/sweep.tsmodified+1−1View file
@@ -24,7 +24,7 @@ export function runSweep(opts: SweepOptions): RunPoint[] {
2424 const p = runPoint({
2525 instance: opts.instance,
2626 n,
27- repeats: opts.repeats ?? 3,
27+ repeats: opts.repeats ?? 5,
2828 sources: opts.sources,
2929 });
3030 points.push(p);