2 * The scene as a movie, with the radiated sound as its audio track.
3 *
4 * The browser's own recorder does the encoding: the composite canvas below is
5 * captured as a video track, the sound is played into a MediaStream audio
6 * track, and MediaRecorder muxes the two into WebM (or MP4 where the browser
7 * writes it). That makes the recording real time, which is the price of using
8 * nothing but the platform: a three-second movie takes three seconds to write,
9 * and a machine too busy to keep up drops frames rather than slowing down.
10 * The result reports how many of the run's frames were actually drawn.
11 *
12 * The composite canvas is the scene, drawn at the movie's own resolution
13 * rather than the one on screen, with a title bar carrying the simulated time
14 * and a caption bar carrying the field, its colour scale and the exaggeration.
15 * Everything a viewer needs to read the movie is therefore in the file.
16 */
17import { colormapTable, getColormap, toAudioBuffer, type AudioPrep } from "@plate-dyn/core";
18import { withWebmDuration } from "./webmDuration";
20export interface MovieLegend {
21 label: string;
22 unit: string;
23 /** Range of the colour scale, in SI; `scale` multiplies it for the labels */
24 min: number;
25 max: number;
26 scale: number;
27 colormap: string;
28}
30export interface MovieOverlay {
31 title: string;
32 /** A second line: what is exaggerated, and by how much */
33 note?: string;
34 legend?: MovieLegend;
35}
37/** The scene the recorder draws from (Plate3DView's imperative handle). */
38export interface MovieSource {
39 canvas: () => HTMLCanvasElement | null;
40 drawFrame: (k: number, size?: { width: number; height: number }) => void;
41}
43export interface MovieOptions {
44 source: MovieSource;
45 /** Simulated time of every frame [s] */
46 frameTimes: Float64Array;
47 /** The sound, normalized as the player and the WAV export have it */
48 audio?: AudioPrep;
49 /** 1 is real time; below that the movie and the sound are slowed together */
50 speed?: number;
51 /** Width of the movie in pixels; the height follows `aspect` plus the bars */
52 width?: number;
53 /** Height of the scene as a fraction of the width */
54 aspect?: number;
55 overlay?: MovieOverlay;
56 /** Frame to leave on screen when the recording ends */
57 restoreFrame?: number;
58 onProgress?: (fraction: number) => void;
59 signal?: AbortSignal;
60}
62export interface MovieResult {
63 blob: Blob;
64 ext: string;
65 mimeType: string;
66 /** Length of the movie [s] */
67 seconds: number;
68 width: number;
69 height: number;
70 /** Frames the run has, and how many of them the recording drew */
71 frames: number;
72 drawn: number;
73}
75/**
76 * Containers to try, in order. WebM with Opus comes first because it is the
77 * one Chrome and Firefox record with the sound: their MP4 recording accepts no
78 * audio codec at all (`isTypeSupported` says no to every `video/mp4` string
79 * that names one), and a movie without its sound is not what was asked for.
80 * The MP4 entries are for Safari, which records MP4 with AAC and no WebM.
81 */
82const FORMATS = [
83 { mimeType: "video/webm;codecs=vp9,opus", ext: "webm" },
84 { mimeType: "video/webm;codecs=vp8,opus", ext: "webm" },
85 { mimeType: "video/webm", ext: "webm" },
86 { mimeType: 'video/mp4;codecs="avc1.42E01E,mp4a.40.2"', ext: "mp4" },
87 { mimeType: "video/mp4", ext: "mp4" },
88];
90const defaultSupported = (mime: string): boolean =>
91 typeof MediaRecorder !== "undefined" && MediaRecorder.isTypeSupported(mime);
93/** The first container this browser can write, or null. */
94export function pickMovieFormat(
95 isSupported: (mime: string) => boolean = defaultSupported,
96): { mimeType: string; ext: string } | null {
97 return FORMATS.find((f) => isSupported(f.mimeType)) ?? null;
98}
100export type MovieSupport =
101 | { status: "available"; mimeType: string; ext: string }
102 | { status: "unavailable"; reason: string };
104/** Whether this browser can record the scene at all, and in what. */
105export function movieSupport(): MovieSupport {
106 if (typeof MediaRecorder === "undefined")
107 return { status: "unavailable", reason: "this browser has no MediaRecorder" };
108 if (
109 typeof HTMLCanvasElement === "undefined" ||
110 typeof HTMLCanvasElement.prototype.captureStream !== "function"
111 )
112 return { status: "unavailable", reason: "this browser cannot capture a canvas as video" };
113 const f = pickMovieFormat();
114 if (!f)
115 return { status: "unavailable", reason: "this browser writes no video format the lab can use" };
116 return { status: "available", mimeType: f.mimeType, ext: f.ext };
117}
119/**
120 * The last frame whose time has arrived at simulated time t, searching forward
121 * from `from`. Frames are monotonic, so a run that falls behind skips to the
122 * frame it should be showing rather than replaying the ones it missed.
123 */
124export function frameAtTime(frameTimes: ArrayLike<number>, t: number, from = 0): number {
125 let k = Math.max(0, Math.min(from, frameTimes.length - 1));
126 while (k + 1 < frameTimes.length && frameTimes[k + 1] <= t) k++;
127 return k;
128}
130/** Simulated span of the movie [s]: the frames, or the sound if it lasts longer. */
131export function movieSpan(frameTimes: ArrayLike<number>, audio?: AudioPrep): number {
132 const last = frameTimes.length ? frameTimes[frameTimes.length - 1] : 0;
133 return Math.max(last, audio?.durationS ?? 0);
134}
136const even = (x: number): number => 2 * Math.round(x / 2);
138const TEXT = "#1c2430";
139const MUTED = "#5b6573";
140const RULE = "#d9dde3";
142function fmt(x: number): string {
143 if (x === 0) return "0";
144 const a = Math.abs(x);
145 if (a >= 1e4 || a < 1e-2) return x.toExponential(2);
146 return Number(x.toPrecision(3)).toString();
147}
149/** Record the scene, resolving with the finished file. */
150export async function recordMovie(opts: MovieOptions): Promise<MovieResult> {
151 const { source, frameTimes, audio, overlay, onProgress, signal } = opts;
152 const speed = opts.speed ?? 1;
153 const support = movieSupport();
154 if (support.status !== "available") throw new Error(support.reason);
155 if (!frameTimes.length) throw new Error("the run kept no frames to record");
156 if (!(speed > 0)) throw new Error("the speed must be positive");
158 const width = even(opts.width ?? 960);
159 const sceneH = even(width * (opts.aspect ?? 0.7));
160 const titleH = overlay ? 44 : 0;
161 const legendH = overlay ? (overlay.legend ? 62 : 32) : 0;
162 const height = even(sceneH + titleH + legendH);
164 const out = document.createElement("canvas");
165 out.width = width;
166 out.height = height;
167 const ctx = out.getContext("2d");
168 if (!ctx) throw new Error("this browser gave no 2-D canvas to compose the movie on");
170 const drawOverlay = (t: number) => {
171 ctx.fillStyle = "#fff";
172 ctx.fillRect(0, 0, width, titleH);
173 ctx.fillRect(0, titleH + sceneH, width, legendH);
174 ctx.strokeStyle = RULE;
175 ctx.lineWidth = 1;
176 ctx.beginPath();
177 ctx.moveTo(0, titleH + 0.5);
178 ctx.lineTo(width, titleH + 0.5);
179 ctx.moveTo(0, titleH + sceneH + 0.5);
180 ctx.lineTo(width, titleH + sceneH + 0.5);
181 ctx.stroke();
182 if (!overlay) return;
183 ctx.textBaseline = "alphabetic";
184 ctx.fillStyle = TEXT;
185 ctx.font = "600 19px system-ui, -apple-system, 'Segoe UI', Roboto, sans-serif";
186 ctx.textAlign = "left";
187 ctx.fillText(overlay.title, 16, 29);
188 ctx.textAlign = "right";
189 ctx.font = "17px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace";
190 ctx.fillText(`t = ${t.toFixed(3)} s`, width - 16, 29);
191 ctx.textAlign = "left";
192 const base = titleH + sceneH;
193 const lg = overlay.legend;
194 if (lg) {
195 ctx.font = "13px system-ui, -apple-system, 'Segoe UI', Roboto, sans-serif";
196 ctx.fillStyle = TEXT;
197 ctx.fillText(`${lg.label} [${lg.unit}]`, 16, base + 20);
198 const bx = 16;
199 const by = base + 30;
200 const bw = 240;
201 const bh = 10;
202 const table = colormapTable(getColormap(lg.colormap));
203 for (let i = 0; i < bw; i++) {
204 const k = Math.round((i / (bw - 1)) * 255) * 3;
205 ctx.fillStyle = `rgb(${table[k]},${table[k + 1]},${table[k + 2]})`;
206 ctx.fillRect(bx + i, by, 1, bh);
207 }
208 ctx.strokeStyle = RULE;
209 ctx.strokeRect(bx + 0.5, by + 0.5, bw - 1, bh - 1);
210 ctx.fillStyle = MUTED;
211 ctx.font = "12px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace";
212 ctx.fillText(fmt(lg.min * lg.scale), bx, by + bh + 14);
213 ctx.textAlign = "right";
214 ctx.fillText(fmt(lg.max * lg.scale), bx + bw, by + bh + 14);
215 ctx.textAlign = "left";
216 }
217 if (overlay.note) {
218 ctx.fillStyle = MUTED;
219 ctx.font = "13px system-ui, -apple-system, 'Segoe UI', Roboto, sans-serif";
220 ctx.textAlign = "right";
221 ctx.fillText(overlay.note, width - 16, base + (lg ? 40 : 21));
222 ctx.textAlign = "left";
223 }
224 };
226 let drawn = 0;
227 const render = (k: number) => {
228 source.drawFrame(k, { width, height: sceneH });
229 const src = source.canvas();
230 ctx.fillStyle = "#fff";
231 ctx.fillRect(0, titleH, width, sceneH);
232 if (src) ctx.drawImage(src, 0, titleH, width, sceneH);
233 drawOverlay(frameTimes[k] ?? 0);
234 drawn++;
235 };
237 const stream = out.captureStream(0);
238 const video = stream.getVideoTracks()[0] as CanvasCaptureMediaStreamTrack;
239 let actx: AudioContext | undefined;
240 let node: AudioBufferSourceNode | undefined;
241 const chunks: Blob[] = [];
242 let rec: MediaRecorder | undefined;
243 try {
244 if (audio && audio.samples.length > 1) {
245 actx = new AudioContext();
246 await actx.resume();
247 const dest = actx.createMediaStreamDestination();
248 node = actx.createBufferSource();
249 node.buffer = toAudioBuffer(actx, audio);
250 // below real time the sound is slowed like a tape, and drops in pitch with it
251 node.playbackRate.value = speed;
252 node.connect(dest);
253 const track = dest.stream.getAudioTracks()[0];
254 if (track) stream.addTrack(track);
255 }
257 rec = new MediaRecorder(stream, { mimeType: support.mimeType, videoBitsPerSecond: 8e6 });
258 rec.ondataavailable = (e) => {
259 if (e.data.size > 0) chunks.push(e.data);
260 };
261 const stopped = new Promise<void>((resolve) => {
262 rec!.onstop = () => resolve();
263 });
265 const span = movieSpan(frameTimes, audio);
266 const wall = span / speed;
267 render(0);
268 video.requestFrame();
269 const tRec = performance.now();
270 rec.start(250);
271 node?.start();
273 const t0 = performance.now();
274 let last = 0;
275 await new Promise<void>((resolve) => {
276 const step = () => {
277 const elapsed = (performance.now() - t0) / 1000;
278 const k = frameAtTime(frameTimes, elapsed * speed, last);
279 if (k !== last) {
280 last = k;
281 render(k);
282 }
283 video.requestFrame();
284 onProgress?.(Math.min(1, elapsed / Math.max(wall, 1e-6)));
285 if (signal?.aborted || elapsed >= wall + 0.15) resolve();
286 else requestAnimationFrame(step);
287 };
288 requestAnimationFrame(step);
289 });
291 rec.stop();
292 await stopped;
293 const seconds = (performance.now() - tRec) / 1000;
294 let blob = new Blob(chunks, { type: support.mimeType });
295 if (blob.size === 0) throw new Error("the browser's recorder produced an empty file");
296 if (support.ext === "webm") {
297 // the recorder leaves the length out, since it does not know it until now
298 const patched = withWebmDuration(new Uint8Array(await blob.arrayBuffer()), seconds);
299 blob = new Blob([patched as BlobPart], { type: support.mimeType });
300 }
301 return {
302 blob,
303 ext: support.ext,
304 mimeType: support.mimeType,
305 seconds,
306 width,
307 height,
308 frames: frameTimes.length,
309 drawn,
310 };
311 } finally {
312 try {
313 node?.stop();
314 } catch {
315 /* already stopped */
316 }
317 void actx?.close();
318 for (const t of stream.getTracks()) t.stop();
319 // back to the size and the frame the app is showing
320 source.drawFrame(opts.restoreFrame ?? 0);
321 }
322}
324/** Hand the finished movie to the browser as a download. */
325export function saveMovie(result: MovieResult, filename: string): void {
326 const url = URL.createObjectURL(result.blob);
327 const a = document.createElement("a");
328 a.href = url;
329 a.download = `${filename}.${result.ext}`;
330 a.click();
331 setTimeout(() => URL.revokeObjectURL(url), 0);
332}