1// Incremental STFT spectrogram drawn on a 2D canvas. The signal is kept as
2// it arrives and turned into dB magnitude columns (one FFT per hop) as soon
3// as a complete window is available. The plot area is an ImageData at
4// device-pixel resolution whose pixel columns are max-pooled over the STFT
5// columns and frequency bins they cover; on append only the pixel columns
6// touched by the new STFT columns are recomputed, so draw() is a putImageData
7// plus the axes and the cursor. The dB scale is relative to the largest
8// magnitude seen so far (0 dB at the peak, down to -dbRange); when the peak
9// rises the whole image is re-rendered from the stored columns.
11import { fft, hann } from "./fft.ts";
13export interface SpectrogramOptions {
14 fs: number;
15 window: number;
16 hop: number;
17 maxHz: number;
18 dbRange: number;
19 logFreq: boolean;
20}
22const LOG_FLOOR_HZ = 20;
23const EMPTY_RGB = [26, 26, 32];
24const FRAME_RGB = "#121216";
25const AXIS_COLOR = "rgba(215, 215, 228, 0.85)";
26const TICK_COLOR = "rgba(110, 225, 255, 0.95)";
27const CURSOR_COLOR = "rgba(255, 255, 255, 0.85)";
28// Re-render everything when the peak has risen by more than this since the last full render.
29const PEAK_TOLERANCE_DB = 0.5;
31export class Spectrogram {
32 private readonly canvas: HTMLCanvasElement;
33 private readonly ctx: CanvasRenderingContext2D;
34 private opts: SpectrogramOptions;
36 // Signal and STFT state.
37 private signal = new Float32Array(1 << 16);
38 private nSamples = 0;
39 private consumed = 0;
40 private columns: Float32Array[] = [];
41 private peakDb = -Infinity;
42 private win: Float32Array;
43 private re: Float32Array;
44 private im: Float32Array;
46 // Display state.
47 private duration = 1;
48 private cursor: number | null = null;
49 private ticks: number[] | null = null;
51 // Geometry in device pixels.
52 private dpr = 1;
53 private W = 1;
54 private H = 1;
55 private left = 0;
56 private top = 0;
57 private pw = 1;
58 private ph = 1;
59 private fmin = 0;
60 private fmax = 1;
61 private image: ImageData | null = null;
62 private renderedPeak = -Infinity;
63 /** Per pixel row, the bin range [2*py, 2*py+1) pooled into it. */
64 private rowBins = new Int32Array(0);
65 /** Per pixel column, the STFT column range [2*px, 2*px+1) pooled into it. */
66 private pixCols = new Int32Array(0);
68 private raf = 0;
69 private observer: ResizeObserver | null = null;
70 private destroyed = false;
72 constructor(canvas: HTMLCanvasElement, opts: SpectrogramOptions) {
73 const ctx = canvas.getContext("2d");
74 if (!ctx) throw new Error("Spectrogram: could not get a 2D canvas context");
75 this.canvas = canvas;
76 this.ctx = ctx;
77 this.opts = { ...opts };
78 validate(this.opts);
79 this.win = hann(this.opts.window);
80 this.re = new Float32Array(this.opts.window);
81 this.im = new Float32Array(this.opts.window);
82 if (typeof ResizeObserver !== "undefined") {
83 this.observer = new ResizeObserver(() => this.resize());
84 this.observer.observe(canvas);
85 }
86 this.resize();
87 }
89 /** Changes options; the columns are recomputed from the signal seen so far when the STFT parameters change. */
90 setOptions(opts: Partial<SpectrogramOptions>): void {
91 const next = { ...this.opts, ...opts };
92 validate(next);
93 const prev = this.opts;
94 this.opts = next;
95 if (next.window !== prev.window || next.hop !== prev.hop || next.fs !== prev.fs) {
96 this.win = hann(next.window);
97 this.re = new Float32Array(next.window);
98 this.im = new Float32Array(next.window);
99 this.columns = [];
100 this.consumed = 0;
101 this.peakDb = -Infinity;
102 this.computeColumns();
103 }
104 this.rebuildGeometry();
105 this.scheduleDraw();
106 }
108 /** Clears the signal and the image. */
109 reset(): void {
110 this.nSamples = 0;
111 this.consumed = 0;
112 this.columns = [];
113 this.peakDb = -Infinity;
114 this.renderAll();
115 this.scheduleDraw();
116 }
118 /** Appends signal; any new complete columns are computed and rendered incrementally. */
119 append(samples: Float32Array): void {
120 if (samples.length === 0) return;
121 const need = this.nSamples + samples.length;
122 if (need > this.signal.length) {
123 let cap = this.signal.length;
124 while (cap < need) cap *= 2;
125 const grown = new Float32Array(cap);
126 grown.set(this.signal.subarray(0, this.nSamples));
127 this.signal = grown;
128 }
129 this.signal.set(samples, this.nSamples);
130 this.nSamples = need;
131 const before = this.columns.length;
132 this.computeColumns();
133 if (this.columns.length > before) this.renderNew(before);
134 this.scheduleDraw();
135 }
137 /** Replaces the whole signal. */
138 setSignal(samples: Float32Array): void {
139 this.nSamples = 0;
140 this.consumed = 0;
141 this.columns = [];
142 this.peakDb = -Infinity;
143 if (samples.length === 0) {
144 this.renderAll();
145 this.scheduleDraw();
146 return;
147 }
148 this.append(samples);
149 }
151 /** Playback cursor [s], drawn as a vertical line; null hides it. */
152 setCursor(seconds: number | null): void {
153 if (seconds === this.cursor) return;
154 this.cursor = seconds;
155 this.scheduleDraw();
156 }
158 /** Short horizontal ticks at these frequencies [Hz] on the left edge of the plot (e.g. modal frequencies). */
159 setTicks(freqs: number[] | null): void {
160 this.ticks = freqs ? freqs.slice() : null;
161 this.scheduleDraw();
162 }
164 /** The time axis spans this duration [s], so the image does not rescale as the signal grows. */
165 setDuration(seconds: number): void {
166 const d = seconds > 0 && Number.isFinite(seconds) ? seconds : 1;
167 if (d === this.duration) return;
168 this.duration = d;
169 this.rebuildGeometry();
170 this.scheduleDraw();
171 }
173 /** Matches the canvas pixel size to its CSS size and the device pixel ratio, then re-renders. */
174 resize(): void {
175 if (this.destroyed) return;
176 const dpr = (typeof window !== "undefined" && window.devicePixelRatio) || 1;
177 const cssW = this.canvas.clientWidth || this.canvas.width / dpr || 1;
178 const cssH = this.canvas.clientHeight || this.canvas.height / dpr || 1;
179 const W = Math.max(1, Math.round(cssW * dpr));
180 const H = Math.max(1, Math.round(cssH * dpr));
181 if (this.canvas.width !== W) this.canvas.width = W;
182 if (this.canvas.height !== H) this.canvas.height = H;
183 if (W === this.W && H === this.H && dpr === this.dpr && this.image) {
184 this.scheduleDraw();
185 return;
186 }
187 this.dpr = dpr;
188 this.W = W;
189 this.H = H;
190 this.rebuildGeometry();
191 this.scheduleDraw();
192 }
194 /** Draws the current image, axes, ticks and cursor immediately. */
195 draw(): void {
196 if (this.raf) {
197 cancelAnimationFrame(this.raf);
198 this.raf = 0;
199 }
200 if (this.destroyed || !this.image) return;
201 const ctx = this.ctx;
202 const { dpr, left, top, pw, ph } = this;
203 ctx.setTransform(1, 0, 0, 1, 0, 0);
204 ctx.fillStyle = FRAME_RGB;
205 ctx.fillRect(0, 0, this.W, this.H);
206 ctx.putImageData(this.image, left, top);
207 this.drawAxes();
208 if (this.ticks) {
209 ctx.fillStyle = TICK_COLOR;
210 const len = Math.round(6 * dpr);
211 const thick = Math.max(1, Math.round(dpr));
212 for (const f of this.ticks) {
213 if (!(f >= this.fmin && f <= this.fmax)) continue;
214 const y = Math.round(top + (1 - this.freqFrac(f)) * ph - thick / 2);
215 ctx.fillRect(left, y, len, thick);
216 }
217 }
218 const c = this.cursor;
219 if (c !== null && c >= 0 && c <= this.duration) {
220 const x = Math.round(left + (c / this.duration) * pw);
221 ctx.fillStyle = CURSOR_COLOR;
222 ctx.fillRect(x, top, Math.max(1, Math.round(dpr)), ph);
223 }
224 }
226 destroy(): void {
227 this.destroyed = true;
228 if (this.raf) {
229 cancelAnimationFrame(this.raf);
230 this.raf = 0;
231 }
232 this.observer?.disconnect();
233 this.observer = null;
234 this.columns = [];
235 this.signal = new Float32Array(0);
236 this.image = null;
237 }
239 // ---- STFT ----
241 private computeColumns(): void {
242 const { window: n, hop } = this.opts;
243 const nBins = (n >> 1) + 1;
244 const re = this.re;
245 const im = this.im;
246 const win = this.win;
247 while (this.consumed + n <= this.nSamples) {
248 const off = this.consumed;
249 for (let k = 0; k < n; k++) {
250 re[k] = this.signal[off + k] * win[k];
251 im[k] = 0;
252 }
253 fft(re, im);
254 const col = new Float32Array(nBins);
255 let peak = this.peakDb;
256 for (let k = 0; k < nBins; k++) {
257 const p = re[k] * re[k] + im[k] * im[k];
258 const db = 10 * Math.log10(p + 1e-30);
259 col[k] = db;
260 if (db > peak) peak = db;
261 }
262 this.peakDb = peak;
263 this.columns.push(col);
264 this.consumed += hop;
265 }
266 }
268 // ---- Geometry and rendering of the plot image ----
270 private freqFrac(f: number): number {
271 if (this.opts.logFreq) {
272 return (Math.log(f) - Math.log(this.fmin)) / (Math.log(this.fmax) - Math.log(this.fmin));
273 }
274 return (f - this.fmin) / (this.fmax - this.fmin);
275 }
277 private fracFreq(frac: number): number {
278 if (this.opts.logFreq) {
279 const lo = Math.log(this.fmin);
280 const hi = Math.log(this.fmax);
281 return Math.exp(lo + frac * (hi - lo));
282 }
283 return this.fmin + frac * (this.fmax - this.fmin);
284 }
286 private rebuildGeometry(): void {
287 const { dpr, W, H } = this;
288 const o = this.opts;
289 this.left = Math.round(50 * dpr);
290 this.top = Math.round(6 * dpr);
291 const bottom = Math.round(20 * dpr);
292 const right = Math.round(8 * dpr);
293 this.pw = Math.max(1, W - this.left - right);
294 this.ph = Math.max(1, H - this.top - bottom);
295 const nyquist = o.fs / 2;
296 if (o.logFreq) {
297 this.fmin = LOG_FLOOR_HZ;
298 this.fmax = Math.max(LOG_FLOOR_HZ * 2, Math.min(o.maxHz, nyquist));
299 } else {
300 this.fmin = 0;
301 this.fmax = Math.max(1, Math.min(o.maxHz, nyquist));
302 }
304 // Bin range per pixel row (row 0 at the top).
305 const binHz = o.fs / o.window;
306 const nBins = (o.window >> 1) + 1;
307 const ph = this.ph;
308 this.rowBins = new Int32Array(2 * ph);
309 for (let py = 0; py < ph; py++) {
310 const fHi = this.fracFreq(1 - py / ph);
311 const fLo = this.fracFreq(1 - (py + 1) / ph);
312 let b0 = Math.round(fLo / binHz);
313 let b1 = Math.round(fHi / binHz);
314 b0 = Math.max(0, Math.min(nBins - 1, b0));
315 b1 = Math.max(b0 + 1, Math.min(nBins, b1));
316 this.rowBins[2 * py] = b0;
317 this.rowBins[2 * py + 1] = b1;
318 }
320 // STFT column range per pixel column. Column c is centered at time
321 // (c * hop + window / 2) / fs; a pixel narrower than a hop takes the
322 // nearest column.
323 const pw = this.pw;
324 const dt = this.duration / pw; // seconds per pixel
325 const half = o.window / 2;
326 const maxCols = Math.max(1, Math.floor((this.duration * o.fs - o.window) / o.hop) + 1);
327 this.pixCols = new Int32Array(2 * pw);
328 for (let px = 0; px < pw; px++) {
329 let c0 = Math.ceil((px * dt * o.fs - half) / o.hop);
330 let c1 = Math.ceil(((px + 1) * dt * o.fs - half) / o.hop);
331 if (c1 <= c0) {
332 c0 = Math.round(((px + 0.5) * dt * o.fs - half) / o.hop);
333 c1 = c0 + 1;
334 }
335 c0 = Math.max(0, Math.min(maxCols - 1, c0));
336 c1 = Math.max(c0 + 1, c1);
337 this.pixCols[2 * px] = c0;
338 this.pixCols[2 * px + 1] = c1;
339 }
341 this.image = this.ctx.createImageData(pw, ph);
342 this.renderAll();
343 }
345 private renderAll(): void {
346 if (!this.image) return;
347 this.renderedPeak = this.peakDb;
348 for (let px = 0; px < this.pw; px++) this.renderPixelColumn(px);
349 }
351 /** Renders the pixel columns touched by STFT columns [firstNew, columns.length). */
352 private renderNew(firstNew: number): void {
353 if (!this.image) return;
354 if (this.peakDb > this.renderedPeak + PEAK_TOLERANCE_DB || !Number.isFinite(this.renderedPeak)) {
355 this.renderAll();
356 return;
357 }
358 const last = this.columns.length;
359 for (let px = 0; px < this.pw; px++) {
360 const c0 = this.pixCols[2 * px];
361 const c1 = this.pixCols[2 * px + 1];
362 if (c1 > firstNew && c0 < last) this.renderPixelColumn(px);
363 }
364 }
366 private renderPixelColumn(px: number): void {
367 const img = this.image;
368 if (!img) return;
369 const data = img.data;
370 const { pw, ph } = this;
371 const cols = this.columns;
372 const c0 = this.pixCols[2 * px];
373 const c1 = Math.min(cols.length, this.pixCols[2 * px + 1]);
374 if (c0 >= c1) {
375 for (let py = 0; py < ph; py++) {
376 const o = (py * pw + px) * 4;
377 data[o] = EMPTY_RGB[0];
378 data[o + 1] = EMPTY_RGB[1];
379 data[o + 2] = EMPTY_RGB[2];
380 data[o + 3] = 255;
381 }
382 return;
383 }
384 const ref = this.renderedPeak;
385 const range = this.opts.dbRange;
386 const lut = colormapLut();
387 for (let py = 0; py < ph; py++) {
388 const b0 = this.rowBins[2 * py];
389 const b1 = this.rowBins[2 * py + 1];
390 let v = -Infinity;
391 for (let c = c0; c < c1; c++) {
392 const col = cols[c];
393 for (let b = b0; b < b1; b++) if (col[b] > v) v = col[b];
394 }
395 let t = (v - ref + range) / range;
396 if (!(t > 0)) t = 0;
397 else if (t > 1) t = 1;
398 const i = Math.round(t * 255) * 3;
399 const o = (py * pw + px) * 4;
400 data[o] = lut[i];
401 data[o + 1] = lut[i + 1];
402 data[o + 2] = lut[i + 2];
403 data[o + 3] = 255;
404 }
405 }
407 // ---- Axes ----
409 private drawAxes(): void {
410 const ctx = this.ctx;
411 const { dpr, left, top, pw, ph, fmin, fmax } = this;
412 ctx.fillStyle = AXIS_COLOR;
413 ctx.font = `${Math.round(10 * dpr)}px system-ui, -apple-system, "Segoe UI", Helvetica, Arial, sans-serif`;
414 ctx.textAlign = "right";
415 ctx.textBaseline = "middle";
416 const fticks = this.opts.logFreq
417 ? LOG_TICKS.filter((f) => f >= fmin && f <= fmax)
418 : niceTicks(0, fmax, Math.max(2, Math.floor(ph / (36 * dpr))));
419 const thin = Math.max(1, Math.round(dpr));
420 let lastY = -Infinity;
421 for (const f of fticks) {
422 const y = Math.round(top + (1 - this.freqFrac(f)) * ph);
423 if (Math.abs(y - lastY) < 12 * dpr) continue;
424 lastY = y;
425 ctx.fillRect(left - 4 * dpr, y, 4 * dpr, thin);
426 ctx.fillText(formatHz(f), left - 6 * dpr, y);
427 }
428 ctx.textAlign = "center";
429 ctx.textBaseline = "top";
430 const tticks = niceTicks(0, this.duration, Math.max(2, Math.floor(pw / (64 * dpr))));
431 const yText = top + ph + 4 * dpr;
432 for (let k = 0; k < tticks.length; k++) {
433 const t = tticks[k];
434 const x = Math.round(left + (t / this.duration) * pw);
435 ctx.fillRect(x, top + ph, thin, 4 * dpr);
436 let label = formatNumber(t);
437 if (k === tticks.length - 1) label += " s";
438 ctx.fillText(label, x, yText);
439 }
440 }
442 private scheduleDraw(): void {
443 if (this.destroyed || this.raf) return;
444 if (typeof requestAnimationFrame === "undefined") {
445 this.draw();
446 return;
447 }
448 this.raf = requestAnimationFrame(() => {
449 this.raf = 0;
450 this.draw();
451 });
452 }
453}
455const LOG_TICKS = [20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000];
457function validate(o: SpectrogramOptions): void {
458 if (!(o.window >= 2) || (o.window & (o.window - 1)) !== 0) throw new Error(`Spectrogram: window ${o.window} is not a power of two`);
459 if (!(o.hop >= 1)) throw new Error(`Spectrogram: hop must be positive`);
460 if (!(o.fs > 0)) throw new Error(`Spectrogram: fs must be positive`);
461 if (!(o.dbRange > 0)) throw new Error(`Spectrogram: dbRange must be positive`);
462 o.window = Math.round(o.window);
463 o.hop = Math.round(o.hop);
464}
466function formatHz(f: number): string {
467 if (f >= 1000) return `${formatNumber(f / 1000)} kHz`;
468 return `${formatNumber(f)} Hz`;
469}
471function formatNumber(x: number): string {
472 const s = x.toFixed(3);
473 return s.replace(/\.?0+$/, "");
474}
476/** Round tick positions covering [lo, hi] with about `count` intervals. */
477function niceTicks(lo: number, hi: number, count: number): number[] {
478 const span = hi - lo;
479 if (!(span > 0)) return [lo];
480 const raw = span / count;
481 const mag = 10 ** Math.floor(Math.log10(raw));
482 const norm = raw / mag;
483 const step = (norm < 1.5 ? 1 : norm < 3.5 ? 2 : norm < 7.5 ? 5 : 10) * mag;
484 const ticks: number[] = [];
485 const first = Math.ceil(lo / step - 1e-9);
486 const last = Math.floor(hi / step + 1e-9);
487 for (let k = first; k <= last; k++) ticks.push(+(k * step).toPrecision(12));
488 return ticks;
489}
491// Inferno-like colormap: anchor colors at t = 0, 0.1, ..., 1 interpolated
492// linearly into a 256-entry RGB table.
493const INFERNO_ANCHORS: [number, number, number][] = [
494 [0, 0, 4],
495 [22, 11, 57],
496 [66, 10, 104],
497 [106, 23, 110],
498 [147, 38, 103],
499 [188, 55, 84],
500 [221, 81, 58],
501 [243, 120, 25],
502 [252, 165, 10],
503 [246, 215, 70],
504 [252, 255, 164],
505];
507let lut: Uint8ClampedArray | null = null;
509function colormapLut(): Uint8ClampedArray {
510 if (lut) return lut;
511 const table = new Uint8ClampedArray(256 * 3);
512 const segs = INFERNO_ANCHORS.length - 1;
513 for (let i = 0; i < 256; i++) {
514 const t = (i / 255) * segs;
515 const s = Math.min(segs - 1, Math.floor(t));
516 const u = t - s;
517 const a = INFERNO_ANCHORS[s];
518 const b = INFERNO_ANCHORS[s + 1];
519 for (let k = 0; k < 3; k++) table[i * 3 + k] = Math.round(a[k] + (b[k] - a[k]) * u);
520 }
521 lut = table;
522 return table;
523}