8354482cymmodes: linear modes of a cymsim cymbal driven in the full nonlinear model, with the energy of every mode over timeJeremy Magland 1// The modal views of a run. The spectrogram is time against mode frequency with colour the modal
2// energy in dB below the loudest value seen so far: each mode is one horizontal trace, so energy
3// arriving in a mode is visible as its trace lighting up, resolved per mode rather than smeared
4// by an STFT window. The bar view is the energy of every mode at the cursor time. Both share the
5// frequency axis (linear or logarithmic, with a selectable top) and the dB range, and clicking
6// near a trace or a bar selects that mode.
7import type { Stft } from "../analysis/spectrogram.ts";
8import type { ModalTracker } from "../modal/tracker.ts";
10export type FrequencyScale = "linear" | "log";
12/** Perceptual-ish dark-to-bright colour ramp for t in [0, 1]. */
13function rampColor(t: number): [number, number, number] {
14 const stops: [number, number, number][] = [
15 [12, 13, 18], [48, 26, 74], [120, 35, 107], [193, 62, 88], [240, 122, 52], [252, 200, 96], [252, 252, 200],
16 ];
17 const x = Math.min(1, Math.max(0, t)) * (stops.length - 1), i = Math.min(stops.length - 2, Math.floor(x)), f = x - i;
18 return [0, 1, 2].map((k) => Math.round(stops[i][k] + f * (stops[i + 1][k] - stops[i][k]))) as [number, number, number];
19}
21class FrequencyAxis {
22 scale: FrequencyScale = "linear";
23 fmax = 20000;
24 fmin = 30;
25 /** Pixel y (0 at the top) of frequency f within a plot of height H. */
26 y(f: number, H: number): number {
27 if (this.scale === "linear") return H * (1 - f / this.fmax);
28 const l0 = Math.log(this.fmin), l1 = Math.log(this.fmax);
29 return H * (1 - (Math.log(Math.max(f, this.fmin)) - l0) / (l1 - l0));
30 }
31 /** Frequency at pixel y, the inverse of ``y``. */
32 f(y: number, H: number): number {
33 if (this.scale === "linear") return this.fmax * (1 - y / H);
34 const l0 = Math.log(this.fmin), l1 = Math.log(this.fmax);
35 return Math.exp(l0 + (1 - y / H) * (l1 - l0));
36 }
37 ticks(): number[] {
38 if (this.scale === "linear") {
39 const step = this.fmax > 10000 ? 2000 : this.fmax > 4000 ? 1000 : this.fmax > 1500 ? 500 : 200;
40 const out = [];
41 for (let f = step; f <= this.fmax; f += step) out.push(f);
42 return out;
43 }
44 return [50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000].filter((f) => f >= this.fmin && f <= this.fmax);
45 }
46}
48const LEFT = 44, BOTTOM = 16;
50export class ModalSpectrogram {
51 private readonly canvas: HTMLCanvasElement;
52 private readonly image = document.createElement("canvas");
53 readonly axis = new FrequencyAxis();
54 dbRange = 80;
55 tracker: ModalTracker | null = null;
56 duration = 1; // x extent [s]
57 cursorTime: number | null = null;
58 selectedMode = -1;
59 onSelect: ((mode: number, time: number) => void) | null = null;
60 onHover: ((text: string) => void) | null = null;
61 private dirty = true;
63 constructor(canvas: HTMLCanvasElement) {
64 this.canvas = canvas;
65 canvas.addEventListener("pointerdown", (e) => {
66 const pick = this.pick(e);
67 if (pick) this.onSelect?.(pick.mode, pick.time);
68 });
69 canvas.addEventListener("pointermove", (e) => {
70 const pick = this.pick(e);
71 if (!pick || !this.tracker) { this.onHover?.(""); return; }
72 const s = this.tracker.indexAtTime(pick.time);
73 const E = s >= 0 ? this.tracker.energies[s][pick.mode] : 0;
74 this.onHover?.(`mode ${pick.mode} at ${this.freqOf(pick.mode).toFixed(1)} Hz, ${this.db(E).toFixed(0)} dB at t = ${pick.time.toFixed(2)} s`);
75 });
76 canvas.addEventListener("pointerleave", () => this.onHover?.(""));
77 }
79 private freqOf(m: number) { return this.tracker ? this.tracker.omega[m] / (2 * Math.PI) : 0; }
80 private db(E: number) { return 10 * Math.log10(Math.max(E, 1e-300) / Math.max(this.tracker?.maxEnergy ?? 1, 1e-300)); }
82 private pick(e: PointerEvent): { mode: number; time: number } | null {
83 const t = this.tracker;
84 if (!t || !t.length) return null;
85 const r = this.canvas.getBoundingClientRect();
86 const W = r.width - LEFT, H = r.height - BOTTOM;
87 const px = e.clientX - r.left - LEFT, py = e.clientY - r.top;
88 if (px < 0 || py > H) return null;
89 const time = Math.max(0, Math.min(this.duration, (px / W) * this.duration));
90 let best = -1, bestD = 8; // within 8 px of a trace
91 for (let m = 0; m < t.nModes; m++) {
92 const d = Math.abs(this.axis.y(this.freqOf(m), H) - py);
93 if (d < bestD) { bestD = d; best = m; }
94 }
95 return best >= 0 ? { mode: best, time } : null;
96 }
98 invalidate() { this.dirty = true; }
100 /** Rebuild the cached image: max-aggregate the snapshots into pixel columns. */
101 private renderImage(W: number, H: number) {
102 const img = this.image;
103 if (img.width !== W || img.height !== H) { img.width = W; img.height = H; }
104 const g = img.getContext("2d")!;
105 g.fillStyle = "#0c0d12"; g.fillRect(0, 0, W, H);
106 const t = this.tracker;
107 if (!t || !t.length) return;
108 const data = g.getImageData(0, 0, W, H), px = data.data;
109 const cols = new Float32Array(W * H).fill(-1e30);
110 const maxE = Math.max(t.maxEnergy, 1e-300);
111 for (let s = 0; s < t.length; s++) {
112 const x = Math.min(W - 1, Math.floor((t.times[s] / this.duration) * W));
113 const E = t.energies[s];
114 for (let m = 0; m < t.nModes; m++) {
115 const f = t.omega[m] / (2 * Math.PI);
116 if (f > this.axis.fmax) continue;
117 const y = Math.round(this.axis.y(f, H));
118 if (y < 0 || y >= H) continue;
119 const dB = 10 * Math.log10(Math.max(E[m], 1e-300) / maxE);
120 const i = y * W + x;
121 if (dB > cols[i]) cols[i] = dB;
122 }
123 }
124 for (let i = 0; i < W * H; i++) {
125 if (cols[i] < -1e29) continue;
126 const [r, gg, b] = rampColor(1 + cols[i] / this.dbRange);
127 px[4 * i] = r; px[4 * i + 1] = gg; px[4 * i + 2] = b; px[4 * i + 3] = 255;
128 }
129 g.putImageData(data, 0, 0);
130 this.dirty = false;
131 }
133 draw() {
134 const cv = this.canvas, dpr = window.devicePixelRatio || 1;
135 const CW = Math.max(1, Math.round(cv.clientWidth * dpr)), CH = Math.max(1, Math.round(cv.clientHeight * dpr));
136 if (cv.width !== CW || cv.height !== CH) { cv.width = CW; cv.height = CH; this.dirty = true; }
137 const ctx = cv.getContext("2d")!;
138 ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
139 const W = cv.clientWidth - LEFT, H = cv.clientHeight - BOTTOM;
140 if (W < 10 || H < 10) return;
141 if (this.dirty || this.image.width !== W || this.image.height !== H) this.renderImage(W, H);
142 ctx.fillStyle = "#14161a"; ctx.fillRect(0, 0, cv.clientWidth, cv.clientHeight);
143 ctx.drawImage(this.image, LEFT, 0);
144 // axes
145 ctx.fillStyle = "#9aa0a8"; ctx.font = "10px system-ui"; ctx.textAlign = "right"; ctx.textBaseline = "middle";
146 for (const f of this.axis.ticks()) {
147 const y = this.axis.y(f, H);
148 ctx.fillText(f >= 1000 ? `${f / 1000}k` : String(f), LEFT - 4, y);
149 ctx.strokeStyle = "rgba(154,160,168,0.12)"; ctx.beginPath(); ctx.moveTo(LEFT, y); ctx.lineTo(LEFT + W, y); ctx.stroke();
150 }
151 ctx.textAlign = "center"; ctx.textBaseline = "top";
152 const tStep = this.duration > 8 ? 2 : this.duration > 3 ? 1 : this.duration > 1.2 ? 0.5 : 0.2;
153 for (let tt = 0; tt <= this.duration + 1e-9; tt += tStep) {
154 const x = LEFT + (tt / this.duration) * W;
155 ctx.fillText(`${tt.toFixed(tt < 1 && tStep < 1 ? 1 : 0)}s`, x, H + 3);
156 }
157 // selected mode trace and cursor
158 if (this.tracker && this.selectedMode >= 0 && this.selectedMode < this.tracker.nModes) {
159 const y = this.axis.y(this.freqOf(this.selectedMode), H);
160 ctx.strokeStyle = "rgba(224,162,58,0.8)"; ctx.setLineDash([3, 4]);
161 ctx.beginPath(); ctx.moveTo(LEFT, y); ctx.lineTo(LEFT + W, y); ctx.stroke(); ctx.setLineDash([]);
162 }
163 if (this.cursorTime !== null) {
164 const x = LEFT + (this.cursorTime / this.duration) * W;
165 ctx.strokeStyle = "rgba(230,230,230,0.7)";
166 ctx.beginPath(); ctx.moveTo(x, 0); ctx.lineTo(x, H); ctx.stroke();
167 }
168 }
169}
171export class EnergyBars {
172 private readonly canvas: HTMLCanvasElement;
173 readonly axis: FrequencyAxis; // shared with the spectrogram
174 dbRange = 80;
175 tracker: ModalTracker | null = null;
176 snapshot = -1;
177 selectedMode = -1;
178 onSelect: ((mode: number) => void) | null = null;
179 onHover: ((text: string) => void) | null = null;
181 constructor(canvas: HTMLCanvasElement, axis: FrequencyAxis) {
182 this.canvas = canvas; this.axis = axis;
183 canvas.addEventListener("pointerdown", (e) => { const m = this.pick(e); if (m >= 0) this.onSelect?.(m); });
184 canvas.addEventListener("pointermove", (e) => {
185 const m = this.pick(e);
186 if (m < 0 || !this.tracker || this.snapshot < 0) { this.onHover?.(""); return; }
187 const E = this.tracker.energies[this.snapshot][m];
188 this.onHover?.(`mode ${m} at ${(this.tracker.omega[m] / (2 * Math.PI)).toFixed(1)} Hz, ${this.db(E).toFixed(0)} dB`);
189 });
190 canvas.addEventListener("pointerleave", () => this.onHover?.(""));
191 }
193 private db(E: number) { return 10 * Math.log10(Math.max(E, 1e-300) / Math.max(this.tracker?.maxEnergy ?? 1, 1e-300)); }
195 /** Frequency is on x here (the transpose of the spectrogram's y). */
196 private x(f: number, W: number): number { return W - this.axis.y(f, W); }
198 private pick(e: PointerEvent): number {
199 const t = this.tracker;
200 if (!t) return -1;
201 const r = this.canvas.getBoundingClientRect();
202 const W = r.width - LEFT, px = e.clientX - r.left - LEFT;
203 let best = -1, bestD = 5;
204 for (let m = 0; m < t.nModes; m++) {
205 const d = Math.abs(this.x(t.omega[m] / (2 * Math.PI), W) - px);
206 if (d < bestD) { bestD = d; best = m; }
207 }
208 return best;
209 }
211 draw() {
212 const cv = this.canvas, dpr = window.devicePixelRatio || 1;
213 const CW = Math.max(1, Math.round(cv.clientWidth * dpr)), CH = Math.max(1, Math.round(cv.clientHeight * dpr));
214 if (cv.width !== CW || cv.height !== CH) { cv.width = CW; cv.height = CH; }
215 const ctx = cv.getContext("2d")!;
216 ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
217 const W = cv.clientWidth - LEFT, H = cv.clientHeight - BOTTOM;
218 ctx.fillStyle = "#0c0d12"; ctx.fillRect(0, 0, cv.clientWidth, cv.clientHeight);
219 if (W < 10 || H < 10) return;
220 ctx.fillStyle = "#9aa0a8"; ctx.font = "10px system-ui"; ctx.textAlign = "center"; ctx.textBaseline = "top";
221 for (const f of this.axis.ticks()) ctx.fillText(f >= 1000 ? `${f / 1000}k` : String(f), LEFT + this.x(f, W), H + 3);
222 ctx.textAlign = "right"; ctx.textBaseline = "middle";
223 for (let dB = 0; dB >= -this.dbRange; dB -= 20) {
224 const y = (-dB / this.dbRange) * H;
225 ctx.fillText(String(dB), LEFT - 4, Math.min(H - 6, Math.max(6, y)));
226 ctx.strokeStyle = "rgba(154,160,168,0.12)"; ctx.beginPath(); ctx.moveTo(LEFT, y); ctx.lineTo(LEFT + W, y); ctx.stroke();
227 }
228 const t = this.tracker;
229 if (!t || this.snapshot < 0 || this.snapshot >= t.length) return;
230 const E = t.energies[this.snapshot];
231 for (let m = 0; m < t.nModes; m++) {
232 const f = t.omega[m] / (2 * Math.PI);
233 if (f > this.axis.fmax) continue;
234 const dB = this.db(E[m]);
235 if (dB < -this.dbRange) continue;
236 const x = LEFT + this.x(f, W), h = (1 + dB / this.dbRange) * H;
237 const driven = m === t.drivenMode, sel = m === this.selectedMode;
238 ctx.strokeStyle = sel ? "#e0a23a" : driven ? "#6fbf73" : "#7f9fd0";
239 ctx.lineWidth = sel || driven ? 2 : 1;
240 ctx.beginPath(); ctx.moveTo(x, H); ctx.lineTo(x, H - h); ctx.stroke();
241 }
242 ctx.lineWidth = 1;
243 }
244}
246/** A true spectrogram (of the listener signal or of one modal coordinate) on the same frequency
247 * axis as the modal view, with the linear mode frequencies overlaid as ticks at the left edge.
248 * This is the complement of the modal-energy view: the modal view says which shapes hold the
249 * energy, pinned at their linear frequencies; this view says at which frequencies the signal
250 * actually oscillates, so combination tones between the modes, amplitude-shifted peaks and the
251 * broadband turbulent floor are visible against the modal grid. */
252export class StftView {
253 private readonly canvas: HTMLCanvasElement;
254 private readonly image = document.createElement("canvas");
255 readonly axis: FrequencyAxis;
256 dbRange = 80;
257 stft: Stft | null = null;
258 duration = 1;
259 cursorTime: number | null = null;
260 modeFreqs: Float64Array | null = null;
261 selectedMode = -1;
262 drivenMode = -1;
263 onSelect: ((time: number) => void) | null = null;
264 onHover: ((text: string) => void) | null = null;
265 private dirty = true;
267 constructor(canvas: HTMLCanvasElement, axis: FrequencyAxis) {
268 this.canvas = canvas; this.axis = axis;
269 canvas.addEventListener("pointerdown", (e) => {
270 const p = this.pos(e);
271 if (p) this.onSelect?.(p.time);
272 });
273 canvas.addEventListener("pointermove", (e) => {
274 const p = this.pos(e);
275 if (!p) { this.onHover?.(""); return; }
276 let text = `t = ${p.time.toFixed(2)} s, f = ${p.f.toFixed(0)} Hz`;
277 if (this.modeFreqs && this.modeFreqs.length) {
278 let best = 0;
279 for (let m = 1; m < this.modeFreqs.length; m++) if (Math.abs(this.modeFreqs[m] - p.f) < Math.abs(this.modeFreqs[best] - p.f)) best = m;
280 const df = p.f - this.modeFreqs[best];
281 text += `; nearest mode ${best} at ${this.modeFreqs[best].toFixed(0)} Hz (${df >= 0 ? "+" : ""}${df.toFixed(0)} Hz)`;
282 }
283 this.onHover?.(text);
284 });
285 canvas.addEventListener("pointerleave", () => this.onHover?.(""));
286 }
288 private pos(e: PointerEvent): { time: number; f: number } | null {
289 const r = this.canvas.getBoundingClientRect();
290 const W = r.width - LEFT, H = r.height - BOTTOM;
291 const px = e.clientX - r.left - LEFT, py = e.clientY - r.top;
292 if (px < 0 || py > H || W < 10) return null;
293 return { time: Math.max(0, Math.min(this.duration, (px / W) * this.duration)), f: this.axis.f(py, H) };
294 }
296 invalidate() { this.dirty = true; }
298 private renderImage(W: number, H: number) {
299 const img = this.image;
300 if (img.width !== W || img.height !== H) { img.width = W; img.height = H; }
301 const g = img.getContext("2d")!;
302 g.fillStyle = "#0c0d12"; g.fillRect(0, 0, W, H);
303 const st = this.stft;
304 if (!st || !st.columns.length) { this.dirty = false; return; }
305 let maxP = 0;
306 for (const col of st.columns) for (const p of col) maxP = Math.max(maxP, p);
307 if (!(maxP > 0)) { this.dirty = false; return; }
308 const data = g.getImageData(0, 0, W, H), px = data.data;
309 const cells = new Float32Array(W * H).fill(-1e30);
310 for (let c = 0; c < st.columns.length; c++) {
311 const x = Math.min(W - 1, Math.floor((st.columnTime(c) / this.duration) * W));
312 if (x < 0) continue;
313 const col = st.columns[c];
314 for (let b = 1; b < st.nBins; b++) {
315 const f = st.binFrequency(b);
316 if (f > this.axis.fmax) break;
317 const y = Math.round(this.axis.y(f, H));
318 if (y < 0 || y >= H) continue;
319 const dB = 10 * Math.log10(col[b] / maxP);
320 const i = y * W + x;
321 if (dB > cells[i]) cells[i] = dB;
322 }
323 }
324 for (let i = 0; i < W * H; i++) {
325 if (cells[i] < -1e29) continue;
326 const [r, gg, b] = rampColor(1 + cells[i] / this.dbRange);
327 px[4 * i] = r; px[4 * i + 1] = gg; px[4 * i + 2] = b; px[4 * i + 3] = 255;
328 }
329 g.putImageData(data, 0, 0);
330 this.dirty = false;
331 }
333 draw() {
334 const cv = this.canvas, dpr = window.devicePixelRatio || 1;
335 const CW = Math.max(1, Math.round(cv.clientWidth * dpr)), CH = Math.max(1, Math.round(cv.clientHeight * dpr));
336 if (cv.width !== CW || cv.height !== CH) { cv.width = CW; cv.height = CH; this.dirty = true; }
337 const ctx = cv.getContext("2d")!;
338 ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
339 const W = cv.clientWidth - LEFT, H = cv.clientHeight - BOTTOM;
340 if (W < 10 || H < 10) return;
341 if (this.dirty || this.image.width !== W || this.image.height !== H) this.renderImage(W, H);
342 ctx.fillStyle = "#14161a"; ctx.fillRect(0, 0, cv.clientWidth, cv.clientHeight);
343 ctx.drawImage(this.image, LEFT, 0);
344 // axes
345 ctx.fillStyle = "#9aa0a8"; ctx.font = "10px system-ui"; ctx.textAlign = "right"; ctx.textBaseline = "middle";
346 for (const f of this.axis.ticks()) {
347 const y = this.axis.y(f, H);
348 ctx.fillText(f >= 1000 ? `${f / 1000}k` : String(f), LEFT - 4, y);
349 ctx.strokeStyle = "rgba(154,160,168,0.12)"; ctx.beginPath(); ctx.moveTo(LEFT, y); ctx.lineTo(LEFT + W, y); ctx.stroke();
350 }
351 ctx.textAlign = "center"; ctx.textBaseline = "top";
352 const tStep = this.duration > 8 ? 2 : this.duration > 3 ? 1 : this.duration > 1.2 ? 0.5 : 0.2;
353 for (let tt = 0; tt <= this.duration + 1e-9; tt += tStep) {
354 const x = LEFT + (tt / this.duration) * W;
355 ctx.fillText(`${tt.toFixed(tt < 1 && tStep < 1 ? 1 : 0)}s`, x, H + 3);
356 }
357 // the linear modal grid: a tick at the left edge for every mode frequency
358 if (this.modeFreqs) {
359 ctx.strokeStyle = "rgba(127,159,208,0.55)";
360 ctx.beginPath();
361 for (let m = 0; m < this.modeFreqs.length; m++) {
362 const f = this.modeFreqs[m];
363 if (f > this.axis.fmax) continue;
364 const y = this.axis.y(f, H);
365 ctx.moveTo(LEFT, y); ctx.lineTo(LEFT + 6, y);
366 }
367 ctx.stroke();
368 const line = (m: number, color: string) => {
369 if (m < 0 || m >= this.modeFreqs!.length) return;
370 const y = this.axis.y(this.modeFreqs![m], H);
371 ctx.strokeStyle = color; ctx.setLineDash([3, 4]);
372 ctx.beginPath(); ctx.moveTo(LEFT, y); ctx.lineTo(LEFT + W, y); ctx.stroke(); ctx.setLineDash([]);
373 };
374 line(this.drivenMode, "rgba(111,191,115,0.7)");
375 if (this.selectedMode !== this.drivenMode) line(this.selectedMode, "rgba(224,162,58,0.8)");
376 }
377 if (this.cursorTime !== null) {
378 const x = LEFT + (this.cursorTime / this.duration) * W;
379 ctx.strokeStyle = "rgba(230,230,230,0.7)";
380 ctx.beginPath(); ctx.moveTo(x, 0); ctx.lineTo(x, H); ctx.stroke();
381 }
382 }
383}