Per-instrument render cache in IndexedDB and .wav download - proof-of-concept/instrawave-compose

proof-of-concept / instrawave-compose

Per-instrument render cache in IndexedDB and .wav download

Rendered instrument tracks are cached against a signature of their own events, tuning, gains, and bake key, in memory and in IndexedDB, so edits elsewhere re-render nothing, unchanged takes replay free across reloads, and a stored track plays without its bake loaded. Each track renders to its last event plus its longest decay rather than the global take length. Save .wav downloads the mix as 16-bit PCM at 44.1 kHz.

Jeremy Magland <jmagland@flatironinstitute.org> committed 1 month ago commit 838af4352486 parent 9f20d5b Browse files

6 changed files+147−25

README.mdmodified+11−2View file

@@ -76,8 +76,17 @@ dulcimer; the equations are linear, so this is a pure scale). Traces come

7676 back at the solver rate, are resampled by linear interpolation, levelled

7777 per instrument (a pressure trace carries no absolute scale worth keeping

7878 against a synth's), and mixed; relative volumes within one instrument stay

79-physical. Mixes are cached against the exact events and bindings, so

80-replaying an unchanged take is free.

79+physical.

80+

81+Rendered tracks are cached per instrument, in memory and in IndexedDB,

82+each stamped with exactly what it was rendered from: its events, tuning,

83+gains, and the bake key. Editing a synth lane or another instrument

84+therefore re-renders nothing, an unchanged take replays for free across

85+reloads, and a stored track plays without its bake even being loaded.

86+Each track runs to its own last event plus its longest decay, so its

87+signature cannot be perturbed by hits on other lanes. Save .wav downloads

88+the current mix as 16-bit PCM at 44.1 kHz, rendered through the same

89+caches.

8190

8291 Loudness to excitation strength is the only feature mapping for now.

8392 Duration to decay, vocal pitch to drum pitch, and brightness to strike

index.htmlmodified+1−0View file

@@ -105,6 +105,7 @@

105105 <button id="take-redetect" disabled>Re-detect & classify</button>

106106 <span style="flex:1"></span>

107107 <button id="take-play" disabled>Play</button>

108+ <button id="take-save" disabled>Save .wav</button>

108109 </div>

109110 <div class="bar" id="bakebar" hidden></div>

110111

scripts/test-node.tsmodified+14−0View file

@@ -9,6 +9,7 @@ import { extractHit } from '../src/analysis/features.ts';

99 import { classify, type TrainingSet } from '../src/analysis/classify.ts';

1010 import { gainFromPeakDb } from '../src/render/types.ts';

1111 import { mixInto, peak, resampleLinear } from '../src/render/mix.ts';

12+import { wavBytes } from '../src/render/wav.ts';

1213

1314 const SR = 48000;

1415 let failures = 0;

@@ -109,5 +110,18 @@ function burst(x: Float32Array, t0: number, freq: number, decay: number, amp: nu

109110 check(Math.abs(peak(new Float32Array([0.1, -0.9, 0.4])) - 0.9) < 1e-6, 'peak: absolute maximum');

110111 }

111112

113+// --- wav encoding ----------------------------------------------------------------

114+

115+{

116+ const x = new Float32Array([0, 0.5, -0.5, 2]); // the last clips

117+ const bytes = wavBytes(x, 44100);

118+ const v = new DataView(bytes.buffer);

119+ check(bytes.length === 44 + 8, `wav: ${bytes.length} bytes for 4 samples`);

120+ check(String.fromCharCode(bytes[0], bytes[1], bytes[2], bytes[3]) === 'RIFF', 'wav: RIFF magic');

121+ check(v.getUint32(24, true) === 44100, 'wav: sample rate field');

122+ check(v.getInt16(46, true) === Math.round(0.5 * 32767), 'wav: 0.5 encodes to half scale');

123+ check(v.getInt16(50, true) === 32767, 'wav: over-range clips to full scale');

124+}

125+

112126 console.log(failures ? `\n${failures} failure(s)` : '\nall checks passed');

113127 process.exit(failures ? 1 : 0);

src/main.tsmodified+79−21View file

@@ -15,11 +15,13 @@ import { classify, type TrainingSet } from './analysis/classify.ts';

1515 import { renderSynth } from './render/synth.ts';

1616 import { gainFromPeakDb, type NoteEvent, type SynthPreset } from './render/types.ts';

1717 import { mixInto, peak, resampleLinear } from './render/mix.ts';

18+import { wavBytes } from './render/wav.ts';

1819 import { IwEngine, parseBake, PhaseCancelled, type StimulusEvent } from './render/instrawave.ts';

1920 import { Viewer, type Camera } from 'instrawave/src/host/view.ts';

2021 import {

2122 clearBulk, clearStore, deleteClipAudio, getBakeBytes, getBakeKey, getClipAudio, getTakeAudio,

22- loadIwOverrides, loadTakeHits, loadUi, loadVoices, putBake, putClipAudio, putTakeAudio,

23+ getRender, loadIwOverrides, loadTakeHits, loadUi, loadVoices, putBake, putClipAudio,

24+ putRender, putTakeAudio,

2325 saveIwOverrides, saveTakeHits, saveUi, saveVoices, trainPool,

2426 type TakeHit, type TrainClip, type Voice,

2527 } from './store.ts';

@@ -683,6 +685,7 @@ function syncTakeBar(): void {

683685 btn.disabled = recorder !== null && !recordingTake;

684686 ($('take-redetect') as HTMLButtonElement).disabled = take.clip === null;

685687 ($('take-play') as HTMLButtonElement).disabled = take.hits.length === 0;

688+ ($('take-save') as HTMLButtonElement).disabled = take.hits.length === 0;

686689 }

687690

688691 function syncPlayButton(): void {

@@ -770,8 +773,53 @@ $('hit-learn').addEventListener('click', () => {

770773

771774 // ---- the mix ----------------------------------------------------------------

772775

776+const iwTrackCache = new Map<string, { sig: string; data: Float32Array }>();

777+

778+/** The rendered track for one instrument's events: from the caches when

779+ * the signature (events, tuning, gains, bake key) matches, memory first,

780+ * then IndexedDB, where a stored track plays without its bake even

781+ * loaded; from the GPU otherwise. Returned at the mix rate, levelled to

782+ * 0.8 peak. */

783+async function iwTrack(name: string, evs: StimulusEvent[]): Promise<Float32Array> {

784+ evs.sort((a, b) => a.t - b.t);

785+ // The track's own length: its last event plus its longest decay, so an

786+ // edit on another lane cannot change this group's signature.

787+ let tail = 1.5;

788+ const hasT60 = iw.tuning(name).some((p) => p.key === 't60');

789+ for (const e of evs) {

790+ const t60 = e.set?.t60 ?? (hasT60 ? iw.defaultValue(name, 't60') : 1.5);

791+ tail = Math.max(tail, Math.min(6, t60));

792+ }

793+ const seconds = evs[evs.length - 1].t + tail;

794+ const sig = JSON.stringify({

795+ key: await iw.currentKey(name),

796+ secs: seconds,

797+ evs: evs.map((e) => [e.t, Object.entries(e.set ?? {}).sort()]),

798+ });

799+ const mem = iwTrackCache.get(name);

800+ if (mem?.sig === sig) return mem.data;

801+ const stored = await getRender(name).catch(() => null);

802+ if (stored?.sig === sig) {

803+ iwTrackCache.set(name, stored);

804+ return stored.data;

805+ }

806+ if (!(await ensureBakeLoaded(name))) {

807+ throw new Error(`precompute the ${name} bake first`);

808+ }

809+ const { trace, dt } = await iw.render(name, seconds, evs, (done, total) =>

810+ status(`rendering ${name} (${evs.length} event${evs.length === 1 ? '' : 's'}): ${Math.round((100 * done) / total)}%`));

811+ // Physical pressure amplitudes carry no absolute scale worth keeping

812+ // across engines; each instrument's track is levelled to sit alongside

813+ // the synth part, while relative volumes WITHIN it stay physical.

814+ const data = resampleLinear(trace, 1 / dt, MIX_SR);

815+ const p = peak(data);

816+ if (p > 0) for (let i = 0; i < data.length; i++) data[i] *= 0.8 / p;

817+ iwTrackCache.set(name, { sig, data });

818+ void putRender(name, sig, data).catch(() => { /* it will re-render next session */ });

819+ return data;

820+}

821+

773822 async function renderMix(): Promise<AudioBuffer> {

774- const seconds = Math.max(...take.hits.map((h) => h.t)) + 1.5;

775823 const sig = JSON.stringify({

776824 hits: take.hits.map((h) => [h.t, h.voiceId, h.gain]),

777825 bindings: voices.map((v) => [v.id, v.binding]),

@@ -798,14 +846,13 @@ async function renderMix(): Promise<AudioBuffer> {

798846 list.push({ t: h.t, set });

799847 }

800848 }

801- const missing: string[] = [];

802- for (const n of iwEvents.keys()) {

803- if (!(await ensureBakeLoaded(n))) missing.push(n);

804- }

805- if (missing.length) {

806- throw new Error(`precompute the ${missing.join(' and ')} bake${missing.length > 1 ? 's' : ''} first`);

807- }

849+ const tracks: Float32Array[] = [];

850+ for (const [name, evs] of iwEvents) tracks.push(await iwTrack(name, evs));

808851

852+ const synthSeconds = synthEvents.length

853+ ? Math.max(...synthEvents.map((e) => e.t)) + 1.5

854+ : 0;

855+ const seconds = Math.max(1, synthSeconds, ...tracks.map((tr) => tr.length / MIX_SR));

809856 const out = new Float32Array(Math.ceil(seconds * MIX_SR));

810857 if (synthEvents.length) {

811858 const buf = await renderSynth(

@@ -814,21 +861,11 @@ async function renderMix(): Promise<AudioBuffer> {

814861 const v = voiceById(id);

815862 return v?.binding.kind === 'synth' ? v.binding.preset : 'kick';

816863 },

817- seconds,

864+ synthSeconds,

818865 );

819866 mixInto(out, buf.getChannelData(0));

820867 }

821- for (const [name, evs] of iwEvents) {

822- evs.sort((a, b) => a.t - b.t);

823- const { trace, dt } = await iw.render(name, seconds, evs, (done, total) =>

824- status(`rendering ${name} (${evs.length} events): ${Math.round((100 * done) / total)}%`));

825- // Physical pressure amplitudes carry no absolute scale worth keeping

826- // across engines; each instrument's trace is levelled to sit alongside

827- // the synth part, while relative volumes WITHIN it stay physical.

828- const res = resampleLinear(trace, 1 / dt, MIX_SR);

829- const p = peak(res);

830- if (p > 0) mixInto(out, res, 0.8 / p);

831- }

868+ for (const tr of tracks) mixInto(out, tr);

832869 const pAll = peak(out);

833870 if (pAll > 0.98) for (let i = 0; i < out.length; i++) out[i] *= 0.95 / pAll;

834871 const buf = audio().createBuffer(1, out.length, MIX_SR);

@@ -837,6 +874,27 @@ async function renderMix(): Promise<AudioBuffer> {

837874 return buf;

838875 }

839876

877+$('take-save').addEventListener('click', async () => {

878+ if (take.hits.length === 0) return;

879+ const btn = $('take-save') as HTMLButtonElement;

880+ btn.disabled = true;

881+ try {

882+ const buf = await renderMix();

883+ const bytes = wavBytes(buf.getChannelData(0), MIX_SR);

884+ const url = URL.createObjectURL(new Blob([bytes as Uint8Array<ArrayBuffer>], { type: 'audio/wav' }));

885+ const a = document.createElement('a');

886+ a.href = url;

887+ a.download = 'instrawave-compose-take.wav';

888+ a.click();

889+ URL.revokeObjectURL(url);

890+ status(`saved ${(buf.duration).toFixed(1)} s of audio`);

891+ } catch (e) {

892+ status(e instanceof Error ? e.message : String(e));

893+ } finally {

894+ btn.disabled = take.hits.length === 0;

895+ }

896+});

897+

840898 $('take-play').addEventListener('click', async () => {

841899 if (playing) { stopPlayback(); return; }

842900 if (take.hits.length === 0) return;

src/render/wav.tsadded+29−0View file

@@ -0,0 +1,29 @@

1+/** A mono float trace as 16-bit PCM .wav bytes, clipped to [-1, 1]. The

2+ * mix is already levelled and at the mix rate, so no resampling or

3+ * normalization happens here. */

4+export function wavBytes(samples: Float32Array, sr: number): Uint8Array {

5+ const n = samples.length;

6+ const buf = new ArrayBuffer(44 + 2 * n);

7+ const view = new DataView(buf);

8+ const str = (off: number, s: string): void => {

9+ for (let i = 0; i < s.length; i++) view.setUint8(off + i, s.charCodeAt(i));

10+ };

11+ str(0, 'RIFF');

12+ view.setUint32(4, 36 + 2 * n, true);

13+ str(8, 'WAVE');

14+ str(12, 'fmt ');

15+ view.setUint32(16, 16, true);

16+ view.setUint16(20, 1, true); // PCM

17+ view.setUint16(22, 1, true); // mono

18+ view.setUint32(24, sr, true);

19+ view.setUint32(28, 2 * sr, true);

20+ view.setUint16(32, 2, true);

21+ view.setUint16(34, 16, true);

22+ str(36, 'data');

23+ view.setUint32(40, 2 * n, true);

24+ for (let i = 0; i < n; i++) {

25+ const v = Math.max(-1, Math.min(1, samples[i]));

26+ view.setInt16(44 + 2 * i, Math.round(v * 32767), true);

27+ }

28+ return new Uint8Array(buf);

29+}

src/store.tsmodified+13−2View file

@@ -104,12 +104,12 @@ export function clearStore(): void {

104104

105105 // ---- bulk data in IndexedDB ---------------------------------------------------

106106

107-const STORES = ['clips', 'takes', 'bakes', 'bakemeta'] as const;

107+const STORES = ['clips', 'takes', 'bakes', 'bakemeta', 'renders'] as const;

108108

109109 let dbPromise: Promise<IDBDatabase> | null = null;

110110 function db(): Promise<IDBDatabase> {

111111 return (dbPromise ??= new Promise((resolve, reject) => {

112- const req = indexedDB.open('instrawave-compose', 2);

112+ const req = indexedDB.open('instrawave-compose', 3);

113113 req.onupgradeneeded = () => {

114114 for (const s of STORES) {

115115 if (!req.result.objectStoreNames.contains(s)) req.result.createObjectStore(s);

@@ -160,6 +160,17 @@ export function getBakeBytes(name: string): Promise<Uint8Array | null> {

160160 return op<Uint8Array | undefined>('bakes', 'readonly', (s) => s.get(name)).then((r) => r ?? null);

161161 }

162162

163+/** A rendered instrument track at the mix rate, keyed by the instrument

164+ * and stamped with the exact events, tuning, and bake key it was rendered

165+ * from; a matching signature means the GPU has nothing to redo. */

166+export function putRender(name: string, sig: string, data: Float32Array): Promise<unknown> {

167+ return op('renders', 'readwrite', (s) => s.put({ sig, data }, name));

168+}

169+export function getRender(name: string): Promise<{ sig: string; data: Float32Array } | null> {

170+ return op<{ sig: string; data: Float32Array } | undefined>('renders', 'readonly', (s) => s.get(name))

171+ .then((r) => r ?? null);

172+}

173+

163174 export async function clearBulk(): Promise<void> {

164175 for (const s of STORES) await op(s, 'readwrite', (st) => st.clear());

165176 }