Per-voice playback volume applied at mix time - proof-of-concept/instrawave-compose

proof-of-concept / instrawave-compose

Per-voice playback volume applied at mix time

Each voice row gains a volume slider. Render caching moves from one track per instrument to one track per voice (same GPU cost, since passes scale with events, and the bake stays shared), so volume scales a finished cached track at mix time and never triggers a render; retuning one voice now also re-renders only its own track.

Jeremy Magland <jmagland@flatironinstitute.org> committed 1 month ago commit 2d56c5e37a94 parent 30039cd Browse files

4 changed files+82−54

README.mdmodified+11−9View file

@@ -80,15 +80,17 @@ per instrument (a pressure trace carries no absolute scale worth keeping

8080 against a synth's), and mixed; relative volumes within one instrument stay

8181 physical.

8282

83-Rendered tracks are cached per instrument, in memory and in IndexedDB,

84-each stamped with exactly what it was rendered from: its events, tuning,

85-gains, and the bake key. Editing a synth lane or another instrument

86-therefore re-renders nothing, an unchanged take replays for free across

87-reloads, and a stored track plays without its bake even being loaded.

88-Each track runs to its own last event plus its longest decay, so its

89-signature cannot be perturbed by hits on other lanes. Save .wav downloads

90-the current mix as 16-bit PCM at 44.1 kHz, rendered through the same

91-caches.

83+Rendered tracks are cached per voice, in memory and in IndexedDB, each

84+stamped with exactly what it was rendered from: its events, tuning, gains,

85+instrument, and the bake key. Editing one voice therefore re-renders only

86+that voice's track, an unchanged take replays for free across reloads, and

87+a stored track plays without its bake even being loaded. Each track runs

88+to its own last event plus its longest decay, so its signature cannot be

89+perturbed by hits on other lanes. Each voice also carries a playback

90+volume slider; volume is deliberately absent from the track signature and

91+scales the finished track at mix time, so moving it never renders

92+anything. Save .wav downloads the current mix as 16-bit PCM at 44.1 kHz,

93+rendered through the same caches.

9294

9395 Loudness to excitation strength is the only feature mapping for now.

9496 Duration to decay, vocal pitch to drum pitch, and brightness to strike

index.htmlmodified+1−0View file

@@ -41,6 +41,7 @@

4141 .tuning-panel label span:last-child { opacity: 0.7; font-variant-numeric: tabular-nums; }

4242 .tuning-panel input[type='range'] { width: 100%; }

4343 .voice-row progress { width: 6rem; }

44+ .voice-row input.vol { width: 5rem; }

4445 #iwedit { border: 1px solid rgba(128,128,128,0.35); border-radius: 8px; padding: 0.6rem 0.8rem; margin: 0.5rem 0; }

4546 .iwedit-grid { display: grid; grid-template-columns: minmax(0, 1fr) 24rem; gap: 0 1.5rem; align-items: start; }

4647 @media (max-width: 56rem) { .iwedit-grid { grid-template-columns: 1fr; } }

src/main.tsmodified+61−44View file

@@ -20,7 +20,7 @@ import { IwEngine, parseBake, PhaseCancelled, type StimulusEvent } from './rende

2020 import { Viewer, type Camera } from 'instrawave/src/host/view.ts';

2121 import {

2222 clearBulk, clearStore, deleteClipAudio, getBakeBytes, getBakeKey, getClipAudio, getTakeAudio,

23- getRender, loadIwOverrides, loadTakeHits, loadUi, loadVoices, putBake, putClipAudio,

23+ deleteRender, getRender, loadIwOverrides, loadTakeHits, loadUi, loadVoices, putBake, putClipAudio,

2424 putRender, putTakeAudio,

2525 saveIwOverrides, saveTakeHits, saveUi, saveVoices, trainPool,

2626 type TakeHit, type TrainClip, type Voice,

@@ -428,6 +428,7 @@ function renderVoiceList(): void {

428428 if (!confirm(`Delete voice "${voice.name}"? Its training and its timeline events go with it.`)) return;

429429 for (const c of voice.clips) void deleteClipAudio(c.id);

430430 voices = voices.filter((v) => v.id !== voice.id);

431+ void deleteRender(voice.id);

431432 take.hits = take.hits.filter((h) => h.voiceId !== voice.id);

432433 saveTakeHits(take.hits);

433434 save();

@@ -478,9 +479,26 @@ function renderVoiceList(): void {

478479 iwControls.push(instBtn);

479480 }

480481

482+ // Playback volume: applied when tracks are mixed, so it never causes

483+ // a render; the cached track is simply scaled.

484+ const vol = document.createElement('input');

485+ vol.type = 'range';

486+ vol.min = '0';

487+ vol.max = '1.5';

488+ vol.step = '0.01';

489+ vol.className = 'vol';

490+ vol.value = String(voice.volume);

491+ const volTitle = () => { vol.title = `playback volume ${Math.round(voice.volume * 100)}%`; };

492+ volTitle();

493+ vol.addEventListener('input', () => {

494+ voice.volume = Number(vol.value);

495+ volTitle();

496+ save();

497+ });

498+

481499 const spacer = document.createElement('span');

482500 spacer.className = 'spacer';

483- row.append(swatch, name, count, rec, editBtn, clear, spacer, binding, tune, ...iwControls, del);

501+ row.append(swatch, name, count, rec, editBtn, clear, spacer, binding, vol, tune, ...iwControls, del);

484502 list.appendChild(row);

485503

486504 if (voice.binding.kind === 'iw' && expandedTuning.has(voice.id)) {

@@ -550,6 +568,7 @@ $('add-voice').addEventListener('click', () => {

550568 name,

551569 color: PALETTE[voices.length % PALETTE.length],

552570 binding: { kind: 'synth', preset: PRESETS[voices.length % PRESETS.length] },

571+ volume: 1,

553572 clips: [],

554573 loose: [],

555574 });

@@ -798,12 +817,14 @@ $('hit-learn').addEventListener('click', () => {

798817

799818 const iwTrackCache = new Map<string, { sig: string; data: Float32Array }>();

800819

801-/** The rendered track for one instrument's events: from the caches when

802- * the signature (events, tuning, gains, bake key) matches, memory first,

803- * then IndexedDB, where a stored track plays without its bake even

820+/** The rendered track for one instrawave voice's events: from the caches

821+ * when the signature (events, tuning, gains, bake key) matches, memory

822+ * first, then IndexedDB, where a stored track plays without its bake even

804823 * loaded; from the GPU otherwise. Returned at the mix rate, levelled to

805- * 0.8 peak. */

806-async function iwTrack(name: string, evs: StimulusEvent[]): Promise<Float32Array> {

824+ * 0.8 peak. Playback volume is deliberately absent from the signature. */

825+async function iwTrack(voice: Voice, evs: StimulusEvent[]): Promise<Float32Array> {

826+ if (voice.binding.kind !== 'iw') throw new Error(`voice "${voice.name}" is not bound to an instrument`);

827+ const name = voice.binding.instrument;

807828 evs.sort((a, b) => a.t - b.t);

808829 // The track's own length: its last event plus its longest decay, so an

809830 // edit on another lane cannot change this group's signature.

@@ -815,80 +836,76 @@ async function iwTrack(name: string, evs: StimulusEvent[]): Promise<Float32Array

815836 }

816837 const seconds = evs[evs.length - 1].t + tail;

817838 const sig = JSON.stringify({

839+ inst: name,

818840 key: await iw.currentKey(name),

819841 secs: seconds,

820842 evs: evs.map((e) => [e.t, Object.entries(e.set ?? {}).sort()]),

821843 });

822- const mem = iwTrackCache.get(name);

844+ const mem = iwTrackCache.get(voice.id);

823845 if (mem?.sig === sig) return mem.data;

824- const stored = await getRender(name).catch(() => null);

846+ const stored = await getRender(voice.id).catch(() => null);

825847 if (stored?.sig === sig) {

826- iwTrackCache.set(name, stored);

848+ iwTrackCache.set(voice.id, stored);

827849 return stored.data;

828850 }

829851 if (!(await ensureBakeLoaded(name))) {

830852 throw new Error(`precompute the ${name} bake first`);

831853 }

832854 const { trace, dt } = await iw.render(name, seconds, evs, (done, total) =>

833- status(`rendering ${name} (${evs.length} event${evs.length === 1 ? '' : 's'}): ${Math.round((100 * done) / total)}%`));

855+ status(`rendering "${voice.name}" on ${name} (${evs.length} event${evs.length === 1 ? '' : 's'}): ${Math.round((100 * done) / total)}%`));

834856 // Physical pressure amplitudes carry no absolute scale worth keeping

835857 // across engines; each instrument's track is levelled to sit alongside

836858 // the synth part, while relative volumes WITHIN it stay physical.

837859 const data = resampleLinear(trace, 1 / dt, MIX_SR);

838860 const p = peak(data);

839861 if (p > 0) for (let i = 0; i < data.length; i++) data[i] *= 0.8 / p;

840- iwTrackCache.set(name, { sig, data });

841- void putRender(name, sig, data).catch(() => { /* it will re-render next session */ });

862+ iwTrackCache.set(voice.id, { sig, data });

863+ void putRender(voice.id, sig, data).catch(() => { /* it will re-render next session */ });

842864 return data;

843865 }

844866

845867 async function renderMix(): Promise<AudioBuffer> {

846868 const sig = JSON.stringify({

847869 hits: take.hits.map((h) => [h.t, h.voiceId, h.gain]),

848- bindings: voices.map((v) => [v.id, v.binding]),

870+ bindings: voices.map((v) => [v.id, v.binding, v.volume]),

849871 iw: iw.dumpOverrides(),

850872 });

851873 if (renderCache?.sig === sig) return renderCache.buf;

852874

853- const synthEvents: NoteEvent[] = [];

854- const iwEvents = new Map<string, StimulusEvent[]>();

875+ // One track per voice, mixed at that voice's playback volume. The

876+ // volume scales a finished track, so moving a slider re-mixes cached

877+ // audio and renders nothing.

878+ const byVoice = new Map<string, TakeHit[]>();

855879 for (const h of take.hits) {

856- const v = voiceById(h.voiceId);

857- if (!v) continue;

880+ const list = byVoice.get(h.voiceId);

881+ if (list) list.push(h);

882+ else if (voiceById(h.voiceId)) byVoice.set(h.voiceId, [h]);

883+ }

884+ const tracks: { data: Float32Array; vol: number }[] = [];

885+ for (const [voiceId, hits] of byVoice) {

886+ const v = voiceById(voiceId)!;

858887 if (v.binding.kind === 'synth') {

859- synthEvents.push({ t: h.t, voiceId: h.voiceId, gain: h.gain });

888+ const preset = v.binding.preset;

889+ const events: NoteEvent[] = hits.map((h) => ({ t: h.t, voiceId, gain: h.gain }));

890+ const secs = Math.max(...events.map((e) => e.t)) + 1.5;

891+ const buf = await renderSynth(events, () => preset, secs);

892+ tracks.push({ data: buf.getChannelData(0), vol: v.volume });

860893 } else {

861894 const b = v.binding;

862895 const allowed = new Set(iw.tuning(b.instrument).map((p) => p.key));

863- const set: Record<string, number> = {};

864- for (const [k, val] of Object.entries(b.set)) if (allowed.has(k)) set[k] = val;

865896 const strength = iw.strengthParam(b.instrument);

866- set[strength] = (b.set[strength] ?? iw.defaultValue(b.instrument, strength)) * h.gain;

867- let list = iwEvents.get(b.instrument);

868- if (!list) iwEvents.set(b.instrument, (list = []));

869- list.push({ t: h.t, set });

897+ const events: StimulusEvent[] = hits.map((h) => {

898+ const set: Record<string, number> = {};

899+ for (const [k, val] of Object.entries(b.set)) if (allowed.has(k)) set[k] = val;

900+ set[strength] = (b.set[strength] ?? iw.defaultValue(b.instrument, strength)) * h.gain;

901+ return { t: h.t, set };

902+ });

903+ tracks.push({ data: await iwTrack(v, events), vol: v.volume });

870904 }

871905 }

872- const tracks: Float32Array[] = [];

873- for (const [name, evs] of iwEvents) tracks.push(await iwTrack(name, evs));

874-

875- const synthSeconds = synthEvents.length

876- ? Math.max(...synthEvents.map((e) => e.t)) + 1.5

877- : 0;

878- const seconds = Math.max(1, synthSeconds, ...tracks.map((tr) => tr.length / MIX_SR));

906+ const seconds = Math.max(1, ...tracks.map((tr) => tr.data.length / MIX_SR));

879907 const out = new Float32Array(Math.ceil(seconds * MIX_SR));

880- if (synthEvents.length) {

881- const buf = await renderSynth(

882- synthEvents,

883- (id) => {

884- const v = voiceById(id);

885- return v?.binding.kind === 'synth' ? v.binding.preset : 'kick';

886- },

887- synthSeconds,

888- );

889- mixInto(out, buf.getChannelData(0));

890- }

891- for (const tr of tracks) mixInto(out, tr);

908+ for (const tr of tracks) mixInto(out, tr.data, tr.vol);

892909 const pAll = peak(out);

893910 if (pAll > 0.98) for (let i = 0; i < out.length; i++) out[i] *= 0.95 / pAll;

894911 const buf = audio().createBuffer(1, out.length, MIX_SR);

src/store.tsmodified+9−1View file

@@ -24,6 +24,7 @@ export interface Voice {

2424 name: string;

2525 color: string;

2626 binding: Binding;

27+ volume: number; // playback level, applied at mix time; never renders

2728 clips: TrainClip[];

2829 loose: TrainHit[]; // hits learned from takes; they belong to no clip

2930 }

@@ -55,13 +56,17 @@ function read<T>(key: string): T | null {

5556

5657 export function loadVoices(): Voice[] {

5758 const doc = read<{ voices: Voice[] }>(KEY);

58- if (doc) return doc.voices;

59+ if (doc) {

60+ for (const v of doc.voices) v.volume ??= 1; // pre-volume stores

61+ return doc.voices;

62+ }

5963 // v1 stored a flat training list and a synth preset per voice.

6064 const v1 = read<{ voices: { id: string; name: string; color: string; preset: string; train: TrainHit[] }[] }>(KEY_V1);

6165 if (v1) {

6266 const voices: Voice[] = v1.voices.map((v) => ({

6367 id: v.id, name: v.name, color: v.color,

6468 binding: { kind: 'synth', preset: v.preset as ('kick' | 'snare' | 'hat' | 'tom') },

69+ volume: 1,

6570 clips: [], loose: v.train,

6671 }));

6772 localStorage.setItem(KEY, JSON.stringify({ voices }));

@@ -170,6 +175,9 @@ export function getRender(name: string): Promise<{ sig: string; data: Float32Arr

170175 return op<{ sig: string; data: Float32Array } | undefined>('renders', 'readonly', (s) => s.get(name))

171176 .then((r) => r ?? null);

172177 }

178+export function deleteRender(name: string): Promise<unknown> {

179+ return op('renders', 'readwrite', (s) => s.delete(name));

180+}

173181

174182 export async function clearBulk(): Promise<void> {

175183 for (const s of STORES) await op(s, 'readwrite', (st) => st.clear());