/** * The page: voices (named, trained by example, each bound to a synth * preset or an instrawave instrument), a take (recorded once, segmented, * classified onto the voices), a timeline (the take as editable events), * and playback through the renderer seam in render/types.ts. Training * clips persist (audio in IndexedDB, detections and features in * localStorage) and reopen in the same editor they were made in. The GPU * is touched only when at least one bound instrument is instrawave. */ import type { Clip } from './audio/recorder.ts'; import { Recorder } from './audio/recorder.ts'; import { detectOnsets } from './analysis/onsets.ts'; import { extractHit } from './analysis/features.ts'; import { classify, type TrainingSet } from './analysis/classify.ts'; import { renderSynth } from './render/synth.ts'; import { gainFromPeakDb, type NoteEvent, type SynthPreset } from './render/types.ts'; import { mixInto, peak, resampleLinear } from './render/mix.ts'; import { IwEngine, parseBake, PhaseCancelled, type StimulusEvent } from './render/instrawave.ts'; import { Viewer, type Camera } from 'instrawave/src/host/view.ts'; import { clearBulk, clearStore, deleteClipAudio, getBakeBytes, getBakeKey, getClipAudio, getTakeAudio, loadIwOverrides, loadTakeHits, loadUi, loadVoices, putBake, putClipAudio, putTakeAudio, saveIwOverrides, saveTakeHits, saveUi, saveVoices, trainPool, type TakeHit, type TrainClip, type Voice, } from './store.ts'; import { WaveView, type Marker } from './ui/waveform.ts'; import { Timeline } from './ui/timeline.ts'; const PALETTE = ['#e4573d', '#3d7de4', '#3dbb6e', '#d9a520', '#9061d9', '#d95fa8', '#4fb8c9', '#96a13a']; const PRESETS: SynthPreset[] = ['kick', 'snare', 'hat', 'tom']; const MIX_SR = 44100; const $ = (id: string) => document.getElementById(id) as T; const uid = () => Math.random().toString(36).slice(2, 10); // ---- state ---------------------------------------------------------------- let voices: Voice[] = loadVoices(); let take: { clip: Clip | null; hits: TakeHit[] } = { clip: null, hits: [] }; let edit: { voiceId: string; clipId: string; samples: Float32Array; sr: number } | null = null; let recTarget: { kind: 'voice'; voiceId: string } | { kind: 'take' } | null = null; const expandedTuning = new Set(); let actx: AudioContext | null = null; const audio = (): AudioContext => (actx ??= new AudioContext()); let recorder: Recorder | null = null; let playing: { src: AudioBufferSourceNode; t0: number; dur: number } | null = null; let renderCache: { sig: string; buf: AudioBuffer } | null = null; const iw = new IwEngine(); iw.setOverrides(loadIwOverrides()); const saveIw = () => saveIwOverrides(iw.dumpOverrides()); const ui = loadUi(); let bakeJob: { name: string; cancel: boolean } | null = null; let iwBusy = false; let bakeProg: HTMLProgressElement | null = null; function save(): void { saveVoices(voices); } function voiceById(id: string): Voice | undefined { return voices.find((v) => v.id === id); } function status(msg: string): void { $('status').textContent = msg; } // ---- audition ------------------------------------------------------------- function playSlice(samples: Float32Array, sr: number, s0: number, s1: number): void { const ctx = audio(); const n = Math.min(s1, s0 + Math.floor(0.5 * sr)) - s0; if (n <= 0) return; const buf = ctx.createBuffer(1, n, sr); buf.getChannelData(0).set(samples.subarray(s0, s0 + n)); const src = ctx.createBufferSource(); src.buffer = buf; src.connect(ctx.destination); src.start(); } async function playBuffer(buf: AudioBuffer, onFrame?: (t: number | null) => void): Promise { const ctx = audio(); if (ctx.state === 'suspended') await ctx.resume(); const src = ctx.createBufferSource(); src.buffer = buf; src.connect(ctx.destination); const t0 = ctx.currentTime; playing = { src, t0, dur: buf.duration }; src.onended = () => { if (playing?.src === src) playing = null; onFrame?.(null); syncPlayButton(); }; src.start(); if (onFrame) { const tick = () => { if (!playing || playing.src !== src) return; onFrame(ctx.currentTime - t0); requestAnimationFrame(tick); }; requestAnimationFrame(tick); } syncPlayButton(); } function stopPlayback(): void { playing?.src.stop(); playing = null; timeline.playhead(null); syncPlayButton(); } // ---- recording ------------------------------------------------------------ let meterRaf = 0; function meterOn(): void { $('meter').hidden = false; const fill = $('meter').firstElementChild as HTMLElement; const tick = () => { fill.style.width = `${(recorder?.level() ?? 0) * 100}%`; if (recorder?.recording) meterRaf = requestAnimationFrame(tick); }; meterRaf = requestAnimationFrame(tick); } function meterOff(): void { cancelAnimationFrame(meterRaf); $('meter').hidden = true; } async function startRecording(target: NonNullable): Promise { recorder = new Recorder(audio()); try { await recorder.start(); } catch (e) { status(`microphone unavailable: ${e instanceof Error ? e.message : e}`); recorder = null; return; } recTarget = target; meterOn(); renderVoiceList(); syncTakeBar(); status(target.kind === 'take' ? 'recording the take; stop when the phrase is done' : `recording examples for "${voiceById(target.voiceId)?.name}"; make the sound several times, then stop`); } async function stopRecording(): Promise { if (!recorder) return; const target = recTarget!; let clip: Clip; try { clip = await recorder.stop(); } finally { recorder = null; recTarget = null; meterOff(); } renderVoiceList(); syncTakeBar(); if (target.kind === 'voice') { const voice = voiceById(target.voiceId); if (!voice) return; const sens = Number(($('capture-sens') as HTMLInputElement).value); const tclip: TrainClip = { id: uid(), sr: clip.sr, sens, onsets: detectOnsets(clip.samples, clip.sr, sens), excluded: [], hits: [], }; recomputeHits(tclip, clip.samples); voice.clips.push(tclip); save(); void putClipAudio(tclip.id, clip.samples, clip.sr).catch(() => status('warning: clip audio could not be stored; this clip will not be re-editable after a reload')); openEditor(voice.id, tclip.id, { samples: clip.samples, sr: clip.sr }); renderVoiceList(); } else { take.clip = clip; void putTakeAudio(clip.samples, clip.sr).catch(() => status('warning: take audio could not be stored; it will not survive a reload')); processTake(); } } // ---- training clip editor ---------------------------------------------------- const waveView = new WaveView($('capture-wave') as HTMLCanvasElement); function editingClip(): TrainClip | null { if (!edit) return null; return voiceById(edit.voiceId)?.clips.find((c) => c.id === edit!.clipId) ?? null; } function recomputeHits(clip: TrainClip, samples: Float32Array): void { const ex = new Set(clip.excluded); clip.hits = []; for (let i = 0; i < clip.onsets.length; i++) { if (ex.has(i)) continue; const s0 = Math.floor(clip.onsets[i] * clip.sr); const s1 = i + 1 < clip.onsets.length ? Math.floor(clip.onsets[i + 1] * clip.sr) : samples.length; const f = extractHit(samples, clip.sr, s0, s1); clip.hits.push({ vec: f.vec, peakDb: f.peakDb }); } } function openEditor(voiceId: string, clipId: string, audioData?: { samples: Float32Array; sr: number }): void { const voice = voiceById(voiceId); const clip = voice?.clips.find((c) => c.id === clipId); if (!voice || !clip) return; const open = (samples: Float32Array, sr: number) => { edit = { voiceId, clipId, samples, sr }; ($('capture-sens') as HTMLInputElement).value = String(clip.sens); $('capture').hidden = false; refreshEditor(); }; if (audioData) open(audioData.samples, audioData.sr); else { void getClipAudio(clipId).then((a) => { if (a) open(a.samples, a.sr); else status('this clip\'s audio is missing (stored before audio persistence, or cleared); its hits still classify'); }); } } function refreshEditor(): void { const voice = edit && voiceById(edit.voiceId); const clip = editingClip(); if (!edit || !voice || !clip) return; $('capture-title').textContent = `"${voice.name}"`; const sel = $('capture-clip') as HTMLSelectElement; sel.textContent = ''; voice.clips.forEach((c, i) => { const o = document.createElement('option'); o.value = c.id; o.textContent = `clip ${i + 1} · ${c.hits.length} hit${c.hits.length === 1 ? '' : 's'}`; sel.appendChild(o); }); sel.value = clip.id; const markers: Marker[] = clip.onsets.map((t, i) => ({ t, excluded: clip.excluded.includes(i) })); waveView.set({ samples: edit.samples, sr: edit.sr }, markers, voice.color); syncEditorButtons(); } function syncEditorButtons(): void { const clip = editingClip(); const n = waveView.selection.size; const btn = $('capture-exclude') as HTMLButtonElement; btn.disabled = n === 0; if (clip && n > 0) { const allExcluded = [...waveView.selection].every((i) => clip.excluded.includes(i)); btn.textContent = allExcluded ? `Include (${n})` : `Exclude (${n})`; } else btn.textContent = 'Exclude'; $('capture-count').textContent = clip ? `${clip.hits.length} of ${clip.onsets.length} hits included${n ? ` · ${n} selected` : ''}` : ''; } function toggleExcluded(): void { const clip = editingClip(); if (!edit || !clip || waveView.selection.size === 0) return; const sel = [...waveView.selection]; const allExcluded = sel.every((i) => clip.excluded.includes(i)); const ex = new Set(clip.excluded); for (const i of sel) { if (allExcluded) ex.delete(i); else ex.add(i); } clip.excluded = [...ex].sort((a, b) => a - b); recomputeHits(clip, edit.samples); save(); waveView.updateMarkers(clip.onsets.map((t, i) => ({ t, excluded: ex.has(i) }))); syncEditorButtons(); renderVoiceList(); } waveView.onAudition = (i) => { const clip = editingClip(); if (!edit || !clip) return; const s0 = Math.floor(clip.onsets[i] * edit.sr); const s1 = i + 1 < clip.onsets.length ? Math.floor(clip.onsets[i + 1] * edit.sr) : edit.samples.length; playSlice(edit.samples, edit.sr, s0, s1); }; waveView.onSelectionChange = syncEditorButtons; $('capture-exclude').addEventListener('click', toggleExcluded); $('capture-sens').addEventListener('input', () => { ui.captureSens = Number(($('capture-sens') as HTMLInputElement).value); saveUi(ui); const clip = editingClip(); if (!edit || !clip) return; clip.sens = Number(($('capture-sens') as HTMLInputElement).value); clip.onsets = detectOnsets(edit.samples, edit.sr, clip.sens); clip.excluded = []; recomputeHits(clip, edit.samples); save(); refreshEditor(); renderVoiceList(); }); $('capture-clip').addEventListener('change', () => { if (!edit) return; openEditor(edit.voiceId, ($('capture-clip') as HTMLSelectElement).value); }); $('capture-delete').addEventListener('click', () => { const voice = edit && voiceById(edit.voiceId); const clip = editingClip(); if (!edit || !voice || !clip) return; if (!confirm(`Delete this clip and its ${clip.hits.length} trained hits?`)) return; void deleteClipAudio(clip.id); voice.clips = voice.clips.filter((c) => c.id !== clip.id); save(); renderVoiceList(); if (voice.clips.length) openEditor(voice.id, voice.clips[voice.clips.length - 1].id); else closeEditor(); }); function closeEditor(): void { edit = null; $('capture').hidden = true; } $('capture-done').addEventListener('click', closeEditor); // ---- voices --------------------------------------------------------------- function bindingValue(v: Voice): string { return v.binding.kind === 'synth' ? `synth:${v.binding.preset}` : `iw:${v.binding.instrument}`; } function renderVoiceList(): void { const list = $('voice-list'); list.textContent = ''; const busy = recorder !== null; for (const voice of voices) { const row = document.createElement('div'); row.className = 'voice-row'; const swatch = document.createElement('span'); swatch.className = 'swatch'; swatch.style.background = voice.color; const name = document.createElement('input'); name.className = 'name'; name.value = voice.name; name.addEventListener('change', () => { voice.name = name.value.trim() || voice.name; name.value = voice.name; save(); refreshTimeline(); if (edit?.voiceId === voice.id) refreshEditor(); }); const n = trainPool(voice).length; const count = document.createElement('span'); count.className = 'count'; count.textContent = `${voice.clips.length} clip${voice.clips.length === 1 ? '' : 's'} · ${n} hit${n === 1 ? '' : 's'}`; const rec = document.createElement('button'); const recordingThis = recTarget?.kind === 'voice' && recTarget.voiceId === voice.id; rec.innerHTML = recordingThis ? '■ Stop' : '● Record'; if (recordingThis) rec.className = 'rec'; rec.disabled = busy && !recordingThis; rec.addEventListener('click', () => { if (recordingThis) void stopRecording(); else void startRecording({ kind: 'voice', voiceId: voice.id }); }); const editBtn = document.createElement('button'); editBtn.textContent = 'Edit'; editBtn.title = 'reopen this voice\'s training clips'; editBtn.disabled = voice.clips.length === 0; editBtn.addEventListener('click', () => { openEditor(voice.id, voice.clips[voice.clips.length - 1].id); }); const binding = document.createElement('select'); const gSynth = document.createElement('optgroup'); gSynth.label = 'synth'; for (const p of PRESETS) { const o = document.createElement('option'); o.value = `synth:${p}`; o.textContent = p; gSynth.appendChild(o); } const gIw = document.createElement('optgroup'); gIw.label = 'instrawave'; for (const nm of iw.names()) { const o = document.createElement('option'); o.value = `iw:${nm}`; o.textContent = nm; gIw.appendChild(o); } binding.append(gSynth, gIw); binding.value = bindingValue(voice); binding.addEventListener('change', () => { const [kind, which] = binding.value.split(':'); if (kind === 'synth') voice.binding = { kind: 'synth', preset: which as SynthPreset }; else { const old = voice.binding; const set = old.kind === 'iw' && old.instrument === which ? old.set : {}; voice.binding = { kind: 'iw', instrument: which, set }; } save(); renderVoiceList(); updateBakeBar(); }); const tune = document.createElement('button'); tune.textContent = expandedTuning.has(voice.id) ? 'Tune ▾' : 'Tune ▸'; tune.disabled = voice.binding.kind !== 'iw'; tune.addEventListener('click', () => { if (expandedTuning.has(voice.id)) expandedTuning.delete(voice.id); else expandedTuning.add(voice.id); renderVoiceList(); }); const clear = document.createElement('button'); clear.textContent = 'Clear'; clear.title = 'forget this voice\'s training'; clear.disabled = n === 0; clear.addEventListener('click', () => { if (!confirm(`Forget all ${n} trained hits of "${voice.name}"?`)) return; for (const c of voice.clips) void deleteClipAudio(c.id); voice.clips = []; voice.loose = []; save(); if (edit?.voiceId === voice.id) closeEditor(); renderVoiceList(); }); const del = document.createElement('button'); del.textContent = '✕'; del.title = 'delete voice'; del.addEventListener('click', () => { if (!confirm(`Delete voice "${voice.name}"? Its training and its timeline events go with it.`)) return; for (const c of voice.clips) void deleteClipAudio(c.id); voices = voices.filter((v) => v.id !== voice.id); take.hits = take.hits.filter((h) => h.voiceId !== voice.id); saveTakeHits(take.hits); save(); if (edit?.voiceId === voice.id) closeEditor(); renderVoiceList(); refreshTimeline(); updateBakeBar(); }); const spacer = document.createElement('span'); spacer.className = 'spacer'; row.append(swatch, name, count, rec, editBtn, clear, spacer, binding, tune, del); list.appendChild(row); if (voice.binding.kind === 'iw' && expandedTuning.has(voice.id)) { list.appendChild(tuningPanel(voice, voice.binding.instrument, voice.binding.set)); } } } function tuningPanel(voice: Voice, instrument: string, set: Record): HTMLElement { const panel = document.createElement('div'); panel.className = 'tuning-panel'; const bar = document.createElement('div'); bar.className = 'bar'; bar.style.gridColumn = '1 / -1'; bar.style.margin = '0'; const hear = document.createElement('button'); hear.textContent = 'Hear'; hear.addEventListener('click', () => void auditionIw(instrument, set)); const hint = document.createElement('span'); hint.className = 'hint'; hint.textContent = 'one hit at these settings, from the current bake'; bar.append(hear, hint); panel.appendChild(bar); for (const p of iw.tuning(instrument)) { const div = document.createElement('div'); div.className = 'param'; const label = document.createElement('label'); const nameSpan = document.createElement('span'); nameSpan.textContent = `${p.label} (${p.unit})`; const valSpan = document.createElement('span'); const input = document.createElement('input'); input.type = 'range'; input.min = String(p.min); input.max = String(p.max); input.step = String(p.step ?? (p.max - p.min) / 200); input.value = String(set[p.key] ?? p.value); const show = () => { valSpan.textContent = Number(input.value).toPrecision(4); }; show(); input.addEventListener('input', () => { set[p.key] = Number(input.value); show(); save(); }); label.append(nameSpan, valSpan); div.append(label, input); panel.appendChild(div); } return panel; } $('add-voice').addEventListener('click', () => { const input = $('voice-name') as HTMLInputElement; const name = input.value.trim() || `voice ${voices.length + 1}`; voices.push({ id: uid(), name, color: PALETTE[voices.length % PALETTE.length], binding: { kind: 'synth', preset: PRESETS[voices.length % PRESETS.length] }, clips: [], loose: [], }); input.value = ''; save(); renderVoiceList(); refreshTimeline(); }); $('reset-all').addEventListener('click', () => { if (!confirm('Forget all voices, training, and the current take?')) return; clearStore(); void clearBulk(); iw.setOverrides({}); saveIw(); voices = []; closeIwEditor(); take = { clip: null, hits: [] }; closeEditor(); renderVoiceList(); refreshTimeline(); syncTakeBar(); updateBakeBar(); status(''); }); // ---- bakes ------------------------------------------------------------------ function neededInstruments(): string[] { return [...new Set(voices.filter((v) => v.binding.kind === 'iw') .map((v) => (v.binding as { instrument: string }).instrument))]; } function updateBakeBar(): void { void refreshBakeBar(); } let bakeBarEpoch = 0; async function refreshBakeBar(): Promise { const epoch = ++bakeBarEpoch; const bar = $('bakebar'); const needed = neededInstruments(); bar.hidden = needed.length === 0; if (needed.length === 0) { bar.textContent = ''; return; } const states = await Promise.all(needed.map(async (name) => { if (await iw.bakeReady(name)) return 'ready' as const; if (bakeJob?.name === name) return 'baking' as const; const stored = await getBakeKey(name).catch(() => null); return stored === (await iw.currentKey(name)) ? ('stored' as const) : ('none' as const); })); if (epoch !== bakeBarEpoch) return; // a newer refresh superseded this one bar.textContent = ''; bakeProg = null; needed.forEach((name, i) => { const label = document.createElement('span'); label.className = 'hint'; label.textContent = `${name} bake:`; bar.appendChild(label); const st = states[i]; if (st === 'ready' || st === 'stored') { const ok = document.createElement('span'); ok.textContent = st; ok.style.color = '#15803d'; if (st === 'stored') ok.title = 'saved in the browser; loads on first use'; bar.appendChild(ok); } else if (st === 'baking') { const prog = document.createElement('progress'); prog.max = 1; prog.value = 0; bakeProg = prog; const cancel = document.createElement('button'); cancel.textContent = 'Cancel'; cancel.addEventListener('click', () => { if (bakeJob) bakeJob.cancel = true; }); bar.append(prog, cancel); } else { const btn = document.createElement('button'); btn.textContent = 'Precompute'; btn.disabled = bakeJob !== null; btn.addEventListener('click', () => void startBake(name)); bar.appendChild(btn); } const inst = document.createElement('button'); inst.textContent = 'Instrument\u2026'; inst.title = 'geometry, accuracy, and the 3D view'; inst.addEventListener('click', () => void openIwEditor(name)); bar.appendChild(inst); }); } /** Bring a bake into memory: already there, or stored in the browser from * an earlier session under the same geometry. */ async function ensureBakeLoaded(name: string): Promise { if (await iw.bakeReady(name)) return true; const key = await iw.currentKey(name); if ((await getBakeKey(name).catch(() => null)) !== key) return false; status(`loading the stored ${name} bake\u2026`); const bytes = await getBakeBytes(name).catch(() => null); if (!bytes) { status(''); return false; } const bake = parseBake(bytes); if (bake.key !== key) return false; iw.adoptBake(name, bake); status(''); updateBakeBar(); return true; } async function startBake(name: string): Promise { if (bakeJob) return; bakeJob = { name, cancel: false }; updateBakeBar(); try { await iw.precompute( name, (done, total) => { if (bakeProg) bakeProg.value = done / total; }, () => bakeJob?.cancel ?? true, ); status(`${name} bake ready; storing it in the browser\u2026`); const ser = iw.serializedBake(name); if (ser) { await putBake(name, ser.key, ser.bytes).then( () => status(`${name} bake ready and stored (${(ser.bytes.length / 1e6).toFixed(0)} MB)`), () => status(`${name} bake ready, but storing it failed; it will be recomputed next session`), ); } } catch (e) { status(e instanceof PhaseCancelled ? `${name} precompute cancelled` : `${name} precompute failed: ${e instanceof Error ? e.message : e}`); } finally { bakeJob = null; updateBakeBar(); } } // ---- take ----------------------------------------------------------------- const timeline = new Timeline($('timeline') as HTMLCanvasElement); function processTake(): void { if (!take.clip) return; stopPlayback(); const { samples, sr } = take.clip; const sens = Number(($('take-sens') as HTMLInputElement).value); const onsets = detectOnsets(samples, sr, sens); const trained = voices.filter((v) => trainPool(v).length > 0); const trainSet: TrainingSet = { voiceIds: trained.flatMap((v) => trainPool(v).map(() => v.id)), vecs: trained.flatMap((v) => trainPool(v).map((h) => h.vec)), }; const hits: TakeHit[] = []; let refDb = -160; for (let i = 0; i < onsets.length; i++) { const s0 = Math.floor(onsets[i] * sr); const s1 = i + 1 < onsets.length ? Math.floor(onsets[i + 1] * sr) : samples.length; const f = extractHit(samples, sr, s0, s1); if (f.peakDb > refDb) refDb = f.peakDb; const cls = classify(f.vec, trainSet); const voiceId = cls?.voiceId ?? voices[0]?.id; if (!voiceId) continue; hits.push({ id: uid(), t: onsets[i], voiceId, gain: 0, peakDb: f.peakDb, vec: f.vec, s0, s1 }); } for (const h of hits) h.gain = gainFromPeakDb(h.peakDb, refDb); take.hits = hits; saveTakeHits(take.hits); refreshTimeline(); syncTakeBar(); if (voices.length === 0) status('no voices defined; add and train a voice, then hit re-detect'); else if (trainSet.vecs.length === 0) status(`no trained hits yet; all ${hits.length} events landed on "${voices[0].name}"`); else status(`${hits.length} hits classified across ${trained.length} trained voice${trained.length === 1 ? '' : 's'}`); } function refreshTimeline(): void { timeline.setData( voices.map((v) => ({ id: v.id, name: v.name, color: v.color })), take.hits, take.clip, ); syncHitButtons(); } function syncTakeBar(): void { const recordingTake = recTarget?.kind === 'take'; const btn = $('take-record') as HTMLButtonElement; btn.innerHTML = recordingTake ? '■ Stop' : '● Record take'; btn.className = recordingTake ? 'rec' : ''; btn.disabled = recorder !== null && !recordingTake; ($('take-redetect') as HTMLButtonElement).disabled = take.clip === null; ($('take-play') as HTMLButtonElement).disabled = take.hits.length === 0; } function syncPlayButton(): void { ($('take-play') as HTMLButtonElement).textContent = playing ? 'Stop' : 'Play'; ($('take-play') as HTMLButtonElement).disabled = take.hits.length === 0 && !playing; } function syncHitButtons(): void { const sel = take.hits.find((h) => h.id === timeline.selected); ($('hit-delete') as HTMLButtonElement).disabled = !sel; ($('hit-learn') as HTMLButtonElement).disabled = !sel || sel.vec === null; } $('take-record').addEventListener('click', () => { if (recTarget?.kind === 'take') void stopRecording(); else void startRecording({ kind: 'take' }); }); $('take-redetect').addEventListener('click', processTake); $('take-sens').addEventListener('change', () => { ui.takeSens = Number(($('take-sens') as HTMLInputElement).value); saveUi(ui); if (take.clip) processTake(); }); timeline.onSelect = (id) => { syncHitButtons(); const hit = take.hits.find((h) => h.id === id); if (!hit) return; const voice = voiceById(hit.voiceId); status(`hit at ${hit.t.toFixed(3)} s · "${voice?.name ?? '?'}" · strength ${hit.gain.toFixed(2)}`); if (hit.s0 >= 0 && take.clip) playSlice(take.clip.samples, take.clip.sr, hit.s0, hit.s1); else if (voice?.binding.kind === 'synth') void auditionSynth(hit, voice.binding.preset); }; async function auditionSynth(hit: TakeHit, preset: SynthPreset): Promise { const buf = await renderSynth([{ t: 0, voiceId: hit.voiceId, gain: hit.gain }], () => preset, 1); void playBuffer(buf); } timeline.onReassign = (id, voiceId) => { saveTakeHits(take.hits); const voice = voiceById(voiceId); status(voice ? `hit moved to "${voice.name}"; Learn adds it to that voice's training` : ''); syncHitButtons(); }; timeline.onAdd = (t, voiceId) => { take.hits.push({ id: uid(), t, voiceId, gain: 0.8, peakDb: 0, vec: null, s0: -1, s1: -1 }); take.hits.sort((a, b) => a.t - b.t); saveTakeHits(take.hits); refreshTimeline(); }; $('hit-delete').addEventListener('click', deleteSelected); document.addEventListener('keydown', (e) => { if (document.activeElement && ['INPUT', 'SELECT', 'TEXTAREA'].includes(document.activeElement.tagName)) return; if (!$('capture').hidden && edit) { if (e.key === 'x' || e.key === 'X' || e.key === 'Delete' || e.key === 'Backspace') { e.preventDefault(); toggleExcluded(); } return; } if (e.key === 'Delete' || e.key === 'Backspace') deleteSelected(); }); function deleteSelected(): void { if (!timeline.selected) return; take.hits = take.hits.filter((h) => h.id !== timeline.selected); saveTakeHits(take.hits); timeline.selected = null; refreshTimeline(); } $('hit-learn').addEventListener('click', () => { const hit = take.hits.find((h) => h.id === timeline.selected); if (!hit || !hit.vec) return; const voice = voiceById(hit.voiceId); if (!voice) return; voice.loose.push({ vec: hit.vec, peakDb: hit.peakDb }); save(); renderVoiceList(); status(`learned this hit as "${voice.name}" (${trainPool(voice).length} trained)`); }); // ---- the mix ---------------------------------------------------------------- async function renderMix(): Promise { const seconds = Math.max(...take.hits.map((h) => h.t)) + 1.5; const sig = JSON.stringify({ hits: take.hits.map((h) => [h.t, h.voiceId, h.gain]), bindings: voices.map((v) => [v.id, v.binding]), iw: iw.dumpOverrides(), }); if (renderCache?.sig === sig) return renderCache.buf; const synthEvents: NoteEvent[] = []; const iwEvents = new Map(); for (const h of take.hits) { const v = voiceById(h.voiceId); if (!v) continue; if (v.binding.kind === 'synth') { synthEvents.push({ t: h.t, voiceId: h.voiceId, gain: h.gain }); } else { const b = v.binding; const allowed = new Set(iw.tuning(b.instrument).map((p) => p.key)); const set: Record = {}; for (const [k, val] of Object.entries(b.set)) if (allowed.has(k)) set[k] = val; const strength = iw.strengthParam(b.instrument); set[strength] = (b.set[strength] ?? iw.defaultValue(b.instrument, strength)) * h.gain; let list = iwEvents.get(b.instrument); if (!list) iwEvents.set(b.instrument, (list = [])); list.push({ t: h.t, set }); } } const missing: string[] = []; for (const n of iwEvents.keys()) { if (!(await ensureBakeLoaded(n))) missing.push(n); } if (missing.length) { throw new Error(`precompute the ${missing.join(' and ')} bake${missing.length > 1 ? 's' : ''} first`); } const out = new Float32Array(Math.ceil(seconds * MIX_SR)); if (synthEvents.length) { const buf = await renderSynth( synthEvents, (id) => { const v = voiceById(id); return v?.binding.kind === 'synth' ? v.binding.preset : 'kick'; }, seconds, ); mixInto(out, buf.getChannelData(0)); } for (const [name, evs] of iwEvents) { evs.sort((a, b) => a.t - b.t); const { trace, dt } = await iw.render(name, seconds, evs, (done, total) => status(`rendering ${name} (${evs.length} events): ${Math.round((100 * done) / total)}%`)); // Physical pressure amplitudes carry no absolute scale worth keeping // across engines; each instrument's trace is levelled to sit alongside // the synth part, while relative volumes WITHIN it stay physical. const res = resampleLinear(trace, 1 / dt, MIX_SR); const p = peak(res); if (p > 0) mixInto(out, res, 0.8 / p); } const pAll = peak(out); if (pAll > 0.98) for (let i = 0; i < out.length; i++) out[i] *= 0.95 / pAll; const buf = audio().createBuffer(1, out.length, MIX_SR); buf.getChannelData(0).set(out); renderCache = { sig, buf }; return buf; } $('take-play').addEventListener('click', async () => { if (playing) { stopPlayback(); return; } if (take.hits.length === 0) return; const btn = $('take-play') as HTMLButtonElement; btn.disabled = true; try { const buf = await renderMix(); status(''); void playBuffer(buf, (t) => timeline.playhead(t)); } catch (e) { status(e instanceof Error ? e.message : String(e)); } finally { btn.disabled = false; syncPlayButton(); } }); // ---- tuning audition ---------------------------------------------------------- async function auditionIw(instrument: string, set: Record): Promise { if (iwBusy || bakeJob) { status('the GPU is busy; try again in a moment'); return; } iwBusy = true; try { if (!(await ensureBakeLoaded(instrument))) { status(`no current ${instrument} bake; precompute it in the bake bar below`); return; } const allowed = new Set(iw.tuning(instrument).map((p) => p.key)); const ev: Record = {}; for (const [k, v] of Object.entries(set)) if (allowed.has(k)) ev[k] = v; const t60 = set.t60 ?? (allowed.has('t60') ? iw.defaultValue(instrument, 't60') : 2); const seconds = Math.min(5, Math.max(1.2, t60 * 0.6 + 0.5)); const { trace, dt } = await iw.render(instrument, seconds, [{ t: 0, set: ev }], (done, total) => status(`rendering ${instrument}: ${Math.round((100 * done) / total)}%`)); const res = resampleLinear(trace, 1 / dt, MIX_SR); const pk = peak(res); if (pk > 0) for (let i = 0; i < res.length; i++) res[i] *= 0.8 / pk; const buf = audio().createBuffer(1, res.length, MIX_SR); buf.getChannelData(0).set(res); status(''); void playBuffer(buf); } catch (e) { status(e instanceof Error ? e.message : String(e)); } finally { iwBusy = false; } } // ---- instrument editor (geometry, accuracy, the 3D view) ---------------------- // Geometry belongs to the instrument, not the voice: it is what the shared // bake depends on, so the editor opens from the bake bar and an edit marks // the bake stale there. The scene is instrawave's own viewer, raymarching // the solver's actual fields, so what it shows is what the bake will see. let iwEdit: { name: string; viewer: Viewer | null } | null = null; const iwCam: Camera = { yaw: -2.2, pitch: 0.45, dist: 1.6 }; const iwVolume = $('iw-volume') as HTMLCanvasElement; const iwOverlay = $('iw-overlay') as HTMLCanvasElement; function sizeSceneCanvas(c: HTMLCanvasElement): void { const r = c.getBoundingClientRect(); const dpr = Math.min(devicePixelRatio || 1, 2); const w = Math.max(1, Math.round(r.width * dpr)); const h = Math.max(1, Math.round(r.height * dpr)); if (c.width !== w || c.height !== h) { c.width = w; c.height = h; } } let iwViewTimer: number | null = null; function scheduleIwView(): void { if (iwViewTimer !== null) clearTimeout(iwViewTimer); iwViewTimer = window.setTimeout(() => { iwViewTimer = null; const e = iwEdit; if (!e?.viewer) return; void (async () => { try { await e.viewer!.update(iw.scope(e.name)); e.viewer!.draw(iwCam); } catch (err) { status(`scene view: ${err instanceof Error ? err.message : err}`); } })(); }, 180); } async function openIwEditor(name: string): Promise { iwEdit = { name, viewer: null }; $('iwedit').hidden = false; $('iwedit-title').textContent = name; buildIwParams(name); try { const device = await iw.ensureDevice(); if (iwEdit?.name !== name) return; sizeSceneCanvas(iwVolume); sizeSceneCanvas(iwOverlay); iwEdit.viewer = new Viewer(device, iw.file(name), iwVolume, iwOverlay); scheduleIwView(); } catch (e) { status(`scene view unavailable: ${e instanceof Error ? e.message : e} (the sliders still work)`); } } function buildIwParams(name: string): void { const box = $('iwedit-params'); box.textContent = ''; const values = iw.values(name); for (const p of iw.geometryParams(name)) { const div = document.createElement('div'); div.className = 'param'; const label = document.createElement('label'); const nameSpan = document.createElement('span'); nameSpan.textContent = `${p.label} (${p.unit})`; if (p.hint) nameSpan.title = p.hint; const valSpan = document.createElement('span'); const input = document.createElement('input'); input.type = 'range'; input.min = String(p.min); input.max = String(p.max); input.step = String(p.step ?? (p.max - p.min) / 200); input.value = String(values[p.key]); const show = () => { valSpan.textContent = Number(input.value).toPrecision(4); }; show(); input.addEventListener('input', () => { iw.setOverride(name, p.key, Number(input.value)); show(); saveIw(); scheduleIwView(); updateBakeBar(); // the bake just went stale (or came back) }); label.append(nameSpan, valSpan); div.append(label, input); box.appendChild(div); } } function closeIwEditor(): void { iwEdit = null; $('iwedit').hidden = true; } $('iwedit-done').addEventListener('click', closeIwEditor); { let dragging = false, px = 0, py = 0; iwOverlay.addEventListener('pointerdown', (e) => { dragging = true; px = e.clientX; py = e.clientY; iwOverlay.setPointerCapture(e.pointerId); }); iwOverlay.addEventListener('pointermove', (e) => { if (!dragging || !iwEdit?.viewer) return; iwCam.yaw -= (e.clientX - px) * 0.008; iwCam.pitch = Math.min(1.45, Math.max(-1.45, iwCam.pitch + (e.clientY - py) * 0.008)); px = e.clientX; py = e.clientY; iwEdit.viewer.draw(iwCam); }); iwOverlay.addEventListener('pointerup', () => { dragging = false; }); iwOverlay.addEventListener('wheel', (e) => { if (!iwEdit?.viewer) return; e.preventDefault(); iwCam.dist = Math.min(6, Math.max(0.3, iwCam.dist * Math.exp(e.deltaY * 0.001))); iwEdit.viewer.draw(iwCam); }, { passive: false }); window.addEventListener('resize', () => { if (!iwEdit?.viewer) return; sizeSceneCanvas(iwVolume); sizeSceneCanvas(iwOverlay); scheduleIwView(); }); } // ---- boot ----------------------------------------------------------------- ($('capture-sens') as HTMLInputElement).value = String(ui.captureSens); ($('take-sens') as HTMLInputElement).value = String(ui.takeSens); take.hits = loadTakeHits() ?? []; void getTakeAudio().then((a) => { if (a) { take.clip = { samples: a.samples, sr: a.sr }; refreshTimeline(); syncTakeBar(); } }).catch(() => { /* the hits still play and edit; re-detect needs the audio */ }); renderVoiceList(); refreshTimeline(); syncTakeBar(); updateBakeBar(); if (voices.length === 0) { status('start by adding a voice, then record yourself making its sound a handful of times'); }