/** * The page: voices (named, trained by example), a take (recorded once, * segmented, classified onto the voices), a timeline (the take as editable * events), and playback through the renderer seam in render/types.ts. All * analysis is offline on the recorded clip; nothing here is realtime. */ import type { Clip } from './audio/recorder.ts'; import { Recorder } from './audio/recorder.ts'; import { detectOnsets } from './analysis/onsets.ts'; import { extractHit } from './analysis/features.ts'; import { classify, type TrainingSet } from './analysis/classify.ts'; import { renderSynth } from './render/synth.ts'; import { gainFromPeakDb, type NoteEvent, type SynthPreset } from './render/types.ts'; import { WaveView, type Marker } from './ui/waveform.ts'; import { Timeline } from './ui/timeline.ts'; const PALETTE = ['#e4573d', '#3d7de4', '#3dbb6e', '#d9a520', '#9061d9', '#d95fa8', '#4fb8c9', '#96a13a']; const PRESETS: SynthPreset[] = ['kick', 'snare', 'hat', 'tom']; const STORE_KEY = 'instrawave-compose/v1'; interface TrainHit { vec: number[]; peakDb: number; } interface Voice { id: string; name: string; color: string; preset: SynthPreset; train: TrainHit[]; } interface TakeHit { id: string; t: number; voiceId: string; gain: number; peakDb: number; vec: number[] | null; // null for hand-added hits s0: number; s1: number; // sample bounds in the take clip; -1 if hand-added } const $ = (id: string) => document.getElementById(id) as T; const uid = () => Math.random().toString(36).slice(2, 10); // ---- state ---------------------------------------------------------------- let voices: Voice[] = loadVoices(); let take: { clip: Clip | null; hits: TakeHit[] } = { clip: null, hits: [] }; let pending: { clip: Clip; voiceId: string; onsets: number[]; excluded: Set } | null = null; let recTarget: { kind: 'voice'; voiceId: string } | { kind: 'take' } | null = null; let actx: AudioContext | null = null; const audio = (): AudioContext => (actx ??= new AudioContext()); let recorder: Recorder | null = null; let playing: { src: AudioBufferSourceNode; t0: number; dur: number } | null = null; function loadVoices(): Voice[] { try { const raw = localStorage.getItem(STORE_KEY); if (raw) return (JSON.parse(raw) as { voices: Voice[] }).voices; } catch { /* a corrupt store starts fresh */ } return []; } function save(): void { localStorage.setItem(STORE_KEY, JSON.stringify({ voices })); } function voiceById(id: string): Voice | undefined { return voices.find((v) => v.id === id); } function status(msg: string): void { $('status').textContent = msg; } // ---- audition ------------------------------------------------------------- function playSlice(clip: Clip, s0: number, s1: number): void { const ctx = audio(); const n = Math.min(s1, s0 + Math.floor(0.5 * clip.sr)) - s0; if (n <= 0) return; const buf = ctx.createBuffer(1, n, clip.sr); buf.getChannelData(0).set(clip.samples.subarray(s0, s0 + n)); const src = ctx.createBufferSource(); src.buffer = buf; src.connect(ctx.destination); src.start(); } async function playBuffer(buf: AudioBuffer, onFrame?: (t: number | null) => void): Promise { const ctx = audio(); if (ctx.state === 'suspended') await ctx.resume(); const src = ctx.createBufferSource(); src.buffer = buf; src.connect(ctx.destination); const t0 = ctx.currentTime; playing = { src, t0, dur: buf.duration }; src.onended = () => { if (playing?.src === src) playing = null; onFrame?.(null); syncPlayButton(); }; src.start(); if (onFrame) { const tick = () => { if (!playing || playing.src !== src) return; onFrame(ctx.currentTime - t0); requestAnimationFrame(tick); }; requestAnimationFrame(tick); } syncPlayButton(); } function stopPlayback(): void { playing?.src.stop(); playing = null; timeline.playhead(null); syncPlayButton(); } // ---- recording ------------------------------------------------------------ let meterRaf = 0; function meterOn(): void { $('meter').hidden = false; const fill = $('meter').firstElementChild as HTMLElement; const tick = () => { fill.style.width = `${(recorder?.level() ?? 0) * 100}%`; if (recorder?.recording) meterRaf = requestAnimationFrame(tick); }; meterRaf = requestAnimationFrame(tick); } function meterOff(): void { cancelAnimationFrame(meterRaf); $('meter').hidden = true; } async function startRecording(target: NonNullable): Promise { recorder = new Recorder(audio()); try { await recorder.start(); } catch (e) { status(`microphone unavailable: ${e instanceof Error ? e.message : e}`); recorder = null; return; } recTarget = target; meterOn(); renderVoiceList(); syncTakeBar(); status(target.kind === 'take' ? 'recording the take; stop when the phrase is done' : `recording examples for "${voiceById(target.voiceId)?.name}"; make the sound several times, then stop`); } async function stopRecording(): Promise { if (!recorder) return; const target = recTarget!; let clip: Clip; try { clip = await recorder.stop(); } finally { recorder = null; recTarget = null; meterOff(); } renderVoiceList(); syncTakeBar(); if (target.kind === 'voice') { pending = { clip, voiceId: target.voiceId, onsets: [], excluded: new Set() }; redetectPending(); $('capture').hidden = false; } else { take.clip = clip; processTake(); } } // ---- capture review (voice training) --------------------------------------- const waveView = new WaveView($('capture-wave') as HTMLCanvasElement); function redetectPending(): void { if (!pending) return; const sens = Number(($('capture-sens') as HTMLInputElement).value); pending.onsets = detectOnsets(pending.clip.samples, pending.clip.sr, sens); pending.excluded = new Set(); refreshCapture(); } function refreshCapture(): void { if (!pending) return; const voice = voiceById(pending.voiceId)!; const markers: Marker[] = pending.onsets.map((t, i) => ({ t, excluded: pending!.excluded.has(i) })); waveView.set(pending.clip, markers, voice.color); const n = pending.onsets.length - pending.excluded.size; $('capture-title').textContent = `${pending.onsets.length} hit${pending.onsets.length === 1 ? '' : 's'} detected`; ($('capture-commit') as HTMLButtonElement).textContent = `Add ${n} hit${n === 1 ? '' : 's'} to "${voice.name}"`; ($('capture-commit') as HTMLButtonElement).disabled = n === 0; ($('capture-exclude') as HTMLButtonElement).disabled = true; } waveView.onMarkerClick = (i) => { if (!pending) return; waveView.selected = i; waveView.draw(); const s0 = Math.floor(pending.onsets[i] * pending.clip.sr); const s1 = i + 1 < pending.onsets.length ? Math.floor(pending.onsets[i + 1] * pending.clip.sr) : pending.clip.samples.length; playSlice(pending.clip, s0, s1); const btn = $('capture-exclude') as HTMLButtonElement; btn.disabled = false; btn.textContent = pending.excluded.has(i) ? 'Include' : 'Exclude'; }; $('capture-exclude').addEventListener('click', () => { const i = waveView.selected; if (!pending || i < 0) return; if (pending.excluded.has(i)) pending.excluded.delete(i); else pending.excluded.add(i); const sel = i; refreshCapture(); waveView.selected = sel; waveView.draw(); const btn = $('capture-exclude') as HTMLButtonElement; btn.disabled = false; btn.textContent = pending.excluded.has(i) ? 'Include' : 'Exclude'; }); $('capture-sens').addEventListener('input', redetectPending); $('capture-commit').addEventListener('click', () => { if (!pending) return; const voice = voiceById(pending.voiceId)!; const { samples, sr } = pending.clip; let added = 0; for (let i = 0; i < pending.onsets.length; i++) { if (pending.excluded.has(i)) continue; const s0 = Math.floor(pending.onsets[i] * sr); const s1 = i + 1 < pending.onsets.length ? Math.floor(pending.onsets[i + 1] * sr) : samples.length; const f = extractHit(samples, sr, s0, s1); voice.train.push({ vec: f.vec, peakDb: f.peakDb }); added++; } save(); pending = null; $('capture').hidden = true; renderVoiceList(); status(`added ${added} hit${added === 1 ? '' : 's'} to "${voice.name}" (${voice.train.length} total)`); }); $('capture-discard').addEventListener('click', () => { pending = null; $('capture').hidden = true; }); // ---- voices --------------------------------------------------------------- function renderVoiceList(): void { const list = $('voice-list'); list.textContent = ''; const busy = recorder !== null; for (const voice of voices) { const row = document.createElement('div'); row.className = 'voice-row'; const swatch = document.createElement('span'); swatch.className = 'swatch'; swatch.style.background = voice.color; const name = document.createElement('input'); name.className = 'name'; name.value = voice.name; name.addEventListener('change', () => { voice.name = name.value.trim() || voice.name; name.value = voice.name; save(); refreshTimeline(); }); const count = document.createElement('span'); count.className = 'count'; count.textContent = `${voice.train.length} trained hit${voice.train.length === 1 ? '' : 's'}`; const preset = document.createElement('select'); for (const p of PRESETS) { const o = document.createElement('option'); o.value = p; o.textContent = p; preset.appendChild(o); } preset.value = voice.preset; preset.addEventListener('change', () => { voice.preset = preset.value as SynthPreset; save(); }); const rec = document.createElement('button'); const recordingThis = recTarget?.kind === 'voice' && recTarget.voiceId === voice.id; rec.innerHTML = recordingThis ? '■ Stop' : '● Record'; if (recordingThis) rec.className = 'rec'; rec.disabled = busy && !recordingThis; rec.addEventListener('click', () => { if (recordingThis) void stopRecording(); else void startRecording({ kind: 'voice', voiceId: voice.id }); }); const clear = document.createElement('button'); clear.textContent = 'Clear'; clear.title = 'forget this voice\'s trained hits'; clear.disabled = voice.train.length === 0; clear.addEventListener('click', () => { voice.train = []; save(); renderVoiceList(); }); const del = document.createElement('button'); del.textContent = '✕'; del.title = 'delete voice'; del.addEventListener('click', () => { if (!confirm(`Delete voice "${voice.name}"? Its trained hits and its timeline events go with it.`)) return; voices = voices.filter((v) => v.id !== voice.id); take.hits = take.hits.filter((h) => h.voiceId !== voice.id); save(); renderVoiceList(); refreshTimeline(); }); const spacer = document.createElement('span'); spacer.className = 'spacer'; row.append(swatch, name, count, rec, clear, spacer, preset, del); list.appendChild(row); } } $('add-voice').addEventListener('click', () => { const input = $('voice-name') as HTMLInputElement; const name = input.value.trim() || `voice ${voices.length + 1}`; voices.push({ id: uid(), name, color: PALETTE[voices.length % PALETTE.length], preset: PRESETS[voices.length % PRESETS.length], train: [], }); input.value = ''; save(); renderVoiceList(); refreshTimeline(); }); $('reset-all').addEventListener('click', () => { if (!confirm('Forget all voices, training, and the current take?')) return; localStorage.removeItem(STORE_KEY); voices = []; take = { clip: null, hits: [] }; pending = null; $('capture').hidden = true; renderVoiceList(); refreshTimeline(); syncTakeBar(); status(''); }); // ---- take ----------------------------------------------------------------- const timeline = new Timeline($('timeline') as HTMLCanvasElement); function processTake(): void { if (!take.clip) return; stopPlayback(); const { samples, sr } = take.clip; const sens = Number(($('take-sens') as HTMLInputElement).value); const onsets = detectOnsets(samples, sr, sens); const trained = voices.filter((v) => v.train.length > 0); const trainSet: TrainingSet = { voiceIds: trained.flatMap((v) => v.train.map(() => v.id)), vecs: trained.flatMap((v) => v.train.map((h) => h.vec)), }; const hits: TakeHit[] = []; let refDb = -160; for (let i = 0; i < onsets.length; i++) { const s0 = Math.floor(onsets[i] * sr); const s1 = i + 1 < onsets.length ? Math.floor(onsets[i + 1] * sr) : samples.length; const f = extractHit(samples, sr, s0, s1); if (f.peakDb > refDb) refDb = f.peakDb; const cls = classify(f.vec, trainSet); const voiceId = cls?.voiceId ?? voices[0]?.id; if (!voiceId) continue; hits.push({ id: uid(), t: onsets[i], voiceId, gain: 0, peakDb: f.peakDb, vec: f.vec, s0, s1 }); } for (const h of hits) h.gain = gainFromPeakDb(h.peakDb, refDb); take.hits = hits; refreshTimeline(); syncTakeBar(); if (voices.length === 0) status('no voices defined; add and train a voice, then hit re-detect'); else if (trainSet.vecs.length === 0) status(`no trained hits yet; all ${hits.length} events landed on "${voices[0].name}"`); else status(`${hits.length} hits classified across ${trained.length} trained voice${trained.length === 1 ? '' : 's'}`); } function refreshTimeline(): void { timeline.setData( voices.map((v) => ({ id: v.id, name: v.name, color: v.color })), take.hits, take.clip, ); syncHitButtons(); } function syncTakeBar(): void { const recordingTake = recTarget?.kind === 'take'; const btn = $('take-record') as HTMLButtonElement; btn.innerHTML = recordingTake ? '■ Stop' : '● Record take'; btn.className = recordingTake ? 'rec' : ''; btn.disabled = recorder !== null && !recordingTake; ($('take-redetect') as HTMLButtonElement).disabled = take.clip === null; ($('take-play') as HTMLButtonElement).disabled = take.hits.length === 0; } function syncPlayButton(): void { ($('take-play') as HTMLButtonElement).textContent = playing ? 'Stop' : 'Play'; ($('take-play') as HTMLButtonElement).disabled = take.hits.length === 0 && !playing; } function syncHitButtons(): void { const sel = take.hits.find((h) => h.id === timeline.selected); ($('hit-delete') as HTMLButtonElement).disabled = !sel; ($('hit-learn') as HTMLButtonElement).disabled = !sel || sel.vec === null; } $('take-record').addEventListener('click', () => { if (recTarget?.kind === 'take') void stopRecording(); else void startRecording({ kind: 'take' }); }); $('take-redetect').addEventListener('click', processTake); $('take-sens').addEventListener('change', () => { if (take.clip) processTake(); }); timeline.onSelect = (id) => { syncHitButtons(); const hit = take.hits.find((h) => h.id === id); if (!hit) return; if (hit.s0 >= 0 && take.clip) playSlice(take.clip, hit.s0, hit.s1); else void auditionSynth(hit); }; async function auditionSynth(hit: TakeHit): Promise { const voice = voiceById(hit.voiceId); if (!voice) return; const buf = await renderSynth([{ t: 0, voiceId: hit.voiceId, gain: hit.gain }], () => voice.preset, 1); void playBuffer(buf); } timeline.onReassign = (id, voiceId) => { const voice = voiceById(voiceId); status(voice ? `hit moved to "${voice.name}"; Learn adds it to that voice's training` : ''); syncHitButtons(); }; timeline.onAdd = (t, voiceId) => { take.hits.push({ id: uid(), t, voiceId, gain: 0.8, peakDb: 0, vec: null, s0: -1, s1: -1 }); take.hits.sort((a, b) => a.t - b.t); refreshTimeline(); }; $('hit-delete').addEventListener('click', deleteSelected); document.addEventListener('keydown', (e) => { if (e.key !== 'Delete' && e.key !== 'Backspace') return; if (document.activeElement && ['INPUT', 'SELECT', 'TEXTAREA'].includes(document.activeElement.tagName)) return; deleteSelected(); }); function deleteSelected(): void { if (!timeline.selected) return; take.hits = take.hits.filter((h) => h.id !== timeline.selected); timeline.selected = null; refreshTimeline(); } $('hit-learn').addEventListener('click', () => { const hit = take.hits.find((h) => h.id === timeline.selected); if (!hit || !hit.vec) return; const voice = voiceById(hit.voiceId); if (!voice) return; voice.train.push({ vec: hit.vec, peakDb: hit.peakDb }); save(); renderVoiceList(); status(`learned this hit as "${voice.name}" (${voice.train.length} trained)`); }); $('take-play').addEventListener('click', async () => { if (playing) { stopPlayback(); return; } if (take.hits.length === 0) return; const events: NoteEvent[] = take.hits.map((h) => ({ t: h.t, voiceId: h.voiceId, gain: h.gain })); const seconds = Math.max(...events.map((e) => e.t)) + 1.5; const buf = await renderSynth(events, (id) => voiceById(id)?.preset ?? 'kick', seconds); void playBuffer(buf, (t) => timeline.playhead(t)); }); // ---- boot ----------------------------------------------------------------- renderVoiceList(); refreshTimeline(); syncTakeBar(); if (voices.length === 0) { status('start by adding a voice, then record yourself making its sound a handful of times'); }