src/main.ts at 21381f9a0c80102d0ffaf55421bd3597e55388e2 - proof-of-concept/instrawave-compose

proof-of-concept / instrawave-compose

1130 lines · 41.7 KBBlame History Raw

1/**

2 * The page: voices (named, trained by example, each bound to a synth

3 * preset or an instrawave instrument), a take (recorded once, segmented,

4 * classified onto the voices), a timeline (the take as editable events),

5 * and playback through the renderer seam in render/types.ts. Training

6 * clips persist (audio in IndexedDB, detections and features in

7 * localStorage) and reopen in the same editor they were made in. The GPU

8 * is touched only when at least one bound instrument is instrawave.

9 */

10import type { Clip } from './audio/recorder.ts';

11import { Recorder } from './audio/recorder.ts';

12import { detectOnsets } from './analysis/onsets.ts';

13import { extractHit } from './analysis/features.ts';

14import { classify, type TrainingSet } from './analysis/classify.ts';

15import { renderSynth } from './render/synth.ts';

16import { gainFromPeakDb, type NoteEvent, type SynthPreset } from './render/types.ts';

17import { mixInto, peak, resampleLinear } from './render/mix.ts';

18import { wavBytes } from './render/wav.ts';

19import { IwEngine, parseBake, PhaseCancelled, type StimulusEvent } from './render/instrawave.ts';

20import { Viewer, type Camera } from 'instrawave/src/host/view.ts';

21import {

22 clearBulk, clearStore, deleteClipAudio, getBakeBytes, getBakeKey, getClipAudio, getTakeAudio,

23 deleteRender, getRender, loadIwOverrides, loadTakeHits, loadUi, loadVoices, putBake, putClipAudio,

24 putRender, putTakeAudio,

25 saveIwOverrides, saveTakeHits, saveUi, saveVoices, trainPool,

26 type TakeHit, type TrainClip, type Voice,

27} from './store.ts';

28import { WaveView, type Marker } from './ui/waveform.ts';

29import { Timeline } from './ui/timeline.ts';

31const PALETTE = ['#e4573d', '#3d7de4', '#3dbb6e', '#d9a520', '#9061d9', '#d95fa8', '#4fb8c9', '#96a13a'];

32const PRESETS: SynthPreset[] = ['kick', 'snare', 'hat', 'tom'];

33const MIX_SR = 44100;

35const $ = <T extends HTMLElement>(id: string) => document.getElementById(id) as T;

36const uid = () => Math.random().toString(36).slice(2, 10);

38// ---- state ----------------------------------------------------------------

40let voices: Voice[] = loadVoices();

41let take: { clip: Clip | null; hits: TakeHit[] } = { clip: null, hits: [] };

42let edit: { voiceId: string; clipId: string; samples: Float32Array; sr: number } | null = null;

43let recTarget: { kind: 'voice'; voiceId: string } | { kind: 'take' } | null = null;

44const expandedTuning = new Set<string>();

46let actx: AudioContext | null = null;

47const audio = (): AudioContext => (actx ??= new AudioContext());

48let recorder: Recorder | null = null;

49let playing: { src: AudioBufferSourceNode; t0: number; dur: number } | null = null;

50let renderCache: { sig: string; buf: AudioBuffer } | null = null;

52const iw = new IwEngine();

53iw.setOverrides(loadIwOverrides());

54const saveIw = () => saveIwOverrides(iw.dumpOverrides());

55const ui = loadUi();

56let bakeJob: { name: string; cancel: boolean } | null = null;

57let iwBusy = false;

58let bakeProgs: HTMLProgressElement[] = [];

60function save(): void { saveVoices(voices); }

61function voiceById(id: string): Voice | undefined { return voices.find((v) => v.id === id); }

62function status(msg: string): void { $('status').textContent = msg; }

64// ---- audition -------------------------------------------------------------

66function playSlice(samples: Float32Array, sr: number, s0: number, s1: number): void {

67 const ctx = audio();

68 const n = Math.min(s1, s0 + Math.floor(0.5 * sr)) - s0;

69 if (n <= 0) return;

70 const buf = ctx.createBuffer(1, n, sr);

71 buf.getChannelData(0).set(samples.subarray(s0, s0 + n));

72 const src = ctx.createBufferSource();

73 src.buffer = buf;

74 src.connect(ctx.destination);

75 src.start();

76}

78async function playBuffer(buf: AudioBuffer, onFrame?: (t: number | null) => void): Promise<void> {

79 const ctx = audio();

80 if (ctx.state === 'suspended') await ctx.resume();

81 const src = ctx.createBufferSource();

82 src.buffer = buf;

83 src.connect(ctx.destination);

84 const t0 = ctx.currentTime;

85 playing = { src, t0, dur: buf.duration };

86 src.onended = () => { if (playing?.src === src) playing = null; onFrame?.(null); syncPlayButton(); };

87 src.start();

88 if (onFrame) {

89 const tick = () => {

90 if (!playing || playing.src !== src) return;

91 onFrame(ctx.currentTime - t0);

92 requestAnimationFrame(tick);

93 };

94 requestAnimationFrame(tick);

95 }

96 syncPlayButton();

97}

99function stopPlayback(): void {

100 playing?.src.stop();

101 playing = null;

102 timeline.playhead(null);

103 syncPlayButton();

104}

105

106// ---- recording ------------------------------------------------------------

107

108let meterRaf = 0;

109function meterOn(): void {

110 $('meter').hidden = false;

111 const fill = $('meter').firstElementChild as HTMLElement;

112 const tick = () => {

113 fill.style.width = `${(recorder?.level() ?? 0) * 100}%`;

114 if (recorder?.recording) meterRaf = requestAnimationFrame(tick);

115 };

116 meterRaf = requestAnimationFrame(tick);

117}

118function meterOff(): void {

119 cancelAnimationFrame(meterRaf);

120 $('meter').hidden = true;

121}

122

123async function startRecording(target: NonNullable<typeof recTarget>): Promise<void> {

124 recorder = new Recorder(audio());

125 try {

126 await recorder.start();

127 } catch (e) {

128 status(`microphone unavailable: ${e instanceof Error ? e.message : e}`);

129 recorder = null;

130 return;

131 }

132 recTarget = target;

133 meterOn();

134 renderVoiceList();

135 syncTakeBar();

136 status(target.kind === 'take'

137 ? 'recording the take; stop when the phrase is done'

138 : `recording examples for "${voiceById(target.voiceId)?.name}"; make the sound several times, then stop`);

139}

140

141async function stopRecording(): Promise<void> {

142 if (!recorder) return;

143 const target = recTarget!;

144 let clip: Clip;

145 try {

146 clip = await recorder.stop();

147 } finally {

148 recorder = null;

149 recTarget = null;

150 meterOff();

151 }

152 renderVoiceList();

153 syncTakeBar();

154 if (target.kind === 'voice') {

155 const voice = voiceById(target.voiceId);

156 if (!voice) return;

157 const sens = Number(($('capture-sens') as HTMLInputElement).value);

158 const tclip: TrainClip = {

159 id: uid(), sr: clip.sr, sens,

160 onsets: detectOnsets(clip.samples, clip.sr, sens),

161 excluded: [], hits: [],

162 };

163 recomputeHits(tclip, clip.samples);

164 voice.clips.push(tclip);

165 save();

166 void putClipAudio(tclip.id, clip.samples, clip.sr).catch(() =>

167 status('warning: clip audio could not be stored; this clip will not be re-editable after a reload'));

168 openEditor(voice.id, tclip.id, { samples: clip.samples, sr: clip.sr });

169 renderVoiceList();

170 } else {

171 take.clip = clip;

172 void putTakeAudio(clip.samples, clip.sr).catch(() =>

173 status('warning: take audio could not be stored; it will not survive a reload'));

174 processTake();

175 }

176}

177

178// ---- training clip editor ----------------------------------------------------

179

180const waveView = new WaveView($('capture-wave') as HTMLCanvasElement);

181

182function editingClip(): TrainClip | null {

183 if (!edit) return null;

184 return voiceById(edit.voiceId)?.clips.find((c) => c.id === edit!.clipId) ?? null;

185}

186

187function recomputeHits(clip: TrainClip, samples: Float32Array): void {

188 const ex = new Set(clip.excluded);

189 clip.hits = [];

190 for (let i = 0; i < clip.onsets.length; i++) {

191 if (ex.has(i)) continue;

192 const s0 = Math.floor(clip.onsets[i] * clip.sr);

193 const s1 = i + 1 < clip.onsets.length ? Math.floor(clip.onsets[i + 1] * clip.sr) : samples.length;

194 const f = extractHit(samples, clip.sr, s0, s1);

195 clip.hits.push({ vec: f.vec, peakDb: f.peakDb });

196 }

197}

198

199function openEditor(voiceId: string, clipId: string, audioData?: { samples: Float32Array; sr: number }): void {

200 const voice = voiceById(voiceId);

201 const clip = voice?.clips.find((c) => c.id === clipId);

202 if (!voice || !clip) return;

203 const open = (samples: Float32Array, sr: number) => {

204 edit = { voiceId, clipId, samples, sr };

205 ($('capture-sens') as HTMLInputElement).value = String(clip.sens);

206 $('capture').hidden = false;

207 refreshEditor();

208 };

209 if (audioData) open(audioData.samples, audioData.sr);

210 else {

211 void getClipAudio(clipId).then((a) => {

212 if (a) open(a.samples, a.sr);

213 else status('this clip\'s audio is missing (stored before audio persistence, or cleared); its hits still classify');

214 });

215 }

216}

217

218function refreshEditor(): void {

219 const voice = edit && voiceById(edit.voiceId);

220 const clip = editingClip();

221 if (!edit || !voice || !clip) return;

222 $('capture-title').textContent = `"${voice.name}"`;

223 const sel = $('capture-clip') as HTMLSelectElement;

224 sel.textContent = '';

225 voice.clips.forEach((c, i) => {

226 const o = document.createElement('option');

227 o.value = c.id;

228 o.textContent = `clip ${i + 1} · ${c.hits.length} hit${c.hits.length === 1 ? '' : 's'}`;

229 sel.appendChild(o);

230 });

231 sel.value = clip.id;

232 const markers: Marker[] = clip.onsets.map((t, i) => ({ t, excluded: clip.excluded.includes(i) }));

233 waveView.set({ samples: edit.samples, sr: edit.sr }, markers, voice.color);

234 syncEditorButtons();

235}

236

237function syncEditorButtons(): void {

238 const clip = editingClip();

239 const n = waveView.selection.size;

240 const btn = $('capture-exclude') as HTMLButtonElement;

241 btn.disabled = n === 0;

242 if (clip && n > 0) {

243 const allExcluded = [...waveView.selection].every((i) => clip.excluded.includes(i));

244 btn.textContent = allExcluded ? `Include (${n})` : `Exclude (${n})`;

245 } else btn.textContent = 'Exclude';

246 $('capture-count').textContent = clip

247 ? `${clip.hits.length} of ${clip.onsets.length} hits included${n ? ` · ${n} selected` : ''}`

248 : '';

249}

250

251function toggleExcluded(): void {

252 const clip = editingClip();

253 if (!edit || !clip || waveView.selection.size === 0) return;

254 const sel = [...waveView.selection];

255 const allExcluded = sel.every((i) => clip.excluded.includes(i));

256 const ex = new Set(clip.excluded);

257 for (const i of sel) {

258 if (allExcluded) ex.delete(i);

259 else ex.add(i);

260 }

261 clip.excluded = [...ex].sort((a, b) => a - b);

262 recomputeHits(clip, edit.samples);

263 save();

264 waveView.updateMarkers(clip.onsets.map((t, i) => ({ t, excluded: ex.has(i) })));

265 syncEditorButtons();

266 renderVoiceList();

267}

268

269waveView.onAudition = (i) => {

270 const clip = editingClip();

271 if (!edit || !clip) return;

272 const s0 = Math.floor(clip.onsets[i] * edit.sr);

273 const s1 = i + 1 < clip.onsets.length ? Math.floor(clip.onsets[i + 1] * edit.sr) : edit.samples.length;

274 playSlice(edit.samples, edit.sr, s0, s1);

275};

276waveView.onSelectionChange = syncEditorButtons;

277

278$('capture-exclude').addEventListener('click', toggleExcluded);

279

280$('capture-sens').addEventListener('input', () => {

281 ui.captureSens = Number(($('capture-sens') as HTMLInputElement).value);

282 saveUi(ui);

283 const clip = editingClip();

284 if (!edit || !clip) return;

285 clip.sens = Number(($('capture-sens') as HTMLInputElement).value);

286 clip.onsets = detectOnsets(edit.samples, edit.sr, clip.sens);

287 clip.excluded = [];

288 recomputeHits(clip, edit.samples);

289 save();

290 refreshEditor();

291 renderVoiceList();

292});

293

294$('capture-clip').addEventListener('change', () => {

295 if (!edit) return;

296 openEditor(edit.voiceId, ($('capture-clip') as HTMLSelectElement).value);

297});

298

299$('capture-delete').addEventListener('click', () => {

300 const voice = edit && voiceById(edit.voiceId);

301 const clip = editingClip();

302 if (!edit || !voice || !clip) return;

303 if (!confirm(`Delete this clip and its ${clip.hits.length} trained hits?`)) return;

304 void deleteClipAudio(clip.id);

305 voice.clips = voice.clips.filter((c) => c.id !== clip.id);

306 save();

307 renderVoiceList();

308 if (voice.clips.length) openEditor(voice.id, voice.clips[voice.clips.length - 1].id);

309 else closeEditor();

310});

311

312function closeEditor(): void {

313 edit = null;

314 $('capture').hidden = true;

315}

316$('capture-done').addEventListener('click', closeEditor);

317

318// ---- voices ---------------------------------------------------------------

319

320function bindingValue(v: Voice): string {

321 return v.binding.kind === 'synth' ? `synth:${v.binding.preset}` : `iw:${v.binding.instrument}`;

322}

323

324function renderVoiceList(): void {

325 const list = $('voice-list');

326 list.textContent = '';

327 bakeProgs = [];

328 const busy = recorder !== null;

329 for (const voice of voices) {

330 const row = document.createElement('div');

331 row.className = 'voice-row';

332

333 const swatch = document.createElement('span');

334 swatch.className = 'swatch';

335 swatch.style.background = voice.color;

336

337 const name = document.createElement('input');

338 name.className = 'name';

339 name.value = voice.name;

340 name.addEventListener('change', () => {

341 voice.name = name.value.trim() || voice.name;

342 name.value = voice.name;

343 save();

344 refreshTimeline();

345 if (edit?.voiceId === voice.id) refreshEditor();

346 });

347

348 const n = trainPool(voice).length;

349 const count = document.createElement('span');

350 count.className = 'count';

351 count.textContent = `${voice.clips.length} clip${voice.clips.length === 1 ? '' : 's'} · ${n} hit${n === 1 ? '' : 's'}`;

352

353 const rec = document.createElement('button');

354 const recordingThis = recTarget?.kind === 'voice' && recTarget.voiceId === voice.id;

355 rec.innerHTML = recordingThis ? '■ Stop' : '● Record';

356 if (recordingThis) rec.className = 'rec';

357 rec.disabled = busy && !recordingThis;

358 rec.addEventListener('click', () => {

359 if (recordingThis) void stopRecording();

360 else void startRecording({ kind: 'voice', voiceId: voice.id });

361 });

362

363 const editBtn = document.createElement('button');

364 editBtn.textContent = 'Edit';

365 editBtn.title = 'reopen this voice\'s training clips';

366 editBtn.disabled = voice.clips.length === 0;

367 editBtn.addEventListener('click', () => {

368 openEditor(voice.id, voice.clips[voice.clips.length - 1].id);

369 });

370

371 const binding = document.createElement('select');

372 const gSynth = document.createElement('optgroup');

373 gSynth.label = 'synth';

374 for (const p of PRESETS) {

375 const o = document.createElement('option');

376 o.value = `synth:${p}`; o.textContent = p;

377 gSynth.appendChild(o);

378 }

379 const gIw = document.createElement('optgroup');

380 gIw.label = 'instrawave';

381 for (const nm of iw.names()) {

382 const o = document.createElement('option');

383 o.value = `iw:${nm}`; o.textContent = nm;

384 gIw.appendChild(o);

385 }

386 binding.append(gSynth, gIw);

387 binding.value = bindingValue(voice);

388 binding.addEventListener('change', () => {

389 const [kind, which] = binding.value.split(':');

390 if (kind === 'synth') voice.binding = { kind: 'synth', preset: which as SynthPreset };

391 else {

392 const old = voice.binding;

393 const set = old.kind === 'iw' && old.instrument === which ? old.set : {};

394 voice.binding = { kind: 'iw', instrument: which, set };

395 }

396 save();

397 renderVoiceList();

398 updateBakeBar();

399 });

400

401 const tune = document.createElement('button');

402 tune.textContent = expandedTuning.has(voice.id) ? 'Tune ▾' : 'Tune ▸';

403 tune.disabled = voice.binding.kind !== 'iw';

404 tune.addEventListener('click', () => {

405 if (expandedTuning.has(voice.id)) expandedTuning.delete(voice.id);

406 else expandedTuning.add(voice.id);

407 renderVoiceList();

408 });

409

410 const clear = document.createElement('button');

411 clear.textContent = 'Clear';

412 clear.title = 'forget this voice\'s training';

413 clear.disabled = n === 0;

414 clear.addEventListener('click', () => {

415 if (!confirm(`Forget all ${n} trained hits of "${voice.name}"?`)) return;

416 for (const c of voice.clips) void deleteClipAudio(c.id);

417 voice.clips = [];

418 voice.loose = [];

419 save();

420 if (edit?.voiceId === voice.id) closeEditor();

421 renderVoiceList();

422 });

423

424 const del = document.createElement('button');

425 del.textContent = '✕';

426 del.title = 'delete voice';

427 del.addEventListener('click', () => {

428 if (!confirm(`Delete voice "${voice.name}"? Its training and its timeline events go with it.`)) return;

429 for (const c of voice.clips) void deleteClipAudio(c.id);

430 voices = voices.filter((v) => v.id !== voice.id);

431 void deleteRender(voice.id);

432 take.hits = take.hits.filter((h) => h.voiceId !== voice.id);

433 saveTakeHits(take.hits);

434 save();

435 if (edit?.voiceId === voice.id) closeEditor();

436 renderVoiceList();

437 refreshTimeline();

438 updateBakeBar();

439 });

440

441 // The instrument's bake state and its editor live inline on the rows

442 // of the voices bound to it (the state is per instrument, so rows

443 // sharing one show the same thing).

444 const iwControls: HTMLElement[] = [];

445 if (voice.binding.kind === 'iw') {

446 const instName = voice.binding.instrument;

447 const st = bakeStates[instName] ?? 'none';

448 if (st === 'ready' || st === 'stored') {

449 const ok = document.createElement('span');

450 ok.className = 'count';

451 ok.style.color = '#15803d';

452 ok.style.minWidth = 'auto';

453 ok.textContent = `bake ${st}`;

454 if (st === 'stored') ok.title = 'saved in the browser; loads on first use';

455 iwControls.push(ok);

456 } else if (st === 'baking') {

457 const prog = document.createElement('progress');

458 prog.max = 1;

459 bakeProgs.push(prog);

460 const cancel = document.createElement('button');

461 cancel.textContent = 'Cancel';

462 cancel.addEventListener('click', () => { if (bakeJob) bakeJob.cancel = true; });

463 iwControls.push(prog, cancel);

464 } else {

465 const btn = document.createElement('button');

466 btn.textContent = 'Precompute';

467 btn.title = 'bake this instrument at its current geometry';

468 btn.disabled = bakeJob !== null;

469 btn.addEventListener('click', () => void startBake(instName));

470 iwControls.push(btn);

471 }

472 const instBtn = document.createElement('button');

473 instBtn.textContent = iwEdit?.anchor === voice.id ? 'Instrument ▾' : 'Instrument ▸';

474 instBtn.title = 'geometry, accuracy, and the 3D view; shared by every voice bound to this instrument';

475 instBtn.addEventListener('click', () => {

476 if (iwEdit?.anchor === voice.id) closeIwEditor();

477 else void openIwEditor(instName, voice.id);

478 });

479 iwControls.push(instBtn);

480 }

481

482 // Playback volume: applied when tracks are mixed, so it never causes

483 // a render; the cached track is simply scaled.

484 const vol = document.createElement('input');

485 vol.type = 'range';

486 vol.min = '0';

487 vol.max = '1.5';

488 vol.step = '0.01';

489 vol.className = 'vol';

490 vol.value = String(voice.volume);

491 const volTitle = () => { vol.title = `playback volume ${Math.round(voice.volume * 100)}%`; };

492 volTitle();

493 vol.addEventListener('input', () => {

494 voice.volume = Number(vol.value);

495 volTitle();

496 save();

497 });

498

499 const spacer = document.createElement('span');

500 spacer.className = 'spacer';

501 row.append(swatch, name, count, rec, editBtn, clear, spacer, binding, vol, tune, ...iwControls, del);

502 list.appendChild(row);

503

504 if (voice.binding.kind === 'iw' && expandedTuning.has(voice.id)) {

505 list.appendChild(tuningPanel(voice, voice.binding.instrument, voice.binding.set));

506 }

507 if (iwEdit?.anchor === voice.id) {

508 if (voice.binding.kind === 'iw' && voice.binding.instrument === iwEdit.name) {

509 iwEditEl.hidden = false;

510 list.appendChild(iwEditEl);

511 } else {

512 // The binding moved out from under the open editor; hide it without

513 // re-rendering (we are mid-render already).

514 iwEdit = null;

515 iwEditEl.hidden = true;

516 }

517 }

518 }

519}

520

521function tuningPanel(voice: Voice, instrument: string, set: Record<string, number>): HTMLElement {

522 const panel = document.createElement('div');

523 panel.className = 'tuning-panel';

524 const bar = document.createElement('div');

525 bar.className = 'bar';

526 bar.style.gridColumn = '1 / -1';

527 bar.style.margin = '0';

528 const hear = document.createElement('button');

529 hear.textContent = 'Hear';

530 hear.addEventListener('click', () => void auditionIw(instrument, set));

531 const hint = document.createElement('span');

532 hint.className = 'hint';

533 hint.textContent = 'one hit at these settings, from the current bake';

534 bar.append(hear, hint);

535 panel.appendChild(bar);

536 for (const p of iw.tuning(instrument)) {

537 const div = document.createElement('div');

538 div.className = 'param';

539 const label = document.createElement('label');

540 const nameSpan = document.createElement('span');

541 nameSpan.textContent = `${p.label} (${p.unit})`;

542 const valSpan = document.createElement('span');

543 const input = document.createElement('input');

544 input.type = 'range';

545 input.min = String(p.min);

546 input.max = String(p.max);

547 input.step = String(p.step ?? (p.max - p.min) / 200);

548 input.value = String(set[p.key] ?? p.value);

549 const show = () => { valSpan.textContent = Number(input.value).toPrecision(4); };

550 show();

551 input.addEventListener('input', () => {

552 set[p.key] = Number(input.value);

553 show();

554 save();

555 });

556 label.append(nameSpan, valSpan);

557 div.append(label, input);

558 panel.appendChild(div);

559 }

560 return panel;

561}

562

563$('add-voice').addEventListener('click', () => {

564 const input = $('voice-name') as HTMLInputElement;

565 const name = input.value.trim() || `voice ${voices.length + 1}`;

566 voices.push({

567 id: uid(),

568 name,

569 color: PALETTE[voices.length % PALETTE.length],

570 binding: { kind: 'synth', preset: PRESETS[voices.length % PRESETS.length] },

571 volume: 1,

572 clips: [],

573 loose: [],

574 });

575 input.value = '';

576 save();

577 renderVoiceList();

578 refreshTimeline();

579});

580

581$('reset-all').addEventListener('click', () => {

582 if (!confirm('Forget all voices, training, and the current take?')) return;

583 clearStore();

584 void clearBulk();

585 iw.setOverrides({});

586 saveIw();

587 voices = [];

588 closeIwEditor();

589 take = { clip: null, hits: [] };

590 closeEditor();

591 renderVoiceList();

592 refreshTimeline();

593 syncTakeBar();

594 updateBakeBar();

595 status('');

596});

597

598// ---- bakes ------------------------------------------------------------------

599

600function neededInstruments(): string[] {

601 return [...new Set(voices.filter((v) => v.binding.kind === 'iw')

602 .map((v) => (v.binding as { instrument: string }).instrument))];

603}

604

605type BakeState = 'ready' | 'stored' | 'baking' | 'none';

606let bakeStates: Record<string, BakeState> = {};

607

608/** Refresh the per-instrument bake states the voice rows render from;

609 * rebuild the rows only when something actually changed. */

610function updateBakeBar(): void {

611 void refreshBakeStates();

612}

613

614let bakeEpoch = 0;

615async function refreshBakeStates(): Promise<void> {

616 const epoch = ++bakeEpoch;

617 const next: Record<string, BakeState> = {};

618 for (const name of neededInstruments()) {

619 if (bakeJob?.name === name) next[name] = 'baking';

620 else if (await iw.bakeReady(name)) next[name] = 'ready';

621 else if ((await getBakeKey(name).catch(() => null)) === (await iw.currentKey(name))) next[name] = 'stored';

622 else next[name] = 'none';

623 }

624 if (epoch !== bakeEpoch) return; // a newer refresh superseded this one

625 if (JSON.stringify(next) !== JSON.stringify(bakeStates)) {

626 bakeStates = next;

627 renderVoiceList();

628 }

629}

630

631/** Bring a bake into memory: already there, or stored in the browser from

632 * an earlier session under the same geometry. */

633async function ensureBakeLoaded(name: string): Promise<boolean> {

634 if (await iw.bakeReady(name)) return true;

635 const key = await iw.currentKey(name);

636 if ((await getBakeKey(name).catch(() => null)) !== key) return false;

637 status(`loading the stored ${name} bake\u2026`);

638 const bytes = await getBakeBytes(name).catch(() => null);

639 if (!bytes) { status(''); return false; }

640 const bake = parseBake(bytes);

641 if (bake.key !== key) return false;

642 iw.adoptBake(name, bake);

643 status('');

644 updateBakeBar();

645 return true;

646}

647

648async function startBake(name: string): Promise<void> {

649 if (bakeJob) return;

650 bakeJob = { name, cancel: false };

651 updateBakeBar();

652 try {

653 await iw.precompute(

654 name,

655 (done, total) => { for (const prog of bakeProgs) prog.value = done / total; },

656 () => bakeJob?.cancel ?? true,

657 );

658 status(`${name} bake ready; storing it in the browser\u2026`);

659 const ser = iw.serializedBake(name);

660 if (ser) {

661 await putBake(name, ser.key, ser.bytes).then(

662 () => status(`${name} bake ready and stored (${(ser.bytes.length / 1e6).toFixed(0)} MB)`),

663 () => status(`${name} bake ready, but storing it failed; it will be recomputed next session`),

664 );

665 }

666 } catch (e) {

667 status(e instanceof PhaseCancelled ? `${name} precompute cancelled`

668 : `${name} precompute failed: ${e instanceof Error ? e.message : e}`);

669 } finally {

670 bakeJob = null;

671 updateBakeBar();

672 }

673}

674

675// ---- take -----------------------------------------------------------------

676

677const timeline = new Timeline($('timeline') as HTMLCanvasElement);

678

679function processTake(): void {

680 if (!take.clip) return;

681 stopPlayback();

682 const { samples, sr } = take.clip;

683 const sens = Number(($('take-sens') as HTMLInputElement).value);

684 const onsets = detectOnsets(samples, sr, sens);

685 const trained = voices.filter((v) => trainPool(v).length > 0);

686 const trainSet: TrainingSet = {

687 voiceIds: trained.flatMap((v) => trainPool(v).map(() => v.id)),

688 vecs: trained.flatMap((v) => trainPool(v).map((h) => h.vec)),

689 };

690

691 const hits: TakeHit[] = [];

692 let refDb = -160;

693 for (let i = 0; i < onsets.length; i++) {

694 const s0 = Math.floor(onsets[i] * sr);

695 const s1 = i + 1 < onsets.length ? Math.floor(onsets[i + 1] * sr) : samples.length;

696 const f = extractHit(samples, sr, s0, s1);

697 if (f.peakDb > refDb) refDb = f.peakDb;

698 const cls = classify(f.vec, trainSet);

699 const voiceId = cls?.voiceId ?? voices[0]?.id;

700 if (!voiceId) continue;

701 hits.push({ id: uid(), t: onsets[i], voiceId, gain: 0, peakDb: f.peakDb, vec: f.vec, s0, s1 });

702 }

703 for (const h of hits) h.gain = gainFromPeakDb(h.peakDb, refDb);

704 take.hits = hits;

705 saveTakeHits(take.hits);

706 refreshTimeline();

707 syncTakeBar();

708 if (voices.length === 0) status('no voices defined; add and train a voice, then hit re-detect');

709 else if (trainSet.vecs.length === 0) status(`no trained hits yet; all ${hits.length} events landed on "${voices[0].name}"`);

710 else status(`${hits.length} hits classified across ${trained.length} trained voice${trained.length === 1 ? '' : 's'}`);

711}

712

713function refreshTimeline(): void {

714 timeline.setData(

715 voices.map((v) => ({ id: v.id, name: v.name, color: v.color })),

716 take.hits,

717 take.clip,

718 );

719 syncHitButtons();

720}

721

722function syncTakeBar(): void {

723 const recordingTake = recTarget?.kind === 'take';

724 const btn = $('take-record') as HTMLButtonElement;

725 btn.innerHTML = recordingTake ? '■ Stop' : '● Record take';

726 btn.className = recordingTake ? 'rec' : '';

727 btn.disabled = recorder !== null && !recordingTake;

728 ($('take-redetect') as HTMLButtonElement).disabled = take.clip === null;

729 ($('take-play') as HTMLButtonElement).disabled = take.hits.length === 0;

730 ($('take-save') as HTMLButtonElement).disabled = take.hits.length === 0;

731}

732

733function syncPlayButton(): void {

734 ($('take-play') as HTMLButtonElement).textContent = playing ? 'Stop' : 'Play';

735 ($('take-play') as HTMLButtonElement).disabled = take.hits.length === 0 && !playing;

736}

737

738function syncHitButtons(): void {

739 const sel = take.hits.find((h) => h.id === timeline.selected);

740 ($('hit-delete') as HTMLButtonElement).disabled = !sel;

741 ($('hit-learn') as HTMLButtonElement).disabled = !sel || sel.vec === null;

742}

743

744$('take-record').addEventListener('click', () => {

745 if (recTarget?.kind === 'take') void stopRecording();

746 else void startRecording({ kind: 'take' });

747});

748

749$('take-redetect').addEventListener('click', processTake);

750$('take-sens').addEventListener('change', () => {

751 ui.takeSens = Number(($('take-sens') as HTMLInputElement).value);

752 saveUi(ui);

753 if (take.clip) processTake();

754});

755

756timeline.onSelect = (id) => {

757 syncHitButtons();

758 const hit = take.hits.find((h) => h.id === id);

759 if (!hit) return;

760 const voice = voiceById(hit.voiceId);

761 status(`hit at ${hit.t.toFixed(3)} s · "${voice?.name ?? '?'}" · strength ${hit.gain.toFixed(2)}`);

762 if (hit.s0 >= 0 && take.clip) playSlice(take.clip.samples, take.clip.sr, hit.s0, hit.s1);

763 else if (voice?.binding.kind === 'synth') void auditionSynth(hit, voice.binding.preset);

764};

765

766async function auditionSynth(hit: TakeHit, preset: SynthPreset): Promise<void> {

767 const buf = await renderSynth([{ t: 0, voiceId: hit.voiceId, gain: hit.gain }], () => preset, 1);

768 void playBuffer(buf);

769}

770

771timeline.onReassign = (id, voiceId) => {

772 saveTakeHits(take.hits);

773 const voice = voiceById(voiceId);

774 status(voice ? `hit moved to "${voice.name}"; Learn adds it to that voice's training` : '');

775 syncHitButtons();

776};

777

778timeline.onAdd = (t, voiceId) => {

779 take.hits.push({ id: uid(), t, voiceId, gain: 0.8, peakDb: 0, vec: null, s0: -1, s1: -1 });

780 take.hits.sort((a, b) => a.t - b.t);

781 saveTakeHits(take.hits);

782 refreshTimeline();

783};

784

785$('hit-delete').addEventListener('click', deleteSelected);

786document.addEventListener('keydown', (e) => {

787 if (document.activeElement && ['INPUT', 'SELECT', 'TEXTAREA'].includes(document.activeElement.tagName)) return;

788 if (!$('capture').hidden && edit) {

789 if (e.key === 'x' || e.key === 'X' || e.key === 'Delete' || e.key === 'Backspace') {

790 e.preventDefault();

791 toggleExcluded();

792 }

793 return;

794 }

795 if (e.key === 'Delete' || e.key === 'Backspace') deleteSelected();

796});

797function deleteSelected(): void {

798 if (!timeline.selected) return;

799 take.hits = take.hits.filter((h) => h.id !== timeline.selected);

800 saveTakeHits(take.hits);

801 timeline.selected = null;

802 refreshTimeline();

803}

804

805$('hit-learn').addEventListener('click', () => {

806 const hit = take.hits.find((h) => h.id === timeline.selected);

807 if (!hit || !hit.vec) return;

808 const voice = voiceById(hit.voiceId);

809 if (!voice) return;

810 voice.loose.push({ vec: hit.vec, peakDb: hit.peakDb });

811 save();

812 renderVoiceList();

813 status(`learned this hit as "${voice.name}" (${trainPool(voice).length} trained)`);

814});

815

816// ---- the mix ----------------------------------------------------------------

817

818const iwTrackCache = new Map<string, { sig: string; data: Float32Array }>();

819

820/** The rendered track for one instrawave voice's events: from the caches

821 * when the signature (events, tuning, gains, bake key) matches, memory

822 * first, then IndexedDB, where a stored track plays without its bake even

823 * loaded; from the GPU otherwise. Returned at the mix rate, levelled to

824 * 0.8 peak. Playback volume is deliberately absent from the signature. */

825async function iwTrack(voice: Voice, evs: StimulusEvent[]): Promise<Float32Array> {

826 if (voice.binding.kind !== 'iw') throw new Error(`voice "${voice.name}" is not bound to an instrument`);

827 const name = voice.binding.instrument;

828 evs.sort((a, b) => a.t - b.t);

829 // The track's own length: its last event plus its longest decay, so an

830 // edit on another lane cannot change this group's signature.

831 let tail = 1.5;

832 const hasT60 = iw.tuning(name).some((p) => p.key === 't60');

833 for (const e of evs) {

834 const t60 = e.set?.t60 ?? (hasT60 ? iw.defaultValue(name, 't60') : 1.5);

835 tail = Math.max(tail, Math.min(6, t60));

836 }

837 const seconds = evs[evs.length - 1].t + tail;

838 const sig = JSON.stringify({

839 inst: name,

840 key: await iw.currentKey(name),

841 secs: seconds,

842 evs: evs.map((e) => [e.t, Object.entries(e.set ?? {}).sort()]),

843 });

844 const mem = iwTrackCache.get(voice.id);

845 if (mem?.sig === sig) return mem.data;

846 const stored = await getRender(voice.id).catch(() => null);

847 if (stored?.sig === sig) {

848 iwTrackCache.set(voice.id, stored);

849 return stored.data;

850 }

851 if (!(await ensureBakeLoaded(name))) {

852 throw new Error(`precompute the ${name} bake first`);

853 }

854 const { trace, dt } = await iw.render(name, seconds, evs, (done, total) =>

855 status(`rendering "${voice.name}" on ${name} (${evs.length} event${evs.length === 1 ? '' : 's'}): ${Math.round((100 * done) / total)}%`));

856 // Physical pressure amplitudes carry no absolute scale worth keeping

857 // across engines; each instrument's track is levelled to sit alongside

858 // the synth part, while relative volumes WITHIN it stay physical.

859 const data = resampleLinear(trace, 1 / dt, MIX_SR);

860 const p = peak(data);

861 if (p > 0) for (let i = 0; i < data.length; i++) data[i] *= 0.8 / p;

862 iwTrackCache.set(voice.id, { sig, data });

863 void putRender(voice.id, sig, data).catch(() => { /* it will re-render next session */ });

864 return data;

865}

866

867async function renderMix(): Promise<AudioBuffer> {

868 const sig = JSON.stringify({

869 hits: take.hits.map((h) => [h.t, h.voiceId, h.gain]),

870 bindings: voices.map((v) => [v.id, v.binding, v.volume]),

871 iw: iw.dumpOverrides(),

872 });

873 if (renderCache?.sig === sig) return renderCache.buf;

874

875 // One track per voice, mixed at that voice's playback volume. The

876 // volume scales a finished track, so moving a slider re-mixes cached

877 // audio and renders nothing.

878 const byVoice = new Map<string, TakeHit[]>();

879 for (const h of take.hits) {

880 const list = byVoice.get(h.voiceId);

881 if (list) list.push(h);

882 else if (voiceById(h.voiceId)) byVoice.set(h.voiceId, [h]);

883 }

884 const tracks: { data: Float32Array; vol: number }[] = [];

885 for (const [voiceId, hits] of byVoice) {

886 const v = voiceById(voiceId)!;

887 if (v.binding.kind === 'synth') {

888 const preset = v.binding.preset;

889 const events: NoteEvent[] = hits.map((h) => ({ t: h.t, voiceId, gain: h.gain }));

890 const secs = Math.max(...events.map((e) => e.t)) + 1.5;

891 const buf = await renderSynth(events, () => preset, secs);

892 tracks.push({ data: buf.getChannelData(0), vol: v.volume });

893 } else {

894 const b = v.binding;

895 const allowed = new Set(iw.tuning(b.instrument).map((p) => p.key));

896 const strength = iw.strengthParam(b.instrument);

897 const events: StimulusEvent[] = hits.map((h) => {

898 const set: Record<string, number> = {};

899 for (const [k, val] of Object.entries(b.set)) if (allowed.has(k)) set[k] = val;

900 set[strength] = (b.set[strength] ?? iw.defaultValue(b.instrument, strength)) * h.gain;

901 return { t: h.t, set };

902 });

903 tracks.push({ data: await iwTrack(v, events), vol: v.volume });

904 }

905 }

906 const seconds = Math.max(1, ...tracks.map((tr) => tr.data.length / MIX_SR));

907 const out = new Float32Array(Math.ceil(seconds * MIX_SR));

908 for (const tr of tracks) mixInto(out, tr.data, tr.vol);

909 const pAll = peak(out);

910 if (pAll > 0.98) for (let i = 0; i < out.length; i++) out[i] *= 0.95 / pAll;

911 const buf = audio().createBuffer(1, out.length, MIX_SR);

912 buf.getChannelData(0).set(out);

913 renderCache = { sig, buf };

914 return buf;

915}

916

917$('take-save').addEventListener('click', async () => {

918 if (take.hits.length === 0) return;

919 const btn = $('take-save') as HTMLButtonElement;

920 btn.disabled = true;

921 try {

922 const buf = await renderMix();

923 const bytes = wavBytes(buf.getChannelData(0), MIX_SR);

924 const url = URL.createObjectURL(new Blob([bytes as Uint8Array<ArrayBuffer>], { type: 'audio/wav' }));

925 const a = document.createElement('a');

926 a.href = url;

927 a.download = 'instrawave-compose-take.wav';

928 a.click();

929 URL.revokeObjectURL(url);

930 status(`saved ${(buf.duration).toFixed(1)} s of audio`);

931 } catch (e) {

932 status(e instanceof Error ? e.message : String(e));

933 } finally {

934 btn.disabled = take.hits.length === 0;

935 }

936});

937

938$('take-play').addEventListener('click', async () => {

939 if (playing) { stopPlayback(); return; }

940 if (take.hits.length === 0) return;

941 const btn = $('take-play') as HTMLButtonElement;

942 btn.disabled = true;

943 try {

944 const buf = await renderMix();

945 status('');

946 void playBuffer(buf, (t) => timeline.playhead(t));

947 } catch (e) {

948 status(e instanceof Error ? e.message : String(e));

949 } finally {

950 btn.disabled = false;

951 syncPlayButton();

952 }

953});

954

955// ---- tuning audition ----------------------------------------------------------

956

957async function auditionIw(instrument: string, set: Record<string, number>): Promise<void> {

958 if (iwBusy || bakeJob) { status('the GPU is busy; try again in a moment'); return; }

959 iwBusy = true;

960 try {

961 if (!(await ensureBakeLoaded(instrument))) {

962 status(`no current ${instrument} bake; precompute it in the voice row`);

963 return;

964 }

965 const allowed = new Set(iw.tuning(instrument).map((p) => p.key));

966 const ev: Record<string, number> = {};

967 for (const [k, v] of Object.entries(set)) if (allowed.has(k)) ev[k] = v;

968 const t60 = set.t60 ?? (allowed.has('t60') ? iw.defaultValue(instrument, 't60') : 2);

969 const seconds = Math.min(5, Math.max(1.2, t60 * 0.6 + 0.5));

970 const { trace, dt } = await iw.render(instrument, seconds, [{ t: 0, set: ev }], (done, total) =>

971 status(`rendering ${instrument}: ${Math.round((100 * done) / total)}%`));

972 const res = resampleLinear(trace, 1 / dt, MIX_SR);

973 const pk = peak(res);

974 if (pk > 0) for (let i = 0; i < res.length; i++) res[i] *= 0.8 / pk;

975 const buf = audio().createBuffer(1, res.length, MIX_SR);

976 buf.getChannelData(0).set(res);

977 status('');

978 void playBuffer(buf);

979 } catch (e) {

980 status(e instanceof Error ? e.message : String(e));

981 } finally {

982 iwBusy = false;

983 }

984}

985

986// ---- instrument editor (geometry, accuracy, the 3D view) ----------------------

987// Geometry belongs to the instrument, not the voice: it is what the shared

988// bake depends on, so the editor opens from the bake bar and an edit marks

989// the bake stale there. The scene is instrawave's own viewer, raymarching

990// the solver's actual fields, so what it shows is what the bake will see.

991

992let iwEdit: { name: string; anchor: string; viewer: Viewer | null } | null = null;

993const iwEditEl = $('iwedit');

994const iwEditTitle = $('iwedit-title');

995const iwEditParams = $('iwedit-params');

996const iwCam: Camera = { yaw: -2.2, pitch: 0.45, dist: 1.6 };

997const iwVolume = $('iw-volume') as HTMLCanvasElement;

998const iwOverlay = $('iw-overlay') as HTMLCanvasElement;

999

1000function sizeSceneCanvas(c: HTMLCanvasElement): void {

1001 const r = c.getBoundingClientRect();

1002 const dpr = Math.min(devicePixelRatio || 1, 2);

1003 const w = Math.max(1, Math.round(r.width * dpr));

1004 const h = Math.max(1, Math.round(r.height * dpr));

1005 if (c.width !== w || c.height !== h) { c.width = w; c.height = h; }

1006}

1007

1008let iwViewTimer: number | null = null;

1009function scheduleIwView(): void {

1010 if (iwViewTimer !== null) clearTimeout(iwViewTimer);

1011 iwViewTimer = window.setTimeout(() => {

1012 iwViewTimer = null;

1013 const e = iwEdit;

1014 if (!e?.viewer) return;

1015 void (async () => {

1016 try {

1017 await e.viewer!.update(iw.scope(e.name));

1018 e.viewer!.draw(iwCam);

1019 } catch (err) {

1020 status(`scene view: ${err instanceof Error ? err.message : err}`);

1021 }

1022 })();

1023 }, 180);

1024}

1025

1026async function openIwEditor(name: string, anchor: string): Promise<void> {

1027 iwEdit = { name, anchor, viewer: null };

1028 iwEditTitle.textContent = name;

1029 buildIwParams(name);

1030 renderVoiceList(); // inserts the panel under its row and shows it

1031 try {

1032 const device = await iw.ensureDevice();

1033 if (iwEdit?.name !== name || iwEdit.anchor !== anchor) return;

1034 sizeSceneCanvas(iwVolume);

1035 sizeSceneCanvas(iwOverlay);

1036 iwEdit.viewer = new Viewer(device, iw.file(name), iwVolume, iwOverlay);

1037 scheduleIwView();

1038 } catch (e) {

1039 status(`scene view unavailable: ${e instanceof Error ? e.message : e} (the sliders still work)`);

1040 }

1041}

1042

1043function buildIwParams(name: string): void {

1044 const box = iwEditParams;

1045 box.textContent = '';

1046 const values = iw.values(name);

1047 for (const p of iw.geometryParams(name)) {

1048 const div = document.createElement('div');

1049 div.className = 'param';

1050 const label = document.createElement('label');

1051 const nameSpan = document.createElement('span');

1052 nameSpan.textContent = `${p.label} (${p.unit})`;

1053 if (p.hint) nameSpan.title = p.hint;

1054 const valSpan = document.createElement('span');

1055 const input = document.createElement('input');

1056 input.type = 'range';

1057 input.min = String(p.min);

1058 input.max = String(p.max);

1059 input.step = String(p.step ?? (p.max - p.min) / 200);

1060 input.value = String(values[p.key]);

1061 const show = () => { valSpan.textContent = Number(input.value).toPrecision(4); };

1062 show();

1063 input.addEventListener('input', () => {

1064 iw.setOverride(name, p.key, Number(input.value));

1065 show();

1066 saveIw();

1067 scheduleIwView();

1068 updateBakeBar(); // the bake just went stale (or came back)

1069 });

1070 label.append(nameSpan, valSpan);

1071 div.append(label, input);

1072 box.appendChild(div);

1073 }

1074}

1075

1076function closeIwEditor(): void {

1077 const wasOpen = iwEdit !== null;

1078 iwEdit = null;

1079 iwEditEl.hidden = true;

1080 if (wasOpen) renderVoiceList();

1081}

1082$('iwedit-done').addEventListener('click', closeIwEditor);

1083

1084{

1085 let dragging = false, px = 0, py = 0;

1086 iwOverlay.addEventListener('pointerdown', (e) => {

1087 dragging = true; px = e.clientX; py = e.clientY;

1088 iwOverlay.setPointerCapture(e.pointerId);

1089 });

1090 iwOverlay.addEventListener('pointermove', (e) => {

1091 if (!dragging || !iwEdit?.viewer) return;

1092 iwCam.yaw -= (e.clientX - px) * 0.008;

1093 iwCam.pitch = Math.min(1.45, Math.max(-1.45, iwCam.pitch + (e.clientY - py) * 0.008));

1094 px = e.clientX; py = e.clientY;

1095 iwEdit.viewer.draw(iwCam);

1096 });

1097 iwOverlay.addEventListener('pointerup', () => { dragging = false; });

1098 iwOverlay.addEventListener('wheel', (e) => {

1099 if (!iwEdit?.viewer) return;

1100 e.preventDefault();

1101 iwCam.dist = Math.min(6, Math.max(0.3, iwCam.dist * Math.exp(e.deltaY * 0.001)));

1102 iwEdit.viewer.draw(iwCam);

1103 }, { passive: false });

1104 window.addEventListener('resize', () => {

1105 if (!iwEdit?.viewer) return;

1106 sizeSceneCanvas(iwVolume);

1107 sizeSceneCanvas(iwOverlay);

1108 scheduleIwView();

1109 });

1110}

1111

1112// ---- boot -----------------------------------------------------------------

1113

1114($('capture-sens') as HTMLInputElement).value = String(ui.captureSens);

1115($('take-sens') as HTMLInputElement).value = String(ui.takeSens);

1116take.hits = loadTakeHits() ?? [];

1117void getTakeAudio().then((a) => {

1118 if (a) {

1119 take.clip = { samples: a.samples, sr: a.sr };

1120 refreshTimeline();

1121 syncTakeBar();

1122 }

1123}).catch(() => { /* the hits still play and edit; re-detect needs the audio */ });

1124renderVoiceList();

1125refreshTimeline();

1126syncTakeBar();

1127updateBakeBar();

1128if (voices.length === 0) {

1129 status('start by adding a voice, then record yourself making its sound a handful of times');

1130}