src/main.ts at 30039cd33bc82c19c80ca298e5c8880d10d9020b - proof-of-concept/instrawave-compose

proof-of-concept / instrawave-compose

1113 lines · 40.7 KBBlame History Raw

1/**

2 * The page: voices (named, trained by example, each bound to a synth

3 * preset or an instrawave instrument), a take (recorded once, segmented,

4 * classified onto the voices), a timeline (the take as editable events),

5 * and playback through the renderer seam in render/types.ts. Training

6 * clips persist (audio in IndexedDB, detections and features in

7 * localStorage) and reopen in the same editor they were made in. The GPU

8 * is touched only when at least one bound instrument is instrawave.

9 */

10import type { Clip } from './audio/recorder.ts';

11import { Recorder } from './audio/recorder.ts';

12import { detectOnsets } from './analysis/onsets.ts';

13import { extractHit } from './analysis/features.ts';

14import { classify, type TrainingSet } from './analysis/classify.ts';

15import { renderSynth } from './render/synth.ts';

16import { gainFromPeakDb, type NoteEvent, type SynthPreset } from './render/types.ts';

17import { mixInto, peak, resampleLinear } from './render/mix.ts';

18import { wavBytes } from './render/wav.ts';

19import { IwEngine, parseBake, PhaseCancelled, type StimulusEvent } from './render/instrawave.ts';

20import { Viewer, type Camera } from 'instrawave/src/host/view.ts';

21import {

22 clearBulk, clearStore, deleteClipAudio, getBakeBytes, getBakeKey, getClipAudio, getTakeAudio,

23 getRender, loadIwOverrides, loadTakeHits, loadUi, loadVoices, putBake, putClipAudio,

24 putRender, putTakeAudio,

25 saveIwOverrides, saveTakeHits, saveUi, saveVoices, trainPool,

26 type TakeHit, type TrainClip, type Voice,

27} from './store.ts';

28import { WaveView, type Marker } from './ui/waveform.ts';

29import { Timeline } from './ui/timeline.ts';

31const PALETTE = ['#e4573d', '#3d7de4', '#3dbb6e', '#d9a520', '#9061d9', '#d95fa8', '#4fb8c9', '#96a13a'];

32const PRESETS: SynthPreset[] = ['kick', 'snare', 'hat', 'tom'];

33const MIX_SR = 44100;

35const $ = <T extends HTMLElement>(id: string) => document.getElementById(id) as T;

36const uid = () => Math.random().toString(36).slice(2, 10);

38// ---- state ----------------------------------------------------------------

40let voices: Voice[] = loadVoices();

41let take: { clip: Clip | null; hits: TakeHit[] } = { clip: null, hits: [] };

42let edit: { voiceId: string; clipId: string; samples: Float32Array; sr: number } | null = null;

43let recTarget: { kind: 'voice'; voiceId: string } | { kind: 'take' } | null = null;

44const expandedTuning = new Set<string>();

46let actx: AudioContext | null = null;

47const audio = (): AudioContext => (actx ??= new AudioContext());

48let recorder: Recorder | null = null;

49let playing: { src: AudioBufferSourceNode; t0: number; dur: number } | null = null;

50let renderCache: { sig: string; buf: AudioBuffer } | null = null;

52const iw = new IwEngine();

53iw.setOverrides(loadIwOverrides());

54const saveIw = () => saveIwOverrides(iw.dumpOverrides());

55const ui = loadUi();

56let bakeJob: { name: string; cancel: boolean } | null = null;

57let iwBusy = false;

58let bakeProgs: HTMLProgressElement[] = [];

60function save(): void { saveVoices(voices); }

61function voiceById(id: string): Voice | undefined { return voices.find((v) => v.id === id); }

62function status(msg: string): void { $('status').textContent = msg; }

64// ---- audition -------------------------------------------------------------

66function playSlice(samples: Float32Array, sr: number, s0: number, s1: number): void {

67 const ctx = audio();

68 const n = Math.min(s1, s0 + Math.floor(0.5 * sr)) - s0;

69 if (n <= 0) return;

70 const buf = ctx.createBuffer(1, n, sr);

71 buf.getChannelData(0).set(samples.subarray(s0, s0 + n));

72 const src = ctx.createBufferSource();

73 src.buffer = buf;

74 src.connect(ctx.destination);

75 src.start();

76}

78async function playBuffer(buf: AudioBuffer, onFrame?: (t: number | null) => void): Promise<void> {

79 const ctx = audio();

80 if (ctx.state === 'suspended') await ctx.resume();

81 const src = ctx.createBufferSource();

82 src.buffer = buf;

83 src.connect(ctx.destination);

84 const t0 = ctx.currentTime;

85 playing = { src, t0, dur: buf.duration };

86 src.onended = () => { if (playing?.src === src) playing = null; onFrame?.(null); syncPlayButton(); };

87 src.start();

88 if (onFrame) {

89 const tick = () => {

90 if (!playing || playing.src !== src) return;

91 onFrame(ctx.currentTime - t0);

92 requestAnimationFrame(tick);

93 };

94 requestAnimationFrame(tick);

95 }

96 syncPlayButton();

97}

99function stopPlayback(): void {

100 playing?.src.stop();

101 playing = null;

102 timeline.playhead(null);

103 syncPlayButton();

104}

105

106// ---- recording ------------------------------------------------------------

107

108let meterRaf = 0;

109function meterOn(): void {

110 $('meter').hidden = false;

111 const fill = $('meter').firstElementChild as HTMLElement;

112 const tick = () => {

113 fill.style.width = `${(recorder?.level() ?? 0) * 100}%`;

114 if (recorder?.recording) meterRaf = requestAnimationFrame(tick);

115 };

116 meterRaf = requestAnimationFrame(tick);

117}

118function meterOff(): void {

119 cancelAnimationFrame(meterRaf);

120 $('meter').hidden = true;

121}

122

123async function startRecording(target: NonNullable<typeof recTarget>): Promise<void> {

124 recorder = new Recorder(audio());

125 try {

126 await recorder.start();

127 } catch (e) {

128 status(`microphone unavailable: ${e instanceof Error ? e.message : e}`);

129 recorder = null;

130 return;

131 }

132 recTarget = target;

133 meterOn();

134 renderVoiceList();

135 syncTakeBar();

136 status(target.kind === 'take'

137 ? 'recording the take; stop when the phrase is done'

138 : `recording examples for "${voiceById(target.voiceId)?.name}"; make the sound several times, then stop`);

139}

140

141async function stopRecording(): Promise<void> {

142 if (!recorder) return;

143 const target = recTarget!;

144 let clip: Clip;

145 try {

146 clip = await recorder.stop();

147 } finally {

148 recorder = null;

149 recTarget = null;

150 meterOff();

151 }

152 renderVoiceList();

153 syncTakeBar();

154 if (target.kind === 'voice') {

155 const voice = voiceById(target.voiceId);

156 if (!voice) return;

157 const sens = Number(($('capture-sens') as HTMLInputElement).value);

158 const tclip: TrainClip = {

159 id: uid(), sr: clip.sr, sens,

160 onsets: detectOnsets(clip.samples, clip.sr, sens),

161 excluded: [], hits: [],

162 };

163 recomputeHits(tclip, clip.samples);

164 voice.clips.push(tclip);

165 save();

166 void putClipAudio(tclip.id, clip.samples, clip.sr).catch(() =>

167 status('warning: clip audio could not be stored; this clip will not be re-editable after a reload'));

168 openEditor(voice.id, tclip.id, { samples: clip.samples, sr: clip.sr });

169 renderVoiceList();

170 } else {

171 take.clip = clip;

172 void putTakeAudio(clip.samples, clip.sr).catch(() =>

173 status('warning: take audio could not be stored; it will not survive a reload'));

174 processTake();

175 }

176}

177

178// ---- training clip editor ----------------------------------------------------

179

180const waveView = new WaveView($('capture-wave') as HTMLCanvasElement);

181

182function editingClip(): TrainClip | null {

183 if (!edit) return null;

184 return voiceById(edit.voiceId)?.clips.find((c) => c.id === edit!.clipId) ?? null;

185}

186

187function recomputeHits(clip: TrainClip, samples: Float32Array): void {

188 const ex = new Set(clip.excluded);

189 clip.hits = [];

190 for (let i = 0; i < clip.onsets.length; i++) {

191 if (ex.has(i)) continue;

192 const s0 = Math.floor(clip.onsets[i] * clip.sr);

193 const s1 = i + 1 < clip.onsets.length ? Math.floor(clip.onsets[i + 1] * clip.sr) : samples.length;

194 const f = extractHit(samples, clip.sr, s0, s1);

195 clip.hits.push({ vec: f.vec, peakDb: f.peakDb });

196 }

197}

198

199function openEditor(voiceId: string, clipId: string, audioData?: { samples: Float32Array; sr: number }): void {

200 const voice = voiceById(voiceId);

201 const clip = voice?.clips.find((c) => c.id === clipId);

202 if (!voice || !clip) return;

203 const open = (samples: Float32Array, sr: number) => {

204 edit = { voiceId, clipId, samples, sr };

205 ($('capture-sens') as HTMLInputElement).value = String(clip.sens);

206 $('capture').hidden = false;

207 refreshEditor();

208 };

209 if (audioData) open(audioData.samples, audioData.sr);

210 else {

211 void getClipAudio(clipId).then((a) => {

212 if (a) open(a.samples, a.sr);

213 else status('this clip\'s audio is missing (stored before audio persistence, or cleared); its hits still classify');

214 });

215 }

216}

217

218function refreshEditor(): void {

219 const voice = edit && voiceById(edit.voiceId);

220 const clip = editingClip();

221 if (!edit || !voice || !clip) return;

222 $('capture-title').textContent = `"${voice.name}"`;

223 const sel = $('capture-clip') as HTMLSelectElement;

224 sel.textContent = '';

225 voice.clips.forEach((c, i) => {

226 const o = document.createElement('option');

227 o.value = c.id;

228 o.textContent = `clip ${i + 1} · ${c.hits.length} hit${c.hits.length === 1 ? '' : 's'}`;

229 sel.appendChild(o);

230 });

231 sel.value = clip.id;

232 const markers: Marker[] = clip.onsets.map((t, i) => ({ t, excluded: clip.excluded.includes(i) }));

233 waveView.set({ samples: edit.samples, sr: edit.sr }, markers, voice.color);

234 syncEditorButtons();

235}

236

237function syncEditorButtons(): void {

238 const clip = editingClip();

239 const n = waveView.selection.size;

240 const btn = $('capture-exclude') as HTMLButtonElement;

241 btn.disabled = n === 0;

242 if (clip && n > 0) {

243 const allExcluded = [...waveView.selection].every((i) => clip.excluded.includes(i));

244 btn.textContent = allExcluded ? `Include (${n})` : `Exclude (${n})`;

245 } else btn.textContent = 'Exclude';

246 $('capture-count').textContent = clip

247 ? `${clip.hits.length} of ${clip.onsets.length} hits included${n ? ` · ${n} selected` : ''}`

248 : '';

249}

250

251function toggleExcluded(): void {

252 const clip = editingClip();

253 if (!edit || !clip || waveView.selection.size === 0) return;

254 const sel = [...waveView.selection];

255 const allExcluded = sel.every((i) => clip.excluded.includes(i));

256 const ex = new Set(clip.excluded);

257 for (const i of sel) {

258 if (allExcluded) ex.delete(i);

259 else ex.add(i);

260 }

261 clip.excluded = [...ex].sort((a, b) => a - b);

262 recomputeHits(clip, edit.samples);

263 save();

264 waveView.updateMarkers(clip.onsets.map((t, i) => ({ t, excluded: ex.has(i) })));

265 syncEditorButtons();

266 renderVoiceList();

267}

268

269waveView.onAudition = (i) => {

270 const clip = editingClip();

271 if (!edit || !clip) return;

272 const s0 = Math.floor(clip.onsets[i] * edit.sr);

273 const s1 = i + 1 < clip.onsets.length ? Math.floor(clip.onsets[i + 1] * edit.sr) : edit.samples.length;

274 playSlice(edit.samples, edit.sr, s0, s1);

275};

276waveView.onSelectionChange = syncEditorButtons;

277

278$('capture-exclude').addEventListener('click', toggleExcluded);

279

280$('capture-sens').addEventListener('input', () => {

281 ui.captureSens = Number(($('capture-sens') as HTMLInputElement).value);

282 saveUi(ui);

283 const clip = editingClip();

284 if (!edit || !clip) return;

285 clip.sens = Number(($('capture-sens') as HTMLInputElement).value);

286 clip.onsets = detectOnsets(edit.samples, edit.sr, clip.sens);

287 clip.excluded = [];

288 recomputeHits(clip, edit.samples);

289 save();

290 refreshEditor();

291 renderVoiceList();

292});

293

294$('capture-clip').addEventListener('change', () => {

295 if (!edit) return;

296 openEditor(edit.voiceId, ($('capture-clip') as HTMLSelectElement).value);

297});

298

299$('capture-delete').addEventListener('click', () => {

300 const voice = edit && voiceById(edit.voiceId);

301 const clip = editingClip();

302 if (!edit || !voice || !clip) return;

303 if (!confirm(`Delete this clip and its ${clip.hits.length} trained hits?`)) return;

304 void deleteClipAudio(clip.id);

305 voice.clips = voice.clips.filter((c) => c.id !== clip.id);

306 save();

307 renderVoiceList();

308 if (voice.clips.length) openEditor(voice.id, voice.clips[voice.clips.length - 1].id);

309 else closeEditor();

310});

311

312function closeEditor(): void {

313 edit = null;

314 $('capture').hidden = true;

315}

316$('capture-done').addEventListener('click', closeEditor);

317

318// ---- voices ---------------------------------------------------------------

319

320function bindingValue(v: Voice): string {

321 return v.binding.kind === 'synth' ? `synth:${v.binding.preset}` : `iw:${v.binding.instrument}`;

322}

323

324function renderVoiceList(): void {

325 const list = $('voice-list');

326 list.textContent = '';

327 bakeProgs = [];

328 const busy = recorder !== null;

329 for (const voice of voices) {

330 const row = document.createElement('div');

331 row.className = 'voice-row';

332

333 const swatch = document.createElement('span');

334 swatch.className = 'swatch';

335 swatch.style.background = voice.color;

336

337 const name = document.createElement('input');

338 name.className = 'name';

339 name.value = voice.name;

340 name.addEventListener('change', () => {

341 voice.name = name.value.trim() || voice.name;

342 name.value = voice.name;

343 save();

344 refreshTimeline();

345 if (edit?.voiceId === voice.id) refreshEditor();

346 });

347

348 const n = trainPool(voice).length;

349 const count = document.createElement('span');

350 count.className = 'count';

351 count.textContent = `${voice.clips.length} clip${voice.clips.length === 1 ? '' : 's'} · ${n} hit${n === 1 ? '' : 's'}`;

352

353 const rec = document.createElement('button');

354 const recordingThis = recTarget?.kind === 'voice' && recTarget.voiceId === voice.id;

355 rec.innerHTML = recordingThis ? '■ Stop' : '● Record';

356 if (recordingThis) rec.className = 'rec';

357 rec.disabled = busy && !recordingThis;

358 rec.addEventListener('click', () => {

359 if (recordingThis) void stopRecording();

360 else void startRecording({ kind: 'voice', voiceId: voice.id });

361 });

362

363 const editBtn = document.createElement('button');

364 editBtn.textContent = 'Edit';

365 editBtn.title = 'reopen this voice\'s training clips';

366 editBtn.disabled = voice.clips.length === 0;

367 editBtn.addEventListener('click', () => {

368 openEditor(voice.id, voice.clips[voice.clips.length - 1].id);

369 });

370

371 const binding = document.createElement('select');

372 const gSynth = document.createElement('optgroup');

373 gSynth.label = 'synth';

374 for (const p of PRESETS) {

375 const o = document.createElement('option');

376 o.value = `synth:${p}`; o.textContent = p;

377 gSynth.appendChild(o);

378 }

379 const gIw = document.createElement('optgroup');

380 gIw.label = 'instrawave';

381 for (const nm of iw.names()) {

382 const o = document.createElement('option');

383 o.value = `iw:${nm}`; o.textContent = nm;

384 gIw.appendChild(o);

385 }

386 binding.append(gSynth, gIw);

387 binding.value = bindingValue(voice);

388 binding.addEventListener('change', () => {

389 const [kind, which] = binding.value.split(':');

390 if (kind === 'synth') voice.binding = { kind: 'synth', preset: which as SynthPreset };

391 else {

392 const old = voice.binding;

393 const set = old.kind === 'iw' && old.instrument === which ? old.set : {};

394 voice.binding = { kind: 'iw', instrument: which, set };

395 }

396 save();

397 renderVoiceList();

398 updateBakeBar();

399 });

400

401 const tune = document.createElement('button');

402 tune.textContent = expandedTuning.has(voice.id) ? 'Tune ▾' : 'Tune ▸';

403 tune.disabled = voice.binding.kind !== 'iw';

404 tune.addEventListener('click', () => {

405 if (expandedTuning.has(voice.id)) expandedTuning.delete(voice.id);

406 else expandedTuning.add(voice.id);

407 renderVoiceList();

408 });

409

410 const clear = document.createElement('button');

411 clear.textContent = 'Clear';

412 clear.title = 'forget this voice\'s training';

413 clear.disabled = n === 0;

414 clear.addEventListener('click', () => {

415 if (!confirm(`Forget all ${n} trained hits of "${voice.name}"?`)) return;

416 for (const c of voice.clips) void deleteClipAudio(c.id);

417 voice.clips = [];

418 voice.loose = [];

419 save();

420 if (edit?.voiceId === voice.id) closeEditor();

421 renderVoiceList();

422 });

423

424 const del = document.createElement('button');

425 del.textContent = '✕';

426 del.title = 'delete voice';

427 del.addEventListener('click', () => {

428 if (!confirm(`Delete voice "${voice.name}"? Its training and its timeline events go with it.`)) return;

429 for (const c of voice.clips) void deleteClipAudio(c.id);

430 voices = voices.filter((v) => v.id !== voice.id);

431 take.hits = take.hits.filter((h) => h.voiceId !== voice.id);

432 saveTakeHits(take.hits);

433 save();

434 if (edit?.voiceId === voice.id) closeEditor();

435 renderVoiceList();

436 refreshTimeline();

437 updateBakeBar();

438 });

439

440 // The instrument's bake state and its editor live inline on the rows

441 // of the voices bound to it (the state is per instrument, so rows

442 // sharing one show the same thing).

443 const iwControls: HTMLElement[] = [];

444 if (voice.binding.kind === 'iw') {

445 const instName = voice.binding.instrument;

446 const st = bakeStates[instName] ?? 'none';

447 if (st === 'ready' || st === 'stored') {

448 const ok = document.createElement('span');

449 ok.className = 'count';

450 ok.style.color = '#15803d';

451 ok.style.minWidth = 'auto';

452 ok.textContent = `bake ${st}`;

453 if (st === 'stored') ok.title = 'saved in the browser; loads on first use';

454 iwControls.push(ok);

455 } else if (st === 'baking') {

456 const prog = document.createElement('progress');

457 prog.max = 1;

458 bakeProgs.push(prog);

459 const cancel = document.createElement('button');

460 cancel.textContent = 'Cancel';

461 cancel.addEventListener('click', () => { if (bakeJob) bakeJob.cancel = true; });

462 iwControls.push(prog, cancel);

463 } else {

464 const btn = document.createElement('button');

465 btn.textContent = 'Precompute';

466 btn.title = 'bake this instrument at its current geometry';

467 btn.disabled = bakeJob !== null;

468 btn.addEventListener('click', () => void startBake(instName));

469 iwControls.push(btn);

470 }

471 const instBtn = document.createElement('button');

472 instBtn.textContent = iwEdit?.anchor === voice.id ? 'Instrument ▾' : 'Instrument ▸';

473 instBtn.title = 'geometry, accuracy, and the 3D view; shared by every voice bound to this instrument';

474 instBtn.addEventListener('click', () => {

475 if (iwEdit?.anchor === voice.id) closeIwEditor();

476 else void openIwEditor(instName, voice.id);

477 });

478 iwControls.push(instBtn);

479 }

480

481 const spacer = document.createElement('span');

482 spacer.className = 'spacer';

483 row.append(swatch, name, count, rec, editBtn, clear, spacer, binding, tune, ...iwControls, del);

484 list.appendChild(row);

485

486 if (voice.binding.kind === 'iw' && expandedTuning.has(voice.id)) {

487 list.appendChild(tuningPanel(voice, voice.binding.instrument, voice.binding.set));

488 }

489 if (iwEdit?.anchor === voice.id) {

490 if (voice.binding.kind === 'iw' && voice.binding.instrument === iwEdit.name) {

491 iwEditEl.hidden = false;

492 list.appendChild(iwEditEl);

493 } else {

494 // The binding moved out from under the open editor; hide it without

495 // re-rendering (we are mid-render already).

496 iwEdit = null;

497 iwEditEl.hidden = true;

498 }

499 }

500 }

501}

502

503function tuningPanel(voice: Voice, instrument: string, set: Record<string, number>): HTMLElement {

504 const panel = document.createElement('div');

505 panel.className = 'tuning-panel';

506 const bar = document.createElement('div');

507 bar.className = 'bar';

508 bar.style.gridColumn = '1 / -1';

509 bar.style.margin = '0';

510 const hear = document.createElement('button');

511 hear.textContent = 'Hear';

512 hear.addEventListener('click', () => void auditionIw(instrument, set));

513 const hint = document.createElement('span');

514 hint.className = 'hint';

515 hint.textContent = 'one hit at these settings, from the current bake';

516 bar.append(hear, hint);

517 panel.appendChild(bar);

518 for (const p of iw.tuning(instrument)) {

519 const div = document.createElement('div');

520 div.className = 'param';

521 const label = document.createElement('label');

522 const nameSpan = document.createElement('span');

523 nameSpan.textContent = `${p.label} (${p.unit})`;

524 const valSpan = document.createElement('span');

525 const input = document.createElement('input');

526 input.type = 'range';

527 input.min = String(p.min);

528 input.max = String(p.max);

529 input.step = String(p.step ?? (p.max - p.min) / 200);

530 input.value = String(set[p.key] ?? p.value);

531 const show = () => { valSpan.textContent = Number(input.value).toPrecision(4); };

532 show();

533 input.addEventListener('input', () => {

534 set[p.key] = Number(input.value);

535 show();

536 save();

537 });

538 label.append(nameSpan, valSpan);

539 div.append(label, input);

540 panel.appendChild(div);

541 }

542 return panel;

543}

544

545$('add-voice').addEventListener('click', () => {

546 const input = $('voice-name') as HTMLInputElement;

547 const name = input.value.trim() || `voice ${voices.length + 1}`;

548 voices.push({

549 id: uid(),

550 name,

551 color: PALETTE[voices.length % PALETTE.length],

552 binding: { kind: 'synth', preset: PRESETS[voices.length % PRESETS.length] },

553 clips: [],

554 loose: [],

555 });

556 input.value = '';

557 save();

558 renderVoiceList();

559 refreshTimeline();

560});

561

562$('reset-all').addEventListener('click', () => {

563 if (!confirm('Forget all voices, training, and the current take?')) return;

564 clearStore();

565 void clearBulk();

566 iw.setOverrides({});

567 saveIw();

568 voices = [];

569 closeIwEditor();

570 take = { clip: null, hits: [] };

571 closeEditor();

572 renderVoiceList();

573 refreshTimeline();

574 syncTakeBar();

575 updateBakeBar();

576 status('');

577});

578

579// ---- bakes ------------------------------------------------------------------

580

581function neededInstruments(): string[] {

582 return [...new Set(voices.filter((v) => v.binding.kind === 'iw')

583 .map((v) => (v.binding as { instrument: string }).instrument))];

584}

585

586type BakeState = 'ready' | 'stored' | 'baking' | 'none';

587let bakeStates: Record<string, BakeState> = {};

588

589/** Refresh the per-instrument bake states the voice rows render from;

590 * rebuild the rows only when something actually changed. */

591function updateBakeBar(): void {

592 void refreshBakeStates();

593}

594

595let bakeEpoch = 0;

596async function refreshBakeStates(): Promise<void> {

597 const epoch = ++bakeEpoch;

598 const next: Record<string, BakeState> = {};

599 for (const name of neededInstruments()) {

600 if (bakeJob?.name === name) next[name] = 'baking';

601 else if (await iw.bakeReady(name)) next[name] = 'ready';

602 else if ((await getBakeKey(name).catch(() => null)) === (await iw.currentKey(name))) next[name] = 'stored';

603 else next[name] = 'none';

604 }

605 if (epoch !== bakeEpoch) return; // a newer refresh superseded this one

606 if (JSON.stringify(next) !== JSON.stringify(bakeStates)) {

607 bakeStates = next;

608 renderVoiceList();

609 }

610}

611

612/** Bring a bake into memory: already there, or stored in the browser from

613 * an earlier session under the same geometry. */

614async function ensureBakeLoaded(name: string): Promise<boolean> {

615 if (await iw.bakeReady(name)) return true;

616 const key = await iw.currentKey(name);

617 if ((await getBakeKey(name).catch(() => null)) !== key) return false;

618 status(`loading the stored ${name} bake\u2026`);

619 const bytes = await getBakeBytes(name).catch(() => null);

620 if (!bytes) { status(''); return false; }

621 const bake = parseBake(bytes);

622 if (bake.key !== key) return false;

623 iw.adoptBake(name, bake);

624 status('');

625 updateBakeBar();

626 return true;

627}

628

629async function startBake(name: string): Promise<void> {

630 if (bakeJob) return;

631 bakeJob = { name, cancel: false };

632 updateBakeBar();

633 try {

634 await iw.precompute(

635 name,

636 (done, total) => { for (const prog of bakeProgs) prog.value = done / total; },

637 () => bakeJob?.cancel ?? true,

638 );

639 status(`${name} bake ready; storing it in the browser\u2026`);

640 const ser = iw.serializedBake(name);

641 if (ser) {

642 await putBake(name, ser.key, ser.bytes).then(

643 () => status(`${name} bake ready and stored (${(ser.bytes.length / 1e6).toFixed(0)} MB)`),

644 () => status(`${name} bake ready, but storing it failed; it will be recomputed next session`),

645 );

646 }

647 } catch (e) {

648 status(e instanceof PhaseCancelled ? `${name} precompute cancelled`

649 : `${name} precompute failed: ${e instanceof Error ? e.message : e}`);

650 } finally {

651 bakeJob = null;

652 updateBakeBar();

653 }

654}

655

656// ---- take -----------------------------------------------------------------

657

658const timeline = new Timeline($('timeline') as HTMLCanvasElement);

659

660function processTake(): void {

661 if (!take.clip) return;

662 stopPlayback();

663 const { samples, sr } = take.clip;

664 const sens = Number(($('take-sens') as HTMLInputElement).value);

665 const onsets = detectOnsets(samples, sr, sens);

666 const trained = voices.filter((v) => trainPool(v).length > 0);

667 const trainSet: TrainingSet = {

668 voiceIds: trained.flatMap((v) => trainPool(v).map(() => v.id)),

669 vecs: trained.flatMap((v) => trainPool(v).map((h) => h.vec)),

670 };

671

672 const hits: TakeHit[] = [];

673 let refDb = -160;

674 for (let i = 0; i < onsets.length; i++) {

675 const s0 = Math.floor(onsets[i] * sr);

676 const s1 = i + 1 < onsets.length ? Math.floor(onsets[i + 1] * sr) : samples.length;

677 const f = extractHit(samples, sr, s0, s1);

678 if (f.peakDb > refDb) refDb = f.peakDb;

679 const cls = classify(f.vec, trainSet);

680 const voiceId = cls?.voiceId ?? voices[0]?.id;

681 if (!voiceId) continue;

682 hits.push({ id: uid(), t: onsets[i], voiceId, gain: 0, peakDb: f.peakDb, vec: f.vec, s0, s1 });

683 }

684 for (const h of hits) h.gain = gainFromPeakDb(h.peakDb, refDb);

685 take.hits = hits;

686 saveTakeHits(take.hits);

687 refreshTimeline();

688 syncTakeBar();

689 if (voices.length === 0) status('no voices defined; add and train a voice, then hit re-detect');

690 else if (trainSet.vecs.length === 0) status(`no trained hits yet; all ${hits.length} events landed on "${voices[0].name}"`);

691 else status(`${hits.length} hits classified across ${trained.length} trained voice${trained.length === 1 ? '' : 's'}`);

692}

693

694function refreshTimeline(): void {

695 timeline.setData(

696 voices.map((v) => ({ id: v.id, name: v.name, color: v.color })),

697 take.hits,

698 take.clip,

699 );

700 syncHitButtons();

701}

702

703function syncTakeBar(): void {

704 const recordingTake = recTarget?.kind === 'take';

705 const btn = $('take-record') as HTMLButtonElement;

706 btn.innerHTML = recordingTake ? '■ Stop' : '● Record take';

707 btn.className = recordingTake ? 'rec' : '';

708 btn.disabled = recorder !== null && !recordingTake;

709 ($('take-redetect') as HTMLButtonElement).disabled = take.clip === null;

710 ($('take-play') as HTMLButtonElement).disabled = take.hits.length === 0;

711 ($('take-save') as HTMLButtonElement).disabled = take.hits.length === 0;

712}

713

714function syncPlayButton(): void {

715 ($('take-play') as HTMLButtonElement).textContent = playing ? 'Stop' : 'Play';

716 ($('take-play') as HTMLButtonElement).disabled = take.hits.length === 0 && !playing;

717}

718

719function syncHitButtons(): void {

720 const sel = take.hits.find((h) => h.id === timeline.selected);

721 ($('hit-delete') as HTMLButtonElement).disabled = !sel;

722 ($('hit-learn') as HTMLButtonElement).disabled = !sel || sel.vec === null;

723}

724

725$('take-record').addEventListener('click', () => {

726 if (recTarget?.kind === 'take') void stopRecording();

727 else void startRecording({ kind: 'take' });

728});

729

730$('take-redetect').addEventListener('click', processTake);

731$('take-sens').addEventListener('change', () => {

732 ui.takeSens = Number(($('take-sens') as HTMLInputElement).value);

733 saveUi(ui);

734 if (take.clip) processTake();

735});

736

737timeline.onSelect = (id) => {

738 syncHitButtons();

739 const hit = take.hits.find((h) => h.id === id);

740 if (!hit) return;

741 const voice = voiceById(hit.voiceId);

742 status(`hit at ${hit.t.toFixed(3)} s · "${voice?.name ?? '?'}" · strength ${hit.gain.toFixed(2)}`);

743 if (hit.s0 >= 0 && take.clip) playSlice(take.clip.samples, take.clip.sr, hit.s0, hit.s1);

744 else if (voice?.binding.kind === 'synth') void auditionSynth(hit, voice.binding.preset);

745};

746

747async function auditionSynth(hit: TakeHit, preset: SynthPreset): Promise<void> {

748 const buf = await renderSynth([{ t: 0, voiceId: hit.voiceId, gain: hit.gain }], () => preset, 1);

749 void playBuffer(buf);

750}

751

752timeline.onReassign = (id, voiceId) => {

753 saveTakeHits(take.hits);

754 const voice = voiceById(voiceId);

755 status(voice ? `hit moved to "${voice.name}"; Learn adds it to that voice's training` : '');

756 syncHitButtons();

757};

758

759timeline.onAdd = (t, voiceId) => {

760 take.hits.push({ id: uid(), t, voiceId, gain: 0.8, peakDb: 0, vec: null, s0: -1, s1: -1 });

761 take.hits.sort((a, b) => a.t - b.t);

762 saveTakeHits(take.hits);

763 refreshTimeline();

764};

765

766$('hit-delete').addEventListener('click', deleteSelected);

767document.addEventListener('keydown', (e) => {

768 if (document.activeElement && ['INPUT', 'SELECT', 'TEXTAREA'].includes(document.activeElement.tagName)) return;

769 if (!$('capture').hidden && edit) {

770 if (e.key === 'x' || e.key === 'X' || e.key === 'Delete' || e.key === 'Backspace') {

771 e.preventDefault();

772 toggleExcluded();

773 }

774 return;

775 }

776 if (e.key === 'Delete' || e.key === 'Backspace') deleteSelected();

777});

778function deleteSelected(): void {

779 if (!timeline.selected) return;

780 take.hits = take.hits.filter((h) => h.id !== timeline.selected);

781 saveTakeHits(take.hits);

782 timeline.selected = null;

783 refreshTimeline();

784}

785

786$('hit-learn').addEventListener('click', () => {

787 const hit = take.hits.find((h) => h.id === timeline.selected);

788 if (!hit || !hit.vec) return;

789 const voice = voiceById(hit.voiceId);

790 if (!voice) return;

791 voice.loose.push({ vec: hit.vec, peakDb: hit.peakDb });

792 save();

793 renderVoiceList();

794 status(`learned this hit as "${voice.name}" (${trainPool(voice).length} trained)`);

795});

796

797// ---- the mix ----------------------------------------------------------------

798

799const iwTrackCache = new Map<string, { sig: string; data: Float32Array }>();

800

801/** The rendered track for one instrument's events: from the caches when

802 * the signature (events, tuning, gains, bake key) matches, memory first,

803 * then IndexedDB, where a stored track plays without its bake even

804 * loaded; from the GPU otherwise. Returned at the mix rate, levelled to

805 * 0.8 peak. */

806async function iwTrack(name: string, evs: StimulusEvent[]): Promise<Float32Array> {

807 evs.sort((a, b) => a.t - b.t);

808 // The track's own length: its last event plus its longest decay, so an

809 // edit on another lane cannot change this group's signature.

810 let tail = 1.5;

811 const hasT60 = iw.tuning(name).some((p) => p.key === 't60');

812 for (const e of evs) {

813 const t60 = e.set?.t60 ?? (hasT60 ? iw.defaultValue(name, 't60') : 1.5);

814 tail = Math.max(tail, Math.min(6, t60));

815 }

816 const seconds = evs[evs.length - 1].t + tail;

817 const sig = JSON.stringify({

818 key: await iw.currentKey(name),

819 secs: seconds,

820 evs: evs.map((e) => [e.t, Object.entries(e.set ?? {}).sort()]),

821 });

822 const mem = iwTrackCache.get(name);

823 if (mem?.sig === sig) return mem.data;

824 const stored = await getRender(name).catch(() => null);

825 if (stored?.sig === sig) {

826 iwTrackCache.set(name, stored);

827 return stored.data;

828 }

829 if (!(await ensureBakeLoaded(name))) {

830 throw new Error(`precompute the ${name} bake first`);

831 }

832 const { trace, dt } = await iw.render(name, seconds, evs, (done, total) =>

833 status(`rendering ${name} (${evs.length} event${evs.length === 1 ? '' : 's'}): ${Math.round((100 * done) / total)}%`));

834 // Physical pressure amplitudes carry no absolute scale worth keeping

835 // across engines; each instrument's track is levelled to sit alongside

836 // the synth part, while relative volumes WITHIN it stay physical.

837 const data = resampleLinear(trace, 1 / dt, MIX_SR);

838 const p = peak(data);

839 if (p > 0) for (let i = 0; i < data.length; i++) data[i] *= 0.8 / p;

840 iwTrackCache.set(name, { sig, data });

841 void putRender(name, sig, data).catch(() => { /* it will re-render next session */ });

842 return data;

843}

844

845async function renderMix(): Promise<AudioBuffer> {

846 const sig = JSON.stringify({

847 hits: take.hits.map((h) => [h.t, h.voiceId, h.gain]),

848 bindings: voices.map((v) => [v.id, v.binding]),

849 iw: iw.dumpOverrides(),

850 });

851 if (renderCache?.sig === sig) return renderCache.buf;

852

853 const synthEvents: NoteEvent[] = [];

854 const iwEvents = new Map<string, StimulusEvent[]>();

855 for (const h of take.hits) {

856 const v = voiceById(h.voiceId);

857 if (!v) continue;

858 if (v.binding.kind === 'synth') {

859 synthEvents.push({ t: h.t, voiceId: h.voiceId, gain: h.gain });

860 } else {

861 const b = v.binding;

862 const allowed = new Set(iw.tuning(b.instrument).map((p) => p.key));

863 const set: Record<string, number> = {};

864 for (const [k, val] of Object.entries(b.set)) if (allowed.has(k)) set[k] = val;

865 const strength = iw.strengthParam(b.instrument);

866 set[strength] = (b.set[strength] ?? iw.defaultValue(b.instrument, strength)) * h.gain;

867 let list = iwEvents.get(b.instrument);

868 if (!list) iwEvents.set(b.instrument, (list = []));

869 list.push({ t: h.t, set });

870 }

871 }

872 const tracks: Float32Array[] = [];

873 for (const [name, evs] of iwEvents) tracks.push(await iwTrack(name, evs));

874

875 const synthSeconds = synthEvents.length

876 ? Math.max(...synthEvents.map((e) => e.t)) + 1.5

877 : 0;

878 const seconds = Math.max(1, synthSeconds, ...tracks.map((tr) => tr.length / MIX_SR));

879 const out = new Float32Array(Math.ceil(seconds * MIX_SR));

880 if (synthEvents.length) {

881 const buf = await renderSynth(

882 synthEvents,

883 (id) => {

884 const v = voiceById(id);

885 return v?.binding.kind === 'synth' ? v.binding.preset : 'kick';

886 },

887 synthSeconds,

888 );

889 mixInto(out, buf.getChannelData(0));

890 }

891 for (const tr of tracks) mixInto(out, tr);

892 const pAll = peak(out);

893 if (pAll > 0.98) for (let i = 0; i < out.length; i++) out[i] *= 0.95 / pAll;

894 const buf = audio().createBuffer(1, out.length, MIX_SR);

895 buf.getChannelData(0).set(out);

896 renderCache = { sig, buf };

897 return buf;

898}

899

900$('take-save').addEventListener('click', async () => {

901 if (take.hits.length === 0) return;

902 const btn = $('take-save') as HTMLButtonElement;

903 btn.disabled = true;

904 try {

905 const buf = await renderMix();

906 const bytes = wavBytes(buf.getChannelData(0), MIX_SR);

907 const url = URL.createObjectURL(new Blob([bytes as Uint8Array<ArrayBuffer>], { type: 'audio/wav' }));

908 const a = document.createElement('a');

909 a.href = url;

910 a.download = 'instrawave-compose-take.wav';

911 a.click();

912 URL.revokeObjectURL(url);

913 status(`saved ${(buf.duration).toFixed(1)} s of audio`);

914 } catch (e) {

915 status(e instanceof Error ? e.message : String(e));

916 } finally {

917 btn.disabled = take.hits.length === 0;

918 }

919});

920

921$('take-play').addEventListener('click', async () => {

922 if (playing) { stopPlayback(); return; }

923 if (take.hits.length === 0) return;

924 const btn = $('take-play') as HTMLButtonElement;

925 btn.disabled = true;

926 try {

927 const buf = await renderMix();

928 status('');

929 void playBuffer(buf, (t) => timeline.playhead(t));

930 } catch (e) {

931 status(e instanceof Error ? e.message : String(e));

932 } finally {

933 btn.disabled = false;

934 syncPlayButton();

935 }

936});

937

938// ---- tuning audition ----------------------------------------------------------

939

940async function auditionIw(instrument: string, set: Record<string, number>): Promise<void> {

941 if (iwBusy || bakeJob) { status('the GPU is busy; try again in a moment'); return; }

942 iwBusy = true;

943 try {

944 if (!(await ensureBakeLoaded(instrument))) {

945 status(`no current ${instrument} bake; precompute it in the voice row`);

946 return;

947 }

948 const allowed = new Set(iw.tuning(instrument).map((p) => p.key));

949 const ev: Record<string, number> = {};

950 for (const [k, v] of Object.entries(set)) if (allowed.has(k)) ev[k] = v;

951 const t60 = set.t60 ?? (allowed.has('t60') ? iw.defaultValue(instrument, 't60') : 2);

952 const seconds = Math.min(5, Math.max(1.2, t60 * 0.6 + 0.5));

953 const { trace, dt } = await iw.render(instrument, seconds, [{ t: 0, set: ev }], (done, total) =>

954 status(`rendering ${instrument}: ${Math.round((100 * done) / total)}%`));

955 const res = resampleLinear(trace, 1 / dt, MIX_SR);

956 const pk = peak(res);

957 if (pk > 0) for (let i = 0; i < res.length; i++) res[i] *= 0.8 / pk;

958 const buf = audio().createBuffer(1, res.length, MIX_SR);

959 buf.getChannelData(0).set(res);

960 status('');

961 void playBuffer(buf);

962 } catch (e) {

963 status(e instanceof Error ? e.message : String(e));

964 } finally {

965 iwBusy = false;

966 }

967}

968

969// ---- instrument editor (geometry, accuracy, the 3D view) ----------------------

970// Geometry belongs to the instrument, not the voice: it is what the shared

971// bake depends on, so the editor opens from the bake bar and an edit marks

972// the bake stale there. The scene is instrawave's own viewer, raymarching

973// the solver's actual fields, so what it shows is what the bake will see.

974

975let iwEdit: { name: string; anchor: string; viewer: Viewer | null } | null = null;

976const iwEditEl = $('iwedit');

977const iwEditTitle = $('iwedit-title');

978const iwEditParams = $('iwedit-params');

979const iwCam: Camera = { yaw: -2.2, pitch: 0.45, dist: 1.6 };

980const iwVolume = $('iw-volume') as HTMLCanvasElement;

981const iwOverlay = $('iw-overlay') as HTMLCanvasElement;

982

983function sizeSceneCanvas(c: HTMLCanvasElement): void {

984 const r = c.getBoundingClientRect();

985 const dpr = Math.min(devicePixelRatio || 1, 2);

986 const w = Math.max(1, Math.round(r.width * dpr));

987 const h = Math.max(1, Math.round(r.height * dpr));

988 if (c.width !== w || c.height !== h) { c.width = w; c.height = h; }

989}

990

991let iwViewTimer: number | null = null;

992function scheduleIwView(): void {

993 if (iwViewTimer !== null) clearTimeout(iwViewTimer);

994 iwViewTimer = window.setTimeout(() => {

995 iwViewTimer = null;

996 const e = iwEdit;

997 if (!e?.viewer) return;

998 void (async () => {

999 try {

1000 await e.viewer!.update(iw.scope(e.name));

1001 e.viewer!.draw(iwCam);

1002 } catch (err) {

1003 status(`scene view: ${err instanceof Error ? err.message : err}`);

1004 }

1005 })();

1006 }, 180);

1007}

1008

1009async function openIwEditor(name: string, anchor: string): Promise<void> {

1010 iwEdit = { name, anchor, viewer: null };

1011 iwEditTitle.textContent = name;

1012 buildIwParams(name);

1013 renderVoiceList(); // inserts the panel under its row and shows it

1014 try {

1015 const device = await iw.ensureDevice();

1016 if (iwEdit?.name !== name || iwEdit.anchor !== anchor) return;

1017 sizeSceneCanvas(iwVolume);

1018 sizeSceneCanvas(iwOverlay);

1019 iwEdit.viewer = new Viewer(device, iw.file(name), iwVolume, iwOverlay);

1020 scheduleIwView();

1021 } catch (e) {

1022 status(`scene view unavailable: ${e instanceof Error ? e.message : e} (the sliders still work)`);

1023 }

1024}

1025

1026function buildIwParams(name: string): void {

1027 const box = iwEditParams;

1028 box.textContent = '';

1029 const values = iw.values(name);

1030 for (const p of iw.geometryParams(name)) {

1031 const div = document.createElement('div');

1032 div.className = 'param';

1033 const label = document.createElement('label');

1034 const nameSpan = document.createElement('span');

1035 nameSpan.textContent = `${p.label} (${p.unit})`;

1036 if (p.hint) nameSpan.title = p.hint;

1037 const valSpan = document.createElement('span');

1038 const input = document.createElement('input');

1039 input.type = 'range';

1040 input.min = String(p.min);

1041 input.max = String(p.max);

1042 input.step = String(p.step ?? (p.max - p.min) / 200);

1043 input.value = String(values[p.key]);

1044 const show = () => { valSpan.textContent = Number(input.value).toPrecision(4); };

1045 show();

1046 input.addEventListener('input', () => {

1047 iw.setOverride(name, p.key, Number(input.value));

1048 show();

1049 saveIw();

1050 scheduleIwView();

1051 updateBakeBar(); // the bake just went stale (or came back)

1052 });

1053 label.append(nameSpan, valSpan);

1054 div.append(label, input);

1055 box.appendChild(div);

1056 }

1057}

1058

1059function closeIwEditor(): void {

1060 const wasOpen = iwEdit !== null;

1061 iwEdit = null;

1062 iwEditEl.hidden = true;

1063 if (wasOpen) renderVoiceList();

1064}

1065$('iwedit-done').addEventListener('click', closeIwEditor);

1066

1067{

1068 let dragging = false, px = 0, py = 0;

1069 iwOverlay.addEventListener('pointerdown', (e) => {

1070 dragging = true; px = e.clientX; py = e.clientY;

1071 iwOverlay.setPointerCapture(e.pointerId);

1072 });

1073 iwOverlay.addEventListener('pointermove', (e) => {

1074 if (!dragging || !iwEdit?.viewer) return;

1075 iwCam.yaw -= (e.clientX - px) * 0.008;

1076 iwCam.pitch = Math.min(1.45, Math.max(-1.45, iwCam.pitch + (e.clientY - py) * 0.008));

1077 px = e.clientX; py = e.clientY;

1078 iwEdit.viewer.draw(iwCam);

1079 });

1080 iwOverlay.addEventListener('pointerup', () => { dragging = false; });

1081 iwOverlay.addEventListener('wheel', (e) => {

1082 if (!iwEdit?.viewer) return;

1083 e.preventDefault();

1084 iwCam.dist = Math.min(6, Math.max(0.3, iwCam.dist * Math.exp(e.deltaY * 0.001)));

1085 iwEdit.viewer.draw(iwCam);

1086 }, { passive: false });

1087 window.addEventListener('resize', () => {

1088 if (!iwEdit?.viewer) return;

1089 sizeSceneCanvas(iwVolume);

1090 sizeSceneCanvas(iwOverlay);

1091 scheduleIwView();

1092 });

1093}

1094

1095// ---- boot -----------------------------------------------------------------

1096

1097($('capture-sens') as HTMLInputElement).value = String(ui.captureSens);

1098($('take-sens') as HTMLInputElement).value = String(ui.takeSens);

1099take.hits = loadTakeHits() ?? [];

1100void getTakeAudio().then((a) => {

1101 if (a) {

1102 take.clip = { samples: a.samples, sr: a.sr };

1103 refreshTimeline();

1104 syncTakeBar();

1105 }

1106}).catch(() => { /* the hits still play and edit; re-detect needs the audio */ });

1107renderVoiceList();

1108refreshTimeline();

1109syncTakeBar();

1110updateBakeBar();

1111if (voices.length === 0) {

1112 status('start by adding a voice, then record yourself making its sound a handful of times');

1113}