src/main.ts at 9f20d5bfab29b32df359ab1957ff1922d9c2a06b - proof-of-concept/instrawave-compose

proof-of-concept / instrawave-compose

1027 lines · 37.0 KBBlame History Raw

1/**

2 * The page: voices (named, trained by example, each bound to a synth

3 * preset or an instrawave instrument), a take (recorded once, segmented,

4 * classified onto the voices), a timeline (the take as editable events),

5 * and playback through the renderer seam in render/types.ts. Training

6 * clips persist (audio in IndexedDB, detections and features in

7 * localStorage) and reopen in the same editor they were made in. The GPU

8 * is touched only when at least one bound instrument is instrawave.

9 */

10import type { Clip } from './audio/recorder.ts';

11import { Recorder } from './audio/recorder.ts';

12import { detectOnsets } from './analysis/onsets.ts';

13import { extractHit } from './analysis/features.ts';

14import { classify, type TrainingSet } from './analysis/classify.ts';

15import { renderSynth } from './render/synth.ts';

16import { gainFromPeakDb, type NoteEvent, type SynthPreset } from './render/types.ts';

17import { mixInto, peak, resampleLinear } from './render/mix.ts';

18import { IwEngine, parseBake, PhaseCancelled, type StimulusEvent } from './render/instrawave.ts';

19import { Viewer, type Camera } from 'instrawave/src/host/view.ts';

20import {

21 clearBulk, clearStore, deleteClipAudio, getBakeBytes, getBakeKey, getClipAudio, getTakeAudio,

22 loadIwOverrides, loadTakeHits, loadUi, loadVoices, putBake, putClipAudio, putTakeAudio,

23 saveIwOverrides, saveTakeHits, saveUi, saveVoices, trainPool,

24 type TakeHit, type TrainClip, type Voice,

25} from './store.ts';

26import { WaveView, type Marker } from './ui/waveform.ts';

27import { Timeline } from './ui/timeline.ts';

29const PALETTE = ['#e4573d', '#3d7de4', '#3dbb6e', '#d9a520', '#9061d9', '#d95fa8', '#4fb8c9', '#96a13a'];

30const PRESETS: SynthPreset[] = ['kick', 'snare', 'hat', 'tom'];

31const MIX_SR = 44100;

33const $ = <T extends HTMLElement>(id: string) => document.getElementById(id) as T;

34const uid = () => Math.random().toString(36).slice(2, 10);

36// ---- state ----------------------------------------------------------------

38let voices: Voice[] = loadVoices();

39let take: { clip: Clip | null; hits: TakeHit[] } = { clip: null, hits: [] };

40let edit: { voiceId: string; clipId: string; samples: Float32Array; sr: number } | null = null;

41let recTarget: { kind: 'voice'; voiceId: string } | { kind: 'take' } | null = null;

42const expandedTuning = new Set<string>();

44let actx: AudioContext | null = null;

45const audio = (): AudioContext => (actx ??= new AudioContext());

46let recorder: Recorder | null = null;

47let playing: { src: AudioBufferSourceNode; t0: number; dur: number } | null = null;

48let renderCache: { sig: string; buf: AudioBuffer } | null = null;

50const iw = new IwEngine();

51iw.setOverrides(loadIwOverrides());

52const saveIw = () => saveIwOverrides(iw.dumpOverrides());

53const ui = loadUi();

54let bakeJob: { name: string; cancel: boolean } | null = null;

55let iwBusy = false;

56let bakeProg: HTMLProgressElement | null = null;

58function save(): void { saveVoices(voices); }

59function voiceById(id: string): Voice | undefined { return voices.find((v) => v.id === id); }

60function status(msg: string): void { $('status').textContent = msg; }

62// ---- audition -------------------------------------------------------------

64function playSlice(samples: Float32Array, sr: number, s0: number, s1: number): void {

65 const ctx = audio();

66 const n = Math.min(s1, s0 + Math.floor(0.5 * sr)) - s0;

67 if (n <= 0) return;

68 const buf = ctx.createBuffer(1, n, sr);

69 buf.getChannelData(0).set(samples.subarray(s0, s0 + n));

70 const src = ctx.createBufferSource();

71 src.buffer = buf;

72 src.connect(ctx.destination);

73 src.start();

74}

76async function playBuffer(buf: AudioBuffer, onFrame?: (t: number | null) => void): Promise<void> {

77 const ctx = audio();

78 if (ctx.state === 'suspended') await ctx.resume();

79 const src = ctx.createBufferSource();

80 src.buffer = buf;

81 src.connect(ctx.destination);

82 const t0 = ctx.currentTime;

83 playing = { src, t0, dur: buf.duration };

84 src.onended = () => { if (playing?.src === src) playing = null; onFrame?.(null); syncPlayButton(); };

85 src.start();

86 if (onFrame) {

87 const tick = () => {

88 if (!playing || playing.src !== src) return;

89 onFrame(ctx.currentTime - t0);

90 requestAnimationFrame(tick);

91 };

92 requestAnimationFrame(tick);

93 }

94 syncPlayButton();

95}

97function stopPlayback(): void {

98 playing?.src.stop();

99 playing = null;

100 timeline.playhead(null);

101 syncPlayButton();

102}

103

104// ---- recording ------------------------------------------------------------

105

106let meterRaf = 0;

107function meterOn(): void {

108 $('meter').hidden = false;

109 const fill = $('meter').firstElementChild as HTMLElement;

110 const tick = () => {

111 fill.style.width = `${(recorder?.level() ?? 0) * 100}%`;

112 if (recorder?.recording) meterRaf = requestAnimationFrame(tick);

113 };

114 meterRaf = requestAnimationFrame(tick);

115}

116function meterOff(): void {

117 cancelAnimationFrame(meterRaf);

118 $('meter').hidden = true;

119}

120

121async function startRecording(target: NonNullable<typeof recTarget>): Promise<void> {

122 recorder = new Recorder(audio());

123 try {

124 await recorder.start();

125 } catch (e) {

126 status(`microphone unavailable: ${e instanceof Error ? e.message : e}`);

127 recorder = null;

128 return;

129 }

130 recTarget = target;

131 meterOn();

132 renderVoiceList();

133 syncTakeBar();

134 status(target.kind === 'take'

135 ? 'recording the take; stop when the phrase is done'

136 : `recording examples for "${voiceById(target.voiceId)?.name}"; make the sound several times, then stop`);

137}

138

139async function stopRecording(): Promise<void> {

140 if (!recorder) return;

141 const target = recTarget!;

142 let clip: Clip;

143 try {

144 clip = await recorder.stop();

145 } finally {

146 recorder = null;

147 recTarget = null;

148 meterOff();

149 }

150 renderVoiceList();

151 syncTakeBar();

152 if (target.kind === 'voice') {

153 const voice = voiceById(target.voiceId);

154 if (!voice) return;

155 const sens = Number(($('capture-sens') as HTMLInputElement).value);

156 const tclip: TrainClip = {

157 id: uid(), sr: clip.sr, sens,

158 onsets: detectOnsets(clip.samples, clip.sr, sens),

159 excluded: [], hits: [],

160 };

161 recomputeHits(tclip, clip.samples);

162 voice.clips.push(tclip);

163 save();

164 void putClipAudio(tclip.id, clip.samples, clip.sr).catch(() =>

165 status('warning: clip audio could not be stored; this clip will not be re-editable after a reload'));

166 openEditor(voice.id, tclip.id, { samples: clip.samples, sr: clip.sr });

167 renderVoiceList();

168 } else {

169 take.clip = clip;

170 void putTakeAudio(clip.samples, clip.sr).catch(() =>

171 status('warning: take audio could not be stored; it will not survive a reload'));

172 processTake();

173 }

174}

175

176// ---- training clip editor ----------------------------------------------------

177

178const waveView = new WaveView($('capture-wave') as HTMLCanvasElement);

179

180function editingClip(): TrainClip | null {

181 if (!edit) return null;

182 return voiceById(edit.voiceId)?.clips.find((c) => c.id === edit!.clipId) ?? null;

183}

184

185function recomputeHits(clip: TrainClip, samples: Float32Array): void {

186 const ex = new Set(clip.excluded);

187 clip.hits = [];

188 for (let i = 0; i < clip.onsets.length; i++) {

189 if (ex.has(i)) continue;

190 const s0 = Math.floor(clip.onsets[i] * clip.sr);

191 const s1 = i + 1 < clip.onsets.length ? Math.floor(clip.onsets[i + 1] * clip.sr) : samples.length;

192 const f = extractHit(samples, clip.sr, s0, s1);

193 clip.hits.push({ vec: f.vec, peakDb: f.peakDb });

194 }

195}

196

197function openEditor(voiceId: string, clipId: string, audioData?: { samples: Float32Array; sr: number }): void {

198 const voice = voiceById(voiceId);

199 const clip = voice?.clips.find((c) => c.id === clipId);

200 if (!voice || !clip) return;

201 const open = (samples: Float32Array, sr: number) => {

202 edit = { voiceId, clipId, samples, sr };

203 ($('capture-sens') as HTMLInputElement).value = String(clip.sens);

204 $('capture').hidden = false;

205 refreshEditor();

206 };

207 if (audioData) open(audioData.samples, audioData.sr);

208 else {

209 void getClipAudio(clipId).then((a) => {

210 if (a) open(a.samples, a.sr);

211 else status('this clip\'s audio is missing (stored before audio persistence, or cleared); its hits still classify');

212 });

213 }

214}

215

216function refreshEditor(): void {

217 const voice = edit && voiceById(edit.voiceId);

218 const clip = editingClip();

219 if (!edit || !voice || !clip) return;

220 $('capture-title').textContent = `"${voice.name}"`;

221 const sel = $('capture-clip') as HTMLSelectElement;

222 sel.textContent = '';

223 voice.clips.forEach((c, i) => {

224 const o = document.createElement('option');

225 o.value = c.id;

226 o.textContent = `clip ${i + 1} · ${c.hits.length} hit${c.hits.length === 1 ? '' : 's'}`;

227 sel.appendChild(o);

228 });

229 sel.value = clip.id;

230 const markers: Marker[] = clip.onsets.map((t, i) => ({ t, excluded: clip.excluded.includes(i) }));

231 waveView.set({ samples: edit.samples, sr: edit.sr }, markers, voice.color);

232 syncEditorButtons();

233}

234

235function syncEditorButtons(): void {

236 const clip = editingClip();

237 const n = waveView.selection.size;

238 const btn = $('capture-exclude') as HTMLButtonElement;

239 btn.disabled = n === 0;

240 if (clip && n > 0) {

241 const allExcluded = [...waveView.selection].every((i) => clip.excluded.includes(i));

242 btn.textContent = allExcluded ? `Include (${n})` : `Exclude (${n})`;

243 } else btn.textContent = 'Exclude';

244 $('capture-count').textContent = clip

245 ? `${clip.hits.length} of ${clip.onsets.length} hits included${n ? ` · ${n} selected` : ''}`

246 : '';

247}

248

249function toggleExcluded(): void {

250 const clip = editingClip();

251 if (!edit || !clip || waveView.selection.size === 0) return;

252 const sel = [...waveView.selection];

253 const allExcluded = sel.every((i) => clip.excluded.includes(i));

254 const ex = new Set(clip.excluded);

255 for (const i of sel) {

256 if (allExcluded) ex.delete(i);

257 else ex.add(i);

258 }

259 clip.excluded = [...ex].sort((a, b) => a - b);

260 recomputeHits(clip, edit.samples);

261 save();

262 waveView.updateMarkers(clip.onsets.map((t, i) => ({ t, excluded: ex.has(i) })));

263 syncEditorButtons();

264 renderVoiceList();

265}

266

267waveView.onAudition = (i) => {

268 const clip = editingClip();

269 if (!edit || !clip) return;

270 const s0 = Math.floor(clip.onsets[i] * edit.sr);

271 const s1 = i + 1 < clip.onsets.length ? Math.floor(clip.onsets[i + 1] * edit.sr) : edit.samples.length;

272 playSlice(edit.samples, edit.sr, s0, s1);

273};

274waveView.onSelectionChange = syncEditorButtons;

275

276$('capture-exclude').addEventListener('click', toggleExcluded);

277

278$('capture-sens').addEventListener('input', () => {

279 ui.captureSens = Number(($('capture-sens') as HTMLInputElement).value);

280 saveUi(ui);

281 const clip = editingClip();

282 if (!edit || !clip) return;

283 clip.sens = Number(($('capture-sens') as HTMLInputElement).value);

284 clip.onsets = detectOnsets(edit.samples, edit.sr, clip.sens);

285 clip.excluded = [];

286 recomputeHits(clip, edit.samples);

287 save();

288 refreshEditor();

289 renderVoiceList();

290});

291

292$('capture-clip').addEventListener('change', () => {

293 if (!edit) return;

294 openEditor(edit.voiceId, ($('capture-clip') as HTMLSelectElement).value);

295});

296

297$('capture-delete').addEventListener('click', () => {

298 const voice = edit && voiceById(edit.voiceId);

299 const clip = editingClip();

300 if (!edit || !voice || !clip) return;

301 if (!confirm(`Delete this clip and its ${clip.hits.length} trained hits?`)) return;

302 void deleteClipAudio(clip.id);

303 voice.clips = voice.clips.filter((c) => c.id !== clip.id);

304 save();

305 renderVoiceList();

306 if (voice.clips.length) openEditor(voice.id, voice.clips[voice.clips.length - 1].id);

307 else closeEditor();

308});

309

310function closeEditor(): void {

311 edit = null;

312 $('capture').hidden = true;

313}

314$('capture-done').addEventListener('click', closeEditor);

315

316// ---- voices ---------------------------------------------------------------

317

318function bindingValue(v: Voice): string {

319 return v.binding.kind === 'synth' ? `synth:${v.binding.preset}` : `iw:${v.binding.instrument}`;

320}

321

322function renderVoiceList(): void {

323 const list = $('voice-list');

324 list.textContent = '';

325 const busy = recorder !== null;

326 for (const voice of voices) {

327 const row = document.createElement('div');

328 row.className = 'voice-row';

329

330 const swatch = document.createElement('span');

331 swatch.className = 'swatch';

332 swatch.style.background = voice.color;

333

334 const name = document.createElement('input');

335 name.className = 'name';

336 name.value = voice.name;

337 name.addEventListener('change', () => {

338 voice.name = name.value.trim() || voice.name;

339 name.value = voice.name;

340 save();

341 refreshTimeline();

342 if (edit?.voiceId === voice.id) refreshEditor();

343 });

344

345 const n = trainPool(voice).length;

346 const count = document.createElement('span');

347 count.className = 'count';

348 count.textContent = `${voice.clips.length} clip${voice.clips.length === 1 ? '' : 's'} · ${n} hit${n === 1 ? '' : 's'}`;

349

350 const rec = document.createElement('button');

351 const recordingThis = recTarget?.kind === 'voice' && recTarget.voiceId === voice.id;

352 rec.innerHTML = recordingThis ? '■ Stop' : '● Record';

353 if (recordingThis) rec.className = 'rec';

354 rec.disabled = busy && !recordingThis;

355 rec.addEventListener('click', () => {

356 if (recordingThis) void stopRecording();

357 else void startRecording({ kind: 'voice', voiceId: voice.id });

358 });

359

360 const editBtn = document.createElement('button');

361 editBtn.textContent = 'Edit';

362 editBtn.title = 'reopen this voice\'s training clips';

363 editBtn.disabled = voice.clips.length === 0;

364 editBtn.addEventListener('click', () => {

365 openEditor(voice.id, voice.clips[voice.clips.length - 1].id);

366 });

367

368 const binding = document.createElement('select');

369 const gSynth = document.createElement('optgroup');

370 gSynth.label = 'synth';

371 for (const p of PRESETS) {

372 const o = document.createElement('option');

373 o.value = `synth:${p}`; o.textContent = p;

374 gSynth.appendChild(o);

375 }

376 const gIw = document.createElement('optgroup');

377 gIw.label = 'instrawave';

378 for (const nm of iw.names()) {

379 const o = document.createElement('option');

380 o.value = `iw:${nm}`; o.textContent = nm;

381 gIw.appendChild(o);

382 }

383 binding.append(gSynth, gIw);

384 binding.value = bindingValue(voice);

385 binding.addEventListener('change', () => {

386 const [kind, which] = binding.value.split(':');

387 if (kind === 'synth') voice.binding = { kind: 'synth', preset: which as SynthPreset };

388 else {

389 const old = voice.binding;

390 const set = old.kind === 'iw' && old.instrument === which ? old.set : {};

391 voice.binding = { kind: 'iw', instrument: which, set };

392 }

393 save();

394 renderVoiceList();

395 updateBakeBar();

396 });

397

398 const tune = document.createElement('button');

399 tune.textContent = expandedTuning.has(voice.id) ? 'Tune ▾' : 'Tune ▸';

400 tune.disabled = voice.binding.kind !== 'iw';

401 tune.addEventListener('click', () => {

402 if (expandedTuning.has(voice.id)) expandedTuning.delete(voice.id);

403 else expandedTuning.add(voice.id);

404 renderVoiceList();

405 });

406

407 const clear = document.createElement('button');

408 clear.textContent = 'Clear';

409 clear.title = 'forget this voice\'s training';

410 clear.disabled = n === 0;

411 clear.addEventListener('click', () => {

412 if (!confirm(`Forget all ${n} trained hits of "${voice.name}"?`)) return;

413 for (const c of voice.clips) void deleteClipAudio(c.id);

414 voice.clips = [];

415 voice.loose = [];

416 save();

417 if (edit?.voiceId === voice.id) closeEditor();

418 renderVoiceList();

419 });

420

421 const del = document.createElement('button');

422 del.textContent = '✕';

423 del.title = 'delete voice';

424 del.addEventListener('click', () => {

425 if (!confirm(`Delete voice "${voice.name}"? Its training and its timeline events go with it.`)) return;

426 for (const c of voice.clips) void deleteClipAudio(c.id);

427 voices = voices.filter((v) => v.id !== voice.id);

428 take.hits = take.hits.filter((h) => h.voiceId !== voice.id);

429 saveTakeHits(take.hits);

430 save();

431 if (edit?.voiceId === voice.id) closeEditor();

432 renderVoiceList();

433 refreshTimeline();

434 updateBakeBar();

435 });

436

437 const spacer = document.createElement('span');

438 spacer.className = 'spacer';

439 row.append(swatch, name, count, rec, editBtn, clear, spacer, binding, tune, del);

440 list.appendChild(row);

441

442 if (voice.binding.kind === 'iw' && expandedTuning.has(voice.id)) {

443 list.appendChild(tuningPanel(voice, voice.binding.instrument, voice.binding.set));

444 }

445 }

446}

447

448function tuningPanel(voice: Voice, instrument: string, set: Record<string, number>): HTMLElement {

449 const panel = document.createElement('div');

450 panel.className = 'tuning-panel';

451 const bar = document.createElement('div');

452 bar.className = 'bar';

453 bar.style.gridColumn = '1 / -1';

454 bar.style.margin = '0';

455 const hear = document.createElement('button');

456 hear.textContent = 'Hear';

457 hear.addEventListener('click', () => void auditionIw(instrument, set));

458 const hint = document.createElement('span');

459 hint.className = 'hint';

460 hint.textContent = 'one hit at these settings, from the current bake';

461 bar.append(hear, hint);

462 panel.appendChild(bar);

463 for (const p of iw.tuning(instrument)) {

464 const div = document.createElement('div');

465 div.className = 'param';

466 const label = document.createElement('label');

467 const nameSpan = document.createElement('span');

468 nameSpan.textContent = `${p.label} (${p.unit})`;

469 const valSpan = document.createElement('span');

470 const input = document.createElement('input');

471 input.type = 'range';

472 input.min = String(p.min);

473 input.max = String(p.max);

474 input.step = String(p.step ?? (p.max - p.min) / 200);

475 input.value = String(set[p.key] ?? p.value);

476 const show = () => { valSpan.textContent = Number(input.value).toPrecision(4); };

477 show();

478 input.addEventListener('input', () => {

479 set[p.key] = Number(input.value);

480 show();

481 save();

482 });

483 label.append(nameSpan, valSpan);

484 div.append(label, input);

485 panel.appendChild(div);

486 }

487 return panel;

488}

489

490$('add-voice').addEventListener('click', () => {

491 const input = $('voice-name') as HTMLInputElement;

492 const name = input.value.trim() || `voice ${voices.length + 1}`;

493 voices.push({

494 id: uid(),

495 name,

496 color: PALETTE[voices.length % PALETTE.length],

497 binding: { kind: 'synth', preset: PRESETS[voices.length % PRESETS.length] },

498 clips: [],

499 loose: [],

500 });

501 input.value = '';

502 save();

503 renderVoiceList();

504 refreshTimeline();

505});

506

507$('reset-all').addEventListener('click', () => {

508 if (!confirm('Forget all voices, training, and the current take?')) return;

509 clearStore();

510 void clearBulk();

511 iw.setOverrides({});

512 saveIw();

513 voices = [];

514 closeIwEditor();

515 take = { clip: null, hits: [] };

516 closeEditor();

517 renderVoiceList();

518 refreshTimeline();

519 syncTakeBar();

520 updateBakeBar();

521 status('');

522});

523

524// ---- bakes ------------------------------------------------------------------

525

526function neededInstruments(): string[] {

527 return [...new Set(voices.filter((v) => v.binding.kind === 'iw')

528 .map((v) => (v.binding as { instrument: string }).instrument))];

529}

530

531function updateBakeBar(): void {

532 void refreshBakeBar();

533}

534

535let bakeBarEpoch = 0;

536async function refreshBakeBar(): Promise<void> {

537 const epoch = ++bakeBarEpoch;

538 const bar = $('bakebar');

539 const needed = neededInstruments();

540 bar.hidden = needed.length === 0;

541 if (needed.length === 0) { bar.textContent = ''; return; }

542 const states = await Promise.all(needed.map(async (name) => {

543 if (await iw.bakeReady(name)) return 'ready' as const;

544 if (bakeJob?.name === name) return 'baking' as const;

545 const stored = await getBakeKey(name).catch(() => null);

546 return stored === (await iw.currentKey(name)) ? ('stored' as const) : ('none' as const);

547 }));

548 if (epoch !== bakeBarEpoch) return; // a newer refresh superseded this one

549 bar.textContent = '';

550 bakeProg = null;

551 needed.forEach((name, i) => {

552 const label = document.createElement('span');

553 label.className = 'hint';

554 label.textContent = `${name} bake:`;

555 bar.appendChild(label);

556 const st = states[i];

557 if (st === 'ready' || st === 'stored') {

558 const ok = document.createElement('span');

559 ok.textContent = st;

560 ok.style.color = '#15803d';

561 if (st === 'stored') ok.title = 'saved in the browser; loads on first use';

562 bar.appendChild(ok);

563 } else if (st === 'baking') {

564 const prog = document.createElement('progress');

565 prog.max = 1;

566 prog.value = 0;

567 bakeProg = prog;

568 const cancel = document.createElement('button');

569 cancel.textContent = 'Cancel';

570 cancel.addEventListener('click', () => { if (bakeJob) bakeJob.cancel = true; });

571 bar.append(prog, cancel);

572 } else {

573 const btn = document.createElement('button');

574 btn.textContent = 'Precompute';

575 btn.disabled = bakeJob !== null;

576 btn.addEventListener('click', () => void startBake(name));

577 bar.appendChild(btn);

578 }

579 const inst = document.createElement('button');

580 inst.textContent = 'Instrument\u2026';

581 inst.title = 'geometry, accuracy, and the 3D view';

582 inst.addEventListener('click', () => void openIwEditor(name));

583 bar.appendChild(inst);

584 });

585}

586

587/** Bring a bake into memory: already there, or stored in the browser from

588 * an earlier session under the same geometry. */

589async function ensureBakeLoaded(name: string): Promise<boolean> {

590 if (await iw.bakeReady(name)) return true;

591 const key = await iw.currentKey(name);

592 if ((await getBakeKey(name).catch(() => null)) !== key) return false;

593 status(`loading the stored ${name} bake\u2026`);

594 const bytes = await getBakeBytes(name).catch(() => null);

595 if (!bytes) { status(''); return false; }

596 const bake = parseBake(bytes);

597 if (bake.key !== key) return false;

598 iw.adoptBake(name, bake);

599 status('');

600 updateBakeBar();

601 return true;

602}

603

604async function startBake(name: string): Promise<void> {

605 if (bakeJob) return;

606 bakeJob = { name, cancel: false };

607 updateBakeBar();

608 try {

609 await iw.precompute(

610 name,

611 (done, total) => { if (bakeProg) bakeProg.value = done / total; },

612 () => bakeJob?.cancel ?? true,

613 );

614 status(`${name} bake ready; storing it in the browser\u2026`);

615 const ser = iw.serializedBake(name);

616 if (ser) {

617 await putBake(name, ser.key, ser.bytes).then(

618 () => status(`${name} bake ready and stored (${(ser.bytes.length / 1e6).toFixed(0)} MB)`),

619 () => status(`${name} bake ready, but storing it failed; it will be recomputed next session`),

620 );

621 }

622 } catch (e) {

623 status(e instanceof PhaseCancelled ? `${name} precompute cancelled`

624 : `${name} precompute failed: ${e instanceof Error ? e.message : e}`);

625 } finally {

626 bakeJob = null;

627 updateBakeBar();

628 }

629}

630

631// ---- take -----------------------------------------------------------------

632

633const timeline = new Timeline($('timeline') as HTMLCanvasElement);

634

635function processTake(): void {

636 if (!take.clip) return;

637 stopPlayback();

638 const { samples, sr } = take.clip;

639 const sens = Number(($('take-sens') as HTMLInputElement).value);

640 const onsets = detectOnsets(samples, sr, sens);

641 const trained = voices.filter((v) => trainPool(v).length > 0);

642 const trainSet: TrainingSet = {

643 voiceIds: trained.flatMap((v) => trainPool(v).map(() => v.id)),

644 vecs: trained.flatMap((v) => trainPool(v).map((h) => h.vec)),

645 };

646

647 const hits: TakeHit[] = [];

648 let refDb = -160;

649 for (let i = 0; i < onsets.length; i++) {

650 const s0 = Math.floor(onsets[i] * sr);

651 const s1 = i + 1 < onsets.length ? Math.floor(onsets[i + 1] * sr) : samples.length;

652 const f = extractHit(samples, sr, s0, s1);

653 if (f.peakDb > refDb) refDb = f.peakDb;

654 const cls = classify(f.vec, trainSet);

655 const voiceId = cls?.voiceId ?? voices[0]?.id;

656 if (!voiceId) continue;

657 hits.push({ id: uid(), t: onsets[i], voiceId, gain: 0, peakDb: f.peakDb, vec: f.vec, s0, s1 });

658 }

659 for (const h of hits) h.gain = gainFromPeakDb(h.peakDb, refDb);

660 take.hits = hits;

661 saveTakeHits(take.hits);

662 refreshTimeline();

663 syncTakeBar();

664 if (voices.length === 0) status('no voices defined; add and train a voice, then hit re-detect');

665 else if (trainSet.vecs.length === 0) status(`no trained hits yet; all ${hits.length} events landed on "${voices[0].name}"`);

666 else status(`${hits.length} hits classified across ${trained.length} trained voice${trained.length === 1 ? '' : 's'}`);

667}

668

669function refreshTimeline(): void {

670 timeline.setData(

671 voices.map((v) => ({ id: v.id, name: v.name, color: v.color })),

672 take.hits,

673 take.clip,

674 );

675 syncHitButtons();

676}

677

678function syncTakeBar(): void {

679 const recordingTake = recTarget?.kind === 'take';

680 const btn = $('take-record') as HTMLButtonElement;

681 btn.innerHTML = recordingTake ? '■ Stop' : '● Record take';

682 btn.className = recordingTake ? 'rec' : '';

683 btn.disabled = recorder !== null && !recordingTake;

684 ($('take-redetect') as HTMLButtonElement).disabled = take.clip === null;

685 ($('take-play') as HTMLButtonElement).disabled = take.hits.length === 0;

686}

687

688function syncPlayButton(): void {

689 ($('take-play') as HTMLButtonElement).textContent = playing ? 'Stop' : 'Play';

690 ($('take-play') as HTMLButtonElement).disabled = take.hits.length === 0 && !playing;

691}

692

693function syncHitButtons(): void {

694 const sel = take.hits.find((h) => h.id === timeline.selected);

695 ($('hit-delete') as HTMLButtonElement).disabled = !sel;

696 ($('hit-learn') as HTMLButtonElement).disabled = !sel || sel.vec === null;

697}

698

699$('take-record').addEventListener('click', () => {

700 if (recTarget?.kind === 'take') void stopRecording();

701 else void startRecording({ kind: 'take' });

702});

703

704$('take-redetect').addEventListener('click', processTake);

705$('take-sens').addEventListener('change', () => {

706 ui.takeSens = Number(($('take-sens') as HTMLInputElement).value);

707 saveUi(ui);

708 if (take.clip) processTake();

709});

710

711timeline.onSelect = (id) => {

712 syncHitButtons();

713 const hit = take.hits.find((h) => h.id === id);

714 if (!hit) return;

715 const voice = voiceById(hit.voiceId);

716 status(`hit at ${hit.t.toFixed(3)} s · "${voice?.name ?? '?'}" · strength ${hit.gain.toFixed(2)}`);

717 if (hit.s0 >= 0 && take.clip) playSlice(take.clip.samples, take.clip.sr, hit.s0, hit.s1);

718 else if (voice?.binding.kind === 'synth') void auditionSynth(hit, voice.binding.preset);

719};

720

721async function auditionSynth(hit: TakeHit, preset: SynthPreset): Promise<void> {

722 const buf = await renderSynth([{ t: 0, voiceId: hit.voiceId, gain: hit.gain }], () => preset, 1);

723 void playBuffer(buf);

724}

725

726timeline.onReassign = (id, voiceId) => {

727 saveTakeHits(take.hits);

728 const voice = voiceById(voiceId);

729 status(voice ? `hit moved to "${voice.name}"; Learn adds it to that voice's training` : '');

730 syncHitButtons();

731};

732

733timeline.onAdd = (t, voiceId) => {

734 take.hits.push({ id: uid(), t, voiceId, gain: 0.8, peakDb: 0, vec: null, s0: -1, s1: -1 });

735 take.hits.sort((a, b) => a.t - b.t);

736 saveTakeHits(take.hits);

737 refreshTimeline();

738};

739

740$('hit-delete').addEventListener('click', deleteSelected);

741document.addEventListener('keydown', (e) => {

742 if (document.activeElement && ['INPUT', 'SELECT', 'TEXTAREA'].includes(document.activeElement.tagName)) return;

743 if (!$('capture').hidden && edit) {

744 if (e.key === 'x' || e.key === 'X' || e.key === 'Delete' || e.key === 'Backspace') {

745 e.preventDefault();

746 toggleExcluded();

747 }

748 return;

749 }

750 if (e.key === 'Delete' || e.key === 'Backspace') deleteSelected();

751});

752function deleteSelected(): void {

753 if (!timeline.selected) return;

754 take.hits = take.hits.filter((h) => h.id !== timeline.selected);

755 saveTakeHits(take.hits);

756 timeline.selected = null;

757 refreshTimeline();

758}

759

760$('hit-learn').addEventListener('click', () => {

761 const hit = take.hits.find((h) => h.id === timeline.selected);

762 if (!hit || !hit.vec) return;

763 const voice = voiceById(hit.voiceId);

764 if (!voice) return;

765 voice.loose.push({ vec: hit.vec, peakDb: hit.peakDb });

766 save();

767 renderVoiceList();

768 status(`learned this hit as "${voice.name}" (${trainPool(voice).length} trained)`);

769});

770

771// ---- the mix ----------------------------------------------------------------

772

773async function renderMix(): Promise<AudioBuffer> {

774 const seconds = Math.max(...take.hits.map((h) => h.t)) + 1.5;

775 const sig = JSON.stringify({

776 hits: take.hits.map((h) => [h.t, h.voiceId, h.gain]),

777 bindings: voices.map((v) => [v.id, v.binding]),

778 iw: iw.dumpOverrides(),

779 });

780 if (renderCache?.sig === sig) return renderCache.buf;

781

782 const synthEvents: NoteEvent[] = [];

783 const iwEvents = new Map<string, StimulusEvent[]>();

784 for (const h of take.hits) {

785 const v = voiceById(h.voiceId);

786 if (!v) continue;

787 if (v.binding.kind === 'synth') {

788 synthEvents.push({ t: h.t, voiceId: h.voiceId, gain: h.gain });

789 } else {

790 const b = v.binding;

791 const allowed = new Set(iw.tuning(b.instrument).map((p) => p.key));

792 const set: Record<string, number> = {};

793 for (const [k, val] of Object.entries(b.set)) if (allowed.has(k)) set[k] = val;

794 const strength = iw.strengthParam(b.instrument);

795 set[strength] = (b.set[strength] ?? iw.defaultValue(b.instrument, strength)) * h.gain;

796 let list = iwEvents.get(b.instrument);

797 if (!list) iwEvents.set(b.instrument, (list = []));

798 list.push({ t: h.t, set });

799 }

800 }

801 const missing: string[] = [];

802 for (const n of iwEvents.keys()) {

803 if (!(await ensureBakeLoaded(n))) missing.push(n);

804 }

805 if (missing.length) {

806 throw new Error(`precompute the ${missing.join(' and ')} bake${missing.length > 1 ? 's' : ''} first`);

807 }

808

809 const out = new Float32Array(Math.ceil(seconds * MIX_SR));

810 if (synthEvents.length) {

811 const buf = await renderSynth(

812 synthEvents,

813 (id) => {

814 const v = voiceById(id);

815 return v?.binding.kind === 'synth' ? v.binding.preset : 'kick';

816 },

817 seconds,

818 );

819 mixInto(out, buf.getChannelData(0));

820 }

821 for (const [name, evs] of iwEvents) {

822 evs.sort((a, b) => a.t - b.t);

823 const { trace, dt } = await iw.render(name, seconds, evs, (done, total) =>

824 status(`rendering ${name} (${evs.length} events): ${Math.round((100 * done) / total)}%`));

825 // Physical pressure amplitudes carry no absolute scale worth keeping

826 // across engines; each instrument's trace is levelled to sit alongside

827 // the synth part, while relative volumes WITHIN it stay physical.

828 const res = resampleLinear(trace, 1 / dt, MIX_SR);

829 const p = peak(res);

830 if (p > 0) mixInto(out, res, 0.8 / p);

831 }

832 const pAll = peak(out);

833 if (pAll > 0.98) for (let i = 0; i < out.length; i++) out[i] *= 0.95 / pAll;

834 const buf = audio().createBuffer(1, out.length, MIX_SR);

835 buf.getChannelData(0).set(out);

836 renderCache = { sig, buf };

837 return buf;

838}

839

840$('take-play').addEventListener('click', async () => {

841 if (playing) { stopPlayback(); return; }

842 if (take.hits.length === 0) return;

843 const btn = $('take-play') as HTMLButtonElement;

844 btn.disabled = true;

845 try {

846 const buf = await renderMix();

847 status('');

848 void playBuffer(buf, (t) => timeline.playhead(t));

849 } catch (e) {

850 status(e instanceof Error ? e.message : String(e));

851 } finally {

852 btn.disabled = false;

853 syncPlayButton();

854 }

855});

856

857// ---- tuning audition ----------------------------------------------------------

858

859async function auditionIw(instrument: string, set: Record<string, number>): Promise<void> {

860 if (iwBusy || bakeJob) { status('the GPU is busy; try again in a moment'); return; }

861 iwBusy = true;

862 try {

863 if (!(await ensureBakeLoaded(instrument))) {

864 status(`no current ${instrument} bake; precompute it in the bake bar below`);

865 return;

866 }

867 const allowed = new Set(iw.tuning(instrument).map((p) => p.key));

868 const ev: Record<string, number> = {};

869 for (const [k, v] of Object.entries(set)) if (allowed.has(k)) ev[k] = v;

870 const t60 = set.t60 ?? (allowed.has('t60') ? iw.defaultValue(instrument, 't60') : 2);

871 const seconds = Math.min(5, Math.max(1.2, t60 * 0.6 + 0.5));

872 const { trace, dt } = await iw.render(instrument, seconds, [{ t: 0, set: ev }], (done, total) =>

873 status(`rendering ${instrument}: ${Math.round((100 * done) / total)}%`));

874 const res = resampleLinear(trace, 1 / dt, MIX_SR);

875 const pk = peak(res);

876 if (pk > 0) for (let i = 0; i < res.length; i++) res[i] *= 0.8 / pk;

877 const buf = audio().createBuffer(1, res.length, MIX_SR);

878 buf.getChannelData(0).set(res);

879 status('');

880 void playBuffer(buf);

881 } catch (e) {

882 status(e instanceof Error ? e.message : String(e));

883 } finally {

884 iwBusy = false;

885 }

886}

887

888// ---- instrument editor (geometry, accuracy, the 3D view) ----------------------

889// Geometry belongs to the instrument, not the voice: it is what the shared

890// bake depends on, so the editor opens from the bake bar and an edit marks

891// the bake stale there. The scene is instrawave's own viewer, raymarching

892// the solver's actual fields, so what it shows is what the bake will see.

893

894let iwEdit: { name: string; viewer: Viewer | null } | null = null;

895const iwCam: Camera = { yaw: -2.2, pitch: 0.45, dist: 1.6 };

896const iwVolume = $('iw-volume') as HTMLCanvasElement;

897const iwOverlay = $('iw-overlay') as HTMLCanvasElement;

898

899function sizeSceneCanvas(c: HTMLCanvasElement): void {

900 const r = c.getBoundingClientRect();

901 const dpr = Math.min(devicePixelRatio || 1, 2);

902 const w = Math.max(1, Math.round(r.width * dpr));

903 const h = Math.max(1, Math.round(r.height * dpr));

904 if (c.width !== w || c.height !== h) { c.width = w; c.height = h; }

905}

906

907let iwViewTimer: number | null = null;

908function scheduleIwView(): void {

909 if (iwViewTimer !== null) clearTimeout(iwViewTimer);

910 iwViewTimer = window.setTimeout(() => {

911 iwViewTimer = null;

912 const e = iwEdit;

913 if (!e?.viewer) return;

914 void (async () => {

915 try {

916 await e.viewer!.update(iw.scope(e.name));

917 e.viewer!.draw(iwCam);

918 } catch (err) {

919 status(`scene view: ${err instanceof Error ? err.message : err}`);

920 }

921 })();

922 }, 180);

923}

924

925async function openIwEditor(name: string): Promise<void> {

926 iwEdit = { name, viewer: null };

927 $('iwedit').hidden = false;

928 $('iwedit-title').textContent = name;

929 buildIwParams(name);

930 try {

931 const device = await iw.ensureDevice();

932 if (iwEdit?.name !== name) return;

933 sizeSceneCanvas(iwVolume);

934 sizeSceneCanvas(iwOverlay);

935 iwEdit.viewer = new Viewer(device, iw.file(name), iwVolume, iwOverlay);

936 scheduleIwView();

937 } catch (e) {

938 status(`scene view unavailable: ${e instanceof Error ? e.message : e} (the sliders still work)`);

939 }

940}

941

942function buildIwParams(name: string): void {

943 const box = $('iwedit-params');

944 box.textContent = '';

945 const values = iw.values(name);

946 for (const p of iw.geometryParams(name)) {

947 const div = document.createElement('div');

948 div.className = 'param';

949 const label = document.createElement('label');

950 const nameSpan = document.createElement('span');

951 nameSpan.textContent = `${p.label} (${p.unit})`;

952 if (p.hint) nameSpan.title = p.hint;

953 const valSpan = document.createElement('span');

954 const input = document.createElement('input');

955 input.type = 'range';

956 input.min = String(p.min);

957 input.max = String(p.max);

958 input.step = String(p.step ?? (p.max - p.min) / 200);

959 input.value = String(values[p.key]);

960 const show = () => { valSpan.textContent = Number(input.value).toPrecision(4); };

961 show();

962 input.addEventListener('input', () => {

963 iw.setOverride(name, p.key, Number(input.value));

964 show();

965 saveIw();

966 scheduleIwView();

967 updateBakeBar(); // the bake just went stale (or came back)

968 });

969 label.append(nameSpan, valSpan);

970 div.append(label, input);

971 box.appendChild(div);

972 }

973}

974

975function closeIwEditor(): void {

976 iwEdit = null;

977 $('iwedit').hidden = true;

978}

979$('iwedit-done').addEventListener('click', closeIwEditor);

980

981{

982 let dragging = false, px = 0, py = 0;

983 iwOverlay.addEventListener('pointerdown', (e) => {

984 dragging = true; px = e.clientX; py = e.clientY;

985 iwOverlay.setPointerCapture(e.pointerId);

986 });

987 iwOverlay.addEventListener('pointermove', (e) => {

988 if (!dragging || !iwEdit?.viewer) return;

989 iwCam.yaw -= (e.clientX - px) * 0.008;

990 iwCam.pitch = Math.min(1.45, Math.max(-1.45, iwCam.pitch + (e.clientY - py) * 0.008));

991 px = e.clientX; py = e.clientY;

992 iwEdit.viewer.draw(iwCam);

993 });

994 iwOverlay.addEventListener('pointerup', () => { dragging = false; });

995 iwOverlay.addEventListener('wheel', (e) => {

996 if (!iwEdit?.viewer) return;

997 e.preventDefault();

998 iwCam.dist = Math.min(6, Math.max(0.3, iwCam.dist * Math.exp(e.deltaY * 0.001)));

999 iwEdit.viewer.draw(iwCam);

1000 }, { passive: false });

1001 window.addEventListener('resize', () => {

1002 if (!iwEdit?.viewer) return;

1003 sizeSceneCanvas(iwVolume);

1004 sizeSceneCanvas(iwOverlay);

1005 scheduleIwView();

1006 });

1007}

1008

1009// ---- boot -----------------------------------------------------------------

1010

1011($('capture-sens') as HTMLInputElement).value = String(ui.captureSens);

1012($('take-sens') as HTMLInputElement).value = String(ui.takeSens);

1013take.hits = loadTakeHits() ?? [];

1014void getTakeAudio().then((a) => {

1015 if (a) {

1016 take.clip = { samples: a.samples, sr: a.sr };

1017 refreshTimeline();

1018 syncTakeBar();

1019 }

1020}).catch(() => { /* the hits still play and edit; re-detect needs the audio */ });

1021renderVoiceList();

1022refreshTimeline();

1023syncTakeBar();

1024updateBakeBar();

1025if (voices.length === 0) {

1026 status('start by adding a voice, then record yourself making its sound a handful of times');

1027}