src/main.ts at 838af43524861f8c5fa0abb91d665b192d78b8a2 - proof-of-concept/instrawave-compose

proof-of-concept / instrawave-compose

1085 lines · 39.3 KBBlame History Raw

1/**

2 * The page: voices (named, trained by example, each bound to a synth

3 * preset or an instrawave instrument), a take (recorded once, segmented,

4 * classified onto the voices), a timeline (the take as editable events),

5 * and playback through the renderer seam in render/types.ts. Training

6 * clips persist (audio in IndexedDB, detections and features in

7 * localStorage) and reopen in the same editor they were made in. The GPU

8 * is touched only when at least one bound instrument is instrawave.

9 */

10import type { Clip } from './audio/recorder.ts';

11import { Recorder } from './audio/recorder.ts';

12import { detectOnsets } from './analysis/onsets.ts';

13import { extractHit } from './analysis/features.ts';

14import { classify, type TrainingSet } from './analysis/classify.ts';

15import { renderSynth } from './render/synth.ts';

16import { gainFromPeakDb, type NoteEvent, type SynthPreset } from './render/types.ts';

17import { mixInto, peak, resampleLinear } from './render/mix.ts';

18import { wavBytes } from './render/wav.ts';

19import { IwEngine, parseBake, PhaseCancelled, type StimulusEvent } from './render/instrawave.ts';

20import { Viewer, type Camera } from 'instrawave/src/host/view.ts';

21import {

22 clearBulk, clearStore, deleteClipAudio, getBakeBytes, getBakeKey, getClipAudio, getTakeAudio,

23 getRender, loadIwOverrides, loadTakeHits, loadUi, loadVoices, putBake, putClipAudio,

24 putRender, putTakeAudio,

25 saveIwOverrides, saveTakeHits, saveUi, saveVoices, trainPool,

26 type TakeHit, type TrainClip, type Voice,

27} from './store.ts';

28import { WaveView, type Marker } from './ui/waveform.ts';

29import { Timeline } from './ui/timeline.ts';

31const PALETTE = ['#e4573d', '#3d7de4', '#3dbb6e', '#d9a520', '#9061d9', '#d95fa8', '#4fb8c9', '#96a13a'];

32const PRESETS: SynthPreset[] = ['kick', 'snare', 'hat', 'tom'];

33const MIX_SR = 44100;

35const $ = <T extends HTMLElement>(id: string) => document.getElementById(id) as T;

36const uid = () => Math.random().toString(36).slice(2, 10);

38// ---- state ----------------------------------------------------------------

40let voices: Voice[] = loadVoices();

41let take: { clip: Clip | null; hits: TakeHit[] } = { clip: null, hits: [] };

42let edit: { voiceId: string; clipId: string; samples: Float32Array; sr: number } | null = null;

43let recTarget: { kind: 'voice'; voiceId: string } | { kind: 'take' } | null = null;

44const expandedTuning = new Set<string>();

46let actx: AudioContext | null = null;

47const audio = (): AudioContext => (actx ??= new AudioContext());

48let recorder: Recorder | null = null;

49let playing: { src: AudioBufferSourceNode; t0: number; dur: number } | null = null;

50let renderCache: { sig: string; buf: AudioBuffer } | null = null;

52const iw = new IwEngine();

53iw.setOverrides(loadIwOverrides());

54const saveIw = () => saveIwOverrides(iw.dumpOverrides());

55const ui = loadUi();

56let bakeJob: { name: string; cancel: boolean } | null = null;

57let iwBusy = false;

58let bakeProg: HTMLProgressElement | null = null;

60function save(): void { saveVoices(voices); }

61function voiceById(id: string): Voice | undefined { return voices.find((v) => v.id === id); }

62function status(msg: string): void { $('status').textContent = msg; }

64// ---- audition -------------------------------------------------------------

66function playSlice(samples: Float32Array, sr: number, s0: number, s1: number): void {

67 const ctx = audio();

68 const n = Math.min(s1, s0 + Math.floor(0.5 * sr)) - s0;

69 if (n <= 0) return;

70 const buf = ctx.createBuffer(1, n, sr);

71 buf.getChannelData(0).set(samples.subarray(s0, s0 + n));

72 const src = ctx.createBufferSource();

73 src.buffer = buf;

74 src.connect(ctx.destination);

75 src.start();

76}

78async function playBuffer(buf: AudioBuffer, onFrame?: (t: number | null) => void): Promise<void> {

79 const ctx = audio();

80 if (ctx.state === 'suspended') await ctx.resume();

81 const src = ctx.createBufferSource();

82 src.buffer = buf;

83 src.connect(ctx.destination);

84 const t0 = ctx.currentTime;

85 playing = { src, t0, dur: buf.duration };

86 src.onended = () => { if (playing?.src === src) playing = null; onFrame?.(null); syncPlayButton(); };

87 src.start();

88 if (onFrame) {

89 const tick = () => {

90 if (!playing || playing.src !== src) return;

91 onFrame(ctx.currentTime - t0);

92 requestAnimationFrame(tick);

93 };

94 requestAnimationFrame(tick);

95 }

96 syncPlayButton();

97}

99function stopPlayback(): void {

100 playing?.src.stop();

101 playing = null;

102 timeline.playhead(null);

103 syncPlayButton();

104}

105

106// ---- recording ------------------------------------------------------------

107

108let meterRaf = 0;

109function meterOn(): void {

110 $('meter').hidden = false;

111 const fill = $('meter').firstElementChild as HTMLElement;

112 const tick = () => {

113 fill.style.width = `${(recorder?.level() ?? 0) * 100}%`;

114 if (recorder?.recording) meterRaf = requestAnimationFrame(tick);

115 };

116 meterRaf = requestAnimationFrame(tick);

117}

118function meterOff(): void {

119 cancelAnimationFrame(meterRaf);

120 $('meter').hidden = true;

121}

122

123async function startRecording(target: NonNullable<typeof recTarget>): Promise<void> {

124 recorder = new Recorder(audio());

125 try {

126 await recorder.start();

127 } catch (e) {

128 status(`microphone unavailable: ${e instanceof Error ? e.message : e}`);

129 recorder = null;

130 return;

131 }

132 recTarget = target;

133 meterOn();

134 renderVoiceList();

135 syncTakeBar();

136 status(target.kind === 'take'

137 ? 'recording the take; stop when the phrase is done'

138 : `recording examples for "${voiceById(target.voiceId)?.name}"; make the sound several times, then stop`);

139}

140

141async function stopRecording(): Promise<void> {

142 if (!recorder) return;

143 const target = recTarget!;

144 let clip: Clip;

145 try {

146 clip = await recorder.stop();

147 } finally {

148 recorder = null;

149 recTarget = null;

150 meterOff();

151 }

152 renderVoiceList();

153 syncTakeBar();

154 if (target.kind === 'voice') {

155 const voice = voiceById(target.voiceId);

156 if (!voice) return;

157 const sens = Number(($('capture-sens') as HTMLInputElement).value);

158 const tclip: TrainClip = {

159 id: uid(), sr: clip.sr, sens,

160 onsets: detectOnsets(clip.samples, clip.sr, sens),

161 excluded: [], hits: [],

162 };

163 recomputeHits(tclip, clip.samples);

164 voice.clips.push(tclip);

165 save();

166 void putClipAudio(tclip.id, clip.samples, clip.sr).catch(() =>

167 status('warning: clip audio could not be stored; this clip will not be re-editable after a reload'));

168 openEditor(voice.id, tclip.id, { samples: clip.samples, sr: clip.sr });

169 renderVoiceList();

170 } else {

171 take.clip = clip;

172 void putTakeAudio(clip.samples, clip.sr).catch(() =>

173 status('warning: take audio could not be stored; it will not survive a reload'));

174 processTake();

175 }

176}

177

178// ---- training clip editor ----------------------------------------------------

179

180const waveView = new WaveView($('capture-wave') as HTMLCanvasElement);

181

182function editingClip(): TrainClip | null {

183 if (!edit) return null;

184 return voiceById(edit.voiceId)?.clips.find((c) => c.id === edit!.clipId) ?? null;

185}

186

187function recomputeHits(clip: TrainClip, samples: Float32Array): void {

188 const ex = new Set(clip.excluded);

189 clip.hits = [];

190 for (let i = 0; i < clip.onsets.length; i++) {

191 if (ex.has(i)) continue;

192 const s0 = Math.floor(clip.onsets[i] * clip.sr);

193 const s1 = i + 1 < clip.onsets.length ? Math.floor(clip.onsets[i + 1] * clip.sr) : samples.length;

194 const f = extractHit(samples, clip.sr, s0, s1);

195 clip.hits.push({ vec: f.vec, peakDb: f.peakDb });

196 }

197}

198

199function openEditor(voiceId: string, clipId: string, audioData?: { samples: Float32Array; sr: number }): void {

200 const voice = voiceById(voiceId);

201 const clip = voice?.clips.find((c) => c.id === clipId);

202 if (!voice || !clip) return;

203 const open = (samples: Float32Array, sr: number) => {

204 edit = { voiceId, clipId, samples, sr };

205 ($('capture-sens') as HTMLInputElement).value = String(clip.sens);

206 $('capture').hidden = false;

207 refreshEditor();

208 };

209 if (audioData) open(audioData.samples, audioData.sr);

210 else {

211 void getClipAudio(clipId).then((a) => {

212 if (a) open(a.samples, a.sr);

213 else status('this clip\'s audio is missing (stored before audio persistence, or cleared); its hits still classify');

214 });

215 }

216}

217

218function refreshEditor(): void {

219 const voice = edit && voiceById(edit.voiceId);

220 const clip = editingClip();

221 if (!edit || !voice || !clip) return;

222 $('capture-title').textContent = `"${voice.name}"`;

223 const sel = $('capture-clip') as HTMLSelectElement;

224 sel.textContent = '';

225 voice.clips.forEach((c, i) => {

226 const o = document.createElement('option');

227 o.value = c.id;

228 o.textContent = `clip ${i + 1} · ${c.hits.length} hit${c.hits.length === 1 ? '' : 's'}`;

229 sel.appendChild(o);

230 });

231 sel.value = clip.id;

232 const markers: Marker[] = clip.onsets.map((t, i) => ({ t, excluded: clip.excluded.includes(i) }));

233 waveView.set({ samples: edit.samples, sr: edit.sr }, markers, voice.color);

234 syncEditorButtons();

235}

236

237function syncEditorButtons(): void {

238 const clip = editingClip();

239 const n = waveView.selection.size;

240 const btn = $('capture-exclude') as HTMLButtonElement;

241 btn.disabled = n === 0;

242 if (clip && n > 0) {

243 const allExcluded = [...waveView.selection].every((i) => clip.excluded.includes(i));

244 btn.textContent = allExcluded ? `Include (${n})` : `Exclude (${n})`;

245 } else btn.textContent = 'Exclude';

246 $('capture-count').textContent = clip

247 ? `${clip.hits.length} of ${clip.onsets.length} hits included${n ? ` · ${n} selected` : ''}`

248 : '';

249}

250

251function toggleExcluded(): void {

252 const clip = editingClip();

253 if (!edit || !clip || waveView.selection.size === 0) return;

254 const sel = [...waveView.selection];

255 const allExcluded = sel.every((i) => clip.excluded.includes(i));

256 const ex = new Set(clip.excluded);

257 for (const i of sel) {

258 if (allExcluded) ex.delete(i);

259 else ex.add(i);

260 }

261 clip.excluded = [...ex].sort((a, b) => a - b);

262 recomputeHits(clip, edit.samples);

263 save();

264 waveView.updateMarkers(clip.onsets.map((t, i) => ({ t, excluded: ex.has(i) })));

265 syncEditorButtons();

266 renderVoiceList();

267}

268

269waveView.onAudition = (i) => {

270 const clip = editingClip();

271 if (!edit || !clip) return;

272 const s0 = Math.floor(clip.onsets[i] * edit.sr);

273 const s1 = i + 1 < clip.onsets.length ? Math.floor(clip.onsets[i + 1] * edit.sr) : edit.samples.length;

274 playSlice(edit.samples, edit.sr, s0, s1);

275};

276waveView.onSelectionChange = syncEditorButtons;

277

278$('capture-exclude').addEventListener('click', toggleExcluded);

279

280$('capture-sens').addEventListener('input', () => {

281 ui.captureSens = Number(($('capture-sens') as HTMLInputElement).value);

282 saveUi(ui);

283 const clip = editingClip();

284 if (!edit || !clip) return;

285 clip.sens = Number(($('capture-sens') as HTMLInputElement).value);

286 clip.onsets = detectOnsets(edit.samples, edit.sr, clip.sens);

287 clip.excluded = [];

288 recomputeHits(clip, edit.samples);

289 save();

290 refreshEditor();

291 renderVoiceList();

292});

293

294$('capture-clip').addEventListener('change', () => {

295 if (!edit) return;

296 openEditor(edit.voiceId, ($('capture-clip') as HTMLSelectElement).value);

297});

298

299$('capture-delete').addEventListener('click', () => {

300 const voice = edit && voiceById(edit.voiceId);

301 const clip = editingClip();

302 if (!edit || !voice || !clip) return;

303 if (!confirm(`Delete this clip and its ${clip.hits.length} trained hits?`)) return;

304 void deleteClipAudio(clip.id);

305 voice.clips = voice.clips.filter((c) => c.id !== clip.id);

306 save();

307 renderVoiceList();

308 if (voice.clips.length) openEditor(voice.id, voice.clips[voice.clips.length - 1].id);

309 else closeEditor();

310});

311

312function closeEditor(): void {

313 edit = null;

314 $('capture').hidden = true;

315}

316$('capture-done').addEventListener('click', closeEditor);

317

318// ---- voices ---------------------------------------------------------------

319

320function bindingValue(v: Voice): string {

321 return v.binding.kind === 'synth' ? `synth:${v.binding.preset}` : `iw:${v.binding.instrument}`;

322}

323

324function renderVoiceList(): void {

325 const list = $('voice-list');

326 list.textContent = '';

327 const busy = recorder !== null;

328 for (const voice of voices) {

329 const row = document.createElement('div');

330 row.className = 'voice-row';

331

332 const swatch = document.createElement('span');

333 swatch.className = 'swatch';

334 swatch.style.background = voice.color;

335

336 const name = document.createElement('input');

337 name.className = 'name';

338 name.value = voice.name;

339 name.addEventListener('change', () => {

340 voice.name = name.value.trim() || voice.name;

341 name.value = voice.name;

342 save();

343 refreshTimeline();

344 if (edit?.voiceId === voice.id) refreshEditor();

345 });

346

347 const n = trainPool(voice).length;

348 const count = document.createElement('span');

349 count.className = 'count';

350 count.textContent = `${voice.clips.length} clip${voice.clips.length === 1 ? '' : 's'} · ${n} hit${n === 1 ? '' : 's'}`;

351

352 const rec = document.createElement('button');

353 const recordingThis = recTarget?.kind === 'voice' && recTarget.voiceId === voice.id;

354 rec.innerHTML = recordingThis ? '■ Stop' : '● Record';

355 if (recordingThis) rec.className = 'rec';

356 rec.disabled = busy && !recordingThis;

357 rec.addEventListener('click', () => {

358 if (recordingThis) void stopRecording();

359 else void startRecording({ kind: 'voice', voiceId: voice.id });

360 });

361

362 const editBtn = document.createElement('button');

363 editBtn.textContent = 'Edit';

364 editBtn.title = 'reopen this voice\'s training clips';

365 editBtn.disabled = voice.clips.length === 0;

366 editBtn.addEventListener('click', () => {

367 openEditor(voice.id, voice.clips[voice.clips.length - 1].id);

368 });

369

370 const binding = document.createElement('select');

371 const gSynth = document.createElement('optgroup');

372 gSynth.label = 'synth';

373 for (const p of PRESETS) {

374 const o = document.createElement('option');

375 o.value = `synth:${p}`; o.textContent = p;

376 gSynth.appendChild(o);

377 }

378 const gIw = document.createElement('optgroup');

379 gIw.label = 'instrawave';

380 for (const nm of iw.names()) {

381 const o = document.createElement('option');

382 o.value = `iw:${nm}`; o.textContent = nm;

383 gIw.appendChild(o);

384 }

385 binding.append(gSynth, gIw);

386 binding.value = bindingValue(voice);

387 binding.addEventListener('change', () => {

388 const [kind, which] = binding.value.split(':');

389 if (kind === 'synth') voice.binding = { kind: 'synth', preset: which as SynthPreset };

390 else {

391 const old = voice.binding;

392 const set = old.kind === 'iw' && old.instrument === which ? old.set : {};

393 voice.binding = { kind: 'iw', instrument: which, set };

394 }

395 save();

396 renderVoiceList();

397 updateBakeBar();

398 });

399

400 const tune = document.createElement('button');

401 tune.textContent = expandedTuning.has(voice.id) ? 'Tune ▾' : 'Tune ▸';

402 tune.disabled = voice.binding.kind !== 'iw';

403 tune.addEventListener('click', () => {

404 if (expandedTuning.has(voice.id)) expandedTuning.delete(voice.id);

405 else expandedTuning.add(voice.id);

406 renderVoiceList();

407 });

408

409 const clear = document.createElement('button');

410 clear.textContent = 'Clear';

411 clear.title = 'forget this voice\'s training';

412 clear.disabled = n === 0;

413 clear.addEventListener('click', () => {

414 if (!confirm(`Forget all ${n} trained hits of "${voice.name}"?`)) return;

415 for (const c of voice.clips) void deleteClipAudio(c.id);

416 voice.clips = [];

417 voice.loose = [];

418 save();

419 if (edit?.voiceId === voice.id) closeEditor();

420 renderVoiceList();

421 });

422

423 const del = document.createElement('button');

424 del.textContent = '✕';

425 del.title = 'delete voice';

426 del.addEventListener('click', () => {

427 if (!confirm(`Delete voice "${voice.name}"? Its training and its timeline events go with it.`)) return;

428 for (const c of voice.clips) void deleteClipAudio(c.id);

429 voices = voices.filter((v) => v.id !== voice.id);

430 take.hits = take.hits.filter((h) => h.voiceId !== voice.id);

431 saveTakeHits(take.hits);

432 save();

433 if (edit?.voiceId === voice.id) closeEditor();

434 renderVoiceList();

435 refreshTimeline();

436 updateBakeBar();

437 });

438

439 const spacer = document.createElement('span');

440 spacer.className = 'spacer';

441 row.append(swatch, name, count, rec, editBtn, clear, spacer, binding, tune, del);

442 list.appendChild(row);

443

444 if (voice.binding.kind === 'iw' && expandedTuning.has(voice.id)) {

445 list.appendChild(tuningPanel(voice, voice.binding.instrument, voice.binding.set));

446 }

447 }

448}

449

450function tuningPanel(voice: Voice, instrument: string, set: Record<string, number>): HTMLElement {

451 const panel = document.createElement('div');

452 panel.className = 'tuning-panel';

453 const bar = document.createElement('div');

454 bar.className = 'bar';

455 bar.style.gridColumn = '1 / -1';

456 bar.style.margin = '0';

457 const hear = document.createElement('button');

458 hear.textContent = 'Hear';

459 hear.addEventListener('click', () => void auditionIw(instrument, set));

460 const hint = document.createElement('span');

461 hint.className = 'hint';

462 hint.textContent = 'one hit at these settings, from the current bake';

463 bar.append(hear, hint);

464 panel.appendChild(bar);

465 for (const p of iw.tuning(instrument)) {

466 const div = document.createElement('div');

467 div.className = 'param';

468 const label = document.createElement('label');

469 const nameSpan = document.createElement('span');

470 nameSpan.textContent = `${p.label} (${p.unit})`;

471 const valSpan = document.createElement('span');

472 const input = document.createElement('input');

473 input.type = 'range';

474 input.min = String(p.min);

475 input.max = String(p.max);

476 input.step = String(p.step ?? (p.max - p.min) / 200);

477 input.value = String(set[p.key] ?? p.value);

478 const show = () => { valSpan.textContent = Number(input.value).toPrecision(4); };

479 show();

480 input.addEventListener('input', () => {

481 set[p.key] = Number(input.value);

482 show();

483 save();

484 });

485 label.append(nameSpan, valSpan);

486 div.append(label, input);

487 panel.appendChild(div);

488 }

489 return panel;

490}

491

492$('add-voice').addEventListener('click', () => {

493 const input = $('voice-name') as HTMLInputElement;

494 const name = input.value.trim() || `voice ${voices.length + 1}`;

495 voices.push({

496 id: uid(),

497 name,

498 color: PALETTE[voices.length % PALETTE.length],

499 binding: { kind: 'synth', preset: PRESETS[voices.length % PRESETS.length] },

500 clips: [],

501 loose: [],

502 });

503 input.value = '';

504 save();

505 renderVoiceList();

506 refreshTimeline();

507});

508

509$('reset-all').addEventListener('click', () => {

510 if (!confirm('Forget all voices, training, and the current take?')) return;

511 clearStore();

512 void clearBulk();

513 iw.setOverrides({});

514 saveIw();

515 voices = [];

516 closeIwEditor();

517 take = { clip: null, hits: [] };

518 closeEditor();

519 renderVoiceList();

520 refreshTimeline();

521 syncTakeBar();

522 updateBakeBar();

523 status('');

524});

525

526// ---- bakes ------------------------------------------------------------------

527

528function neededInstruments(): string[] {

529 return [...new Set(voices.filter((v) => v.binding.kind === 'iw')

530 .map((v) => (v.binding as { instrument: string }).instrument))];

531}

532

533function updateBakeBar(): void {

534 void refreshBakeBar();

535}

536

537let bakeBarEpoch = 0;

538async function refreshBakeBar(): Promise<void> {

539 const epoch = ++bakeBarEpoch;

540 const bar = $('bakebar');

541 const needed = neededInstruments();

542 bar.hidden = needed.length === 0;

543 if (needed.length === 0) { bar.textContent = ''; return; }

544 const states = await Promise.all(needed.map(async (name) => {

545 if (await iw.bakeReady(name)) return 'ready' as const;

546 if (bakeJob?.name === name) return 'baking' as const;

547 const stored = await getBakeKey(name).catch(() => null);

548 return stored === (await iw.currentKey(name)) ? ('stored' as const) : ('none' as const);

549 }));

550 if (epoch !== bakeBarEpoch) return; // a newer refresh superseded this one

551 bar.textContent = '';

552 bakeProg = null;

553 needed.forEach((name, i) => {

554 const label = document.createElement('span');

555 label.className = 'hint';

556 label.textContent = `${name} bake:`;

557 bar.appendChild(label);

558 const st = states[i];

559 if (st === 'ready' || st === 'stored') {

560 const ok = document.createElement('span');

561 ok.textContent = st;

562 ok.style.color = '#15803d';

563 if (st === 'stored') ok.title = 'saved in the browser; loads on first use';

564 bar.appendChild(ok);

565 } else if (st === 'baking') {

566 const prog = document.createElement('progress');

567 prog.max = 1;

568 prog.value = 0;

569 bakeProg = prog;

570 const cancel = document.createElement('button');

571 cancel.textContent = 'Cancel';

572 cancel.addEventListener('click', () => { if (bakeJob) bakeJob.cancel = true; });

573 bar.append(prog, cancel);

574 } else {

575 const btn = document.createElement('button');

576 btn.textContent = 'Precompute';

577 btn.disabled = bakeJob !== null;

578 btn.addEventListener('click', () => void startBake(name));

579 bar.appendChild(btn);

580 }

581 const inst = document.createElement('button');

582 inst.textContent = 'Instrument\u2026';

583 inst.title = 'geometry, accuracy, and the 3D view';

584 inst.addEventListener('click', () => void openIwEditor(name));

585 bar.appendChild(inst);

586 });

587}

588

589/** Bring a bake into memory: already there, or stored in the browser from

590 * an earlier session under the same geometry. */

591async function ensureBakeLoaded(name: string): Promise<boolean> {

592 if (await iw.bakeReady(name)) return true;

593 const key = await iw.currentKey(name);

594 if ((await getBakeKey(name).catch(() => null)) !== key) return false;

595 status(`loading the stored ${name} bake\u2026`);

596 const bytes = await getBakeBytes(name).catch(() => null);

597 if (!bytes) { status(''); return false; }

598 const bake = parseBake(bytes);

599 if (bake.key !== key) return false;

600 iw.adoptBake(name, bake);

601 status('');

602 updateBakeBar();

603 return true;

604}

605

606async function startBake(name: string): Promise<void> {

607 if (bakeJob) return;

608 bakeJob = { name, cancel: false };

609 updateBakeBar();

610 try {

611 await iw.precompute(

612 name,

613 (done, total) => { if (bakeProg) bakeProg.value = done / total; },

614 () => bakeJob?.cancel ?? true,

615 );

616 status(`${name} bake ready; storing it in the browser\u2026`);

617 const ser = iw.serializedBake(name);

618 if (ser) {

619 await putBake(name, ser.key, ser.bytes).then(

620 () => status(`${name} bake ready and stored (${(ser.bytes.length / 1e6).toFixed(0)} MB)`),

621 () => status(`${name} bake ready, but storing it failed; it will be recomputed next session`),

622 );

623 }

624 } catch (e) {

625 status(e instanceof PhaseCancelled ? `${name} precompute cancelled`

626 : `${name} precompute failed: ${e instanceof Error ? e.message : e}`);

627 } finally {

628 bakeJob = null;

629 updateBakeBar();

630 }

631}

632

633// ---- take -----------------------------------------------------------------

634

635const timeline = new Timeline($('timeline') as HTMLCanvasElement);

636

637function processTake(): void {

638 if (!take.clip) return;

639 stopPlayback();

640 const { samples, sr } = take.clip;

641 const sens = Number(($('take-sens') as HTMLInputElement).value);

642 const onsets = detectOnsets(samples, sr, sens);

643 const trained = voices.filter((v) => trainPool(v).length > 0);

644 const trainSet: TrainingSet = {

645 voiceIds: trained.flatMap((v) => trainPool(v).map(() => v.id)),

646 vecs: trained.flatMap((v) => trainPool(v).map((h) => h.vec)),

647 };

648

649 const hits: TakeHit[] = [];

650 let refDb = -160;

651 for (let i = 0; i < onsets.length; i++) {

652 const s0 = Math.floor(onsets[i] * sr);

653 const s1 = i + 1 < onsets.length ? Math.floor(onsets[i + 1] * sr) : samples.length;

654 const f = extractHit(samples, sr, s0, s1);

655 if (f.peakDb > refDb) refDb = f.peakDb;

656 const cls = classify(f.vec, trainSet);

657 const voiceId = cls?.voiceId ?? voices[0]?.id;

658 if (!voiceId) continue;

659 hits.push({ id: uid(), t: onsets[i], voiceId, gain: 0, peakDb: f.peakDb, vec: f.vec, s0, s1 });

660 }

661 for (const h of hits) h.gain = gainFromPeakDb(h.peakDb, refDb);

662 take.hits = hits;

663 saveTakeHits(take.hits);

664 refreshTimeline();

665 syncTakeBar();

666 if (voices.length === 0) status('no voices defined; add and train a voice, then hit re-detect');

667 else if (trainSet.vecs.length === 0) status(`no trained hits yet; all ${hits.length} events landed on "${voices[0].name}"`);

668 else status(`${hits.length} hits classified across ${trained.length} trained voice${trained.length === 1 ? '' : 's'}`);

669}

670

671function refreshTimeline(): void {

672 timeline.setData(

673 voices.map((v) => ({ id: v.id, name: v.name, color: v.color })),

674 take.hits,

675 take.clip,

676 );

677 syncHitButtons();

678}

679

680function syncTakeBar(): void {

681 const recordingTake = recTarget?.kind === 'take';

682 const btn = $('take-record') as HTMLButtonElement;

683 btn.innerHTML = recordingTake ? '■ Stop' : '● Record take';

684 btn.className = recordingTake ? 'rec' : '';

685 btn.disabled = recorder !== null && !recordingTake;

686 ($('take-redetect') as HTMLButtonElement).disabled = take.clip === null;

687 ($('take-play') as HTMLButtonElement).disabled = take.hits.length === 0;

688 ($('take-save') as HTMLButtonElement).disabled = take.hits.length === 0;

689}

690

691function syncPlayButton(): void {

692 ($('take-play') as HTMLButtonElement).textContent = playing ? 'Stop' : 'Play';

693 ($('take-play') as HTMLButtonElement).disabled = take.hits.length === 0 && !playing;

694}

695

696function syncHitButtons(): void {

697 const sel = take.hits.find((h) => h.id === timeline.selected);

698 ($('hit-delete') as HTMLButtonElement).disabled = !sel;

699 ($('hit-learn') as HTMLButtonElement).disabled = !sel || sel.vec === null;

700}

701

702$('take-record').addEventListener('click', () => {

703 if (recTarget?.kind === 'take') void stopRecording();

704 else void startRecording({ kind: 'take' });

705});

706

707$('take-redetect').addEventListener('click', processTake);

708$('take-sens').addEventListener('change', () => {

709 ui.takeSens = Number(($('take-sens') as HTMLInputElement).value);

710 saveUi(ui);

711 if (take.clip) processTake();

712});

713

714timeline.onSelect = (id) => {

715 syncHitButtons();

716 const hit = take.hits.find((h) => h.id === id);

717 if (!hit) return;

718 const voice = voiceById(hit.voiceId);

719 status(`hit at ${hit.t.toFixed(3)} s · "${voice?.name ?? '?'}" · strength ${hit.gain.toFixed(2)}`);

720 if (hit.s0 >= 0 && take.clip) playSlice(take.clip.samples, take.clip.sr, hit.s0, hit.s1);

721 else if (voice?.binding.kind === 'synth') void auditionSynth(hit, voice.binding.preset);

722};

723

724async function auditionSynth(hit: TakeHit, preset: SynthPreset): Promise<void> {

725 const buf = await renderSynth([{ t: 0, voiceId: hit.voiceId, gain: hit.gain }], () => preset, 1);

726 void playBuffer(buf);

727}

728

729timeline.onReassign = (id, voiceId) => {

730 saveTakeHits(take.hits);

731 const voice = voiceById(voiceId);

732 status(voice ? `hit moved to "${voice.name}"; Learn adds it to that voice's training` : '');

733 syncHitButtons();

734};

735

736timeline.onAdd = (t, voiceId) => {

737 take.hits.push({ id: uid(), t, voiceId, gain: 0.8, peakDb: 0, vec: null, s0: -1, s1: -1 });

738 take.hits.sort((a, b) => a.t - b.t);

739 saveTakeHits(take.hits);

740 refreshTimeline();

741};

742

743$('hit-delete').addEventListener('click', deleteSelected);

744document.addEventListener('keydown', (e) => {

745 if (document.activeElement && ['INPUT', 'SELECT', 'TEXTAREA'].includes(document.activeElement.tagName)) return;

746 if (!$('capture').hidden && edit) {

747 if (e.key === 'x' || e.key === 'X' || e.key === 'Delete' || e.key === 'Backspace') {

748 e.preventDefault();

749 toggleExcluded();

750 }

751 return;

752 }

753 if (e.key === 'Delete' || e.key === 'Backspace') deleteSelected();

754});

755function deleteSelected(): void {

756 if (!timeline.selected) return;

757 take.hits = take.hits.filter((h) => h.id !== timeline.selected);

758 saveTakeHits(take.hits);

759 timeline.selected = null;

760 refreshTimeline();

761}

762

763$('hit-learn').addEventListener('click', () => {

764 const hit = take.hits.find((h) => h.id === timeline.selected);

765 if (!hit || !hit.vec) return;

766 const voice = voiceById(hit.voiceId);

767 if (!voice) return;

768 voice.loose.push({ vec: hit.vec, peakDb: hit.peakDb });

769 save();

770 renderVoiceList();

771 status(`learned this hit as "${voice.name}" (${trainPool(voice).length} trained)`);

772});

773

774// ---- the mix ----------------------------------------------------------------

775

776const iwTrackCache = new Map<string, { sig: string; data: Float32Array }>();

777

778/** The rendered track for one instrument's events: from the caches when

779 * the signature (events, tuning, gains, bake key) matches, memory first,

780 * then IndexedDB, where a stored track plays without its bake even

781 * loaded; from the GPU otherwise. Returned at the mix rate, levelled to

782 * 0.8 peak. */

783async function iwTrack(name: string, evs: StimulusEvent[]): Promise<Float32Array> {

784 evs.sort((a, b) => a.t - b.t);

785 // The track's own length: its last event plus its longest decay, so an

786 // edit on another lane cannot change this group's signature.

787 let tail = 1.5;

788 const hasT60 = iw.tuning(name).some((p) => p.key === 't60');

789 for (const e of evs) {

790 const t60 = e.set?.t60 ?? (hasT60 ? iw.defaultValue(name, 't60') : 1.5);

791 tail = Math.max(tail, Math.min(6, t60));

792 }

793 const seconds = evs[evs.length - 1].t + tail;

794 const sig = JSON.stringify({

795 key: await iw.currentKey(name),

796 secs: seconds,

797 evs: evs.map((e) => [e.t, Object.entries(e.set ?? {}).sort()]),

798 });

799 const mem = iwTrackCache.get(name);

800 if (mem?.sig === sig) return mem.data;

801 const stored = await getRender(name).catch(() => null);

802 if (stored?.sig === sig) {

803 iwTrackCache.set(name, stored);

804 return stored.data;

805 }

806 if (!(await ensureBakeLoaded(name))) {

807 throw new Error(`precompute the ${name} bake first`);

808 }

809 const { trace, dt } = await iw.render(name, seconds, evs, (done, total) =>

810 status(`rendering ${name} (${evs.length} event${evs.length === 1 ? '' : 's'}): ${Math.round((100 * done) / total)}%`));

811 // Physical pressure amplitudes carry no absolute scale worth keeping

812 // across engines; each instrument's track is levelled to sit alongside

813 // the synth part, while relative volumes WITHIN it stay physical.

814 const data = resampleLinear(trace, 1 / dt, MIX_SR);

815 const p = peak(data);

816 if (p > 0) for (let i = 0; i < data.length; i++) data[i] *= 0.8 / p;

817 iwTrackCache.set(name, { sig, data });

818 void putRender(name, sig, data).catch(() => { /* it will re-render next session */ });

819 return data;

820}

821

822async function renderMix(): Promise<AudioBuffer> {

823 const sig = JSON.stringify({

824 hits: take.hits.map((h) => [h.t, h.voiceId, h.gain]),

825 bindings: voices.map((v) => [v.id, v.binding]),

826 iw: iw.dumpOverrides(),

827 });

828 if (renderCache?.sig === sig) return renderCache.buf;

829

830 const synthEvents: NoteEvent[] = [];

831 const iwEvents = new Map<string, StimulusEvent[]>();

832 for (const h of take.hits) {

833 const v = voiceById(h.voiceId);

834 if (!v) continue;

835 if (v.binding.kind === 'synth') {

836 synthEvents.push({ t: h.t, voiceId: h.voiceId, gain: h.gain });

837 } else {

838 const b = v.binding;

839 const allowed = new Set(iw.tuning(b.instrument).map((p) => p.key));

840 const set: Record<string, number> = {};

841 for (const [k, val] of Object.entries(b.set)) if (allowed.has(k)) set[k] = val;

842 const strength = iw.strengthParam(b.instrument);

843 set[strength] = (b.set[strength] ?? iw.defaultValue(b.instrument, strength)) * h.gain;

844 let list = iwEvents.get(b.instrument);

845 if (!list) iwEvents.set(b.instrument, (list = []));

846 list.push({ t: h.t, set });

847 }

848 }

849 const tracks: Float32Array[] = [];

850 for (const [name, evs] of iwEvents) tracks.push(await iwTrack(name, evs));

851

852 const synthSeconds = synthEvents.length

853 ? Math.max(...synthEvents.map((e) => e.t)) + 1.5

854 : 0;

855 const seconds = Math.max(1, synthSeconds, ...tracks.map((tr) => tr.length / MIX_SR));

856 const out = new Float32Array(Math.ceil(seconds * MIX_SR));

857 if (synthEvents.length) {

858 const buf = await renderSynth(

859 synthEvents,

860 (id) => {

861 const v = voiceById(id);

862 return v?.binding.kind === 'synth' ? v.binding.preset : 'kick';

863 },

864 synthSeconds,

865 );

866 mixInto(out, buf.getChannelData(0));

867 }

868 for (const tr of tracks) mixInto(out, tr);

869 const pAll = peak(out);

870 if (pAll > 0.98) for (let i = 0; i < out.length; i++) out[i] *= 0.95 / pAll;

871 const buf = audio().createBuffer(1, out.length, MIX_SR);

872 buf.getChannelData(0).set(out);

873 renderCache = { sig, buf };

874 return buf;

875}

876

877$('take-save').addEventListener('click', async () => {

878 if (take.hits.length === 0) return;

879 const btn = $('take-save') as HTMLButtonElement;

880 btn.disabled = true;

881 try {

882 const buf = await renderMix();

883 const bytes = wavBytes(buf.getChannelData(0), MIX_SR);

884 const url = URL.createObjectURL(new Blob([bytes as Uint8Array<ArrayBuffer>], { type: 'audio/wav' }));

885 const a = document.createElement('a');

886 a.href = url;

887 a.download = 'instrawave-compose-take.wav';

888 a.click();

889 URL.revokeObjectURL(url);

890 status(`saved ${(buf.duration).toFixed(1)} s of audio`);

891 } catch (e) {

892 status(e instanceof Error ? e.message : String(e));

893 } finally {

894 btn.disabled = take.hits.length === 0;

895 }

896});

897

898$('take-play').addEventListener('click', async () => {

899 if (playing) { stopPlayback(); return; }

900 if (take.hits.length === 0) return;

901 const btn = $('take-play') as HTMLButtonElement;

902 btn.disabled = true;

903 try {

904 const buf = await renderMix();

905 status('');

906 void playBuffer(buf, (t) => timeline.playhead(t));

907 } catch (e) {

908 status(e instanceof Error ? e.message : String(e));

909 } finally {

910 btn.disabled = false;

911 syncPlayButton();

912 }

913});

914

915// ---- tuning audition ----------------------------------------------------------

916

917async function auditionIw(instrument: string, set: Record<string, number>): Promise<void> {

918 if (iwBusy || bakeJob) { status('the GPU is busy; try again in a moment'); return; }

919 iwBusy = true;

920 try {

921 if (!(await ensureBakeLoaded(instrument))) {

922 status(`no current ${instrument} bake; precompute it in the bake bar below`);

923 return;

924 }

925 const allowed = new Set(iw.tuning(instrument).map((p) => p.key));

926 const ev: Record<string, number> = {};

927 for (const [k, v] of Object.entries(set)) if (allowed.has(k)) ev[k] = v;

928 const t60 = set.t60 ?? (allowed.has('t60') ? iw.defaultValue(instrument, 't60') : 2);

929 const seconds = Math.min(5, Math.max(1.2, t60 * 0.6 + 0.5));

930 const { trace, dt } = await iw.render(instrument, seconds, [{ t: 0, set: ev }], (done, total) =>

931 status(`rendering ${instrument}: ${Math.round((100 * done) / total)}%`));

932 const res = resampleLinear(trace, 1 / dt, MIX_SR);

933 const pk = peak(res);

934 if (pk > 0) for (let i = 0; i < res.length; i++) res[i] *= 0.8 / pk;

935 const buf = audio().createBuffer(1, res.length, MIX_SR);

936 buf.getChannelData(0).set(res);

937 status('');

938 void playBuffer(buf);

939 } catch (e) {

940 status(e instanceof Error ? e.message : String(e));

941 } finally {

942 iwBusy = false;

943 }

944}

945

946// ---- instrument editor (geometry, accuracy, the 3D view) ----------------------

947// Geometry belongs to the instrument, not the voice: it is what the shared

948// bake depends on, so the editor opens from the bake bar and an edit marks

949// the bake stale there. The scene is instrawave's own viewer, raymarching

950// the solver's actual fields, so what it shows is what the bake will see.

951

952let iwEdit: { name: string; viewer: Viewer | null } | null = null;

953const iwCam: Camera = { yaw: -2.2, pitch: 0.45, dist: 1.6 };

954const iwVolume = $('iw-volume') as HTMLCanvasElement;

955const iwOverlay = $('iw-overlay') as HTMLCanvasElement;

956

957function sizeSceneCanvas(c: HTMLCanvasElement): void {

958 const r = c.getBoundingClientRect();

959 const dpr = Math.min(devicePixelRatio || 1, 2);

960 const w = Math.max(1, Math.round(r.width * dpr));

961 const h = Math.max(1, Math.round(r.height * dpr));

962 if (c.width !== w || c.height !== h) { c.width = w; c.height = h; }

963}

964

965let iwViewTimer: number | null = null;

966function scheduleIwView(): void {

967 if (iwViewTimer !== null) clearTimeout(iwViewTimer);

968 iwViewTimer = window.setTimeout(() => {

969 iwViewTimer = null;

970 const e = iwEdit;

971 if (!e?.viewer) return;

972 void (async () => {

973 try {

974 await e.viewer!.update(iw.scope(e.name));

975 e.viewer!.draw(iwCam);

976 } catch (err) {

977 status(`scene view: ${err instanceof Error ? err.message : err}`);

978 }

979 })();

980 }, 180);

981}

982

983async function openIwEditor(name: string): Promise<void> {

984 iwEdit = { name, viewer: null };

985 $('iwedit').hidden = false;

986 $('iwedit-title').textContent = name;

987 buildIwParams(name);

988 try {

989 const device = await iw.ensureDevice();

990 if (iwEdit?.name !== name) return;

991 sizeSceneCanvas(iwVolume);

992 sizeSceneCanvas(iwOverlay);

993 iwEdit.viewer = new Viewer(device, iw.file(name), iwVolume, iwOverlay);

994 scheduleIwView();

995 } catch (e) {

996 status(`scene view unavailable: ${e instanceof Error ? e.message : e} (the sliders still work)`);

997 }

998}

999

1000function buildIwParams(name: string): void {

1001 const box = $('iwedit-params');

1002 box.textContent = '';

1003 const values = iw.values(name);

1004 for (const p of iw.geometryParams(name)) {

1005 const div = document.createElement('div');

1006 div.className = 'param';

1007 const label = document.createElement('label');

1008 const nameSpan = document.createElement('span');

1009 nameSpan.textContent = `${p.label} (${p.unit})`;

1010 if (p.hint) nameSpan.title = p.hint;

1011 const valSpan = document.createElement('span');

1012 const input = document.createElement('input');

1013 input.type = 'range';

1014 input.min = String(p.min);

1015 input.max = String(p.max);

1016 input.step = String(p.step ?? (p.max - p.min) / 200);

1017 input.value = String(values[p.key]);

1018 const show = () => { valSpan.textContent = Number(input.value).toPrecision(4); };

1019 show();

1020 input.addEventListener('input', () => {

1021 iw.setOverride(name, p.key, Number(input.value));

1022 show();

1023 saveIw();

1024 scheduleIwView();

1025 updateBakeBar(); // the bake just went stale (or came back)

1026 });

1027 label.append(nameSpan, valSpan);

1028 div.append(label, input);

1029 box.appendChild(div);

1030 }

1031}

1032

1033function closeIwEditor(): void {

1034 iwEdit = null;

1035 $('iwedit').hidden = true;

1036}

1037$('iwedit-done').addEventListener('click', closeIwEditor);

1038

1039{

1040 let dragging = false, px = 0, py = 0;

1041 iwOverlay.addEventListener('pointerdown', (e) => {

1042 dragging = true; px = e.clientX; py = e.clientY;

1043 iwOverlay.setPointerCapture(e.pointerId);

1044 });

1045 iwOverlay.addEventListener('pointermove', (e) => {

1046 if (!dragging || !iwEdit?.viewer) return;

1047 iwCam.yaw -= (e.clientX - px) * 0.008;

1048 iwCam.pitch = Math.min(1.45, Math.max(-1.45, iwCam.pitch + (e.clientY - py) * 0.008));

1049 px = e.clientX; py = e.clientY;

1050 iwEdit.viewer.draw(iwCam);

1051 });

1052 iwOverlay.addEventListener('pointerup', () => { dragging = false; });

1053 iwOverlay.addEventListener('wheel', (e) => {

1054 if (!iwEdit?.viewer) return;

1055 e.preventDefault();

1056 iwCam.dist = Math.min(6, Math.max(0.3, iwCam.dist * Math.exp(e.deltaY * 0.001)));

1057 iwEdit.viewer.draw(iwCam);

1058 }, { passive: false });

1059 window.addEventListener('resize', () => {

1060 if (!iwEdit?.viewer) return;

1061 sizeSceneCanvas(iwVolume);

1062 sizeSceneCanvas(iwOverlay);

1063 scheduleIwView();

1064 });

1065}

1066

1067// ---- boot -----------------------------------------------------------------

1068

1069($('capture-sens') as HTMLInputElement).value = String(ui.captureSens);

1070($('take-sens') as HTMLInputElement).value = String(ui.takeSens);

1071take.hits = loadTakeHits() ?? [];

1072void getTakeAudio().then((a) => {

1073 if (a) {

1074 take.clip = { samples: a.samples, sr: a.sr };

1075 refreshTimeline();

1076 syncTakeBar();

1077 }

1078}).catch(() => { /* the hits still play and edit; re-detect needs the audio */ });

1079renderVoiceList();

1080refreshTimeline();

1081syncTakeBar();

1082updateBakeBar();

1083if (voices.length === 0) {

1084 status('start by adding a voice, then record yourself making its sound a handful of times');

1085}