1// voicenote — fully client-side voice dictation.
2// Mic capture (AudioWorklet, 16 kHz) → energy-based VAD segments utterances →
3// a speech model in a Web Worker transcribes them (interim results while you
4// speak, final text appended on silence). Document lives in localStorage.
6import { MODELS, DEFAULT_MODEL, cachedModels, detectDevice } from './models.js';
8const SR = 16000;
9const DOC_KEY = 'voicenote-doc';
10const MODEL_KEY = 'voicenote-model';
12// VAD tuning (all sample counts at 16 kHz)
13const PREROLL_SAMPLES = 0.32 * SR; // audio kept before speech onset
14const ENTER_FRAMES = 2; // consecutive loud frames to trigger speech
15// Long enough that a dramatic mid-sentence pause does not split the sentence:
16// short fragments transcribe badly, because the model loses the surrounding
17// words it needs for context.
18const SILENCE_FINAL = 0.9 * SR; // trailing silence that ends an utterance
19const SOFT_MAX = 20 * SR; // after this, finalize at the next brief dip
20const SOFT_DIP = 0.15 * SR;
21const HARD_MAX = 28 * SR; // hard cut (whisper's window is 30 s)
22const MIN_SPEECH = 0.35 * SR; // discard blips shorter than this
23// Generous, because a trailing fricative ("...Americans") is quiet enough to
24// read as silence, and trimming into it costs the end of the word.
25const KEEP_TAIL = 0.4 * SR; // trailing silence kept when trimming
26const INTERIM_MIN = 0.8 * SR; // utterance length before the first interim
27const INTERIM_EVERY = 1.2 * SR; // new audio between interim runs
29// These models emit stock filler on near-silent audio; drop it when it is the
30// entire result rather than letting it accumulate in the document.
31const JUNK = new Set(['you', 'thank you.', 'thanks for watching!', 'thank you for watching!',
32 '[blank_audio]', '(silence)', '.', 'bye.', 'the', 'so']);
34const $ = (id) => document.getElementById(id);
35const recordBtn = $('recordBtn');
36const recordLabel = $('recordLabel');
37const copyBtn = $('copyBtn');
38const clearBtn = $('clearBtn');
39const statusEl = $('status');
40const modelSel = $('model');
41const modelNote = $('modelNote');
42const progressWrap = $('progressWrap');
43const progressBar = $('progressBar');
44const progressText = $('progressText');
45const doc = $('doc');
46const interimEl = $('interim');
47const wordcountEl = $('wordcount');
48const engineEl = $('engine');
50// ---------- document ----------
52doc.value = localStorage.getItem(DOC_KEY) || '';
54let saveTimer = null;
55function saveDoc() {
56 clearTimeout(saveTimer);
57 saveTimer = setTimeout(() => localStorage.setItem(DOC_KEY, doc.value), 250);
58}
60function updateWordCount() {
61 const n = doc.value.trim().split(/\s+/).filter(Boolean).length;
62 wordcountEl.textContent = `${n} word${n === 1 ? '' : 's'}`;
63}
65function autoresize() {
66 doc.style.height = 'auto';
67 doc.style.height = doc.scrollHeight + 'px';
68}
70doc.addEventListener('input', () => {
71 saveDoc();
72 updateWordCount();
73 autoresize();
74});
76function appendFinal(text) {
77 text = (text || '').trim();
78 if (!text || JUNK.has(text.toLowerCase())) return;
79 const sep = doc.value && !/\s$/.test(doc.value) ? ' ' : '';
80 const focused = document.activeElement === doc;
81 const selStart = doc.selectionStart;
82 const selEnd = doc.selectionEnd;
83 doc.value += sep + text;
84 if (focused) doc.setSelectionRange(selStart, selEnd);
85 saveDoc();
86 updateWordCount();
87 autoresize();
88 if (!focused) window.scrollTo({ top: document.body.scrollHeight });
89}
91copyBtn.addEventListener('click', async () => {
92 try {
93 await navigator.clipboard.writeText(doc.value);
94 copyBtn.textContent = 'Copied ✓';
95 setTimeout(() => (copyBtn.textContent = 'Copy'), 1200);
96 } catch {
97 setStatus('clipboard access denied');
98 }
99});
101clearBtn.addEventListener('click', () => {
102 if (!doc.value.trim() || confirm('Clear the document? This cannot be undone.')) {
103 doc.value = '';
104 interimEl.textContent = '';
105 localStorage.setItem(DOC_KEY, '');
106 updateWordCount();
107 autoresize();
108 doc.focus();
109 }
110});
112// ---------- model picker ----------
114let modelId = localStorage.getItem(MODEL_KEY) || DEFAULT_MODEL;
115if (!MODELS.some((m) => m.id === modelId)) modelId = DEFAULT_MODEL;
117for (const m of MODELS) {
118 const opt = document.createElement('option');
119 opt.value = m.id;
120 opt.textContent = `${m.label} — ${m.mb} MB`;
121 modelSel.appendChild(opt);
122}
123modelSel.value = modelId;
125function showModelNote() {
126 modelNote.textContent = MODELS.find((m) => m.id === modelId)?.note ?? '';
127}
129// Label each option with the download it would actually cost on the backend
130// we expect to use, and mark the ones already sitting in the browser cache.
131async function refreshCacheLabels() {
132 const cached = await cachedModels(device);
133 for (const opt of modelSel.options) {
134 const m = MODELS.find((x) => x.id === opt.value);
135 opt.textContent = `${m.label} — ${cached.has(m.id) ? 'downloaded' : `${m.mb[device]} MB`}`;
136 }
137}
139modelSel.addEventListener('change', () => {
140 modelId = modelSel.value;
141 localStorage.setItem(MODEL_KEY, modelId);
142 showModelNote();
144 // Drop the loaded model; anything queued for it is abandoned with the worker.
145 if (worker) {
146 worker.terminate();
147 worker = null;
148 }
149 modelState = 'unloaded';
150 outstanding = 0;
151 interimEl.textContent = '';
152 interimUtt = -1;
153 engineEl.textContent = '';
154 progressWrap.classList.add('hidden');
156 if (recording) ensureModel(); // keep listening; load the new model right away
157 updateStatus();
158});
160// ---------- transcription worker ----------
162let worker = null;
163let modelState = 'unloaded'; // unloaded | loading | ready | error
164let outstanding = 0; // transcription requests in flight
165let nextReqId = 1;
166let device = 'wasm'; // backend we expect to use; confirmed on 'ready'
167let forceCpu = false; // set once WebGPU has proven it cannot run here
169// Finished utterances captured before a model was ready. They live here rather
170// than in the worker so that restarting the worker — which is how the WebGPU
171// fallback works — never costs the user a phrase they already spoke.
172const queued = [];
173let queuedSamples = 0;
174const MAX_QUEUED = 120 * SR; // ~2 minutes of speech; drop the oldest beyond it
176function fmtMB(b) { return (b / 1024 / 1024).toFixed(0); }
178// Creates the worker and starts the model load. The worker is constructed
179// synchronously and the load message queued first, so nothing races it.
180function ensureModel() {
181 if (modelState === 'ready' || modelState === 'loading') return;
182 modelState = 'loading';
183 worker = new Worker('worker.js', { type: 'module' });
184 worker.onmessage = onWorkerMessage;
185 worker.onerror = (e) => failLoad(e.message || 'worker failed to start');
186 worker.postMessage({ type: 'load', model: modelId, device: forceCpu ? 'cpu' : 'auto' });
187}
189// A WebGPU failure needs a brand-new worker: ONNX Runtime will not hand back a
190// working CPU backend in a process where WebGPU already failed to start.
191function retryOnCpu(message) {
192 console.warn('voicenote: WebGPU unavailable, restarting on CPU —', message);
193 forceCpu = true;
194 device = 'wasm';
195 if (worker) { worker.terminate(); worker = null; }
196 modelState = 'unloaded';
197 outstanding = 0;
198 progressWrap.classList.add('hidden');
199 refreshCacheLabels();
200 ensureModel();
201 setStatus('WebGPU unavailable — loading on CPU instead…');
202}
204function failLoad(message) {
205 modelState = 'error';
206 progressWrap.classList.add('hidden');
207 stopMic();
208 syncButton();
209 setStatus(`error: ${message}`);
210}
212function onWorkerMessage(e) {
213 const msg = e.data;
214 if (msg.type === 'progress') {
215 progressWrap.classList.remove('hidden');
216 progressBar.style.width = `${(100 * msg.loaded) / msg.total}%`;
217 progressText.textContent = `${fmtMB(msg.loaded)} / ${fmtMB(msg.total)} MB`;
218 } else if (msg.type === 'status') {
219 setStatus(msg.message);
220 } else if (msg.type === 'ready') {
221 progressWrap.classList.add('hidden');
222 modelState = 'ready';
223 device = msg.device;
224 engineEl.textContent =
225 `${MODELS.find((m) => m.id === modelId).label} · ${msg.device === 'webgpu' ? 'WebGPU' : 'CPU (wasm)'}`;
226 refreshCacheLabels();
227 flushQueued();
228 updateStatus();
229 } else if (msg.type === 'result') {
230 outstanding = Math.max(0, outstanding - 1);
231 if (msg.final) {
232 if (msg.utt === interimUtt) {
233 interimEl.textContent = '';
234 interimUtt = -1;
235 }
236 appendFinal(msg.text);
237 } else if (msg.utt === currentUtt && inSpeech) {
238 const t = (msg.text || '').trim();
239 if (t && !JUNK.has(t.toLowerCase())) {
240 interimEl.textContent = t + ' …';
241 interimUtt = msg.utt;
242 }
243 }
244 updateStatus();
245 } else if (msg.type === 'error') {
246 if (msg.retryOnCpu && !forceCpu) retryOnCpu(msg.message);
247 else if (msg.fatal || modelState === 'loading') failLoad(msg.message);
248 else setStatus(`transcription error: ${msg.message}`);
249 }
250}
252function postAudio(samples, final, uttId) {
253 outstanding++;
254 const buf = samples.buffer;
255 worker.postMessage({ type: 'transcribe', id: nextReqId++, utt: uttId, final, audio: buf }, [buf]);
256}
258function sendAudio(samples, final, uttId) {
259 if (modelState === 'error') return;
260 if (modelState === 'ready' && worker) {
261 postAudio(samples, final, uttId);
262 return;
263 }
264 if (!final) return; // interims are disposable; only hold on to finished phrases
265 queued.push({ samples, uttId });
266 queuedSamples += samples.length;
267 while (queuedSamples > MAX_QUEUED && queued.length > 1) {
268 queuedSamples -= queued.shift().samples.length;
269 }
270}
272function flushQueued() {
273 while (queued.length) {
274 const q = queued.shift();
275 queuedSamples -= q.samples.length;
276 postAudio(q.samples, true, q.uttId);
277 }
278 queuedSamples = 0;
279}
281// ---------- mic capture + VAD ----------
283let recording = false;
284let stream = null;
285let audioCtx = null;
287let inSpeech = false;
288let enterCount = 0;
289let noiseFloor = 0.002;
290let preroll = []; // frames seen before speech onset
291let prerollSamples = 0;
292let utt = []; // frames of the current utterance
293let uttRms = []; // rms per frame, for the trailing-silence trim
294let uttSamples = 0;
295let silenceRun = 0; // samples of trailing sub-threshold audio
296let sinceInterim = 0;
297let currentUtt = 0; // id of the open utterance
298let interimUtt = -1; // utterance whose interim text is on screen
300function concatFrames(frames, totalLen) {
301 const out = new Float32Array(totalLen);
302 let o = 0;
303 for (const f of frames) { out.set(f, o); o += f.length; }
304 return out;
305}
307function resample(frame, fromRate) {
308 if (fromRate === SR) return frame;
309 const ratio = fromRate / SR;
310 const n = Math.floor(frame.length / ratio);
311 const out = new Float32Array(n);
312 for (let i = 0; i < n; i++) {
313 const pos = i * ratio;
314 const i0 = Math.floor(pos);
315 const frac = pos - i0;
316 const i1 = Math.min(i0 + 1, frame.length - 1);
317 out[i] = frame[i0] * (1 - frac) + frame[i1] * frac;
318 }
319 return out;
320}
322function onFrame(frame) {
323 let sum = 0;
324 for (let i = 0; i < frame.length; i++) sum += frame[i] * frame[i];
325 const rms = Math.sqrt(sum / frame.length);
327 const enterTh = Math.max(noiseFloor * 3, 0.006);
328 const exitTh = Math.max(noiseFloor * 2, 0.004);
330 if (!inSpeech) {
331 // adapt the noise floor from quiet frames only
332 if (rms < enterTh) noiseFloor = Math.max(0.0001, 0.95 * noiseFloor + 0.05 * rms);
333 preroll.push(frame);
334 prerollSamples += frame.length;
335 while (prerollSamples > PREROLL_SAMPLES && preroll.length > 1) {
336 prerollSamples -= preroll[0].length;
337 preroll.shift();
338 }
339 enterCount = rms > enterTh ? enterCount + 1 : 0;
340 if (enterCount >= ENTER_FRAMES) {
341 inSpeech = true;
342 enterCount = 0;
343 currentUtt++;
344 utt = preroll.slice();
345 uttRms = preroll.map(() => rms); // approximate; only the tail is used
346 uttSamples = prerollSamples;
347 preroll = [];
348 prerollSamples = 0;
349 silenceRun = 0;
350 sinceInterim = 0;
351 updateStatus();
352 }
353 return;
354 }
356 utt.push(frame);
357 uttRms.push(rms);
358 uttSamples += frame.length;
359 sinceInterim += frame.length;
360 silenceRun = rms < exitTh ? silenceRun + frame.length : 0;
362 const done =
363 silenceRun >= SILENCE_FINAL ||
364 (uttSamples >= SOFT_MAX && silenceRun >= SOFT_DIP) ||
365 uttSamples >= HARD_MAX;
367 if (done) {
368 finalizeUtterance();
369 } else if (
370 modelState === 'ready' &&
371 outstanding === 0 &&
372 uttSamples >= INTERIM_MIN &&
373 sinceInterim >= INTERIM_EVERY
374 ) {
375 sinceInterim = 0;
376 sendAudio(concatFrames(utt, uttSamples), false, currentUtt);
377 }
378}
380function finalizeUtterance() {
381 if (!inSpeech) return;
382 inSpeech = false;
384 // trim trailing silence down to KEEP_TAIL
385 let trailing = 0;
386 const exitTh = Math.max(noiseFloor * 2, 0.004);
387 for (let i = uttRms.length - 1; i >= 0 && uttRms[i] < exitTh; i--) {
388 trailing += utt[i].length;
389 }
390 while (utt.length > 1 && trailing - utt[utt.length - 1].length >= KEEP_TAIL) {
391 const f = utt.pop();
392 uttRms.pop();
393 uttSamples -= f.length;
394 trailing -= f.length;
395 }
397 const voiced = uttSamples - Math.min(trailing, uttSamples);
398 if (voiced >= MIN_SPEECH) {
399 sendAudio(concatFrames(utt, uttSamples), true, currentUtt);
400 } else if (interimUtt === currentUtt) {
401 interimEl.textContent = '';
402 interimUtt = -1;
403 }
404 utt = [];
405 uttRms = [];
406 uttSamples = 0;
407 silenceRun = 0;
408 updateStatus();
409}
411async function startMic() {
412 stream = await navigator.mediaDevices.getUserMedia({
413 audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true },
414 });
415 audioCtx = new AudioContext({ sampleRate: SR });
416 if (audioCtx.state === 'suspended') await audioCtx.resume();
417 await audioCtx.audioWorklet.addModule('worklet.js');
418 const src = audioCtx.createMediaStreamSource(stream);
419 const node = new AudioWorkletNode(audioCtx, 'capture');
420 const rate = audioCtx.sampleRate;
421 node.port.onmessage = (e) => {
422 if (recording) onFrame(resample(e.data, rate));
423 };
424 src.connect(node);
425 node.connect(audioCtx.destination); // worklet outputs silence; keeps the graph pulling
426 recording = true;
427}
429function stopMic() {
430 recording = false;
431 if (inSpeech) finalizeUtterance();
432 if (stream) { stream.getTracks().forEach((t) => t.stop()); stream = null; }
433 if (audioCtx) { audioCtx.close(); audioCtx = null; }
434 inSpeech = false;
435 enterCount = 0;
436 preroll = [];
437 prerollSamples = 0;
438}
440// ---------- UI state ----------
442function setStatus(text) { statusEl.textContent = text; }
444function syncButton() {
445 recordBtn.classList.toggle('live', recording);
446 recordBtn.classList.toggle('speech', recording && inSpeech);
447 recordLabel.textContent = recording ? 'Pause' : 'Record';
448}
450function updateStatus() {
451 syncButton();
452 if (modelState === 'error') return; // keep the error message on screen
453 if (modelState === 'loading') {
454 const n = queued.length;
455 setStatus(
456 'loading speech model (one-time download, then cached)…' +
457 (n ? ` — ${n} phrase${n === 1 ? '' : 's'} waiting` : ''),
458 );
459 } else if (recording) {
460 setStatus(inSpeech ? 'hearing speech…' : outstanding > 0 ? 'transcribing…' : 'listening…');
461 } else if (outstanding > 0) {
462 setStatus('transcribing…');
463 } else if (modelState === 'ready') {
464 setStatus('paused — press Record to continue');
465 } else {
466 setStatus('');
467 }
468}
470recordBtn.addEventListener('click', async () => {
471 if (recording) {
472 stopMic();
473 updateStatus();
474 return;
475 }
476 recordBtn.disabled = true;
477 try {
478 // Open the mic first, while still close to the user gesture, then kick off
479 // the model load in parallel; audio captured meanwhile is queued.
480 await startMic();
481 ensureModel();
482 updateStatus();
483 } catch (err) {
484 console.error(err);
485 stopMic();
486 syncButton();
487 setStatus(
488 err.name === 'NotAllowedError'
489 ? 'microphone access denied — allow it and try again'
490 : `error: ${err.message}`,
491 );
492 } finally {
493 recordBtn.disabled = false;
494 }
495});
497window.addEventListener('beforeunload', () => {
498 clearTimeout(saveTimer);
499 localStorage.setItem(DOC_KEY, doc.value);
500});
502updateWordCount();
503autoresize();
504showModelNote();
505setStatus('');
507// Sizes depend on the backend, so settle that before labelling the picker.
508detectDevice().then((d) => {
509 device = d;
510 refreshCacheLabels();
511});