// Speech capture: turn one participant's audio into fixed-size linear16 frames // and decide which of those frames actually carry speech. // // The point of the gate is cost. Deepgram bills the audio you send it, so an // eight-person room left open for an hour would be billed for eight hours of // mostly silence. Sending only while someone is talking cuts that to roughly // the time people actually speak; the socket stays open across the gaps on // KeepAlive messages instead (see deepgram.ts). // // The detector is deliberately simple: frame energy against an adaptive noise // floor, with hysteresis in both level and time. We emphasize that this is not // a speech/non-speech classifier — a slammed door or a burst of typing will // open the gate, and whatever Deepgram makes of it (usually nothing) lands in // the transcript. What it does do reliably is stay shut through ordinary room // tone, which is where the savings are. A learned detector such as Silero // would reject non-speech far better, at the cost of a few megabytes of model // and runtime; that trade did not seem worth it here. /** Frame length. Short enough for responsive gating, long enough that the * message rate stays modest with eight participants. */ const FRAME_MS = 40 /** Audio retained before the gate opens, so word onsets are not clipped. */ const PREROLL_FRAMES = 8 // 320 ms /** Consecutive loud frames before speech is declared. */ const OPEN_FRAMES = 2 // 80 ms /** Consecutive quiet frames before it ends. Generous, because a mid-sentence * pause that closed the gate would clip the word after it. */ const CLOSE_FRAMES = 20 // 800 ms // The noise floor adapts only while the gate is shut, so a long utterance can // never raise the bar against itself. It falls quickly and rises slowly, since // a level that persists is the room and a level that does not is a person. const FALL_ALPHA = 0.3 const RISE_ALPHA = 0.05 /** A rise is capped at this multiple of the current estimate per step, so one * slammed door cannot lift the floor and deafen the gate behind it. Only a * sustained change in the room moves it. */ const RISE_CLAMP = 4 const OPEN_FACTOR = 2.5 const CLOSE_FACTOR = 1.6 /** Absolute floor on the open threshold (RMS of normalized samples). Without * it, a silent room whose noise estimate has decayed toward zero would open * on anything at all. */ const MIN_RMS = 0.005 /** Frames at the start, and after the watchdog below, during which the gate * is held shut so the floor can be learned before it is used. Without this * the estimate would start at zero, and someone joining from a noisy room * would open the gate on their first frame and never close it. */ const CALIBRATE_FRAMES = 25 // 1 s /** A gate that has been open this long is almost certainly stuck on a room * that got loud after we learned it, which is the expensive failure. Force it * shut and re-learn. A genuine monologue with no 800 ms pause in a full * minute loses the second of audio that recalibration costs. */ const MAX_OPEN_FRAMES = 1500 // 60 s const WORKLET_NAME = 'commonroom-frames' // The worklet is shipped as a source string rather than a separate module so // there is no extra build configuration: it becomes a blob URL at runtime. // It does the framing and the energy measurement (both cheap, both per // sample); the state machine lives on the main thread where it is easier to // reason about and adjust. const WORKLET_SRC = ` class FrameProcessor extends AudioWorkletProcessor { constructor(options) { super() this.size = options.processorOptions.frameSize this.buf = new Float32Array(this.size) this.n = 0 this.done = false // Any message means "you are finished": returning false releases the // processor instead of leaving it in the graph for the rest of the call. this.port.onmessage = () => { this.done = true } } process(inputs) { if (this.done) return false const ch = inputs[0] && inputs[0][0] if (!ch) return true for (let i = 0; i < ch.length; i++) { this.buf[this.n++] = ch[i] if (this.n < this.size) continue const pcm = new Int16Array(this.size) let sum = 0 for (let j = 0; j < this.size; j++) { let s = this.buf[j] if (s > 1) s = 1 else if (s < -1) s = -1 sum += s * s pcm[j] = s < 0 ? s * 0x8000 : s * 0x7fff } this.port.postMessage( {pcm: pcm, rms: Math.sqrt(sum / this.size)}, [pcm.buffer] ) this.n = 0 } return true } } registerProcessor(${JSON.stringify(WORKLET_NAME)}, FrameProcessor) ` let workletUrl: string | null = null /** addModule is per-context and must not be repeated, so the promise is cached * against the context rather than re-issued for every participant. */ const registered = new WeakMap>() export function registerCaptureWorklet(ctx: BaseAudioContext): Promise { let p = registered.get(ctx) if (!p) { if (!workletUrl) { workletUrl = URL.createObjectURL( new Blob([WORKLET_SRC], {type: 'application/javascript'}) ) } p = ctx.audioWorklet.addModule(workletUrl) registered.set(ctx, p) } return p } /** The gate on its own, with no audio plumbing: feed it frame energies and it * reports the frame speech starts on and the frame it ends on. Keeping it * separable is what makes the tuning above testable outside a browser. */ export class SpeechGate { private noise = 0 private loud = 0 private quiet = 0 private open = 0 private active = false private started = false private calibrating = CALIBRATE_FRAMES get speaking(): boolean { return this.active } /** The noise floor the thresholds are currently derived from. */ get noiseFloor(): number { return this.noise } push(rms: number): 'open' | 'close' | null { if (!this.started) { // Start from the room as we find it rather than from zero, so a loud // room is recognized as loud on the first frame instead of being // mistaken for an hour of speech. this.started = true this.noise = rms } if (this.calibrating > 0) { this.calibrating-- this.track(rms) return null } if (!this.active) { this.track(rms) this.loud = rms >= Math.max(this.noise * OPEN_FACTOR, MIN_RMS) ? this.loud + 1 : 0 if (this.loud < OPEN_FRAMES) return null this.active = true this.quiet = 0 this.open = 0 return 'open' } this.open++ this.quiet = rms < Math.max(this.noise * CLOSE_FACTOR, MIN_RMS * 0.7) ? this.quiet + 1 : 0 if (this.quiet >= CLOSE_FRAMES) { this.active = false this.loud = 0 return 'close' } if (this.open >= MAX_OPEN_FRAMES) { this.active = false this.loud = 0 this.calibrating = CALIBRATE_FRAMES return 'close' } return null } /** Move the noise floor toward the current frame. Only called while the gate * is shut, so what it learns is the room and not the speaker. */ private track(rms: number) { if (rms < this.noise) { this.noise = this.noise * (1 - FALL_ALPHA) + rms * FALL_ALPHA return } const target = Math.min(rms, this.noise * RISE_CLAMP + MIN_RMS) this.noise = this.noise * (1 - RISE_ALPHA) + target * RISE_ALPHA } } export interface SpeechHandlers { /** A frame to transcribe, in order. Pre-roll frames arrive in a burst at the * moment speech is declared. */ frame: (pcm: Int16Array) => void /** The gate shut — flush whatever the transcriber has buffered. */ end: () => void } /** Gated capture of one MediaStream's first audio track. Call * `registerCaptureWorklet(ctx)` and await it before constructing. */ export class SpeechCapture { private source: MediaStreamAudioSourceNode private node: AudioWorkletNode private sink: GainNode private gate = new SpeechGate() private ring: Int16Array[] = [] private closed = false constructor( ctx: AudioContext, stream: MediaStream, private handlers: SpeechHandlers ) { const frameSize = Math.round((ctx.sampleRate * FRAME_MS) / 1000) this.source = ctx.createMediaStreamSource(stream) this.node = new AudioWorkletNode(ctx, WORKLET_NAME, { numberOfInputs: 1, numberOfOutputs: 1, outputChannelCount: [1], processorOptions: {frameSize} }) this.node.port.onmessage = e => { const {pcm, rms} = e.data as {pcm: Int16Array; rms: number} this.onFrame(pcm, rms) } // A worklet is only pulled if it reaches the destination, so it goes there // through a muted gain node — the participant's audio is already being // played by their