1// Speech capture: turn one participant's audio into fixed-size linear16 frames
2// and decide which of those frames actually carry speech.
3//
4// The point of the gate is cost. Deepgram bills the audio you send it, so an
5// eight-person room left open for an hour would be billed for eight hours of
6// mostly silence. Sending only while someone is talking cuts that to roughly
7// the time people actually speak; the socket stays open across the gaps on
8// KeepAlive messages instead (see deepgram.ts).
9//
10// The detector is deliberately simple: frame energy against an adaptive noise
11// floor, with hysteresis in both level and time. We emphasize that this is not
12// a speech/non-speech classifier — a slammed door or a burst of typing will
13// open the gate, and whatever Deepgram makes of it (usually nothing) lands in
14// the transcript. What it does do reliably is stay shut through ordinary room
15// tone, which is where the savings are. A learned detector such as Silero
16// would reject non-speech far better, at the cost of a few megabytes of model
17// and runtime; that trade did not seem worth it here.
19/** Frame length. Short enough for responsive gating, long enough that the
20 * message rate stays modest with eight participants. */
21const FRAME_MS = 40
22/** Audio retained before the gate opens, so word onsets are not clipped. */
23const PREROLL_FRAMES = 8 // 320 ms
24/** Consecutive loud frames before speech is declared. */
25const OPEN_FRAMES = 2 // 80 ms
26/** Consecutive quiet frames before it ends. Generous, because a mid-sentence
27 * pause that closed the gate would clip the word after it. */
28const CLOSE_FRAMES = 20 // 800 ms
29// The noise floor adapts only while the gate is shut, so a long utterance can
30// never raise the bar against itself. It falls quickly and rises slowly, since
31// a level that persists is the room and a level that does not is a person.
32const FALL_ALPHA = 0.3
33const RISE_ALPHA = 0.05
34/** A rise is capped at this multiple of the current estimate per step, so one
35 * slammed door cannot lift the floor and deafen the gate behind it. Only a
36 * sustained change in the room moves it. */
37const RISE_CLAMP = 4
38const OPEN_FACTOR = 2.5
39const CLOSE_FACTOR = 1.6
40/** Absolute floor on the open threshold (RMS of normalized samples). Without
41 * it, a silent room whose noise estimate has decayed toward zero would open
42 * on anything at all. */
43const MIN_RMS = 0.005
44/** Frames at the start, and after the watchdog below, during which the gate
45 * is held shut so the floor can be learned before it is used. Without this
46 * the estimate would start at zero, and someone joining from a noisy room
47 * would open the gate on their first frame and never close it. */
48const CALIBRATE_FRAMES = 25 // 1 s
49/** A gate that has been open this long is almost certainly stuck on a room
50 * that got loud after we learned it, which is the expensive failure. Force it
51 * shut and re-learn. A genuine monologue with no 800 ms pause in a full
52 * minute loses the second of audio that recalibration costs. */
53const MAX_OPEN_FRAMES = 1500 // 60 s
55const WORKLET_NAME = 'commonroom-frames'
57// The worklet is shipped as a source string rather than a separate module so
58// there is no extra build configuration: it becomes a blob URL at runtime.
59// It does the framing and the energy measurement (both cheap, both per
60// sample); the state machine lives on the main thread where it is easier to
61// reason about and adjust.
62const WORKLET_SRC = `
63class FrameProcessor extends AudioWorkletProcessor {
64 constructor(options) {
65 super()
66 this.size = options.processorOptions.frameSize
67 this.buf = new Float32Array(this.size)
68 this.n = 0
69 this.done = false
70 // Any message means "you are finished": returning false releases the
71 // processor instead of leaving it in the graph for the rest of the call.
72 this.port.onmessage = () => { this.done = true }
73 }
74 process(inputs) {
75 if (this.done) return false
76 const ch = inputs[0] && inputs[0][0]
77 if (!ch) return true
78 for (let i = 0; i < ch.length; i++) {
79 this.buf[this.n++] = ch[i]
80 if (this.n < this.size) continue
81 const pcm = new Int16Array(this.size)
82 let sum = 0
83 for (let j = 0; j < this.size; j++) {
84 let s = this.buf[j]
85 if (s > 1) s = 1
86 else if (s < -1) s = -1
87 sum += s * s
88 pcm[j] = s < 0 ? s * 0x8000 : s * 0x7fff
89 }
90 this.port.postMessage(
91 {pcm: pcm, rms: Math.sqrt(sum / this.size)},
92 [pcm.buffer]
93 )
94 this.n = 0
95 }
96 return true
97 }
98}
99registerProcessor(${JSON.stringify(WORKLET_NAME)}, FrameProcessor)
100`
102let workletUrl: string | null = null
103/** addModule is per-context and must not be repeated, so the promise is cached
104 * against the context rather than re-issued for every participant. */
105const registered = new WeakMap<BaseAudioContext, Promise<void>>()
107export function registerCaptureWorklet(ctx: BaseAudioContext): Promise<void> {
108 let p = registered.get(ctx)
109 if (!p) {
110 if (!workletUrl) {
111 workletUrl = URL.createObjectURL(
112 new Blob([WORKLET_SRC], {type: 'application/javascript'})
113 )
114 }
115 p = ctx.audioWorklet.addModule(workletUrl)
116 registered.set(ctx, p)
117 }
118 return p
119}
121/** The gate on its own, with no audio plumbing: feed it frame energies and it
122 * reports the frame speech starts on and the frame it ends on. Keeping it
123 * separable is what makes the tuning above testable outside a browser. */
124export class SpeechGate {
125 private noise = 0
126 private loud = 0
127 private quiet = 0
128 private open = 0
129 private active = false
130 private started = false
131 private calibrating = CALIBRATE_FRAMES
133 get speaking(): boolean {
134 return this.active
135 }
137 /** The noise floor the thresholds are currently derived from. */
138 get noiseFloor(): number {
139 return this.noise
140 }
142 push(rms: number): 'open' | 'close' | null {
143 if (!this.started) {
144 // Start from the room as we find it rather than from zero, so a loud
145 // room is recognized as loud on the first frame instead of being
146 // mistaken for an hour of speech.
147 this.started = true
148 this.noise = rms
149 }
150 if (this.calibrating > 0) {
151 this.calibrating--
152 this.track(rms)
153 return null
154 }
155 if (!this.active) {
156 this.track(rms)
157 this.loud = rms >= Math.max(this.noise * OPEN_FACTOR, MIN_RMS)
158 ? this.loud + 1
159 : 0
160 if (this.loud < OPEN_FRAMES) return null
161 this.active = true
162 this.quiet = 0
163 this.open = 0
164 return 'open'
165 }
166 this.open++
167 this.quiet = rms < Math.max(this.noise * CLOSE_FACTOR, MIN_RMS * 0.7)
168 ? this.quiet + 1
169 : 0
170 if (this.quiet >= CLOSE_FRAMES) {
171 this.active = false
172 this.loud = 0
173 return 'close'
174 }
175 if (this.open >= MAX_OPEN_FRAMES) {
176 this.active = false
177 this.loud = 0
178 this.calibrating = CALIBRATE_FRAMES
179 return 'close'
180 }
181 return null
182 }
184 /** Move the noise floor toward the current frame. Only called while the gate
185 * is shut, so what it learns is the room and not the speaker. */
186 private track(rms: number) {
187 if (rms < this.noise) {
188 this.noise = this.noise * (1 - FALL_ALPHA) + rms * FALL_ALPHA
189 return
190 }
191 const target = Math.min(rms, this.noise * RISE_CLAMP + MIN_RMS)
192 this.noise = this.noise * (1 - RISE_ALPHA) + target * RISE_ALPHA
193 }
194}
196export interface SpeechHandlers {
197 /** A frame to transcribe, in order. Pre-roll frames arrive in a burst at the
198 * moment speech is declared. */
199 frame: (pcm: Int16Array) => void
200 /** The gate shut — flush whatever the transcriber has buffered. */
201 end: () => void
202}
204/** Gated capture of one MediaStream's first audio track. Call
205 * `registerCaptureWorklet(ctx)` and await it before constructing. */
206export class SpeechCapture {
207 private source: MediaStreamAudioSourceNode
208 private node: AudioWorkletNode
209 private sink: GainNode
210 private gate = new SpeechGate()
211 private ring: Int16Array[] = []
212 private closed = false
214 constructor(
215 ctx: AudioContext,
216 stream: MediaStream,
217 private handlers: SpeechHandlers
218 ) {
219 const frameSize = Math.round((ctx.sampleRate * FRAME_MS) / 1000)
220 this.source = ctx.createMediaStreamSource(stream)
221 this.node = new AudioWorkletNode(ctx, WORKLET_NAME, {
222 numberOfInputs: 1,
223 numberOfOutputs: 1,
224 outputChannelCount: [1],
225 processorOptions: {frameSize}
226 })
227 this.node.port.onmessage = e => {
228 const {pcm, rms} = e.data as {pcm: Int16Array; rms: number}
229 this.onFrame(pcm, rms)
230 }
231 // A worklet is only pulled if it reaches the destination, so it goes there
232 // through a muted gain node — the participant's audio is already being
233 // played by their <video> element and must not be played twice.
234 this.sink = ctx.createGain()
235 this.sink.gain.value = 0
236 this.source.connect(this.node)
237 this.node.connect(this.sink)
238 this.sink.connect(ctx.destination)
239 }
241 private onFrame(pcm: Int16Array, rms: number) {
242 if (this.closed) return
243 const event = this.gate.push(rms)
244 if (event === 'open') {
245 // The frame that tripped the gate, and the pre-roll behind it, are the
246 // beginning of the utterance.
247 this.ring.push(pcm)
248 for (const f of this.ring) this.handlers.frame(f)
249 this.ring = []
250 return
251 }
252 if (event === 'close') {
253 this.handlers.frame(pcm)
254 this.handlers.end()
255 return
256 }
257 if (this.gate.speaking) {
258 this.handlers.frame(pcm)
259 return
260 }
261 this.ring.push(pcm)
262 if (this.ring.length > PREROLL_FRAMES) this.ring.shift()
263 }
265 close() {
266 if (this.closed) return
267 this.closed = true
268 this.node.port.onmessage = null
269 // Ask the processor to retire itself. This is best effort — a disconnected
270 // node is no longer pulled, so the stop may never be acted on — but a node
271 // with no references and no connections is collectable either way.
272 try {
273 this.node.port.postMessage('stop')
274 } catch {
275 /* ignore */
276 }
277 // Disconnecting a node that was never fully connected throws in some
278 // browsers; nothing here is worth failing a teardown over.
279 try {
280 this.source.disconnect()
281 this.node.disconnect()
282 this.sink.disconnect()
283 } catch {
284 /* ignore */
285 }
286 }
287}