1// One Deepgram streaming connection, carrying one participant's speech.
2//
3// Deepgram offers three speech-to-text paths. The pre-recorded REST endpoint is
4// the cheaper one per minute, but its responses carry no CORS headers, so a
5// browser cannot call it without a proxy server; the streaming WebSocket
6// endpoint is reachable directly. Since CommonRoom has no server of its own,
7// streaming is what we use.
8//
9// Browsers cannot set an Authorization header on a WebSocket, so the API key
10// travels as a subprotocol — `Sec-WebSocket-Protocol: token, <key>`, which is
11// Deepgram's documented scheme for client-side connections. The key therefore
12// leaves the browser only in the TLS handshake with Deepgram itself, and it is
13// never shared with the room.
14//
15// The connection is opened lazily on the first frame of speech and then held
16// open across pauses with KeepAlive messages, which are not billed. A separate
17// connection per speaker is what gives the transcript its attribution: there is
18// no diarization to interpret, since each socket only ever hears one person.
20const ENDPOINT = 'wss://api.deepgram.com/v1/listen'
22/** Deepgram closes an idle socket after 10 s; the docs ask for a KeepAlive
23 * every 3-5 s. */
24const TICK_MS = 3000
25const KEEPALIVE_AFTER_MS = 2500
26/** Give the socket back after a long silence rather than pinging it forever. */
27const IDLE_CLOSE_MS = 120_000
28/** Frames buffered while the socket is still opening (~4 s of speech). */
29const QUEUE_CAP = 100
30/** How long to wait for trailing results after asking the stream to close. */
31const DRAIN_MS = 3000
33const QUERY = {
34 model: 'nova-3',
35 language: 'en',
36 smart_format: 'true',
37 // Our own gate already segments speech, and interim results would triple the
38 // message rate for text we would only overwrite.
39 interim_results: 'false',
40 encoding: 'linear16',
41 channels: '1',
42 endpointing: '400'
43}
45export interface DeepgramHandlers {
46 transcript: (text: string) => void
47 /** Samples actually written to the socket, for the cost readout. */
48 sent: (samples: number) => void
49 /** A problem worth surfacing. `fatal` means stop trying entirely. */
50 failure: (message: string, fatal: boolean) => void
51}
53/** A subprotocol must be an RFC 7230 token, which excludes spaces and most
54 * punctuation. A key with a stray character would make the WebSocket
55 * constructor throw rather than fail as a rejected key, so it is checked up
56 * front where a comprehensible message can still be given. Deepgram keys are
57 * hexadecimal and pass comfortably. */
58export const isUsableApiKey = (key: string): boolean =>
59 /^[A-Za-z0-9!#$%&'*+\-.^_`|~]+$/.test(key)
61export class DeepgramStream {
62 private ws: WebSocket | null = null
63 private queue: Int16Array[] = []
64 private opened = false
65 private failures = 0
66 private lastAudioAt = 0
67 private timer: number | null = null
68 /** Set once the stream is winding down: no new audio, but results still
69 * arriving from the server are delivered. */
70 private draining = false
72 constructor(
73 private apiKey: string,
74 private sampleRate: number,
75 private handlers: DeepgramHandlers
76 ) {}
78 send(pcm: Int16Array) {
79 if (this.draining) return
80 this.lastAudioAt = Date.now()
81 if (!this.ws) this.open()
82 const ws = this.ws
83 if (!ws) return
84 if (ws.readyState === WebSocket.CONNECTING) {
85 // Everything up to here is speech we decided was worth paying for, so it
86 // waits for the socket rather than being dropped — but only up to a
87 // point, in case the connection never comes up.
88 if (this.queue.length < QUEUE_CAP) this.queue.push(pcm)
89 return
90 }
91 if (ws.readyState !== WebSocket.OPEN) return
92 ws.send(pcm)
93 this.handlers.sent(pcm.length)
94 }
96 /** End of utterance: ask Deepgram to flush what it has rather than waiting
97 * for its own endpointing to expire. */
98 finalize() {
99 const ws = this.ws
100 if (ws && ws.readyState === WebSocket.OPEN) {
101 ws.send(JSON.stringify({type: 'Finalize'}))
102 }
103 }
105 /** Stop sending, but stay open briefly so the last utterance still lands. */
106 stop() {
107 if (this.draining) return
108 this.draining = true
109 this.queue = []
110 if (this.timer !== null) clearInterval(this.timer)
111 this.timer = null
112 const ws = this.ws
113 if (!ws) return
114 if (ws.readyState === WebSocket.OPEN) {
115 ws.send(JSON.stringify({type: 'CloseStream'}))
116 window.setTimeout(() => ws.close(), DRAIN_MS)
117 } else {
118 ws.close()
119 }
120 }
122 private open() {
123 const params = new URLSearchParams({
124 ...QUERY,
125 sample_rate: String(this.sampleRate)
126 })
127 let ws: WebSocket
128 try {
129 ws = new WebSocket(`${ENDPOINT}?${params}`, ['token', this.apiKey])
130 } catch {
131 this.handlers.failure(
132 'That Deepgram API key could not be used to open a connection — check it for stray characters.',
133 true
134 )
135 return
136 }
137 ws.binaryType = 'arraybuffer'
138 this.ws = ws
139 this.opened = false
141 ws.onopen = () => {
142 this.opened = true
143 this.failures = 0
144 for (const pcm of this.queue.splice(0)) {
145 ws.send(pcm)
146 this.handlers.sent(pcm.length)
147 }
148 }
149 ws.onmessage = ev => this.onMessage(ev)
150 // An error is always followed by a close, which carries more information.
151 ws.onerror = () => undefined
152 ws.onclose = ev => this.onClose(ws, ev)
154 if (this.timer === null) {
155 this.timer = window.setInterval(() => this.tick(), TICK_MS)
156 }
157 }
159 private onMessage(ev: MessageEvent) {
160 if (typeof ev.data !== 'string') return
161 let msg: {
162 type?: string
163 channel?: {alternatives?: {transcript?: string}[]}
164 description?: string
165 message?: string
166 }
167 try {
168 msg = JSON.parse(ev.data)
169 } catch {
170 return
171 }
172 if (msg.type === 'Results') {
173 const text = msg.channel?.alternatives?.[0]?.transcript
174 if (typeof text === 'string' && text.trim()) this.handlers.transcript(text)
175 return
176 }
177 if (msg.type === 'Error') {
178 const why = msg.description ?? msg.message ?? 'unspecified'
179 this.handlers.failure(`Deepgram reported an error: ${why}`, false)
180 }
181 }
183 private onClose(ws: WebSocket, ev: CloseEvent) {
184 // A socket we already gave up on (see the idle path in `tick`) may close
185 // after its replacement is up; it has nothing left to say.
186 if (this.ws !== ws) return
187 const wasOpen = this.opened
188 this.ws = null
189 this.opened = false
190 this.queue = []
191 if (this.timer !== null) clearInterval(this.timer)
192 this.timer = null
193 if (this.draining) return
195 if (!wasOpen) {
196 // A browser deliberately hides the HTTP status of a failed WebSocket
197 // handshake, so a rejected key and an unreachable network are
198 // indistinguishable here. Say so, and stop after the second attempt
199 // rather than reconnecting into a wall on every utterance.
200 this.failures++
201 this.handlers.failure(
202 `Could not open a Deepgram connection${
203 ev.reason ? ` (${ev.reason})` : ''
204 } — check the API key and your network.`,
205 this.failures >= 2
206 )
207 return
208 }
209 if (ev.code !== 1000) {
210 this.handlers.failure(
211 `The Deepgram connection dropped${
212 ev.reason ? ` (${ev.reason})` : ''
213 }; it will reopen when someone next speaks.`,
214 false
215 )
216 }
217 }
219 private tick() {
220 const ws = this.ws
221 if (!ws || ws.readyState !== WebSocket.OPEN) return
222 const idle = Date.now() - this.lastAudioAt
223 if (idle > IDLE_CLOSE_MS) {
224 // Nothing can be in flight after two minutes of silence, so this one is
225 // released outright; the next utterance opens a fresh socket.
226 ws.send(JSON.stringify({type: 'CloseStream'}))
227 ws.close()
228 this.ws = null
229 if (this.timer !== null) clearInterval(this.timer)
230 this.timer = null
231 return
232 }
233 if (idle > KEEPALIVE_AFTER_MS) ws.send(JSON.stringify({type: 'KeepAlive'}))
234 }
235}