1// Computer ("system") audio: what the speakers play, captured by ffmpeg from the PipeWire/PulseAudio
2// monitor of the default output while a take records.
3//
4// The browser's recordings and this capture are separate, so they are aligned by wall-clock time,
5// which both share since they run on the same machine. ffmpeg reports the wall-clock time of the first
6// captured sample, and the browser reports when its recorders started, paused, and resumed. When
7// the take is finalized, the capture is cut to the same intervals and written as system.webm.
9import { spawn, type ChildProcess } from 'node:child_process';
10import fs from 'node:fs/promises';
11import path from 'node:path';
12import { run } from './media.ts';
13import { exists, takeDir, writeJsonAtomic } from './store.ts';
15export const SYSTEM_SOURCE = '@DEFAULT_MONITOR@';
16// Without an event from the recorder for this long (it sends one every few seconds), the capture is stopped.
17const IDLE_MS = 60_000;
19export type RecEvent = { type: 'start' | 'pause' | 'resume'; at: number }; // at: ms since the epoch
21// Persisted next to the capture, so that an interrupted take can still be aligned when recovered.
22interface Timing {
23 audioStart: number; // wall-clock time of the first captured sample, ms since the epoch
24 events: RecEvent[];
25}
27interface Capture {
28 proc: ChildProcess;
29 exited: Promise<void>;
30 lastSeen: number;
31 watchdog: NodeJS.Timeout;
32}
34const captures = new Map<string, Capture>();
35const partFile = (videoId: string, takeId: string) => path.join(takeDir(videoId, takeId), 'system.part.mka');
36const timingFile = (videoId: string, takeId: string) => path.join(takeDir(videoId, takeId), 'system-timing.json');
38async function readTiming(videoId: string, takeId: string): Promise<Timing | null> {
39 try {
40 return JSON.parse(await fs.readFile(timingFile(videoId, takeId), 'utf8')) as Timing;
41 } catch {
42 return null;
43 }
44}
46// Starts capturing and resolves once the first samples have arrived (so the capture always
47// begins before the recording does), or rejects if ffmpeg cannot open the audio server.
48export async function startSystemCapture(videoId: string, takeId: string): Promise<void> {
49 const key = `${videoId}/${takeId}`;
50 if (captures.has(key)) return;
51 const args = ['-hide_banner', '-nostats', '-f', 'pulse', '-i', SYSTEM_SOURCE, '-ac', '2', '-ar', '48000', '-c:a', 'libopus', '-b:a', '128k', '-f', 'matroska', '-y', partFile(videoId, takeId)];
52 const proc = spawn('ffmpeg', args, { stdio: ['pipe', 'ignore', 'pipe'] });
53 let stderr = '';
54 const exited = new Promise<void>((resolve) => proc.on('close', () => resolve()));
55 const audioStart = await new Promise<number>((resolve, reject) => {
56 const fail = (msg: string) => {
57 proc.kill('SIGKILL');
58 reject(new Error(`Could not capture computer audio: ${msg}`));
59 };
60 const timer = setTimeout(() => fail('no audio arrived from the sound server within 5 s'), 5000);
61 proc.stderr!.setEncoding('utf8');
62 proc.stderr!.on('data', (d: string) => {
63 stderr = (stderr + d).slice(-20_000);
64 const m = /Input #0, pulse.*\n.*start: ([\d.]+)/.exec(stderr);
65 if (m) {
66 clearTimeout(timer);
67 resolve(parseFloat(m[1]) * 1000);
68 }
69 });
70 proc.on('error', (e) => {
71 clearTimeout(timer);
72 fail(e.message);
73 });
74 proc.on('close', (code) => {
75 clearTimeout(timer);
76 fail(`ffmpeg exited with code ${code}: ${stderr.trim().split('\n').slice(-3).join(' ')}`);
77 });
78 });
79 await writeJsonAtomic(timingFile(videoId, takeId), { audioStart, events: [] } satisfies Timing);
80 const cap = { proc, exited, lastSeen: Date.now() } as Capture;
81 cap.watchdog = setInterval(() => {
82 if (Date.now() - cap.lastSeen > IDLE_MS) {
83 console.warn(`system audio: no activity for take ${takeId}; stopping the capture`);
84 stopSystemCapture(videoId, takeId);
85 }
86 }, 5000);
87 captures.set(key, cap);
88}
90// Records a recorder event (or, with no event, just that the recorder is still alive).
91export async function systemCaptureEvent(videoId: string, takeId: string, ev: RecEvent | null) {
92 const cap = captures.get(`${videoId}/${takeId}`);
93 if (cap) cap.lastSeen = Date.now();
94 if (!ev) return;
95 const t = await readTiming(videoId, takeId);
96 if (!t) return;
97 t.events.push(ev);
98 await writeJsonAtomic(timingFile(videoId, takeId), t);
99}
101export async function stopSystemCapture(videoId: string, takeId: string) {
102 const key = `${videoId}/${takeId}`;
103 const cap = captures.get(key);
104 if (!cap) return;
105 captures.delete(key);
106 clearInterval(cap.watchdog);
107 // 'q' lets ffmpeg finish the file cleanly; kill it if it does not exit promptly.
108 cap.proc.stdin?.end('q');
109 const t = setTimeout(() => cap.proc.kill('SIGKILL'), 5000);
110 await cap.exited;
111 clearTimeout(t);
112}
114// The parts of the capture that correspond to the recording, in capture time (seconds); a negative
115// start means the recording began before the capture did. The last interval is open-ended.
116export function keptIntervals(t: Timing): [number, number | null][] {
117 const out: [number, number | null][] = [];
118 let open: number | null = null;
119 for (const ev of t.events) {
120 const s = (ev.at - t.audioStart) / 1000;
121 if ((ev.type === 'start' || ev.type === 'resume') && open === null) open = s;
122 else if (ev.type === 'pause' && open !== null) {
123 out.push([open, s]);
124 open = null;
125 }
126 }
127 if (open !== null) out.push([open, null]);
128 return out;
129}
131// Stops the capture and writes system.webm, aligned with the take and exactly `duration` long.
132// Returns false if there is nothing usable (e.g. the capture never started).
133export async function finalizeSystemAudio(videoId: string, takeId: string, duration: number): Promise<boolean> {
134 await stopSystemCapture(videoId, takeId);
135 const part = partFile(videoId, takeId);
136 const timing = await readTiming(videoId, takeId);
137 if (!(await exists(part)) || !timing) return false;
138 const intervals = keptIntervals(timing);
139 if (intervals.length === 0) return false;
140 const f = (x: number) => x.toFixed(3);
141 const filters = intervals.map(([a, b], i) => {
142 const trim = `atrim=start=${f(Math.max(0, a))}${b !== null ? `:end=${f(Math.max(0, b))}` : ''}`;
143 const lead = a < 0 ? `,adelay=${Math.round(-a * 1000)}:all=1` : '';
144 return `[0:a]${trim},asetpts=PTS-STARTPTS${lead}[i${i}]`;
145 });
146 filters.push(`${intervals.map((_, i) => `[i${i}]`).join('')}concat=n=${intervals.length}:v=0:a=1,apad,atrim=0:${f(duration)}[out]`);
147 const out = path.join(takeDir(videoId, takeId), 'system.webm');
148 await run('ffmpeg', ['-y', '-v', 'error', '-i', part, '-filter_complex', filters.join(';'), '-map', '[out]', '-c:a', 'libopus', '-b:a', '128k', out]);
149 await fs.rm(part, { force: true });
150 await fs.rm(timingFile(videoId, takeId), { force: true });
151 return true;
152}
154// Do not leave ffmpeg capturing after the server exits.
155process.on('exit', () => {
156 for (const cap of captures.values()) cap.proc.kill('SIGKILL');
157});
158for (const sig of ['SIGINT', 'SIGTERM'] as const) {
159 process.once(sig, () => {
160 for (const cap of captures.values()) cap.proc.kill('SIGKILL');
161 process.exit(0);
162 });
163}