// Computer ("system") audio: what the speakers play, captured by ffmpeg from the PipeWire/PulseAudio // monitor of the default output while a take records. // // The browser's recordings and this capture are separate, so they are aligned by wall-clock time, // which both share since they run on the same machine. ffmpeg reports the wall-clock time of the first // captured sample, and the browser reports when its recorders started, paused, and resumed. When // the take is finalized, the capture is cut to the same intervals and written as system.webm. import { spawn, type ChildProcess } from 'node:child_process'; import fs from 'node:fs/promises'; import path from 'node:path'; import { run } from './media.ts'; import { exists, takeDir, writeJsonAtomic } from './store.ts'; export const SYSTEM_SOURCE = '@DEFAULT_MONITOR@'; // Without an event from the recorder for this long (it sends one every few seconds), the capture is stopped. const IDLE_MS = 60_000; export type RecEvent = { type: 'start' | 'pause' | 'resume'; at: number }; // at: ms since the epoch // Persisted next to the capture, so that an interrupted take can still be aligned when recovered. interface Timing { audioStart: number; // wall-clock time of the first captured sample, ms since the epoch events: RecEvent[]; } interface Capture { proc: ChildProcess; exited: Promise; lastSeen: number; watchdog: NodeJS.Timeout; } const captures = new Map(); const partFile = (videoId: string, takeId: string) => path.join(takeDir(videoId, takeId), 'system.part.mka'); const timingFile = (videoId: string, takeId: string) => path.join(takeDir(videoId, takeId), 'system-timing.json'); async function readTiming(videoId: string, takeId: string): Promise { try { return JSON.parse(await fs.readFile(timingFile(videoId, takeId), 'utf8')) as Timing; } catch { return null; } } // Starts capturing and resolves once the first samples have arrived (so the capture always // begins before the recording does), or rejects if ffmpeg cannot open the audio server. export async function startSystemCapture(videoId: string, takeId: string): Promise { const key = `${videoId}/${takeId}`; if (captures.has(key)) return; const args = ['-hide_banner', '-nostats', '-f', 'pulse', '-i', SYSTEM_SOURCE, '-ac', '2', '-ar', '48000', '-c:a', 'libopus', '-b:a', '128k', '-f', 'matroska', '-y', partFile(videoId, takeId)]; const proc = spawn('ffmpeg', args, { stdio: ['pipe', 'ignore', 'pipe'] }); let stderr = ''; const exited = new Promise((resolve) => proc.on('close', () => resolve())); const audioStart = await new Promise((resolve, reject) => { const fail = (msg: string) => { proc.kill('SIGKILL'); reject(new Error(`Could not capture computer audio: ${msg}`)); }; const timer = setTimeout(() => fail('no audio arrived from the sound server within 5 s'), 5000); proc.stderr!.setEncoding('utf8'); proc.stderr!.on('data', (d: string) => { stderr = (stderr + d).slice(-20_000); const m = /Input #0, pulse.*\n.*start: ([\d.]+)/.exec(stderr); if (m) { clearTimeout(timer); resolve(parseFloat(m[1]) * 1000); } }); proc.on('error', (e) => { clearTimeout(timer); fail(e.message); }); proc.on('close', (code) => { clearTimeout(timer); fail(`ffmpeg exited with code ${code}: ${stderr.trim().split('\n').slice(-3).join(' ')}`); }); }); await writeJsonAtomic(timingFile(videoId, takeId), { audioStart, events: [] } satisfies Timing); const cap = { proc, exited, lastSeen: Date.now() } as Capture; cap.watchdog = setInterval(() => { if (Date.now() - cap.lastSeen > IDLE_MS) { console.warn(`system audio: no activity for take ${takeId}; stopping the capture`); stopSystemCapture(videoId, takeId); } }, 5000); captures.set(key, cap); } // Records a recorder event (or, with no event, just that the recorder is still alive). export async function systemCaptureEvent(videoId: string, takeId: string, ev: RecEvent | null) { const cap = captures.get(`${videoId}/${takeId}`); if (cap) cap.lastSeen = Date.now(); if (!ev) return; const t = await readTiming(videoId, takeId); if (!t) return; t.events.push(ev); await writeJsonAtomic(timingFile(videoId, takeId), t); } export async function stopSystemCapture(videoId: string, takeId: string) { const key = `${videoId}/${takeId}`; const cap = captures.get(key); if (!cap) return; captures.delete(key); clearInterval(cap.watchdog); // 'q' lets ffmpeg finish the file cleanly; kill it if it does not exit promptly. cap.proc.stdin?.end('q'); const t = setTimeout(() => cap.proc.kill('SIGKILL'), 5000); await cap.exited; clearTimeout(t); } // The parts of the capture that correspond to the recording, in capture time (seconds); a negative // start means the recording began before the capture did. The last interval is open-ended. export function keptIntervals(t: Timing): [number, number | null][] { const out: [number, number | null][] = []; let open: number | null = null; for (const ev of t.events) { const s = (ev.at - t.audioStart) / 1000; if ((ev.type === 'start' || ev.type === 'resume') && open === null) open = s; else if (ev.type === 'pause' && open !== null) { out.push([open, s]); open = null; } } if (open !== null) out.push([open, null]); return out; } // Stops the capture and writes system.webm, aligned with the take and exactly `duration` long. // Returns false if there is nothing usable (e.g. the capture never started). export async function finalizeSystemAudio(videoId: string, takeId: string, duration: number): Promise { await stopSystemCapture(videoId, takeId); const part = partFile(videoId, takeId); const timing = await readTiming(videoId, takeId); if (!(await exists(part)) || !timing) return false; const intervals = keptIntervals(timing); if (intervals.length === 0) return false; const f = (x: number) => x.toFixed(3); const filters = intervals.map(([a, b], i) => { const trim = `atrim=start=${f(Math.max(0, a))}${b !== null ? `:end=${f(Math.max(0, b))}` : ''}`; const lead = a < 0 ? `,adelay=${Math.round(-a * 1000)}:all=1` : ''; return `[0:a]${trim},asetpts=PTS-STARTPTS${lead}[i${i}]`; }); filters.push(`${intervals.map((_, i) => `[i${i}]`).join('')}concat=n=${intervals.length}:v=0:a=1,apad,atrim=0:${f(duration)}[out]`); const out = path.join(takeDir(videoId, takeId), 'system.webm'); await run('ffmpeg', ['-y', '-v', 'error', '-i', part, '-filter_complex', filters.join(';'), '-map', '[out]', '-c:a', 'libopus', '-b:a', '128k', out]); await fs.rm(part, { force: true }); await fs.rm(timingFile(videoId, takeId), { force: true }); return true; } // Do not leave ffmpeg capturing after the server exits. process.on('exit', () => { for (const cap of captures.values()) cap.proc.kill('SIGKILL'); }); for (const sig of ['SIGINT', 'SIGTERM'] as const) { process.once(sig, () => { for (const cap of captures.values()) cap.proc.kill('SIGKILL'); process.exit(0); }); }