// Data model shared by the server and the browser UI. export type TrackKind = 'screen' | 'camera'; export interface Take { id: string; createdAt: string; status: 'recording' | 'processing' | 'ready' | 'failed'; tracks: { screen: boolean; camera: boolean }; // Which track file carries the microphone audio (null if no mic was recorded). audio: TrackKind | null; // Computer audio (what the speakers played), in system.webm, aligned with the other tracks. systemAudio?: boolean; // Seconds; 0 until the take is finalized. duration: number; transcript?: { status: 'pending' | 'done' | 'failed'; error?: string; updatedAt: string }; // Background removal of the camera track (cutout.webm, VP9 with alpha). queued/running are live // states reported by the server; only done/failed/cancelled are persisted. While running, `stage` // says what is happening and `progress` (absent when unknown) is the fraction of that stage done. // `height` is the number // of lines the cutout was processed at, and `capped` says whether that was limited by the // processing resolution setting rather than by the camera (both absent for older cutouts). matte?: { status: 'queued' | 'running' | 'done' | 'failed' | 'cancelled'; stage?: string; progress?: number; error?: string; height?: number; capped?: boolean; updatedAt: string }; // Pixel size of the screen track, as recorded (absent for takes from before it was stored). screenSize?: { width: number; height: number }; error?: string; } // A piece of a take, [start, end) in take time (seconds). The timeline is an ordered list of these. export interface Segment { id: string; takeId: string; start: number; end: number; } export type LayoutMode = 'pip' | 'side-by-side' | 'screen-only' | 'camera-only'; export type Corner = 'tl' | 'tr' | 'bl' | 'br'; export type CameraShape = 'rect' | 'rounded' | 'circle'; export type BackgroundMode = 'none' | 'blur' | 'color' | 'cutout'; // What replaces the room behind the speaker (requires background removal of the camera takes). export interface Background { mode: BackgroundMode; color: string; // #rrggbb, for 'color' blur: number; // Gaussian sigma in pixels at 1080p output, for 'blur' // Background removal runs on the camera scaled down to at most this many lines. A small // picture-in-picture needs far fewer lines than the camera records, and the processing time is // roughly proportional to the number of pixels. resolution: MatteResolution; } export const MATTE_RESOLUTIONS = [180, 240, 360, 480, 720, 1080] as const; export type MatteResolution = (typeof MATTE_RESOLUTIONS)[number]; export const DEFAULT_BACKGROUND: Background = { mode: 'none', color: '#1e2530', blur: 18, resolution: 480 }; // A cutout processed at fewer lines than the current setting asks for (and fewer than the camera has). export const matteBelowSetting = (m: Take['matte'], bg: Background) => m?.status === 'done' && !!m.capped && (m.height ?? 0) < bg.resolution - 2; export const backgroundOf = (l: Layout): Background => ({ ...DEFAULT_BACKGROUND, ...l.background }); // How screen and camera are combined in takes that have both. export interface Layout { mode: LayoutMode; corner: Corner; // PiP corner; for side-by-side, the left/right side of the camera column size: number; // PiP width as a fraction of the output width shape: CameraShape; background?: Background; // absent in videos created before background removal existed } // A crop rectangle as fractions of the source frame, so it does not depend on the recorded resolution. export interface Crop { x: number; y: number; w: number; h: number; } // Per-take crops of the raw tracks; a missing track means the full frame. export type TakeCrop = Partial>; export const cropOf = (v: Pick, takeId: string, track: TrackKind): Crop | undefined => v.crops?.[takeId]?.[track]; export const isFullFrame = (c: Crop) => c.x < 0.001 && c.y < 0.001 && c.w > 0.999 && c.h > 0.999; // Export heights; the width is always 16:9. 1440p and 2160p show a high-resolution screen // recording without scaling it down, at the cost of slower exports and larger files. export const OUTPUT_HEIGHTS = [720, 1080, 1440, 2160] as const; export type OutputHeight = (typeof OUTPUT_HEIGHTS)[number]; export const isOutputHeight = (h: unknown): h is OutputHeight => OUTPUT_HEIGHTS.includes(h as OutputHeight); export interface OutputSettings { height: OutputHeight; normalizeAudio: boolean; systemVolume?: number; // gain applied to computer audio in the mix, 0..1 (default 1) } export const systemVolumeOf = (o: OutputSettings) => o.systemVolume ?? 1; export interface ExportInfo { file: string; exportedAt: string; signature: string; // exportSignature() at the time of export, used to detect stale exports duration: number; bytes: number; } export interface Video { id: string; title: string; description: string; notes: string; createdAt: string; updatedAt: string; takes: Take[]; timeline: Segment[]; layout: Layout; crops?: Record; // by take id; absent in videos created before cropping existed output: OutputSettings; thumbnail: { file: string; custom: boolean; updatedAt: string } | null; export: ExportInfo | null; } export interface VideoSummary { id: string; title: string; description: string; createdAt: string; updatedAt: string; duration: number; takeCount: number; hasScreen: boolean; hasCamera: boolean; thumbnailUrl: string | null; exportState: 'none' | 'current' | 'stale'; } export interface ExportJob { status: 'running' | 'done' | 'failed'; progress: number; // 0..1 startedAt: string; error?: string; } export interface AppConfig { deepgramModel: string; autoTranscribe: boolean; defaultCountdown: number; defaultCameraHeight: 720 | 1080; defaultOutputHeight: OutputHeight; matteHost: string; // ssh destination for background removal on a GPU; empty for this computer matteSetup: string; // shell commands run on that host first, e.g. "module load python ffmpeg" } export interface PublicConfig extends AppConfig { hasDeepgramKey: boolean; libraryDir: string; } export interface Word { word: string; text: string; // punctuated form for display start: number; end: number; } export interface Transcript { takeId: string; model: string; createdAt: string; text: string; words: Word[]; } // Audio peaks for waveform display: `rate` values per second, each 0..255, base64-encoded bytes. export interface Peaks { rate: number; data: string; } export function exportSignature(v: Pick): string { const seg = v.timeline.map((s) => [s.takeId, round3(s.start), round3(s.end)]); const l = v.layout; const o = v.output; const b = backgroundOf(l); // The background is omitted when off so that exports made before it existed stay current. const bg = b.mode === 'color' ? ['color', b.color] : b.mode === 'blur' ? ['blur', b.blur] : b.mode; // Likewise, crops are included only for takes in the timeline that have one. const crop: Record = {}; for (const id of new Set(v.timeline.map((s) => s.takeId))) { const c = v.crops?.[id]; const r = (x?: Crop) => (x ? [round3(x.x), round3(x.y), round3(x.w), round3(x.h)] : null); if (c?.screen || c?.camera) crop[id] = [r(c.screen), r(c.camera)]; } return JSON.stringify({ seg, layout: [l.mode, l.corner, round3(l.size), l.shape], ...(b.mode === 'none' ? {} : { bg }), ...(Object.keys(crop).length ? { crop } : {}), // And the computer audio level only matters when a take in the timeline has computer audio. ...(v.timeline.some((s) => v.takes.find((t) => t.id === s.takeId)?.systemAudio) ? { sys: round3(systemVolumeOf(o)) } : {}), output: [o.height, o.normalizeAudio], }); } function round3(x: number) { return Math.round(x * 1000) / 1000; } // Input gain of a microphone (see server/micgain.ts). export interface MicGain { available: boolean; // false without wpctl, or when the microphone is not a PipeWire source reason?: string; node?: string; // wpctl target: a node id, or @DEFAULT_AUDIO_SOURCE@ description?: string; volume?: number; // wpctl scale: 1.0 = 100%; a factor k is about 60·log10(k) dB muted?: boolean; } export const MAX_MIC_GAIN = 1.5;