1// Data model shared by the server and the browser UI.
3export type TrackKind = 'screen' | 'camera';
5export interface Take {
6 id: string;
7 createdAt: string;
8 status: 'recording' | 'processing' | 'ready' | 'failed';
9 tracks: { screen: boolean; camera: boolean };
10 // Which track file carries the microphone audio (null if no mic was recorded).
11 audio: TrackKind | null;
12 // Computer audio (what the speakers played), in system.webm, aligned with the other tracks.
13 systemAudio?: boolean;
14 // Seconds; 0 until the take is finalized.
15 duration: number;
16 transcript?: { status: 'pending' | 'done' | 'failed'; error?: string; updatedAt: string };
17 // Background removal of the camera track (cutout.webm, VP9 with alpha). queued/running are live
18 // states reported by the server; only done/failed/cancelled are persisted. While running, `stage`
19 // says what is happening and `progress` (absent when unknown) is the fraction of that stage done.
20 // `height` is the number
21 // of lines the cutout was processed at, and `capped` says whether that was limited by the
22 // processing resolution setting rather than by the camera (both absent for older cutouts).
23 matte?: { status: 'queued' | 'running' | 'done' | 'failed' | 'cancelled'; stage?: string; progress?: number; error?: string; height?: number; capped?: boolean; updatedAt: string };
24 error?: string;
25}
27// A piece of a take, [start, end) in take time (seconds). The timeline is an ordered list of these.
28export interface Segment {
29 id: string;
30 takeId: string;
31 start: number;
32 end: number;
33}
35export type LayoutMode = 'pip' | 'side-by-side' | 'screen-only' | 'camera-only';
36export type Corner = 'tl' | 'tr' | 'bl' | 'br';
37export type CameraShape = 'rect' | 'rounded' | 'circle';
38export type BackgroundMode = 'none' | 'blur' | 'color' | 'cutout';
40// What replaces the room behind the speaker (requires background removal of the camera takes).
41export interface Background {
42 mode: BackgroundMode;
43 color: string; // #rrggbb, for 'color'
44 blur: number; // Gaussian sigma in pixels at 1080p output, for 'blur'
45 // Background removal runs on the camera scaled down to at most this many lines. A small
46 // picture-in-picture needs far fewer lines than the camera records, and the processing time is
47 // roughly proportional to the number of pixels.
48 resolution: MatteResolution;
49}
51export const MATTE_RESOLUTIONS = [180, 240, 360, 480, 720, 1080] as const;
52export type MatteResolution = (typeof MATTE_RESOLUTIONS)[number];
54export const DEFAULT_BACKGROUND: Background = { mode: 'none', color: '#1e2530', blur: 18, resolution: 480 };
56// A cutout processed at fewer lines than the current setting asks for (and fewer than the camera has).
57export const matteBelowSetting = (m: Take['matte'], bg: Background) => m?.status === 'done' && !!m.capped && (m.height ?? 0) < bg.resolution - 2;
59export const backgroundOf = (l: Layout): Background => ({ ...DEFAULT_BACKGROUND, ...l.background });
61// How screen and camera are combined in takes that have both.
62export interface Layout {
63 mode: LayoutMode;
64 corner: Corner; // PiP corner; for side-by-side, the left/right side of the camera column
65 size: number; // PiP width as a fraction of the output width
66 shape: CameraShape;
67 background?: Background; // absent in videos created before background removal existed
68}
70// A crop rectangle as fractions of the source frame, so it does not depend on the recorded resolution.
71export interface Crop {
72 x: number;
73 y: number;
74 w: number;
75 h: number;
76}
78// Per-take crops of the raw tracks; a missing track means the full frame.
79export type TakeCrop = Partial<Record<TrackKind, Crop>>;
81export const cropOf = (v: Pick<Video, 'crops'>, takeId: string, track: TrackKind): Crop | undefined => v.crops?.[takeId]?.[track];
83export const isFullFrame = (c: Crop) => c.x < 0.001 && c.y < 0.001 && c.w > 0.999 && c.h > 0.999;
85export interface OutputSettings {
86 height: 720 | 1080;
87 normalizeAudio: boolean;
88 systemVolume?: number; // gain applied to computer audio in the mix, 0..1 (default 1)
89}
91export const systemVolumeOf = (o: OutputSettings) => o.systemVolume ?? 1;
93export interface ExportInfo {
94 file: string;
95 exportedAt: string;
96 signature: string; // exportSignature() at the time of export, used to detect stale exports
97 duration: number;
98 bytes: number;
99}
101export interface Video {
102 id: string;
103 title: string;
104 description: string;
105 notes: string;
106 createdAt: string;
107 updatedAt: string;
108 takes: Take[];
109 timeline: Segment[];
110 layout: Layout;
111 crops?: Record<string, TakeCrop>; // by take id; absent in videos created before cropping existed
112 output: OutputSettings;
113 thumbnail: { file: string; custom: boolean; updatedAt: string } | null;
114 export: ExportInfo | null;
115}
117export interface VideoSummary {
118 id: string;
119 title: string;
120 description: string;
121 createdAt: string;
122 updatedAt: string;
123 duration: number;
124 takeCount: number;
125 hasScreen: boolean;
126 hasCamera: boolean;
127 thumbnailUrl: string | null;
128 exportState: 'none' | 'current' | 'stale';
129}
131export interface ExportJob {
132 status: 'running' | 'done' | 'failed';
133 progress: number; // 0..1
134 startedAt: string;
135 error?: string;
136}
138export interface AppConfig {
139 deepgramModel: string;
140 autoTranscribe: boolean;
141 defaultCountdown: number;
142 defaultCameraHeight: 720 | 1080;
143 defaultOutputHeight: 720 | 1080;
144 matteHost: string; // ssh destination for background removal on a GPU; empty for this computer
145 matteSetup: string; // shell commands run on that host first, e.g. "module load python ffmpeg"
146}
148export interface PublicConfig extends AppConfig {
149 hasDeepgramKey: boolean;
150 libraryDir: string;
151}
153export interface Word {
154 word: string;
155 text: string; // punctuated form for display
156 start: number;
157 end: number;
158}
160export interface Transcript {
161 takeId: string;
162 model: string;
163 createdAt: string;
164 text: string;
165 words: Word[];
166}
168// Audio peaks for waveform display: `rate` values per second, each 0..255, base64-encoded bytes.
169export interface Peaks {
170 rate: number;
171 data: string;
172}
174export function exportSignature(v: Pick<Video, 'timeline' | 'layout' | 'output' | 'crops' | 'takes'>): string {
175 const seg = v.timeline.map((s) => [s.takeId, round3(s.start), round3(s.end)]);
176 const l = v.layout;
177 const o = v.output;
178 const b = backgroundOf(l);
179 // The background is omitted when off so that exports made before it existed stay current.
180 const bg = b.mode === 'color' ? ['color', b.color] : b.mode === 'blur' ? ['blur', b.blur] : b.mode;
181 // Likewise, crops are included only for takes in the timeline that have one.
182 const crop: Record<string, (number[] | null)[]> = {};
183 for (const id of new Set(v.timeline.map((s) => s.takeId))) {
184 const c = v.crops?.[id];
185 const r = (x?: Crop) => (x ? [round3(x.x), round3(x.y), round3(x.w), round3(x.h)] : null);
186 if (c?.screen || c?.camera) crop[id] = [r(c.screen), r(c.camera)];
187 }
188 return JSON.stringify({
189 seg,
190 layout: [l.mode, l.corner, round3(l.size), l.shape],
191 ...(b.mode === 'none' ? {} : { bg }),
192 ...(Object.keys(crop).length ? { crop } : {}),
193 // And the computer audio level only matters when a take in the timeline has computer audio.
194 ...(v.timeline.some((s) => v.takes.find((t) => t.id === s.takeId)?.systemAudio) ? { sys: round3(systemVolumeOf(o)) } : {}),
195 output: [o.height, o.normalizeAudio],
196 });
197}
199function round3(x: number) {
200 return Math.round(x * 1000) / 1000;
201}
203// Input gain of a microphone (see server/micgain.ts).
204export interface MicGain {
205 available: boolean; // false without wpctl, or when the microphone is not a PipeWire source
206 reason?: string;
207 node?: string; // wpctl target: a node id, or @DEFAULT_AUDIO_SOURCE@
208 description?: string;
209 volume?: number; // wpctl scale: 1.0 = 100%; a factor k is about 60·log10(k) dB
210 muted?: boolean;
211}
213export const MAX_MIC_GAIN = 1.5;