f3253d0Takes: record, organize, and edit explainer videosJeremy Magland 1// Data model shared by the server and the browser UI.
3export type TrackKind = 'screen' | 'camera';
5export interface Take {
6 id: string;
7 createdAt: string;
8 status: 'recording' | 'processing' | 'ready' | 'failed';
9 tracks: { screen: boolean; camera: boolean };
10 // Which track file carries the microphone audio (null if no mic was recorded).
11 audio: TrackKind | null;
6805bfcRecord computer audio alongside the microphoneJeremy Magland 12 // Computer audio (what the speakers played), in system.webm, aligned with the other tracks.
13 systemAudio?: boolean;
f3253d0Takes: record, organize, and edit explainer videosJeremy Magland 14 // Seconds; 0 until the take is finalized.
15 duration: number;
16 transcript?: { status: 'pending' | 'done' | 'failed'; error?: string; updatedAt: string };
17 // Background removal of the camera track (cutout.webm, VP9 with alpha). queued/running are live
be26c78Update matte pipeline, server API and video pageJeremy Magland 18 // states reported by the server; only done/failed/cancelled are persisted. While running, `stage`
19 // says what is happening and `progress` (absent when unknown) is the fraction of that stage done.
20 // `height` is the number
21 // of lines the cutout was processed at, and `capped` says whether that was limited by the
22 // processing resolution setting rather than by the camera (both absent for older cutouts).
23 matte?: { status: 'queued' | 'running' | 'done' | 'failed' | 'cancelled'; stage?: string; progress?: number; error?: string; height?: number; capped?: boolean; updatedAt: string };
6a456f2Export at 1440p and 2160p as well as 720p and 1080p; the output settings say which height shows the screen recording at full size (screen size stored per take, probed once for older takes without touching updatedAt); the screen recording bitrate grows with the pixel count, 6 to 20 Mbit/sJeremy Magland 24 // Pixel size of the screen track, as recorded (absent for takes from before it was stored).
25 screenSize?: { width: number; height: number };
27}
29// A piece of a take, [start, end) in take time (seconds). The timeline is an ordered list of these.
30export interface Segment {
31 id: string;
32 takeId: string;
33 start: number;
34 end: number;
35}
37export type LayoutMode = 'pip' | 'side-by-side' | 'screen-only' | 'camera-only';
38export type Corner = 'tl' | 'tr' | 'bl' | 'br';
39export type CameraShape = 'rect' | 'rounded' | 'circle';
40export type BackgroundMode = 'none' | 'blur' | 'color' | 'cutout';
42// What replaces the room behind the speaker (requires background removal of the camera takes).
43export interface Background {
44 mode: BackgroundMode;
45 color: string; // #rrggbb, for 'color'
46 blur: number; // Gaussian sigma in pixels at 1080p output, for 'blur'
be26c78Update matte pipeline, server API and video pageJeremy Magland 47 // Background removal runs on the camera scaled down to at most this many lines. A small
48 // picture-in-picture needs far fewer lines than the camera records, and the processing time is
49 // roughly proportional to the number of pixels.
50 resolution: MatteResolution;
969e06cSkip repeated and low-motion frames in background removal; add 180-line processingJeremy Magland 53export const MATTE_RESOLUTIONS = [180, 240, 360, 480, 720, 1080] as const;
be26c78Update matte pipeline, server API and video pageJeremy Magland 54export type MatteResolution = (typeof MATTE_RESOLUTIONS)[number];
56export const DEFAULT_BACKGROUND: Background = { mode: 'none', color: '#1e2530', blur: 18, resolution: 480 };
58// A cutout processed at fewer lines than the current setting asks for (and fewer than the camera has).
59export const matteBelowSetting = (m: Take['matte'], bg: Background) => m?.status === 'done' && !!m.capped && (m.height ?? 0) < bg.resolution - 2;
f3253d0Takes: record, organize, and edit explainer videosJeremy Magland 60
61export const backgroundOf = (l: Layout): Background => ({ ...DEFAULT_BACKGROUND, ...l.background });
63// How screen and camera are combined in takes that have both.
64export interface Layout {
65 mode: LayoutMode;
66 corner: Corner; // PiP corner; for side-by-side, the left/right side of the camera column
67 size: number; // PiP width as a fraction of the output width
68 shape: CameraShape;
69 background?: Background; // absent in videos created before background removal existed
70}
5bb2925Crop the screen and camera of each take; default output to 720pJeremy Magland 72// A crop rectangle as fractions of the source frame, so it does not depend on the recorded resolution.
73export interface Crop {
74 x: number;
75 y: number;
76 w: number;
77 h: number;
78}
80// Per-take crops of the raw tracks; a missing track means the full frame.
81export type TakeCrop = Partial<Record<TrackKind, Crop>>;
83export const cropOf = (v: Pick<Video, 'crops'>, takeId: string, track: TrackKind): Crop | undefined => v.crops?.[takeId]?.[track];
85export const isFullFrame = (c: Crop) => c.x < 0.001 && c.y < 0.001 && c.w > 0.999 && c.h > 0.999;
6a456f2Export at 1440p and 2160p as well as 720p and 1080p; the output settings say which height shows the screen recording at full size (screen size stored per take, probed once for older takes without touching updatedAt); the screen recording bitrate grows with the pixel count, 6 to 20 Mbit/sJeremy Magland 87// Export heights; the width is always 16:9. 1440p and 2160p show a high-resolution screen
88// recording without scaling it down, at the cost of slower exports and larger files.
89export const OUTPUT_HEIGHTS = [720, 1080, 1440, 2160] as const;
90export type OutputHeight = (typeof OUTPUT_HEIGHTS)[number];
91export const isOutputHeight = (h: unknown): h is OutputHeight => OUTPUT_HEIGHTS.includes(h as OutputHeight);
f3253d0Takes: record, organize, and edit explainer videosJeremy Magland 93export interface OutputSettings {
6a456f2Export at 1440p and 2160p as well as 720p and 1080p; the output settings say which height shows the screen recording at full size (screen size stored per take, probed once for older takes without touching updatedAt); the screen recording bitrate grows with the pixel count, 6 to 20 Mbit/sJeremy Magland 94 height: OutputHeight;
6805bfcRecord computer audio alongside the microphoneJeremy Magland 96 systemVolume?: number; // gain applied to computer audio in the mix, 0..1 (default 1)
6805bfcRecord computer audio alongside the microphoneJeremy Magland 99export const systemVolumeOf = (o: OutputSettings) => o.systemVolume ?? 1;
f3253d0Takes: record, organize, and edit explainer videosJeremy Magland 101export interface ExportInfo {
102 file: string;
103 exportedAt: string;
104 signature: string; // exportSignature() at the time of export, used to detect stale exports
105 duration: number;
106 bytes: number;
107}
109export interface Video {
110 id: string;
111 title: string;
112 description: string;
113 notes: string;
114 createdAt: string;
115 updatedAt: string;
116 takes: Take[];
117 timeline: Segment[];
118 layout: Layout;
5bb2925Crop the screen and camera of each take; default output to 720pJeremy Magland 119 crops?: Record<string, TakeCrop>; // by take id; absent in videos created before cropping existed
121 thumbnail: { file: string; custom: boolean; updatedAt: string } | null;
122 export: ExportInfo | null;
123}
125export interface VideoSummary {
126 id: string;
127 title: string;
128 description: string;
129 createdAt: string;
130 updatedAt: string;
131 duration: number;
132 takeCount: number;
133 hasScreen: boolean;
134 hasCamera: boolean;
135 thumbnailUrl: string | null;
136 exportState: 'none' | 'current' | 'stale';
137}
139export interface ExportJob {
140 status: 'running' | 'done' | 'failed';
141 progress: number; // 0..1
142 startedAt: string;
143 error?: string;
144}
146export interface AppConfig {
147 deepgramModel: string;
148 autoTranscribe: boolean;
149 defaultCountdown: number;
150 defaultCameraHeight: 720 | 1080;
6a456f2Export at 1440p and 2160p as well as 720p and 1080p; the output settings say which height shows the screen recording at full size (screen size stored per take, probed once for older takes without touching updatedAt); the screen recording bitrate grows with the pixel count, 6 to 20 Mbit/sJeremy Magland 151 defaultOutputHeight: OutputHeight;
307b4e2Run background removal on a remote GPU over sshJeremy Magland 152 matteHost: string; // ssh destination for background removal on a GPU; empty for this computer
153 matteSetup: string; // shell commands run on that host first, e.g. "module load python ffmpeg"
156export interface PublicConfig extends AppConfig {
157 hasDeepgramKey: boolean;
158 libraryDir: string;
159}
161export interface Word {
162 word: string;
163 text: string; // punctuated form for display
164 start: number;
165 end: number;
166}
168export interface Transcript {
169 takeId: string;
170 model: string;
171 createdAt: string;
172 text: string;
173 words: Word[];
174}
176// Audio peaks for waveform display: `rate` values per second, each 0..255, base64-encoded bytes.
177export interface Peaks {
178 rate: number;
179 data: string;
180}
6805bfcRecord computer audio alongside the microphoneJeremy Magland 182export function exportSignature(v: Pick<Video, 'timeline' | 'layout' | 'output' | 'crops' | 'takes'>): string {
f3253d0Takes: record, organize, and edit explainer videosJeremy Magland 183 const seg = v.timeline.map((s) => [s.takeId, round3(s.start), round3(s.end)]);
184 const l = v.layout;
185 const o = v.output;
186 const b = backgroundOf(l);
187 // The background is omitted when off so that exports made before it existed stay current.
188 const bg = b.mode === 'color' ? ['color', b.color] : b.mode === 'blur' ? ['blur', b.blur] : b.mode;
5bb2925Crop the screen and camera of each take; default output to 720pJeremy Magland 189 // Likewise, crops are included only for takes in the timeline that have one.
190 const crop: Record<string, (number[] | null)[]> = {};
191 for (const id of new Set(v.timeline.map((s) => s.takeId))) {
192 const c = v.crops?.[id];
193 const r = (x?: Crop) => (x ? [round3(x.x), round3(x.y), round3(x.w), round3(x.h)] : null);
194 if (c?.screen || c?.camera) crop[id] = [r(c.screen), r(c.camera)];
195 }
196 return JSON.stringify({
197 seg,
198 layout: [l.mode, l.corner, round3(l.size), l.shape],
199 ...(b.mode === 'none' ? {} : { bg }),
200 ...(Object.keys(crop).length ? { crop } : {}),
6805bfcRecord computer audio alongside the microphoneJeremy Magland 201 // And the computer audio level only matters when a take in the timeline has computer audio.
202 ...(v.timeline.some((s) => v.takes.find((t) => t.id === s.takeId)?.systemAudio) ? { sys: round3(systemVolumeOf(o)) } : {}),
5bb2925Crop the screen and camera of each take; default output to 720pJeremy Magland 203 output: [o.height, o.normalizeAudio],
204 });
207function round3(x: number) {
208 return Math.round(x * 1000) / 1000;
209}
211// Input gain of a microphone (see server/micgain.ts).
212export interface MicGain {
213 available: boolean; // false without wpctl, or when the microphone is not a PipeWire source
214 reason?: string;
215 node?: string; // wpctl target: a node id, or @DEFAULT_AUDIO_SOURCE@
216 description?: string;
217 volume?: number; // wpctl scale: 1.0 = 100%; a factor k is about 60·log10(k) dB
218 muted?: boolean;
219}
221export const MAX_MIC_GAIN = 1.5;