// Render the timeline to an MP4 with ffmpeg. // // Each segment becomes a sub-graph: a black canvas of the output size, the screen track (cropped, if set) fit // into its box, the camera track (optionally masked to a rounded rect or circle) overlaid, and // the mic audio trimmed/padded to exactly the segment length. The segments are then concatenated. import fs from 'node:fs/promises'; import path from 'node:path'; import type { Crop, ExportJob, Video } from '../shared/types.ts'; import { cropOf, exportSignature } from '../shared/types.ts'; import { FPS, computeGeometry } from '../shared/layout.ts'; import { timelineDuration } from '../shared/timeline.ts'; import { fileSize, run } from './media.ts'; import { HttpError, getVideo, slugify, takeDir, updateVideo, videoDir } from './store.ts'; const jobs = new Map(); export const getExportJob = (videoId: string) => jobs.get(videoId) ?? null; const f3 = (x: number) => x.toFixed(3); // Crop filter for a rectangle given as fractions of the frame (ffmpeg rounds to the chroma grid), or nothing. const cropFilter = (c: Crop | undefined) => (c ? `crop=w=iw*${c.w.toFixed(5)}:h=ih*${c.h.toFixed(5)}:x=iw*${c.x.toFixed(5)}:y=ih*${c.y.toFixed(5)},` : ''); // Alpha mask for a w x h box with corner radius r (r = w/2 on a square gives a circle), anti-aliased. function alphaExpr(r: number) { return `255*clip(${r}-hypot(max(abs(X+0.5-W/2)-(W/2-${r}),0),max(abs(Y+0.5-H/2)-(H/2-${r}),0))+0.5,0,1)`; } export function buildExportArgs(v: Video, outFile: string): string[] { if (v.timeline.length === 0) throw new HttpError(400, 'The timeline is empty'); const inputs: string[] = []; const filters: string[] = []; let concat = ''; let nIn = 0; const addInput = (file: string, start: number, dur: number, decoder?: string) => { inputs.push(...(decoder ? ['-c:v', decoder] : []), '-ss', f3(start), '-t', f3(dur), '-i', file); return nIn++; }; v.timeline.forEach((seg, i) => { const take = v.takes.find((t) => t.id === seg.takeId); if (!take || take.status !== 'ready') throw new HttpError(400, `Timeline refers to a missing or unfinished take (${seg.takeId})`); const D = seg.end - seg.start; const dir = takeDir(v.id, take.id); const g = computeGeometry(v.layout, v.output, take.tracks.screen, take.tracks.camera); const screenCrop = cropFilter(cropOf(v, take.id, 'screen')); const cameraCrop = cropFilter(cropOf(v, take.id, 'camera')); const idx: Record = {}; const bgMode = g.camera?.background ?? 'none'; if (bgMode !== 'none' && take.matte?.status !== 'done') { throw new HttpError(409, `Background removal has not finished for take ${v.takes.indexOf(take) + 1}`); } if (g.screen) idx.screen = addInput(path.join(dir, 'screen.webm'), seg.start, D); // The cutout has an alpha channel, which only the libvpx decoder preserves. if (bgMode !== 'none') idx.cutout = addInput(path.join(dir, 'cutout.webm'), seg.start, D, 'libvpx-vp9'); if (g.camera && (bgMode === 'none' || bgMode === 'blur')) idx.camera = addInput(path.join(dir, 'camera.webm'), seg.start, D); if (take.audio && idx[take.audio] === undefined) idx[take.audio] = addInput(path.join(dir, `${take.audio}.webm`), seg.start, D); filters.push(`color=c=black:s=${g.W}x${g.H}:r=${FPS}:d=${f3(D)}[bg${i}]`); let cur = `bg${i}`; if (g.screen) { const b = g.screen; filters.push( `[${idx.screen}:v]fps=${FPS}:start_time=0,${screenCrop}scale=${b.w}:${b.h}:force_original_aspect_ratio=decrease:force_divisible_by=2,` + `pad=${b.w}:${b.h}:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1[s${i}]`, ); filters.push(`[${cur}][s${i}]overlay=${b.x}:${b.y}:eof_action=repeat[os${i}]`); cur = `os${i}`; } if (g.camera) { const { box: b, fit, radius, background, color, blurSigma } = g.camera; const fitTo = (transparentPad: boolean) => fit === 'cover' ? `scale=${b.w}:${b.h}:force_original_aspect_ratio=increase,crop=${b.w}:${b.h}` : `scale=${b.w}:${b.h}:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=${b.w}:${b.h}:(ow-iw)/2:(oh-ih)/2:color=${transparentPad ? 'black@0' : 'black'}`; if (background === 'none') { filters.push(`[${idx.camera}:v]fps=${FPS}:start_time=0,${cameraCrop}${fitTo(false)},setsar=1[cam${i}]`); } else { filters.push(`[${idx.cutout}:v]fps=${FPS}:start_time=0,format=yuva420p,${cameraCrop}${fitTo(true)},setsar=1[fg${i}]`); if (background === 'blur') filters.push(`[${idx.camera}:v]fps=${FPS}:start_time=0,${cameraCrop}${fitTo(false)},setsar=1,gblur=sigma=${blurSigma}[cb${i}]`); if (background === 'color') filters.push(`color=c=0x${color.replace('#', '')}:s=${b.w}x${b.h}:r=${FPS}:d=${f3(D)},setsar=1[cb${i}]`); if (background === 'cutout') filters.push(`[fg${i}]null[cam${i}]`); else filters.push(`[cb${i}][fg${i}]overlay=0:0:eof_action=repeat[cam${i}]`); } if (radius > 0) { // Compute the mask for a single frame and loop it (a per-frame geq is several times slower). filters.push( `color=c=black:s=${b.w}x${b.h}:r=${FPS}:d=1,format=gray,geq=lum='${alphaExpr(radius)}',trim=end_frame=1,` + `loop=loop=-1:size=1:start=0,setpts=N/${FPS}/TB,trim=duration=${f3(D)}[m${i}]`, ); filters.push(`[cam${i}]format=yuva420p[cm${i}]`); filters.push(`[cm${i}][m${i}]alphamerge[c${i}]`); } else { filters.push(`[cam${i}]null[c${i}]`); } filters.push(`[${cur}][c${i}]overlay=${b.x}:${b.y}:eof_action=repeat[oc${i}]`); cur = `oc${i}`; } filters.push(`[${cur}]format=yuv420p[v${i}]`); const fade = Math.min(0.015, D / 4); if (take.audio) { filters.push( `[${idx[take.audio]}:a]aresample=48000,aformat=sample_fmts=fltp:channel_layouts=stereo,apad,atrim=0:${f3(D)},asetpts=PTS-STARTPTS,` + `afade=t=in:d=${f3(fade)},afade=t=out:st=${f3(D - fade)}:d=${f3(fade)}[a${i}]`, ); } else { filters.push(`anullsrc=r=48000:cl=stereo,atrim=0:${f3(D)}[a${i}]`); } concat += `[v${i}][a${i}]`; }); filters.push(`${concat}concat=n=${v.timeline.length}:v=1:a=1[vout][acat]`); filters.push(v.output.normalizeAudio ? `[acat]loudnorm=I=-16:TP=-1.5:LRA=11,aresample=48000[aout]` : `[acat]anull[aout]`); return [ '-y', '-v', 'error', '-nostats', '-progress', 'pipe:1', ...inputs, '-filter_complex', filters.join(';'), '-map', '[vout]', '-map', '[aout]', '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '20', '-pix_fmt', 'yuv420p', '-r', String(FPS), '-c:a', 'aac', '-b:a', '160k', '-movflags', '+faststart', outFile, ]; } export async function startExport(videoId: string): Promise { const running = jobs.get(videoId); if (running?.status === 'running') return running; const v = await getVideo(videoId); const signature = exportSignature(v); const duration = timelineDuration(v.timeline); const name = `${slugify(v.title) || 'video'}.mp4`; const dir = videoDir(videoId); const tmp = path.join(dir, `.export-${Date.now()}.mp4`); const args = buildExportArgs(v, tmp); const job: ExportJob = { status: 'running', progress: 0, startedAt: new Date().toISOString() }; jobs.set(videoId, job); (async () => { try { let buf = ''; await run('ffmpeg', args, { onStdout: (d) => { buf += d; const lines = buf.split('\n'); buf = lines.pop() ?? ''; for (const line of lines) { const m = /^out_time_us=(\d+)/.exec(line); if (m && duration > 0) job.progress = Math.min(0.99, parseInt(m[1]) / 1e6 / duration); } }, }); const prev = (await getVideo(videoId)).export; await fs.rename(tmp, path.join(dir, name)); if (prev && prev.file !== name) await fs.rm(path.join(dir, prev.file), { force: true }); const bytes = await fileSize(path.join(dir, name)); await updateVideo(videoId, (v) => { v.export = { file: name, exportedAt: new Date().toISOString(), signature, duration, bytes }; }); job.status = 'done'; job.progress = 1; } catch (e) { job.status = 'failed'; job.error = (e as Error).message; console.warn('export failed:', job.error); await fs.rm(tmp, { force: true }); } })(); return job; }