1/**
2 * Lossless codecs run in the browser, on int16 sample data.
3 *
4 * The baseline throughout is the uncompressed int16 stream: 2 bytes = 16 bits
5 * per sample. A ratio is (baseline bytes) / (compressed bytes).
6 */
7import { zlibSync } from 'fflate'
8import { init as zstdInit, compress as zstdCompress } from '@bokuweb/zstd-wasm'
9import { ansEncode, ansDecode, encodedSize } from './ans'
10import { fitLpc, lpcResidual, lpcRestore, modelSize } from './lpc'
11// Bundled as an asset so the page needs no network at run time. The path is
12// relative rather than a bare specifier because the package's `exports` map
13// has no entry for the wasm file, so `@bokuweb/zstd-wasm/dist/web/zstd.wasm`
14// cannot be resolved.
15import zstdWasmUrl from '../../node_modules/@bokuweb/zstd-wasm/dist/web/zstd.wasm?url'
17export interface Codec {
18 name: string
19 /** Longer description, shown on hover. */
20 note: string
21 /** Compressed size in bytes, including any table the decoder needs. */
22 size: (samples: Int16Array, bytes: Uint8Array) => number
23}
25/**
26 * First differences, in int16 with wraparound — exactly invertible, and the
27 * standard prefilter for signals whose neighbouring samples are related.
28 */
29function delta(samples: Int16Array): Int16Array {
30 const d = new Int16Array(samples.length)
31 d[0] = samples[0]
32 for (let i = 1; i < samples.length; i++) {
33 d[i] = ((samples[i] - samples[i - 1]) << 16) >> 16
34 }
35 return d
36}
38function undelta(d: Int16Array): Int16Array {
39 const s = new Int16Array(d.length)
40 s[0] = d[0]
41 for (let i = 1; i < d.length; i++) {
42 s[i] = ((s[i - 1] + d[i]) << 16) >> 16
43 }
44 return s
45}
47function asBytes(samples: Int16Array): Uint8Array {
48 return new Uint8Array(samples.buffer, samples.byteOffset, samples.byteLength)
49}
51/**
52 * Size of an ANS encoding of `coded`, after decoding it, running `inverse` to
53 * undo whatever prefilter produced it, and checking the result against the
54 * original samples — so a reported size always belongs to an encoding that
55 * actually round-trips. `extraBytes` covers anything else the decoder needs,
56 * such as predictor coefficients.
57 */
58function ansSize(
59 samples: Int16Array,
60 coded: Int16Array,
61 inverse: (coded: Int16Array) => Int16Array,
62 extraBytes = 0,
63): number {
64 const encoded = ansEncode(coded)
65 const decoded = inverse(ansDecode(encoded))
66 if (decoded.length !== samples.length) throw new Error('ANS round-trip length mismatch')
67 for (let i = 0; i < samples.length; i++) {
68 if (decoded[i] !== samples[i]) throw new Error(`ANS round-trip mismatch at ${i}`)
69 }
70 return encodedSize(encoded) + extraBytes
71}
73/** Predictor order. Going past 32 buys well under a percent on this data. */
74const LPC_ORDER = 32
76/** Fit the predictor and take the residual, with the coefficients' cost. */
77function lpcTransform(samples: Int16Array): {
78 residual: Int16Array
79 restore: (residual: Int16Array) => Int16Array
80 extraBytes: number
81} {
82 const model = fitLpc(samples, LPC_ORDER)
83 if (!model) throw new Error('LPC fit failed')
84 return {
85 residual: lpcResidual(samples, model),
86 restore: residual => lpcRestore(residual, model),
87 extraBytes: modelSize(model),
88 }
89}
91export const ZLIB: Codec = {
92 name: 'zlib -9',
93 note: 'DEFLATE at maximum level — the HDF5 gzip filter',
94 size: (_s, bytes) => zlibSync(bytes, { level: 9 }).length,
95}
97export const ZSTD: Codec = {
98 name: 'zstd -19',
99 note: 'Zstandard at level 19 — the Blosc/Zarr default family',
100 size: (_s, bytes) => zstdCompress(bytes, 19).length,
101}
103export const ANS: Codec = {
104 name: 'ANS',
105 note: 'rANS entropy coder over the sample histogram, ported from simple_ans; the size includes the symbol table',
106 size: samples => ansSize(samples, samples, x => x),
107}
109export const DELTA_ZLIB: Codec = {
110 name: 'delta + zlib -9',
111 note: 'First differences, then DEFLATE',
112 size: samples => zlibSync(asBytes(delta(samples)), { level: 9 }).length,
113}
115export const DELTA_ZSTD: Codec = {
116 name: 'delta + zstd -19',
117 note: 'First differences, then Zstandard 19',
118 size: samples => zstdCompress(asBytes(delta(samples)), 19).length,
119}
121export const DELTA_ANS: Codec = {
122 name: 'delta + ANS',
123 note: 'First differences, then the rANS entropy coder',
124 size: samples => ansSize(samples, delta(samples), undelta),
125}
127const LPC_NOTE = `Order-${LPC_ORDER} linear prediction with integer coefficients; the size includes the coefficients`
129export const LPC_ZLIB: Codec = {
130 name: `LPC(${LPC_ORDER}) + zlib -9`,
131 note: `${LPC_NOTE}, then DEFLATE`,
132 size: samples => {
133 const { residual, extraBytes } = lpcTransform(samples)
134 return zlibSync(asBytes(residual), { level: 9 }).length + extraBytes
135 },
136}
138export const LPC_ZSTD: Codec = {
139 name: `LPC(${LPC_ORDER}) + zstd -19`,
140 note: `${LPC_NOTE}, then Zstandard 19`,
141 size: samples => {
142 const { residual, extraBytes } = lpcTransform(samples)
143 return zstdCompress(asBytes(residual), 19).length + extraBytes
144 },
145}
147export const LPC_ANS: Codec = {
148 name: `LPC(${LPC_ORDER}) + ANS`,
149 note: `${LPC_NOTE}, then the rANS entropy coder`,
150 size: samples => {
151 const { residual, restore, extraBytes } = lpcTransform(samples)
152 return ansSize(samples, residual, restore, extraBytes)
153 },
154}
156/** The general-purpose compressors, which know nothing about the data. */
157export const GENERAL_CODECS: Codec[] = [ZLIB, ZSTD]
159/**
160 * Order-0 (memoryless) entropy of an int16 stream, in bits per sample: what a
161 * perfect entropy coder for the sample histogram would spend, with nothing
162 * charged for describing that histogram. The ANS bars sit above this by the
163 * symbol table plus the coder's own arithmetic loss.
164 */
165export function order0Entropy(samples: Int16Array): number {
166 const counts = new Int32Array(65536)
167 for (let i = 0; i < samples.length; i++) counts[samples[i] + 32768]++
168 let bits = 0
169 for (const c of counts) {
170 if (c > 0) {
171 const p = c / samples.length
172 bits -= p * Math.log2(p)
173 }
174 }
175 return bits
176}
178export interface BoundResult {
179 /** The prefilter group this bounds, matching the codec groups. */
180 group: string
181 note: string
182 bitsPerSample: number
183 ratio: number
184 bytes: number
185}
187/** The order-0 bound for each prefilter: raw samples, delta, LPC residual. */
188export function entropyBounds(samples: Int16Array): BoundResult[] {
189 const streams: { group: string; what: string; data: Int16Array }[] = [
190 { group: 'no prefilter', what: 'the samples themselves', data: samples },
191 { group: 'delta', what: 'the first differences', data: delta(samples) },
192 { group: 'LPC', what: `the order-${LPC_ORDER} prediction residual`, data: lpcTransform(samples).residual },
193 ]
194 return streams.map(({ group, what, data }) => {
195 const bitsPerSample = order0Entropy(data)
196 return {
197 group,
198 note: `order-0 entropy of ${what} — the limit for a per-sample entropy coder, with no symbol table or coefficients charged`,
199 bitsPerSample,
200 ratio: 16 / bitsPerSample,
201 bytes: Math.ceil((bitsPerSample * samples.length) / 8),
202 }
203 })
204}
206let ready: Promise<void> | null = null
208// zstd-wasm publishes the *node* build's types while the bundler resolves the
209// browser build (its `exports` map has a "browser" condition). Only the browser
210// build's `init` takes a wasm URL, so the signature has to be restated here.
211const initWithUrl = zstdInit as unknown as (path?: string) => Promise<void>
213/** Load the zstd wasm module. Safe to call repeatedly. */
214export function initCodecs(): Promise<void> {
215 if (!ready) {
216 ready = initWithUrl(zstdWasmUrl).catch((err: unknown) => {
217 ready = null
218 throw err
219 })
220 }
221 return ready
222}
224export interface CodecResult {
225 codec: string
226 note: string
227 bytes: number
228 ratio: number
229 bitsPerSample: number
230}
232/** Compress one int16 buffer with each of the given codecs. */
233export function compressAll(buffer: Uint8Array, codecs: Codec[]): CodecResult[] {
234 // The session's buffer may sit at an odd offset; align before viewing as int16.
235 const aligned = buffer.byteOffset % 2 === 0 ? buffer : new Uint8Array(buffer)
236 const samples = new Int16Array(aligned.buffer, aligned.byteOffset, aligned.byteLength / 2)
237 return codecs.map(codec => {
238 const bytes = codec.size(samples, aligned)
239 return {
240 codec: codec.name,
241 note: codec.note,
242 bytes,
243 ratio: aligned.byteLength / bytes,
244 bitsPerSample: (8 * bytes) / samples.length,
245 }
246 })
247}