/ concept-collection / timeseries-compressibility
Sign in
concept-collection / timeseries-compressibility
Offer LPC orders to 2048, caching the transform across codecs
Jeremy Magland <jmagland@flatironinstitute.org> committed commit 1d751dd3258c parent e411dff Browse files
2 changed files+37−19
src/compress/codecs.tsmodified+36−17View file
@@ -71,25 +71,42 @@ function ansSize(
7171 }
7272
7373 /** Predictor orders the UI offers; 32 is the FLAC default and ours. */
74-export const LPC_ORDERS = [1, 2, 4, 8, 16, 32, 64, 128]
74+export const LPC_ORDERS = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048]
7575 export const DEFAULT_LPC_ORDER = 32
7676
77-/** Fit the predictor and take the residual, with the coefficients' cost. */
78-function lpcTransform(
79- samples: Int16Array,
80- order: number,
81-): {
77+interface LpcTransform {
8278 residual: Int16Array
8379 restore: (residual: Int16Array) => Int16Array
8480 extraBytes: number
85-} {
81+}
82+
83+/**
84+ * The fit and the residual are wanted four times per measurement — once by
85+ * each LPC codec and once by the entropy bound — and cost O(order · N) each,
86+ * which is seconds at the top of the order range. Compute once per
87+ * (samples, order); the entry dies with the sample block.
88+ */
89+const transformCache = new WeakMap<Int16Array, Map<number, LpcTransform>>()
90+
91+/** Fit the predictor and take the residual, with the coefficients' cost. */
92+function lpcTransform(samples: Int16Array, order: number): LpcTransform {
93+ let byOrder = transformCache.get(samples)
94+ if (!byOrder) {
95+ byOrder = new Map()
96+ transformCache.set(samples, byOrder)
97+ }
98+ const cached = byOrder.get(order)
99+ if (cached) return cached
100+
86101 const model = fitLpc(samples, order)
87102 if (!model) throw new Error('LPC fit failed')
88- return {
103+ const transform: LpcTransform = {
89104 residual: lpcResidual(samples, model),
90105 restore: residual => lpcRestore(residual, model),
91106 extraBytes: modelSize(model),
92107 }
108+ byOrder.set(order, transform)
109+ return transform
93110 }
94111
95112 export const ZLIB: Codec = {
@@ -239,19 +256,21 @@ export interface CodecResult {
239256 bitsPerSample: number
240257 }
241258
242-/** Compress one int16 buffer with each of the given codecs. */
243-export function compressAll(buffer: Uint8Array, codecs: Codec[]): CodecResult[] {
244- // The session's buffer may sit at an odd offset; align before viewing as int16.
245- const aligned = buffer.byteOffset % 2 === 0 ? buffer : new Uint8Array(buffer)
246- const samples = new Int16Array(aligned.buffer, aligned.byteOffset, aligned.byteLength / 2)
259+/**
260+ * Compress one int16 block with each of the given codecs. Takes the samples
261+ * rather than raw bytes so every codec — and the entropy bounds — key the
262+ * LPC cache off the same array.
263+ */
264+export function compressAll(samples: Int16Array, codecs: Codec[]): CodecResult[] {
265+ const bytes = asBytes(samples)
247266 return codecs.map(codec => {
248- const bytes = codec.size(samples, aligned)
267+ const size = codec.size(samples, bytes)
249268 return {
250269 codec: codec.name,
251270 note: codec.note,
252- bytes,
253- ratio: aligned.byteLength / bytes,
254- bitsPerSample: (8 * bytes) / samples.length,
271+ bytes: size,
272+ ratio: bytes.byteLength / size,
273+ bitsPerSample: (8 * size) / samples.length,
255274 }
256275 })
257276 }
src/worker/compressWorker.tsmodified+1−2View file
@@ -56,10 +56,9 @@ self.onmessage = async (e: MessageEvent<CompressRequest>) => {
5656 }
5757 const mean = sum / samples.length
5858 const empiricalStd = Math.sqrt(Math.max(0, sumSq / samples.length - mean * mean))
59- const bytes = new Uint8Array(samples.buffer, 0, samples.byteLength)
6059 post({
6160 id,
62- results: compressAll(bytes, [...PLAIN_CODECS, ...lpcCodecs(lpcOrder)]),
61+ results: compressAll(samples, [...PLAIN_CODECS, ...lpcCodecs(lpcOrder)]),
6362 bounds: entropyBounds(samples, lpcOrder),
6463 empiricalStd,
6564 })
moveopenescclose