/ concept-collection / benchcompress
Sign in
concept-collection / benchcompress
markov -> lpc
Jeremy Magland <jmagland@flatironinstitute.org> committed commit 4d8d16f23591 parent 6071a2a Browse files
19 changed files+184−184
benchcompress/setup.pymodified+2−2View file
@@ -33,9 +33,9 @@ setup(
3333 packages=find_packages(where="src"),
3434 package_dir={"": "src"},
3535 ext_modules=[
36- CMakeExtension("benchcompress.algorithms.ans.markov_reconstruct_cpp_ext",
36+ CMakeExtension("benchcompress.algorithms.ans.lpc_reconstruct_cpp_ext",
3737 sourcedir="src/benchcompress/algorithms/ans"),
38- CMakeExtension("benchcompress.algorithms.ans.markov_predict_cpp_ext",
38+ CMakeExtension("benchcompress.algorithms.ans.lpc_predict_cpp_ext",
3939 sourcedir="src/benchcompress/algorithms/ans"),
4040 CMakeExtension("benchcompress.algorithms.ans.get_run_lengths_cpp_ext",
4141 sourcedir="src/benchcompress/algorithms/ans")
benchcompress/src/benchcompress/algorithms/ans/CMakeLists.txtmodified+7−7View file
@@ -1,5 +1,5 @@
11 cmake_minimum_required(VERSION 3.15)
2-project(markov_cpp)
2+project(lpc_cpp)
33
44 set(CMAKE_CXX_STANDARD 14)
55 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -24,16 +24,16 @@ FetchContent_Declare(
2424
2525 FetchContent_MakeAvailable(pybind11 eigen)
2626
27-# Build markov_reconstruct module
28-pybind11_add_module(markov_reconstruct_cpp_ext markov_reconstruct.cpp)
27+# Build lpc_reconstruct module
28+pybind11_add_module(lpc_reconstruct_cpp_ext lpc_reconstruct.cpp)
2929
30-# Build markov_predict module
31-pybind11_add_module(markov_predict_cpp_ext markov_predict.cpp)
32-target_link_libraries(markov_predict_cpp_ext PRIVATE Eigen3::Eigen)
30+# Build lpc_predict module
31+pybind11_add_module(lpc_predict_cpp_ext lpc_predict.cpp)
32+target_link_libraries(lpc_predict_cpp_ext PRIVATE Eigen3::Eigen)
3333
3434 # Build get_run_lengths module
3535 pybind11_add_module(get_run_lengths_cpp_ext get_run_lengths.cpp)
3636
3737 # Install all modules
38-install(TARGETS markov_reconstruct_cpp_ext markov_predict_cpp_ext get_run_lengths_cpp_ext
38+install(TARGETS lpc_reconstruct_cpp_ext lpc_predict_cpp_ext get_run_lengths_cpp_ext
3939 DESTINATION benchcompress/algorithms/ans)
benchcompress/src/benchcompress/algorithms/ans/__init__.pymodified+56−56View file
@@ -1,10 +1,10 @@
11 import numpy as np
22 import os
3-from .markov_reconstruct import (
4- markov_reconstruct as markov_reconstruct_cpp,
3+from .lpc_reconstruct import (
4+ lpc_reconstruct as lpc_reconstruct_cpp,
55 )
6-from .markov_predict import (
7- markov_predict as markov_predict_cpp,
6+from .lpc_predict import (
7+ lpc_predict as lpc_predict_cpp,
88 )
99 from .get_run_lengths import get_run_lengths
1010
@@ -41,7 +41,7 @@ def ans_encode(x: np.ndarray) -> bytes:
4141 header = (
4242 [
4343 dtype_code,
44- encoded.num_bits,
44+ len(encoded.words),
4545 encoded.signal_length,
4646 encoded.state,
4747 len(encoded.symbol_counts),
@@ -51,7 +51,7 @@ def ans_encode(x: np.ndarray) -> bytes:
5151 )
5252 header_bytes = np.array(header, dtype=np.int64).tobytes()
5353 header_size = np.uint32(len(header_bytes))
54- return header_size.tobytes() + header_bytes + encoded.bitstream
54+ return header_size.tobytes() + header_bytes + encoded.words.tobytes()
5555
5656
5757 def ans0_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
@@ -59,10 +59,10 @@ def ans0_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
5959
6060 header_size = np.frombuffer(x[:4], dtype=np.uint32)[0]
6161 header = np.frombuffer(x[4 : 4 + header_size], dtype=np.int64)
62- dtype_code, num_bits, signal_length, state, num_symbols = header[:5]
62+ dtype_code, num_words, signal_length, state, num_symbols = header[:5]
6363 symbol_counts = header[5 : 5 + num_symbols]
6464 symbol_values = header[5 + num_symbols :]
65- bitstream = x[4 + header_size :]
65+ words_bytes = x[4 + header_size :]
6666 if dtype_code == 0:
6767 assert dtype == "uint8"
6868 elif dtype_code == 1:
@@ -77,12 +77,11 @@ def ans0_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
7777 raise ValueError(f"Unsupported dtype code: {dtype_code}")
7878
7979 encoded = EncodedSignal(
80- num_bits=int(num_bits),
8180 signal_length=int(signal_length),
8281 state=int(state),
8382 symbol_counts=symbol_counts.astype(np.uint32),
8483 symbol_values=symbol_values.astype(dtype),
85- bitstream=bitstream,
84+ words=np.frombuffer(words_bytes, dtype=np.uint32, count=num_words),
8685 )
8786 return ans_decode(encoded).reshape(shape)
8887
@@ -111,7 +110,7 @@ def ans_delta_encode(x: np.ndarray) -> bytes:
111110 header = (
112111 [
113112 dtype_code,
114- encoded.num_bits,
113+ len(encoded.words),
115114 encoded.signal_length,
116115 encoded.state,
117116 len(encoded.symbol_counts),
@@ -122,7 +121,7 @@ def ans_delta_encode(x: np.ndarray) -> bytes:
122121 )
123122 header_bytes = np.array(header, dtype=np.int64).tobytes()
124123 header_size = np.uint32(len(header_bytes))
125- return header_size.tobytes() + header_bytes + encoded.bitstream
124+ return header_size.tobytes() + header_bytes + encoded.words.tobytes()
126125
127126
128127 def ans_delta_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
@@ -132,12 +131,12 @@ def ans_delta_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
132131
133132 header_size = np.frombuffer(x[:4], dtype=np.uint32)[0]
134133 header = np.frombuffer(x[4 : 4 + header_size], dtype=np.int64)
135- dtype_code, num_bits, signal_length, state, num_symbols, x0 = header[
134+ dtype_code, num_words, signal_length, state, num_symbols, x0 = header[
136135 :6
137136 ] # Extract x0 from header
138137 symbol_counts = header[6 : 6 + num_symbols]
139138 symbol_values = header[6 + num_symbols :]
140- bitstream = x[4 + header_size :]
139+ words_bytes = x[4 + header_size :]
141140 if dtype_code == 0:
142141 assert dtype == "uint8"
143142 elif dtype_code == 1:
@@ -152,12 +151,11 @@ def ans_delta_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
152151 raise ValueError(f"Unsupported dtype code: {dtype_code}")
153152
154153 encoded = EncodedSignal(
155- num_bits=int(num_bits),
156154 signal_length=int(signal_length),
157155 state=int(state),
158156 symbol_counts=symbol_counts.astype(np.uint32),
159157 symbol_values=symbol_values.astype(dtype),
160- bitstream=bitstream,
158+ words=np.frombuffer(words_bytes, dtype=np.uint32, count=num_words),
161159 )
162160 # Decode the differences
163161 diffs = ans_decode(encoded)
@@ -165,12 +163,12 @@ def ans_delta_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
165163 return np.cumsum(np.insert(diffs, 0, x0))
166164
167165
168-def ans_markov_encode(x: np.ndarray) -> bytes:
166+def ans_lpc_encode(x: np.ndarray) -> bytes:
169167 from simple_ans import ans_encode
170168
171169 assert x.ndim == 1
172170
173- coeffs, initial, resid = markov_predict_cpp(x, M=6, num_training_samples=10000)
171+ coeffs, initial, resid = lpc_predict_cpp(x, M=6, num_training_samples=10000)
174172 # Encode just the differences
175173 encoded = ans_encode(resid)
176174 if x.dtype == np.uint8:
@@ -189,7 +187,7 @@ def ans_markov_encode(x: np.ndarray) -> bytes:
189187 header = (
190188 [
191189 dtype_code,
192- encoded.num_bits,
190+ len(encoded.words),
193191 encoded.signal_length,
194192 encoded.state,
195193 len(encoded.symbol_counts),
@@ -203,21 +201,26 @@ def ans_markov_encode(x: np.ndarray) -> bytes:
203201 )
204202 header_bytes = np.array(header, dtype=np.float64).tobytes()
205203 header_size = np.uint32(len(header_bytes))
206- return header_size.tobytes() + header_bytes + encoded.bitstream
204+ return header_size.tobytes() + header_bytes + encoded.words.tobytes()
207205
208206
209-def ans_markov_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
207+def ans_lpc_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
210208 from simple_ans import ans_decode, EncodedSignal
211209
212210 assert len(shape) == 1
213211
214212 header_size = np.frombuffer(x[:4], dtype=np.uint32)[0]
215213 header = np.frombuffer(x[4 : 4 + header_size], dtype=np.float64)
216- dtype_code, num_bits, signal_length, state, num_symbols, num_coeffs, num_initial = (
217- header[:7]
218- )
214+ (
215+ dtype_code,
216+ num_words,
217+ signal_length,
218+ state,
219+ num_symbols,
220+ num_coeffs,
221+ num_initial,
222+ ) = header[:7]
219223 dtype_code = int(dtype_code)
220- num_bits = int(num_bits)
221224 signal_length = int(signal_length)
222225 state = int(state)
223226 num_symbols = int(num_symbols)
@@ -248,20 +251,19 @@ def ans_markov_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
248251 raise ValueError(f"Unsupported dtype code: {dtype_code}")
249252
250253 encoded = EncodedSignal(
251- num_bits=int(num_bits),
252254 signal_length=int(signal_length),
253255 state=int(state),
254256 symbol_counts=symbol_counts.astype(np.uint32),
255257 symbol_values=symbol_values.astype(dtype),
256- bitstream=bitstream,
258+ words=np.frombuffer(bitstream, dtype=np.uint32, count=num_words),
257259 )
258260
259261 resid = ans_decode(encoded)
260- output = markov_reconstruct_cpp(coeffs, initial, resid)
262+ output = lpc_reconstruct_cpp(coeffs, initial, resid)
261263 return output
262264
263265
264-def ans_markov_sparse_encode(x: np.ndarray) -> bytes:
266+def ans_lpc_sparse_encode(x: np.ndarray) -> bytes:
265267 from simple_ans import ans_encode
266268
267269 assert x.ndim == 1
@@ -284,7 +286,7 @@ def ans_markov_sparse_encode(x: np.ndarray) -> bytes:
284286
285287 non_zero_data = np.concatenate(non_zero_arrays)
286288 assert len(non_zero_data) == np.sum(run_lengths[::2])
287- coeffs, initial, resid = markov_predict_cpp(
289+ coeffs, initial, resid = lpc_predict_cpp(
288290 non_zero_data, M=6, num_training_samples=10000
289291 )
290292 encoded = ans_encode(resid)
@@ -314,8 +316,7 @@ def ans_markov_sparse_encode(x: np.ndarray) -> bytes:
314316 header = (
315317 [
316318 dtype_code,
317- encoded.num_bits,
318- len(encoded.bitstream),
319+ len(encoded.words),
319320 encoded.signal_length,
320321 encoded.state,
321322 len(encoded.symbol_counts),
@@ -333,11 +334,14 @@ def ans_markov_sparse_encode(x: np.ndarray) -> bytes:
333334 header_size = np.uint32(len(header_bytes))
334335
335336 return (
336- header_size.tobytes() + header_bytes + encoded.bitstream + run_lengths.tobytes()
337+ header_size.tobytes()
338+ + header_bytes
339+ + encoded.words.tobytes()
340+ + run_lengths.tobytes()
337341 )
338342
339343
340-def ans_markov_sparse_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
344+def ans_lpc_sparse_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
341345 from simple_ans import ans_decode, EncodedSignal
342346
343347 assert len(shape) == 1
@@ -346,8 +350,7 @@ def ans_markov_sparse_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
346350 header = np.frombuffer(x[4 : 4 + header_size], dtype=np.float64)
347351 (
348352 dtype_code,
349- num_bits,
350- bitstream_length,
353+ num_words,
351354 signal_length,
352355 state,
353356 num_symbols,
@@ -357,8 +360,6 @@ def ans_markov_sparse_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
357360 num_run_lengths,
358361 ) = header[:10]
359362 dtype_code = int(dtype_code)
360- num_bits = int(num_bits)
361- bitstream_length = int(bitstream_length)
362363 signal_length = int(signal_length)
363364 state = int(state)
364365 num_symbols = int(num_symbols)
@@ -377,16 +378,16 @@ def ans_markov_sparse_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
377378 initial = header[pos : pos + num_initial].astype(dtype)
378379 pos += num_initial
379380
380- bitstream_end = 4 + header_size + bitstream_length
381- bitstream = x[4 + header_size : bitstream_end]
381+ words_end = 4 + header_size + num_words * 4
382+ words_bytes = x[4 + header_size : words_end]
382383
383384 # Get run lengths from the remaining bytes
384385 if run_length_dtype_code == 0:
385- run_lengths = np.frombuffer(x[bitstream_end:], dtype=np.uint8)
386+ run_lengths = np.frombuffer(x[words_end:], dtype=np.uint8)
386387 elif run_length_dtype_code == 1:
387- run_lengths = np.frombuffer(x[bitstream_end:], dtype=np.uint16)
388+ run_lengths = np.frombuffer(x[words_end:], dtype=np.uint16)
388389 elif run_length_dtype_code == 2:
389- run_lengths = np.frombuffer(x[bitstream_end:], dtype=np.uint32)
390+ run_lengths = np.frombuffer(x[words_end:], dtype=np.uint32)
390391 else:
391392 raise ValueError(f"Unsupported run length dtype code: {run_length_dtype_code}")
392393
@@ -409,17 +410,16 @@ def ans_markov_sparse_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
409410 raise ValueError(f"Unsupported dtype code: {dtype_code}")
410411
411412 encoded = EncodedSignal(
412- num_bits=int(num_bits),
413413 signal_length=int(signal_length),
414414 state=int(state),
415415 symbol_counts=symbol_counts.astype(np.uint32),
416416 symbol_values=symbol_values.astype(dtype),
417- bitstream=bitstream,
417+ words=np.frombuffer(words_bytes, dtype=np.uint32, count=num_words),
418418 )
419419
420420 # Decode residuals and reconstruct non-zero data
421421 resid = ans_decode(encoded)
422- non_zero_data = markov_reconstruct_cpp(coeffs, initial, resid)
422+ non_zero_data = lpc_reconstruct_cpp(coeffs, initial, resid)
423423
424424 assert len(non_zero_data) == np.sum(run_lengths[::2])
425425
@@ -467,22 +467,22 @@ algorithms = [
467467 "long_description": LONG_DESCRIPTION,
468468 },
469469 {
470- "name": "ANS-markov",
470+ "name": "ANS-lpc",
471471 "version": "6",
472- "encode": lambda x: ans_markov_encode(x),
473- "decode": lambda x, dtype, shape: ans_markov_decode(x, dtype, shape),
474- "description": "ANS compression via simple_ans with Markov prediction for exploiting temporal correlations in the data.",
475- "tags": ["ANS", "integer", "markov_prediction", "1d"],
472+ "encode": lambda x: ans_lpc_encode(x),
473+ "decode": lambda x, dtype, shape: ans_lpc_decode(x, dtype, shape),
474+ "description": "ANS compression via simple_ans with linear predictive codingn for exploiting temporal correlations in the data.",
475+ "tags": ["ANS", "integer", "lpc_prediction", "1d"],
476476 "source_file": SOURCE_FILE,
477477 "long_description": LONG_DESCRIPTION,
478478 },
479479 {
480- "name": "ANS-markov-zrle",
480+ "name": "ANS-lpc-zrle",
481481 "version": "6",
482- "encode": lambda x: ans_markov_sparse_encode(x),
483- "decode": lambda x, dtype, shape: ans_markov_sparse_decode(x, dtype, shape),
484- "description": "ANS compression via simple_ans with Markov prediction and zero run-length encoding for sparse data.",
485- "tags": ["ANS", "integer", "markov_prediction", "zero_rle", "1d"],
482+ "encode": lambda x: ans_lpc_sparse_encode(x),
483+ "decode": lambda x, dtype, shape: ans_lpc_sparse_decode(x, dtype, shape),
484+ "description": "ANS compression via simple_ans with linear predictive codingn and zero run-length encoding for sparse data.",
485+ "tags": ["ANS", "integer", "lpc_prediction", "zero_rle", "1d"],
486486 "source_file": SOURCE_FILE,
487487 "long_description": LONG_DESCRIPTION,
488488 },
benchcompress/src/benchcompress/algorithms/ans/ans.mdmodified+4−4View file
@@ -13,14 +13,14 @@ ANS is a modern entropy coding method that achieves a compression ratio near ari
1313 - Stores differences between consecutive values
1414 - Effective for sequences where adjacent values are similar
1515
16-### Markov Prediction
17-- ANS-markov: ANS with Markov prediction
16+### Linear Predictive Coding (LPC)
17+- ANS-lpc: ANS with linear predictive codingn
1818 - Uses a 6th-order Markov model to predict values based on previous samples
1919 - Compresses the prediction residuals
2020 - Particularly effective for data with temporal correlations
2121
22-### Markov with Zero RLE
23-- ANS-markov-zrle: Combines Markov prediction with zero run-length encoding
22+### Linear Predictive Coding with Zero RLE
23+- ANS-lpc-zrle: Combines linear predictive coding with zero run-length encoding
2424 - Identifies runs of zero values and encodes their lengths
2525 - Applies Markov prediction to the non-zero regions
2626 - Ideal for sparse data with many zeros interspersed with correlated non-zero values
benchcompress/src/benchcompress/algorithms/ans/markov_predict.cpp →benchcompress/src/benchcompress/algorithms/ans/lpc_predict.cpprenamed+11−11View file
@@ -1,28 +1,28 @@
1-#include "markov_predict.hpp"
1+#include "lpc_predict.hpp"
22
33 namespace py = pybind11;
44
55 // Explicit instantiation for int16_t
66 std::tuple<py::array_t<float>, py::array_t<int16_t>, py::array_t<int16_t>>
7-markov_predict_int16(py::array_t<int16_t> x, size_t M,
8- size_t num_training_samples) {
9- return markov_predict_impl<int16_t>(x, M, num_training_samples);
7+lpc_predict_int16(py::array_t<int16_t> x, size_t M,
8+ size_t num_training_samples) {
9+ return lpc_predict_impl<int16_t>(x, M, num_training_samples);
1010 }
1111
1212 // Explicit instantiation for int32_t
1313 std::tuple<py::array_t<float>, py::array_t<int32_t>, py::array_t<int32_t>>
14-markov_predict_int32(py::array_t<int32_t> x, size_t M,
15- size_t num_training_samples) {
16- return markov_predict_impl<int32_t>(x, M, num_training_samples);
14+lpc_predict_int32(py::array_t<int32_t> x, size_t M,
15+ size_t num_training_samples) {
16+ return lpc_predict_impl<int32_t>(x, M, num_training_samples);
1717 }
1818
19-PYBIND11_MODULE(markov_predict_cpp_ext, m) {
20- m.doc() = "C++ implementation of markov_predict using pybind11";
21- m.def("markov_predict_int16", &markov_predict_int16,
19+PYBIND11_MODULE(lpc_predict_cpp_ext, m) {
20+ m.doc() = "C++ implementation of lpc_predict using pybind11";
21+ m.def("lpc_predict_int16", &lpc_predict_int16,
2222 "Predict signal using Markov model and return coefficients, initial "
2323 "values and residuals (int16)",
2424 py::arg("x"), py::arg("M"), py::arg("num_training_samples") = 10000);
25- m.def("markov_predict_int32", &markov_predict_int32,
25+ m.def("lpc_predict_int32", &lpc_predict_int32,
2626 "Predict signal using Markov model and return coefficients, initial "
2727 "values and residuals (int32)",
2828 py::arg("x"), py::arg("M"), py::arg("num_training_samples") = 10000);
benchcompress/src/benchcompress/algorithms/ans/markov_predict.hpp →benchcompress/src/benchcompress/algorithms/ans/lpc_predict.hpprenamed+1−1View file
@@ -11,7 +11,7 @@ namespace py = pybind11;
1111
1212 template <typename T>
1313 std::tuple<py::array_t<float>, py::array_t<T>, py::array_t<T>>
14-markov_predict_impl(py::array_t<T> x, size_t M, size_t num_training_samples) {
14+lpc_predict_impl(py::array_t<T> x, size_t M, size_t num_training_samples) {
1515 // Get array buffer
1616 auto x_buf = x.request();
1717 T *x_ptr = static_cast<T *>(x_buf.ptr);
benchcompress/src/benchcompress/algorithms/ans/markov_predict.py →benchcompress/src/benchcompress/algorithms/ans/lpc_predict.pyrenamed+4−4View file
@@ -1,8 +1,8 @@
11 import numpy as np
2-from .markov_predict_cpp_ext import markov_predict_int16, markov_predict_int32
2+from .lpc_predict_cpp_ext import lpc_predict_int16, lpc_predict_int32
33
44
5-def markov_predict(x: np.ndarray, M: int, num_training_samples: int = 10000) -> tuple:
5+def lpc_predict(x: np.ndarray, M: int, num_training_samples: int = 10000) -> tuple:
66 """Predict signal using Markov model and return coefficients, initial values and residuals using C++ implementation.
77
88 Args:
@@ -19,8 +19,8 @@ def markov_predict(x: np.ndarray, M: int, num_training_samples: int = 10000) ->
1919 """
2020 # Check input dtype and call appropriate implementation
2121 if x.dtype == np.int16:
22- return markov_predict_int16(x, M, num_training_samples)
22+ return lpc_predict_int16(x, M, num_training_samples)
2323 elif x.dtype == np.int32:
24- return markov_predict_int32(x, M, num_training_samples)
24+ return lpc_predict_int32(x, M, num_training_samples)
2525 else:
2626 raise ValueError(f"Input array must be int16 or int32, got {x.dtype}")
benchcompress/src/benchcompress/algorithms/ans/lpc_reconstruct.cppadded+27−0View file
@@ -0,0 +1,27 @@
1+#include "lpc_reconstruct.hpp"
2+
3+namespace py = pybind11;
4+
5+// Explicit instantiation for int16_t
6+py::array_t<int16_t> lpc_reconstruct_int16(py::array_t<float> coeffs,
7+ py::array_t<int16_t> initial,
8+ py::array_t<int16_t> resid) {
9+ return lpc_reconstruct_impl<int16_t>(coeffs, initial, resid);
10+}
11+
12+// Explicit instantiation for int32_t
13+py::array_t<int32_t> lpc_reconstruct_int32(py::array_t<float> coeffs,
14+ py::array_t<int32_t> initial,
15+ py::array_t<int32_t> resid) {
16+ return lpc_reconstruct_impl<int32_t>(coeffs, initial, resid);
17+}
18+
19+PYBIND11_MODULE(lpc_reconstruct_cpp_ext, m) {
20+ m.doc() = "C++ implementation of lpc_reconstruct using pybind11";
21+ m.def("lpc_reconstruct_int16", &lpc_reconstruct_int16,
22+ "Reconstruct signal from Markov model parameters and residuals (int16)",
23+ py::arg("coeffs"), py::arg("initial"), py::arg("resid"));
24+ m.def("lpc_reconstruct_int32", &lpc_reconstruct_int32,
25+ "Reconstruct signal from Markov model parameters and residuals (int32)",
26+ py::arg("coeffs"), py::arg("initial"), py::arg("resid"));
27+}
benchcompress/src/benchcompress/algorithms/ans/markov_reconstruct.hpp →benchcompress/src/benchcompress/algorithms/ans/lpc_reconstruct.hpprenamed+3−3View file
@@ -8,9 +8,9 @@
88 namespace py = pybind11;
99
1010 template <typename T>
11-py::array_t<T> markov_reconstruct_impl(py::array_t<float> coeffs,
12- py::array_t<T> initial,
13- py::array_t<T> resid) {
11+py::array_t<T> lpc_reconstruct_impl(py::array_t<float> coeffs,
12+ py::array_t<T> initial,
13+ py::array_t<T> resid) {
1414 // Get array buffers
1515 auto coeffs_buf = coeffs.request();
1616 auto initial_buf = initial.request();
benchcompress/src/benchcompress/algorithms/ans/markov_reconstruct.py →benchcompress/src/benchcompress/algorithms/ans/lpc_reconstruct.pyrenamed+6−6View file
@@ -1,11 +1,11 @@
11 import numpy as np
2-from .markov_reconstruct_cpp_ext import (
3- markov_reconstruct_int16,
4- markov_reconstruct_int32,
2+from .lpc_reconstruct_cpp_ext import (
3+ lpc_reconstruct_int16,
4+ lpc_reconstruct_int32,
55 )
66
77
8-def markov_reconstruct(coeffs, initial, resid):
8+def lpc_reconstruct(coeffs, initial, resid):
99 """Reconstruct signal from Markov model parameters and residuals using C++ implementation.
1010
1111 Args:
@@ -30,9 +30,9 @@ def markov_reconstruct(coeffs, initial, resid):
3030
3131 # Call appropriate implementation based on dtype
3232 if initial.dtype == np.int16:
33- return markov_reconstruct_int16(coeffs, initial, resid)
33+ return lpc_reconstruct_int16(coeffs, initial, resid)
3434 elif initial.dtype == np.int32:
35- return markov_reconstruct_int32(coeffs, initial, resid)
35+ return lpc_reconstruct_int32(coeffs, initial, resid)
3636 else:
3737 raise ValueError(
3838 f"Initial/residual arrays must be int16 or int32, got {initial.dtype}"
benchcompress/src/benchcompress/algorithms/ans/markov_reconstruct.cppdeleted+0−27View file
@@ -1,27 +0,0 @@
1-#include "markov_reconstruct.hpp"
2-
3-namespace py = pybind11;
4-
5-// Explicit instantiation for int16_t
6-py::array_t<int16_t> markov_reconstruct_int16(py::array_t<float> coeffs,
7- py::array_t<int16_t> initial,
8- py::array_t<int16_t> resid) {
9- return markov_reconstruct_impl<int16_t>(coeffs, initial, resid);
10-}
11-
12-// Explicit instantiation for int32_t
13-py::array_t<int32_t> markov_reconstruct_int32(py::array_t<float> coeffs,
14- py::array_t<int32_t> initial,
15- py::array_t<int32_t> resid) {
16- return markov_reconstruct_impl<int32_t>(coeffs, initial, resid);
17-}
18-
19-PYBIND11_MODULE(markov_reconstruct_cpp_ext, m) {
20- m.doc() = "C++ implementation of markov_reconstruct using pybind11";
21- m.def("markov_reconstruct_int16", &markov_reconstruct_int16,
22- "Reconstruct signal from Markov model parameters and residuals (int16)",
23- py::arg("coeffs"), py::arg("initial"), py::arg("resid"));
24- m.def("markov_reconstruct_int32", &markov_reconstruct_int32,
25- "Reconstruct signal from Markov model parameters and residuals (int32)",
26- py::arg("coeffs"), py::arg("initial"), py::arg("resid"));
27-}
benchcompress/src/benchcompress/algorithms/blosc2/__init__.pymodified+31−31View file
@@ -1,7 +1,7 @@
11 import numpy as np
22 import os
3-from ..ans.markov_reconstruct import markov_reconstruct as markov_reconstruct_cpp
4-from ..ans.markov_predict import markov_predict as markov_predict_cpp
3+from ..ans.lpc_reconstruct import lpc_reconstruct as lpc_reconstruct_cpp
4+from ..ans.lpc_predict import lpc_predict as lpc_predict_cpp
55 from ..ans.get_run_lengths import get_run_lengths
66
77 SOURCE_FILE = "blosc2/__init__.py"
@@ -90,12 +90,12 @@ def blosc2_delta_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
9090 return np.cumsum(y)
9191
9292
93-def blosc2_markov_encode(x: np.ndarray, clevel: int) -> bytes:
93+def blosc2_lpc_encode(x: np.ndarray, clevel: int) -> bytes:
9494 import blosc2
9595 import struct
9696
9797 assert x.ndim == 1
98- coeffs, initial, resid = markov_predict_cpp(x, M=6, num_training_samples=10000)
98+ coeffs, initial, resid = lpc_predict_cpp(x, M=6, num_training_samples=10000)
9999
100100 # Convert coeffs and initial to bytes
101101 coeffs_bytes = coeffs.tobytes()
@@ -123,7 +123,7 @@ def blosc2_markov_encode(x: np.ndarray, clevel: int) -> bytes:
123123 return header + coeffs_bytes + initial_bytes + compressed
124124
125125
126-def blosc2_markov_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
126+def blosc2_lpc_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
127127 import blosc2
128128 import struct
129129
@@ -146,11 +146,11 @@ def blosc2_markov_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
146146 resid = np.frombuffer(decompressed, dtype=dtype)
147147
148148 # Reconstruct signal
149- output = markov_reconstruct_cpp(coeffs, initial, resid)
149+ output = lpc_reconstruct_cpp(coeffs, initial, resid)
150150 return output
151151
152152
153-def blosc2_markov_zrle_encode(x: np.ndarray, clevel: int) -> bytes:
153+def blosc2_lpc_zrle_encode(x: np.ndarray, clevel: int) -> bytes:
154154 import blosc2
155155 import struct
156156
@@ -185,8 +185,8 @@ def blosc2_markov_zrle_encode(x: np.ndarray, clevel: int) -> bytes:
185185
186186 non_zero_data = np.concatenate(non_zero_arrays)
187187
188- # Apply Markov prediction on non-zero data
189- coeffs, initial, resid = markov_predict_cpp(
188+ # Apply linear predictive coding on non-zero data
189+ coeffs, initial, resid = lpc_predict_cpp(
190190 non_zero_data, M=6, num_training_samples=10000
191191 )
192192
@@ -224,7 +224,7 @@ def blosc2_markov_zrle_encode(x: np.ndarray, clevel: int) -> bytes:
224224 return header + coeffs_bytes + initial_bytes + run_lengths_bytes + compressed
225225
226226
227-def blosc2_markov_zrle_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
227+def blosc2_lpc_zrle_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
228228 import blosc2
229229 import struct
230230
@@ -262,7 +262,7 @@ def blosc2_markov_zrle_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
262262 resid = np.frombuffer(decompressed, dtype=dtype)
263263
264264 # Reconstruct non-zero data
265- non_zero_data = markov_reconstruct_cpp(coeffs, initial, resid)
265+ non_zero_data = lpc_reconstruct_cpp(coeffs, initial, resid)
266266
267267 # Reconstruct full array using run lengths
268268 non_zero_pos = 0
@@ -324,42 +324,42 @@ algorithms = [
324324 "long_description": LONG_DESCRIPTION,
325325 },
326326 {
327- "name": "blosc2-1-markov",
327+ "name": "blosc2-1-lpc",
328328 "version": "1",
329- "encode": lambda x: blosc2_markov_encode(x, clevel=1),
330- "decode": lambda x, dtype, shape: blosc2_markov_decode(x, dtype, shape),
331- "description": "Blosc2 compression at level 1 with Markov prediction for exploiting temporal correlations in the data.",
332- "tags": ["blosc2", "markov_prediction", "1d"],
329+ "encode": lambda x: blosc2_lpc_encode(x, clevel=1),
330+ "decode": lambda x, dtype, shape: blosc2_lpc_decode(x, dtype, shape),
331+ "description": "Blosc2 compression at level 1 with linear predictive codingn for exploiting temporal correlations in the data.",
332+ "tags": ["blosc2", "lpc_prediction", "1d"],
333333 "source_file": SOURCE_FILE,
334334 "long_description": LONG_DESCRIPTION,
335335 },
336336 {
337- "name": "blosc2-5-markov",
337+ "name": "blosc2-5-lpc",
338338 "version": "1",
339- "encode": lambda x: blosc2_markov_encode(x, clevel=1),
340- "decode": lambda x, dtype, shape: blosc2_markov_decode(x, dtype, shape),
341- "description": "Blosc2 compression at level 5 with Markov prediction for exploiting temporal correlations in the data.",
342- "tags": ["blosc2", "markov_prediction", "1d"],
339+ "encode": lambda x: blosc2_lpc_encode(x, clevel=1),
340+ "decode": lambda x, dtype, shape: blosc2_lpc_decode(x, dtype, shape),
341+ "description": "Blosc2 compression at level 5 with linear predictive codingn for exploiting temporal correlations in the data.",
342+ "tags": ["blosc2", "lpc_prediction", "1d"],
343343 "source_file": SOURCE_FILE,
344344 "long_description": LONG_DESCRIPTION,
345345 },
346346 {
347- "name": "blosc2-9-markov",
347+ "name": "blosc2-9-lpc",
348348 "version": "1",
349- "encode": lambda x: blosc2_markov_encode(x, clevel=9),
350- "decode": lambda x, dtype, shape: blosc2_markov_decode(x, dtype, shape),
351- "description": "Blosc2 compression at level 9 with Markov prediction for exploiting temporal correlations in the data.",
352- "tags": ["blosc2", "markov_prediction", "1d"],
349+ "encode": lambda x: blosc2_lpc_encode(x, clevel=9),
350+ "decode": lambda x, dtype, shape: blosc2_lpc_decode(x, dtype, shape),
351+ "description": "Blosc2 compression at level 9 with linear predictive codingn for exploiting temporal correlations in the data.",
352+ "tags": ["blosc2", "lpc_prediction", "1d"],
353353 "source_file": SOURCE_FILE,
354354 "long_description": LONG_DESCRIPTION,
355355 },
356356 {
357- "name": "blosc2-9-markov-zrle",
357+ "name": "blosc2-9-lpc-zrle",
358358 "version": "1",
359- "encode": lambda x: blosc2_markov_zrle_encode(x, clevel=9),
360- "decode": lambda x, dtype, shape: blosc2_markov_zrle_decode(x, dtype, shape),
361- "description": "Blosc2 compression at level 9 with Markov prediction and zero run-length encoding for sparse data.",
362- "tags": ["blosc2", "markov_prediction", "zero_rle", "1d"],
359+ "encode": lambda x: blosc2_lpc_zrle_encode(x, clevel=9),
360+ "decode": lambda x, dtype, shape: blosc2_lpc_zrle_decode(x, dtype, shape),
361+ "description": "Blosc2 compression at level 9 with linear predictive codingn and zero run-length encoding for sparse data.",
362+ "tags": ["blosc2", "lpc_prediction", "zero_rle", "1d"],
363363 "source_file": SOURCE_FILE,
364364 "long_description": LONG_DESCRIPTION,
365365 },
benchcompress/src/benchcompress/algorithms/zstd/__init__.pymodified+21−21View file
@@ -1,7 +1,7 @@
11 import numpy as np
22 import os
3-from ..ans.markov_reconstruct import markov_reconstruct as markov_reconstruct_cpp
4-from ..ans.markov_predict import markov_predict as markov_predict_cpp
3+from ..ans.lpc_reconstruct import lpc_reconstruct as lpc_reconstruct_cpp
4+from ..ans.lpc_predict import lpc_predict as lpc_predict_cpp
55 from ..ans.get_run_lengths import get_run_lengths
66
77
@@ -60,12 +60,12 @@ def zstd_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
6060 return y.reshape(shape)
6161
6262
63-def zstd_markov_encode(x: np.ndarray, level: int) -> bytes:
63+def zstd_lpc_encode(x: np.ndarray, level: int) -> bytes:
6464 import zstandard as zstd
6565 import struct
6666
6767 assert x.ndim == 1
68- coeffs, initial, resid = markov_predict_cpp(x, M=6, num_training_samples=10000)
68+ coeffs, initial, resid = lpc_predict_cpp(x, M=6, num_training_samples=10000)
6969
7070 # Convert coeffs and initial to bytes
7171 coeffs_bytes = coeffs.tobytes()
@@ -83,7 +83,7 @@ def zstd_markov_encode(x: np.ndarray, level: int) -> bytes:
8383 return header + coeffs_bytes + initial_bytes + compressed_resid
8484
8585
86-def zstd_markov_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
86+def zstd_lpc_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
8787 import zstandard as zstd
8888 import struct
8989
@@ -106,11 +106,11 @@ def zstd_markov_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
106106 resid = np.frombuffer(resid_buf, dtype=dtype)
107107
108108 # Reconstruct signal
109- output = markov_reconstruct_cpp(coeffs, initial, resid)
109+ output = lpc_reconstruct_cpp(coeffs, initial, resid)
110110 return output
111111
112112
113-def zstd_markov_zrle_encode(x: np.ndarray, level: int) -> bytes:
113+def zstd_lpc_zrle_encode(x: np.ndarray, level: int) -> bytes:
114114 import zstandard as zstd
115115 import struct
116116
@@ -145,8 +145,8 @@ def zstd_markov_zrle_encode(x: np.ndarray, level: int) -> bytes:
145145
146146 non_zero_data = np.concatenate(non_zero_arrays)
147147
148- # Apply Markov prediction on non-zero data
149- coeffs, initial, resid = markov_predict_cpp(
148+ # Apply linear predictive coding on non-zero data
149+ coeffs, initial, resid = lpc_predict_cpp(
150150 non_zero_data, M=6, num_training_samples=10000
151151 )
152152
@@ -174,7 +174,7 @@ def zstd_markov_zrle_encode(x: np.ndarray, level: int) -> bytes:
174174 return header + coeffs_bytes + initial_bytes + run_lengths_bytes + compressed_resid
175175
176176
177-def zstd_markov_zrle_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
177+def zstd_lpc_zrle_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
178178 import zstandard as zstd
179179 import struct
180180
@@ -212,7 +212,7 @@ def zstd_markov_zrle_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
212212 resid = np.frombuffer(resid_buf, dtype=dtype)
213213
214214 # Reconstruct non-zero data
215- non_zero_data = markov_reconstruct_cpp(coeffs, initial, resid)
215+ non_zero_data = lpc_reconstruct_cpp(coeffs, initial, resid)
216216
217217 # Reconstruct full array using run lengths
218218 non_zero_pos = 0
@@ -314,22 +314,22 @@ algorithms = [
314314 "long_description": LONG_DESCRIPTION,
315315 },
316316 {
317- "name": "zstd-22-markov",
317+ "name": "zstd-22-lpc",
318318 "version": "1",
319- "encode": lambda x: zstd_markov_encode(x, level=22),
320- "decode": lambda x, dtype, shape: zstd_markov_decode(x, dtype, shape),
321- "description": "Zstandard compression at level 22 with Markov prediction for exploiting temporal correlations in the data.",
322- "tags": ["zstd", "markov_prediction", "1d"],
319+ "encode": lambda x: zstd_lpc_encode(x, level=22),
320+ "decode": lambda x, dtype, shape: zstd_lpc_decode(x, dtype, shape),
321+ "description": "Zstandard compression at level 22 with linear predictive codingn for exploiting temporal correlations in the data.",
322+ "tags": ["zstd", "lpc_prediction", "1d"],
323323 "source_file": SOURCE_FILE,
324324 "long_description": LONG_DESCRIPTION,
325325 },
326326 {
327- "name": "zstd-22-markov-zrle",
327+ "name": "zstd-22-lpc-zrle",
328328 "version": "1",
329- "encode": lambda x: zstd_markov_zrle_encode(x, level=22),
330- "decode": lambda x, dtype, shape: zstd_markov_zrle_decode(x, dtype, shape),
331- "description": "Zstandard compression at level 22 with Markov prediction and zero run-length encoding for sparse data.",
332- "tags": ["zstd", "markov_prediction", "zero_rle", "1d"],
329+ "encode": lambda x: zstd_lpc_zrle_encode(x, level=22),
330+ "decode": lambda x, dtype, shape: zstd_lpc_zrle_decode(x, dtype, shape),
331+ "description": "Zstandard compression at level 22 with linear predictive codingn and zero run-length encoding for sparse data.",
332+ "tags": ["zstd", "lpc_prediction", "zero_rle", "1d"],
333333 "source_file": SOURCE_FILE,
334334 "long_description": LONG_DESCRIPTION,
335335 },
benchcompress/src/benchcompress/algorithms/zstd/zstd.mdmodified+2−2View file
@@ -19,8 +19,8 @@ Different compression levels trading off speed vs compression ratio:
1919 #### Delta Encoding (zstd-22-delta)
2020 Stores differences between consecutive values. Effective for sequences where adjacent values are similar, like time series data.
2121
22-#### Markov Prediction (zstd-22-markov)
22+#### Linear Predictive Coding (zstd-22-lpc)
2323 Uses a Markov model to predict values based on previous samples. The prediction residuals are then compressed using zstd. This can significantly improve compression for data with temporal correlations.
2424
25-#### Markov with Zero RLE (zstd-22-markov-zrle)
25+#### Linear Predictive Coding with Zero RLE (zstd-22-lpc-zrle)
2626 Combines Markov prediction with zero run-length encoding. Particularly effective for sparse data where many values are zero, as it efficiently encodes runs of zeros while using Markov prediction for the non-zero regions.
benchcompress/src/benchcompress/run_benchmarks/is_compatible.pymodified+2−2View file
@@ -11,8 +11,8 @@ def is_compatible(algorithm_tags: List[str], dataset_tags: List[str]) -> bool:
1111 Returns:
1212 True if the algorithm should be applied to the dataset
1313 """
14- # If algorithm has delta_encoding or markov_prediction, dataset must have continuous, timeseries, 1d, integer
15- if "delta_encoding" in algorithm_tags or "markov_prediction" in algorithm_tags:
14+ # If algorithm has delta_encoding or lpc_prediction, dataset must have continuous, timeseries, 1d, integer
15+ if "delta_encoding" in algorithm_tags or "lpc_prediction" in algorithm_tags:
1616 if (
1717 "correlated" not in dataset_tags
1818 or "timeseries" not in dataset_tags
devel/compile_markov_bench_cpp.shmodified+1−1View file
@@ -1,3 +1,3 @@
11 # sudo apt-get install libeigen3-dev
22
3-g++ -std=c++17 -O2 -I /usr/include/eigen3 markov_bench.cpp -o markov_bench.out
\ No newline at end of file
3+g++ -std=c++17 -O2 -I /usr/include/eigen3 lpc_bench.cpp -o lpc_bench.out
\ No newline at end of file
devel/markov_bench.cppmodified+1−1View file
@@ -1,5 +1,5 @@
11 /*************************************************
2- * markov_bench.cpp
2+ * lpc_bench.cpp
33 *
44 * A C++ program demonstrating:
55 * 1. Generating int16_t data (size N).
devel/markov_bench.pymodified+4−4View file
@@ -94,7 +94,7 @@ try:
9494 from numba import njit
9595
9696 @njit
97- def _predict_markov_numba(pred, coeffs, M):
97+ def _predict_lpc_numba(pred, coeffs, M):
9898 for j in range(M, len(pred)):
9999 val = coeffs[0]
100100 for k in range(1, M + 1):
@@ -109,7 +109,7 @@ try:
109109 M = len(coeffs) - 1
110110 pred = np.zeros(N, dtype=np.float32)
111111 pred[:M] = seed.astype(np.float32)
112- _predict_markov_numba(pred, coeffs, M)
112+ _predict_lpc_numba(pred, coeffs, M)
113113 return np.round(pred).astype(np.int16)
114114
115115 except ImportError:
@@ -124,7 +124,7 @@ except ImportError:
124124 # 4) Main benchmarking function
125125 ########################################################################
126126
127-def benchmark_markov_model(N=1_000_000, M=5, seed=0):
127+def benchmark_lpc_model(N=1_000_000, M=5, seed=0):
128128 """
129129 1) Generate random data x of length N as int16.
130130 2) Fit the Markov model in multiple ways:
@@ -261,5 +261,5 @@ def benchmark_markov_model(N=1_000_000, M=5, seed=0):
261261 ########################################################################
262262 if __name__ == "__main__":
263263 # Example usage
264- benchmark_markov_model(N=5_000_000, M=5, seed=42)
264+ benchmark_lpc_model(N=5_000_000, M=5, seed=42)
265265 # Adjust N, M, and seed as needed.
paper/paper.mdmodified+1−1View file
@@ -57,7 +57,7 @@ In practice, achieving this theoretical compression ratio requires sophisticated
5757
5858 The efficiency of pure entropy coders (such as ANS) diminishes when handling more structured data, such as continuous signals (e.g., voltage traces in electrophysiology). Applying delta encoding partially mitigates this limitation by leveraging the continuity properties of the data through differencing. This reversible preprocessing step enhances ANS performance because the deltas are typically smaller than the original samples, leading to lower entropy when assuming independence between samples.
5959
60-## Linear Markov prediction
60+## Linear Predictive Coding
6161
6262 Even with delta encoding for real or realistic datasets, ANS still can fall short of the compression achieved by methods like ZStandard. To improve its performance by further exploiting temporal correlations in the data, we consider a generalization of delta encoding which we call linear Markov predictive modeling. The Markov prediction scheme employs a linear autoregressive model where each sample is predicted as a linear combination of $M$ previous samples. For a given integer time series $x[t]$, the prediction $\hat{x}[t]$ is computed as:
6363
↑↓ move↵openescclose