add zstd-markov
2 changed files+61−2
zia_benchmark/src/zia_benchmark/algorithms/simple_ans/__init__.pymodified+2−2View file
@@ -1,8 +1,8 @@
11 import numpy as np
2-from zia_benchmark.algorithms.simple_ans.markov_reconstruct import (
2+from .markov_reconstruct import (
33 markov_reconstruct as markov_reconstruct_cpp,
44 )
5-from zia_benchmark.algorithms.simple_ans.markov_predict import (
5+from .markov_predict import (
66 markov_predict as markov_predict_cpp,
77 )
88
zia_benchmark/src/zia_benchmark/algorithms/zstd/__init__.pymodified+59−0View file
@@ -1,4 +1,6 @@
11 import numpy as np
2+from ..simple_ans.markov_reconstruct import markov_reconstruct as markov_reconstruct_cpp
3+from ..simple_ans.markov_predict import markov_predict as markov_predict_cpp
24
35
46 SOURCE_FILE = "zstd/__init__.py"
@@ -44,6 +46,54 @@ def zstd_decode(x: bytes, dtype: str) -> np.ndarray:
4446 return y
4547
4648
49+def zstd_markov_encode(x: np.ndarray, level: int) -> bytes:
50+ import zstandard as zstd
51+ import struct
52+
53+ assert x.ndim == 1
54+ coeffs, initial, resid = markov_predict_cpp(x, M=6, num_training_samples=10000)
55+
56+ # Convert coeffs and initial to bytes
57+ coeffs_bytes = coeffs.tobytes()
58+ initial_bytes = initial.tobytes()
59+
60+ # Create header with lengths
61+ header = struct.pack("QQ", len(coeffs_bytes), len(initial_bytes))
62+
63+ # Compress residuals
64+ resid_bytes = resid.tobytes()
65+ compressor = zstd.ZstdCompressor(level=level)
66+ compressed_resid = compressor.compress(resid_bytes)
67+
68+ # Combine all parts
69+ return header + coeffs_bytes + initial_bytes + compressed_resid
70+
71+
72+def zstd_markov_decode(x: bytes, dtype: str) -> np.ndarray:
73+ import zstandard as zstd
74+ import struct
75+
76+ # Extract header
77+ header_size = struct.calcsize("QQ")
78+ coeffs_len, initial_len = struct.unpack("QQ", x[:header_size])
79+
80+ # Extract coefficients and initial values
81+ pos = header_size
82+ coeffs = np.frombuffer(x[pos : pos + coeffs_len], dtype=np.float64)
83+ pos += coeffs_len
84+ initial = np.frombuffer(x[pos : pos + initial_len], dtype=dtype)
85+ pos += initial_len
86+
87+ # Decompress residuals
88+ decompressor = zstd.ZstdDecompressor()
89+ resid_buf = decompressor.decompress(x[pos:])
90+ resid = np.frombuffer(resid_buf, dtype=dtype)
91+
92+ # Reconstruct signal
93+ output = markov_reconstruct_cpp(coeffs, initial, resid)
94+ return output
95+
96+
4797 algorithms = [
4898 {
4999 "name": "zstd-4",
@@ -110,4 +160,13 @@ algorithms = [
110160 "tags": ["delta_encoding"],
111161 "source_file": SOURCE_FILE,
112162 },
163+ {
164+ "name": "zstd-22-markov",
165+ "version": "1",
166+ "encode": lambda x: zstd_markov_encode(x, level=22),
167+ "decode": lambda x, dtype: zstd_markov_decode(x, dtype),
168+ "description": "Zstandard compression at level 22 with Markov prediction for exploiting temporal correlations in the data.",
169+ "tags": ["markov_prediction"],
170+ "source_file": SOURCE_FILE,
171+ },
113172 ]