markov prediction
4 changed files+197−10
web-ui/src/App.tsxmodified+13−0View file
@@ -22,6 +22,19 @@ function App() {
2222 Comparing different integer array compression algorithms and their
2323 performance
2424 </p>
25+ <a
26+ href="https://github.com/magland/zia"
27+ target="_blank"
28+ rel="noopener noreferrer"
29+ style={{
30+ color: "#0066cc",
31+ textDecoration: "none",
32+ display: "inline-block",
33+ marginTop: "0.5rem",
34+ }}
35+ >
36+ View source on GitHub
37+ </a>
2538 </header>
2639 <main>
2740 <BenchmarkTable />
zia_benchmark/src/zia_benchmark/algorithms/simple_ans/__init__.pymodified+108−10View file
@@ -1,11 +1,77 @@
11 import numpy as np
2+from .markov import markov_predict, markov_reconstruct
3+
4+
5+def simple_ans_encode(x: np.ndarray) -> bytes:
6+ from simple_ans import ans_encode
7+
8+ assert x.ndim == 1
9+ encoded = ans_encode(x)
10+ if x.dtype == np.uint8:
11+ dtype_code = 0
12+ elif x.dtype == np.uint16:
13+ dtype_code = 1
14+ elif x.dtype == np.uint32:
15+ dtype_code = 2
16+ elif x.dtype == np.int16:
17+ dtype_code = 3
18+ elif x.dtype == np.int32:
19+ dtype_code = 4
20+ else:
21+ raise ValueError(f"Unsupported dtype: {x.dtype}")
22+ header = (
23+ [
24+ dtype_code,
25+ encoded.num_bits,
26+ encoded.signal_length,
27+ encoded.state,
28+ len(encoded.symbol_counts),
29+ ]
30+ + [c for c in encoded.symbol_counts]
31+ + [v for v in encoded.symbol_values]
32+ )
33+ header_bytes = np.array(header, dtype=np.int64).tobytes()
34+ header_size = np.uint32(len(header_bytes))
35+ return header_size.tobytes() + header_bytes + encoded.bitstream
36+
37+
38+def simple_ans_decode(x: bytes, dtype: str) -> np.ndarray:
39+ from simple_ans import ans_decode, EncodedSignal
40+
41+ header_size = np.frombuffer(x[:4], dtype=np.uint32)[0]
42+ header = np.frombuffer(x[4 : 4 + header_size], dtype=np.int64)
43+ dtype_code, num_bits, signal_length, state, num_symbols = header[:5]
44+ symbol_counts = header[5 : 5 + num_symbols]
45+ symbol_values = header[5 + num_symbols :]
46+ bitstream = x[4 + header_size :]
47+ if dtype_code == 0:
48+ assert dtype == "uint8"
49+ elif dtype_code == 1:
50+ assert dtype == "uint16"
51+ elif dtype_code == 2:
52+ assert dtype == "uint32"
53+ elif dtype_code == 3:
54+ assert dtype == "int16"
55+ elif dtype_code == 4:
56+ assert dtype == "int32"
57+ else:
58+ raise ValueError(f"Unsupported dtype code: {dtype_code}")
59+
60+ encoded = EncodedSignal(
61+ num_bits=int(num_bits),
62+ signal_length=int(signal_length),
63+ state=int(state),
64+ symbol_counts=symbol_counts.astype(np.uint32),
65+ symbol_values=symbol_values.astype(dtype),
66+ bitstream=bitstream,
67+ )
68+ return ans_decode(encoded)
269
370
471 def simple_ans_delta_encode(x: np.ndarray) -> bytes:
572 from simple_ans import ans_encode
673
774 assert x.ndim == 1
8- # Calculate differences without inserting x[0]
975 y = np.diff(x)
1076 # Encode just the differences
1177 encoded = ans_encode(y)
@@ -77,11 +143,13 @@ def simple_ans_delta_decode(x: bytes, dtype: str) -> np.ndarray:
77143 return np.cumsum(np.insert(diffs, 0, x0))
78144
79145
80-def simple_ans_encode(x: np.ndarray) -> bytes:
146+def simple_ans_markov_encode(x: np.ndarray) -> bytes:
81147 from simple_ans import ans_encode
82148
83149 assert x.ndim == 1
84- encoded = ans_encode(x)
150+ coeffs, initial, resid = markov_predict(x, M=6)
151+ # Encode just the differences
152+ encoded = ans_encode(resid)
85153 if x.dtype == np.uint8:
86154 dtype_code = 0
87155 elif x.dtype == np.uint16:
@@ -94,6 +162,7 @@ def simple_ans_encode(x: np.ndarray) -> bytes:
94162 dtype_code = 4
95163 else:
96164 raise ValueError(f"Unsupported dtype: {x.dtype}")
165+ # Include x[0] in the header
97166 header = (
98167 [
99168 dtype_code,
@@ -101,23 +170,44 @@ def simple_ans_encode(x: np.ndarray) -> bytes:
101170 encoded.signal_length,
102171 encoded.state,
103172 len(encoded.symbol_counts),
173+ len(coeffs),
174+ len(initial),
104175 ]
105176 + [c for c in encoded.symbol_counts]
106177 + [v for v in encoded.symbol_values]
178+ + [c for c in coeffs]
179+ + [v for v in initial]
107180 )
108- header_bytes = np.array(header, dtype=np.int64).tobytes()
181+ header_bytes = np.array(header, dtype=np.float64).tobytes()
109182 header_size = np.uint32(len(header_bytes))
110183 return header_size.tobytes() + header_bytes + encoded.bitstream
111184
112185
113-def simple_ans_decode(x: bytes, dtype: str) -> np.ndarray:
186+def simple_ans_markov_decode(x: bytes, dtype: str) -> np.ndarray:
114187 from simple_ans import ans_decode, EncodedSignal
115188
116189 header_size = np.frombuffer(x[:4], dtype=np.uint32)[0]
117- header = np.frombuffer(x[4 : 4 + header_size], dtype=np.int64)
118- dtype_code, num_bits, signal_length, state, num_symbols = header[:5]
119- symbol_counts = header[5 : 5 + num_symbols]
120- symbol_values = header[5 + num_symbols :]
190+ header = np.frombuffer(x[4 : 4 + header_size], dtype=np.float64)
191+ dtype_code, num_bits, signal_length, state, num_symbols, num_coeffs, num_initial = (
192+ header[:7]
193+ )
194+ dtype_code = int(dtype_code)
195+ num_bits = int(num_bits)
196+ signal_length = int(signal_length)
197+ state = int(state)
198+ num_symbols = int(num_symbols)
199+ num_coeffs = int(num_coeffs)
200+ num_initial = int(num_initial)
201+
202+ pos = 7
203+ symbol_counts = header[pos : pos + num_symbols]
204+ pos += num_symbols
205+ symbol_values = header[pos : pos + num_symbols]
206+ pos += num_symbols
207+ coeffs = header[pos : pos + num_coeffs]
208+ pos += num_coeffs
209+ initial = header[pos : pos + num_initial]
210+ pos += num_initial
121211 bitstream = x[4 + header_size :]
122212 if dtype_code == 0:
123213 assert dtype == "uint8"
@@ -140,7 +230,8 @@ def simple_ans_decode(x: bytes, dtype: str) -> np.ndarray:
140230 symbol_values=symbol_values.astype(dtype),
141231 bitstream=bitstream,
142232 )
143- return ans_decode(encoded)
233+ resid = ans_decode(encoded)
234+ return markov_reconstruct(coeffs, initial, resid)
144235
145236
146237 algorithms = [
@@ -157,4 +248,11 @@ algorithms = [
157248 "decode": lambda x, dtype: simple_ans_delta_decode(x, dtype),
158249 "tags": ["delta_encoding"],
159250 },
251+ {
252+ "name": "simple-ans-markov",
253+ "version": "1",
254+ "encode": lambda x: simple_ans_markov_encode(x),
255+ "decode": lambda x, dtype: simple_ans_markov_decode(x, dtype),
256+ "tags": ["markov_prediction"],
257+ },
160258 ]
zia_benchmark/src/zia_benchmark/algorithms/simple_ans/markov.pyadded+64−0View file
@@ -0,0 +1,64 @@
1+import numpy as np
2+from zia_benchmark._analysis import linear_fit
3+
4+
5+def markov_predict(x: np.ndarray, M: int) -> tuple:
6+ """Predict signal using Markov model and return coefficients, initial values and residuals.
7+
8+ Args:
9+ x: Input signal
10+ M: Number of previous samples to use for prediction (default: 20)
11+
12+ Returns:
13+ tuple: (coefficients, initial_values, residuals)
14+ """
15+ # Keep initial values for reconstruction
16+ initial = x[: M - 1]
17+
18+ # Create sequences of M consecutive samples
19+ sequences = np.array([x[i : i + M] for i in range(len(x) - M + 1)])
20+ predictors = sequences[:, : M - 1] # Use M-1 previous samples to predict
21+ target = sequences[:, M - 1] # The value to predict
22+
23+ # Get coefficients and prediction function using linear regression
24+ coeffs, predict = linear_fit(predictors, target)
25+
26+ # Make predictions using the linear model
27+ predictions = predict(predictors)
28+ predictions = np.round(predictions)
29+
30+ # Calculate residuals (difference between actual and predicted values)
31+ residuals = target - predictions
32+ residuals = residuals.astype(x.dtype)
33+
34+ return coeffs, initial, residuals
35+
36+
37+def markov_reconstruct(
38+ coeffs: np.ndarray, initial: np.ndarray, resid: np.ndarray
39+) -> np.ndarray:
40+ """Reconstruct signal from Markov model parameters and residuals.
41+
42+ Args:
43+ coeffs: Model coefficients from linear regression
44+ initial: Initial values needed for prediction
45+ resid: Prediction residuals
46+
47+ Returns:
48+ np.ndarray: Reconstructed signal
49+ """
50+ M = len(initial) + 1 # Number of samples used in prediction
51+ output = np.zeros(len(resid) + len(initial), dtype=resid.dtype)
52+ output[: len(initial)] = initial # Set initial values
53+
54+ # Reconstruct signal iteratively
55+ for i in range(len(resid)):
56+ # Get previous M-1 values to make prediction
57+ prev_values = output[i : i + M - 1]
58+ # Make prediction using coefficients
59+ prediction = np.sum(coeffs[1:] * prev_values) + coeffs[0]
60+ prediction = np.round(prediction)
61+ # Add residual to get actual value
62+ output[i + M - 1] = prediction + resid[i]
63+
64+ return output
zia_benchmark/src/zia_benchmark/run_benchmarks.pymodified+12−0View file
@@ -25,6 +25,8 @@ def is_compatible(algorithm_tags: List[str], dataset_tags: List[str]) -> bool:
2525 # If algorithm has delta_encoding tag, dataset must have continuous tag
2626 if "delta_encoding" in algorithm_tags and "continuous" not in dataset_tags:
2727 return False
28+ if "markov_prediction" in algorithm_tags and "continuous" not in dataset_tags:
29+ return False
2830 return True
2931
3032
@@ -169,8 +171,18 @@ def run_benchmarks(
169171 print(f" Decode time: {decode_time*1000:.2f}ms")
170172 print(f" Decode throughput: {decode_mb_per_sec:.2f} MB/s")
171173
174+ if len(data) != len(decoded):
175+ raise ValueError(
176+ f"Decompression failed: decoded length {len(decoded)} != original length {len(data)}"
177+ )
178+
172179 # Verify correctness
173180 if not np.array_equal(data, decoded):
181+ print(data[:100])
182+ print(decoded[:100])
183+ for j in range(len(data)):
184+ if data[j] != decoded[j]:
185+ print(f"Error at index {j}: {data[j]} != {decoded[j]}")
174186 raise ValueError(
175187 f"Decompression verification failed for {alg_name} on {dataset['name']}"
176188 )