run length encoding
9 changed files+319−13
benchcompress/setup.pymodified+2−0View file
@@ -47,6 +47,8 @@ setup(
4747 CMakeExtension("benchcompress.algorithms.simple_ans.markov_reconstruct_cpp_ext",
4848 sourcedir="src/benchcompress/algorithms/simple_ans"),
4949 CMakeExtension("benchcompress.algorithms.simple_ans.markov_predict_cpp_ext",
50+ sourcedir="src/benchcompress/algorithms/simple_ans"),
51+ CMakeExtension("benchcompress.algorithms.simple_ans.get_run_lengths_cpp_ext",
5052 sourcedir="src/benchcompress/algorithms/simple_ans")
5153 ],
5254 cmdclass={
benchcompress/src/benchcompress/algorithms/simple_ans/CMakeLists.txtmodified+5−2View file
@@ -31,6 +31,9 @@ pybind11_add_module(markov_reconstruct_cpp_ext markov_reconstruct.cpp)
3131 pybind11_add_module(markov_predict_cpp_ext markov_predict.cpp)
3232 target_link_libraries(markov_predict_cpp_ext PRIVATE Eigen3::Eigen)
3333
34-# Install both modules
35-install(TARGETS markov_reconstruct_cpp_ext markov_predict_cpp_ext
34+# Build get_run_lengths module
35+pybind11_add_module(get_run_lengths_cpp_ext get_run_lengths.cpp)
36+
37+# Install all modules
38+install(TARGETS markov_reconstruct_cpp_ext markov_predict_cpp_ext get_run_lengths_cpp_ext
3639 DESTINATION benchcompress/algorithms/simple_ans)
benchcompress/src/benchcompress/algorithms/simple_ans/__init__.pymodified+195−3View file
@@ -5,6 +5,7 @@ from .markov_reconstruct import (
55 from .markov_predict import (
66 markov_predict as markov_predict_cpp,
77 )
8+from .get_run_lengths import get_run_lengths
89
910 SOURCE_FILE = "simple_ans/__init__.py"
1011
@@ -234,7 +235,7 @@ def simple_ans_markov_decode(x: bytes, dtype: str) -> np.ndarray:
234235 signal_length=int(signal_length),
235236 state=int(state),
236237 symbol_counts=symbol_counts.astype(np.uint32),
237- symbol_values=symbol_values.astype(dtype),
238+ symbol_values=symbol_values.astype(np.int16), # resid is always int16
238239 bitstream=bitstream,
239240 )
240241 import time
@@ -244,6 +245,187 @@ def simple_ans_markov_decode(x: bytes, dtype: str) -> np.ndarray:
244245 return output
245246
246247
248+def simple_ans_markov_sparse_encode(x: np.ndarray) -> bytes:
249+ from simple_ans import ans_encode
250+
251+ assert x.ndim == 1
252+
253+ run_lengths = get_run_lengths(x)
254+
255+ non_zero_arrays = []
256+ array_pos = 0
257+ i = 0
258+ while i < len(run_lengths):
259+ non_zero_len = int(run_lengths[i])
260+ if non_zero_len > 0:
261+ non_zero_arrays.append(x[array_pos : array_pos + non_zero_len])
262+ array_pos += non_zero_len
263+ i += 1
264+ if i < len(run_lengths):
265+ zero_len = int(run_lengths[i])
266+ array_pos += zero_len
267+ i += 1
268+
269+ non_zero_data = np.concatenate(non_zero_arrays)
270+ assert len(non_zero_data) == np.sum(run_lengths[::2])
271+ coeffs, initial, resid = markov_predict_cpp(
272+ non_zero_data, M=6, num_training_samples=10000
273+ )
274+ encoded = ans_encode(resid)
275+
276+ if x.dtype == np.uint8:
277+ dtype_code = 0
278+ elif x.dtype == np.uint16:
279+ dtype_code = 1
280+ elif x.dtype == np.uint32:
281+ dtype_code = 2
282+ elif x.dtype == np.int16:
283+ dtype_code = 3
284+ elif x.dtype == np.int32:
285+ dtype_code = 4
286+ else:
287+ raise ValueError(f"Unsupported dtype: {x.dtype}")
288+
289+ if run_lengths.dtype == np.uint8:
290+ run_length_dtype_code = 0
291+ elif run_lengths.dtype == np.uint16:
292+ run_length_dtype_code = 1
293+ else:
294+ raise ValueError(f"Unsupported run length dtype: {run_lengths.dtype}")
295+
296+ header = (
297+ [
298+ dtype_code,
299+ encoded.num_bits,
300+ len(encoded.bitstream),
301+ encoded.signal_length,
302+ encoded.state,
303+ len(encoded.symbol_counts),
304+ len(coeffs),
305+ len(initial),
306+ run_length_dtype_code,
307+ len(run_lengths),
308+ ]
309+ + [c for c in encoded.symbol_counts]
310+ + [v for v in encoded.symbol_values]
311+ + [c for c in coeffs]
312+ + [v for v in initial]
313+ )
314+ header_bytes = np.array(header, dtype=np.float64).tobytes()
315+ header_size = np.uint32(len(header_bytes))
316+
317+ print(f"Elapsed 5: {time.time() - timer}")
318+ timer = time.time()
319+
320+ return (
321+ header_size.tobytes() + header_bytes + encoded.bitstream + run_lengths.tobytes()
322+ )
323+
324+
325+def simple_ans_markov_sparse_decode(x: bytes, dtype: str) -> np.ndarray:
326+ from simple_ans import ans_decode, EncodedSignal
327+
328+ header_size = np.frombuffer(x[:4], dtype=np.uint32)[0]
329+ header = np.frombuffer(x[4 : 4 + header_size], dtype=np.float64)
330+ (
331+ dtype_code,
332+ num_bits,
333+ bitstream_length,
334+ signal_length,
335+ state,
336+ num_symbols,
337+ num_coeffs,
338+ num_initial,
339+ run_length_dtype_code,
340+ num_run_lengths,
341+ ) = header[:10]
342+ dtype_code = int(dtype_code)
343+ num_bits = int(num_bits)
344+ bitstream_length = int(bitstream_length)
345+ signal_length = int(signal_length)
346+ state = int(state)
347+ num_symbols = int(num_symbols)
348+ num_coeffs = int(num_coeffs)
349+ num_initial = int(num_initial)
350+ run_length_dtype_code = int(run_length_dtype_code)
351+ num_run_lengths = int(num_run_lengths)
352+
353+ pos = 10
354+ symbol_counts = header[pos : pos + num_symbols]
355+ pos += num_symbols
356+ symbol_values = header[pos : pos + num_symbols]
357+ pos += num_symbols
358+ coeffs = header[pos : pos + num_coeffs]
359+ pos += num_coeffs
360+ initial = header[pos : pos + num_initial]
361+ pos += num_initial
362+
363+ bitstream_end = 4 + header_size + bitstream_length
364+ bitstream = x[4 + header_size : bitstream_end]
365+
366+ # Get run lengths from the remaining bytes
367+ if run_length_dtype_code == 0:
368+ run_lengths = np.frombuffer(x[bitstream_end:], dtype=np.uint8)
369+ elif run_length_dtype_code == 1:
370+ run_lengths = np.frombuffer(x[bitstream_end:], dtype=np.uint16)
371+ else:
372+ raise ValueError(f"Unsupported run length dtype code: {run_length_dtype_code}")
373+
374+ if len(run_lengths) != num_run_lengths:
375+ raise ValueError(
376+ f"Expected {num_run_lengths} run lengths, got {len(run_lengths)}"
377+ )
378+
379+ if dtype_code == 0:
380+ assert dtype == "uint8"
381+ elif dtype_code == 1:
382+ assert dtype == "uint16"
383+ elif dtype_code == 2:
384+ assert dtype == "uint32"
385+ elif dtype_code == 3:
386+ assert dtype == "int16"
387+ elif dtype_code == 4:
388+ assert dtype == "int32"
389+ else:
390+ raise ValueError(f"Unsupported dtype code: {dtype_code}")
391+
392+ encoded = EncodedSignal(
393+ num_bits=int(num_bits),
394+ signal_length=int(signal_length),
395+ state=int(state),
396+ symbol_counts=symbol_counts.astype(np.uint32),
397+ symbol_values=symbol_values.astype(np.int16), # resid is always int16
398+ bitstream=bitstream,
399+ )
400+
401+ # Decode residuals and reconstruct non-zero data
402+ resid = ans_decode(encoded)
403+ non_zero_data = markov_reconstruct_cpp(coeffs, initial, resid)
404+
405+ assert len(non_zero_data) == np.sum(run_lengths[::2])
406+
407+ # Reconstruct full array using run lengths
408+ non_zero_pos = 0 # Position in non_zero_data
409+ i = 0
410+ segments = []
411+ while i < len(run_lengths):
412+ non_zero_len = int(run_lengths[i])
413+ if non_zero_len > 0:
414+ # print(f'--- b non_zero_len={non_zero_len} non_zero_pos={non_zero_pos} pos={pos} len(non_zero_data)={len(non_zero_data)} len(output)={len(output)}')
415+ # print('---- y')
416+ segment = non_zero_data[non_zero_pos : non_zero_pos + non_zero_len]
417+ assert len(segment) == non_zero_len
418+ segments.append(segment)
419+ non_zero_pos += non_zero_len
420+ i += 1
421+ if i < len(run_lengths):
422+ segments.append(np.zeros(int(run_lengths[i]), dtype=non_zero_data.dtype))
423+ i += 1
424+ output = np.concatenate(segments)
425+
426+ return output
427+
428+
247429 algorithms = [
248430 {
249431 "name": "simple-ans",
@@ -251,6 +433,7 @@ algorithms = [
251433 "encode": lambda x: simple_ans_encode(x),
252434 "decode": lambda x, dtype: simple_ans_decode(x, dtype),
253435 "description": "Basic Asymmetric Numeral Systems (ANS) entropy coding for efficient data compression.",
436+ "tags": ["ANS"],
254437 "source_file": SOURCE_FILE,
255438 },
256439 {
@@ -259,7 +442,7 @@ algorithms = [
259442 "encode": lambda x: simple_ans_delta_encode(x),
260443 "decode": lambda x, dtype: simple_ans_delta_decode(x, dtype),
261444 "description": "ANS compression with delta encoding for improved compression of sequential data.",
262- "tags": ["delta_encoding"],
445+ "tags": ["ANS", "delta_encoding"],
263446 "source_file": SOURCE_FILE,
264447 },
265448 {
@@ -268,7 +451,16 @@ algorithms = [
268451 "encode": lambda x: simple_ans_markov_encode(x),
269452 "decode": lambda x, dtype: simple_ans_markov_decode(x, dtype),
270453 "description": "ANS compression with Markov prediction for exploiting temporal correlations in the data.",
271- "tags": ["markov_prediction"],
454+ "tags": ["ANS", "markov_prediction"],
455+ "source_file": SOURCE_FILE,
456+ },
457+ {
458+ "name": "simple-ans-markov-sparse",
459+ "version": "6",
460+ "encode": lambda x: simple_ans_markov_sparse_encode(x),
461+ "decode": lambda x, dtype: simple_ans_markov_sparse_decode(x, dtype),
462+ "description": "ANS compression with Markov prediction and run-length encoding for sparse data.",
463+ "tags": ["ANS", "markov_prediction", "run_length_encoding"],
272464 "source_file": SOURCE_FILE,
273465 },
274466 ]
benchcompress/src/benchcompress/algorithms/simple_ans/get_run_lengths.cppadded+92−0View file
@@ -0,0 +1,92 @@
1+#include <cstdint>
2+#include <pybind11/numpy.h>
3+#include <pybind11/pybind11.h>
4+#include <vector>
5+
6+namespace py = pybind11;
7+
8+py::array get_run_lengths_cpp(py::array_t<int16_t> x) {
9+ auto x_buf = x.request();
10+ int16_t *x_ptr = static_cast<int16_t *>(x_buf.ptr);
11+ size_t N = x_buf.shape[0];
12+
13+ std::vector<uint32_t> runs;
14+ size_t i = 0;
15+ uint32_t current_nonzero_run_length = 0;
16+
17+ while (i < N) {
18+ // Check for a sequence of at least 10 zeros
19+ bool has_zeros = true;
20+ for (size_t j = 0; j < 10 && i + j < N; j++) {
21+ if (x_ptr[i + j] != 0) {
22+ has_zeros = false;
23+ break;
24+ }
25+ }
26+
27+ if (has_zeros) {
28+ // Add current non-zero run if any
29+ runs.push_back(current_nonzero_run_length);
30+ current_nonzero_run_length = 0;
31+
32+ // Count consecutive zeros
33+ size_t j = i;
34+ while (j < N && x_ptr[j] == 0) {
35+ j++;
36+ }
37+ runs.push_back(j - i);
38+ i = j;
39+ } else {
40+ current_nonzero_run_length++;
41+ i++;
42+ }
43+ }
44+
45+ // Add final non-zero run if any
46+ if (current_nonzero_run_length > 0) {
47+ runs.push_back(current_nonzero_run_length);
48+ }
49+
50+ // Determine appropriate dtype based on max run length
51+ uint32_t max_run = 0;
52+ for (const auto &run : runs) {
53+ if (run > max_run) {
54+ max_run = run;
55+ }
56+ }
57+
58+ // Create numpy array with appropriate dtype
59+ std::vector<ssize_t> shape = {static_cast<ssize_t>(runs.size())};
60+
61+ if (max_run < 256) {
62+ py::array_t<uint8_t> result(shape);
63+ auto result_buf = result.request();
64+ uint8_t *result_ptr = static_cast<uint8_t *>(result_buf.ptr);
65+ for (size_t i = 0; i < runs.size(); i++) {
66+ result_ptr[i] = static_cast<uint8_t>(runs[i]);
67+ }
68+ return result;
69+ } else if (max_run < 65536) {
70+ py::array_t<uint16_t> result(shape);
71+ auto result_buf = result.request();
72+ uint16_t *result_ptr = static_cast<uint16_t *>(result_buf.ptr);
73+ for (size_t i = 0; i < runs.size(); i++) {
74+ result_ptr[i] = static_cast<uint16_t>(runs[i]);
75+ }
76+ return result;
77+ } else {
78+ py::array_t<uint32_t> result(shape);
79+ auto result_buf = result.request();
80+ uint32_t *result_ptr = static_cast<uint32_t *>(result_buf.ptr);
81+ for (size_t i = 0; i < runs.size(); i++) {
82+ result_ptr[i] = runs[i];
83+ }
84+ return result;
85+ }
86+}
87+
88+PYBIND11_MODULE(get_run_lengths_cpp_ext, m) {
89+ m.doc() = "C++ implementation of get_run_lengths using pybind11";
90+ m.def("get_run_lengths_cpp", &get_run_lengths_cpp,
91+ "Calculate run lengths of zeros and non-zeros in a signal");
92+}
benchcompress/src/benchcompress/algorithms/simple_ans/get_run_lengths.pyadded+16−0View file
@@ -0,0 +1,16 @@
1+import numpy as np
2+from .get_run_lengths_cpp_ext import get_run_lengths_cpp
3+
4+
5+def get_run_lengths(x: np.ndarray) -> np.ndarray:
6+ """Calculate run lengths of zeros and non-zeros in a signal using C++ implementation.
7+
8+ Args:
9+ x: Input signal (will be converted to int16)
10+
11+ Returns:
12+ np.ndarray: Array of run lengths alternating between non-zero and zero runs.
13+ The dtype will be uint8, uint16, or uint32 depending on the maximum run length.
14+ """
15+ # Call C++ implementation with proper type conversion
16+ return get_run_lengths_cpp(x.astype(np.int16))
benchcompress/src/benchcompress/datasets/bernoulli/__init__.pymodified+1−1View file
@@ -10,7 +10,7 @@ def create_bernoulli(*, n_samples: int, p: float, seed: int) -> np.ndarray:
1010 return x
1111
1212
13-tags = ["binary", "integer", "discrete", "synthetic"]
13+tags = ["binary", "integer", "discrete", "synthetic", "i.i.d."]
1414
1515 datasets = [
1616 {
benchcompress/src/benchcompress/datasets/gaussian/__init__.pymodified+1−1View file
@@ -10,7 +10,7 @@ def create_gaussian(*, n_samples: int, stddev: float, seed: int) -> np.ndarray:
1010 return x
1111
1212
13-tags = ["integer", "discrete", "synthetic"]
13+tags = ["integer", "discrete", "synthetic", "i.i.d."]
1414
1515 datasets = [
1616 {
benchcompress/src/benchcompress/datasets/real/__init__.pymodified+6−6View file
@@ -208,7 +208,7 @@ datasets = [
208208 num_samples=500_000, num_channels=1, start_channel=45
209209 ).flatten()
210210 ),
211- "tags": ["continuous", "neurophysiology", "filtered"],
211+ "tags": tags + ["filtered"],
212212 "source_file": SOURCE_FILE,
213213 },
214214 {
@@ -220,7 +220,7 @@ datasets = [
220220 num_samples=500_000, num_channels=1, start_channel=101
221221 ).flatten()
222222 ),
223- "tags": ["continuous", "neurophysiology", "filtered"],
223+ "tags": tags + ["filtered"],
224224 "source_file": SOURCE_FILE,
225225 },
226226 {
@@ -232,7 +232,7 @@ datasets = [
232232 num_samples=500_000, num_channels=1, start_channel=0
233233 ).flatten()
234234 ),
235- "tags": ["continuous", "neurophysiology", "filtered"],
235+ "tags": tags + ["filtered"],
236236 "source_file": SOURCE_FILE,
237237 },
238238 {
@@ -244,7 +244,7 @@ datasets = [
244244 num_samples=500_000, num_channels=1, start_channel=45
245245 ).flatten()
246246 ),
247- "tags": ["continuous", "neurophysiology", "filtered", "sparse"],
247+ "tags": tags + ["filtered", "sparse"],
248248 "source_file": SOURCE_FILE,
249249 },
250250 {
@@ -256,7 +256,7 @@ datasets = [
256256 num_samples=500_000, num_channels=1, start_channel=101
257257 ).flatten()
258258 ),
259- "tags": ["continuous", "neurophysiology", "filtered", "sparse"],
259+ "tags": tags + ["filtered", "sparse"],
260260 "source_file": SOURCE_FILE,
261261 },
262262 {
@@ -268,7 +268,7 @@ datasets = [
268268 num_samples=500_000, num_channels=1, start_channel=0
269269 ).flatten()
270270 ),
271- "tags": ["continuous", "neurophysiology", "filtered", "sparse"],
271+ "tags": tags + ["filtered", "sparse"],
272272 "source_file": SOURCE_FILE,
273273 },
274274 ]
web-ui/src/components/benchmark/charts/BenchmarkCharts.tsxmodified+1−0View file
@@ -51,6 +51,7 @@ export function BenchmarkCharts({ chartData }: BenchmarkChartsProps) {
5151 height: 400,
5252 margin: { t: 5, r: 30, l: 250, b: 30 },
5353 xaxis: { title: "Ratio" },
54+ dragmode: false,
5455 }}
5556 config={{ displayModeBar: false }}
5657 />