add blosc2
4 changed files+377−1
benchcompress/pyproject.tomlmodified+2−1View file
@@ -25,7 +25,8 @@ dependencies = [
2525 "segyio",
2626 "lz4",
2727 "pyedflib",
28- "nibabel"
28+ "nibabel",
29+ "blosc2"
2930 ]
3031
3132 [tool.scikit-build]
benchcompress/src/benchcompress/algorithms/__init__.pymodified+2−0View file
@@ -5,6 +5,7 @@ from .ans import algorithms as ans_algorithms
55 from .lzma import algorithms as lzma_algorithms
66 from .brotli import algorithms as brotli_algorithms
77 from .lz4 import algorithms as lz4_algorithms
8+from .blosc2 import algorithms as blosc2_algorithms
89
910 algorithms = (
1011 bzip2_algorithms
@@ -14,4 +15,5 @@ algorithms = (
1415 + lzma_algorithms
1516 + brotli_algorithms
1617 + lz4_algorithms
18+ + blosc2_algorithms
1719 )
benchcompress/src/benchcompress/algorithms/blosc2/__init__.pyadded+346−0View file
@@ -0,0 +1,346 @@
1+import numpy as np
2+import os
3+from ..ans.markov_reconstruct import markov_reconstruct as markov_reconstruct_cpp
4+from ..ans.markov_predict import markov_predict as markov_predict_cpp
5+from ..ans.get_run_lengths import get_run_lengths
6+
7+SOURCE_FILE = "blosc2/__init__.py"
8+
9+
10+def _load_long_description():
11+ current_dir = os.path.dirname(os.path.abspath(__file__))
12+ md_path = os.path.join(current_dir, "blosc2.md")
13+ with open(md_path, "r", encoding="utf-8") as f:
14+ return f.read()
15+
16+
17+LONG_DESCRIPTION = _load_long_description()
18+
19+
20+def blosc2_encode(x: np.ndarray, clevel: int, filter: int = 2) -> bytes:
21+ import blosc2
22+
23+ # Convert filter int to proper enum
24+ if filter == 2:
25+ blosc_filter = blosc2.Filter.BITSHUFFLE
26+ elif filter == 1:
27+ blosc_filter = blosc2.Filter.SHUFFLE
28+ else:
29+ blosc_filter = blosc2.Filter.NOFILTER
30+
31+ # Get typesize from numpy array
32+ typesize = x.dtype.itemsize
33+
34+ # Compress data
35+ compressed = blosc2.compress(
36+ x, # numpy arrays support buffer interface
37+ typesize=typesize,
38+ clevel=clevel,
39+ filter=blosc_filter,
40+ codec=blosc2.Codec.ZSTD,
41+ )
42+ assert isinstance(compressed, bytes) # Type assertion
43+ return compressed
44+
45+
46+def blosc2_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
47+ import blosc2
48+
49+ decompressed = blosc2.decompress(x)
50+ assert isinstance(decompressed, (bytes, bytearray)) # Type assertion
51+ arr = np.frombuffer(decompressed, dtype=np.dtype(dtype))
52+ return arr.reshape(shape)
53+
54+
55+def blosc2_delta_encode(x: np.ndarray, clevel: int) -> bytes:
56+ import blosc2
57+
58+ assert x.ndim == 1
59+
60+ y = np.diff(x)
61+ y = np.insert(y, 0, x[0])
62+ buf = y.tobytes()
63+
64+ # Convert default filter int to proper enum
65+ blosc_filter = blosc2.Filter.BITSHUFFLE
66+
67+ # Get typesize from numpy array
68+ typesize = y.dtype.itemsize
69+
70+ # Compress data
71+ compressed = blosc2.compress(
72+ buf,
73+ typesize=typesize,
74+ clevel=clevel,
75+ filter=blosc_filter,
76+ codec=blosc2.Codec.ZSTD,
77+ )
78+ assert isinstance(compressed, bytes) # Type assertion
79+ return compressed
80+
81+
82+def blosc2_delta_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
83+ import blosc2
84+
85+ assert len(shape) == 1
86+
87+ decompressed = blosc2.decompress(x)
88+ assert isinstance(decompressed, (bytes, bytearray)) # Type assertion
89+ y = np.frombuffer(decompressed, dtype=np.dtype(dtype))
90+ return np.cumsum(y)
91+
92+
93+def blosc2_markov_encode(x: np.ndarray, clevel: int) -> bytes:
94+ import blosc2
95+ import struct
96+
97+ assert x.ndim == 1
98+ coeffs, initial, resid = markov_predict_cpp(x, M=6, num_training_samples=10000)
99+
100+ # Convert coeffs and initial to bytes
101+ coeffs_bytes = coeffs.tobytes()
102+ initial_bytes = initial.tobytes()
103+
104+ # Create header with lengths
105+ header = struct.pack("QQ", len(coeffs_bytes), len(initial_bytes))
106+
107+ # Convert default filter int to proper enum
108+ blosc_filter = blosc2.Filter.BITSHUFFLE
109+
110+ # Get typesize from numpy array
111+ typesize = resid.dtype.itemsize
112+
113+ # Compress data
114+ compressed = blosc2.compress(
115+ resid.tobytes(),
116+ typesize=typesize,
117+ clevel=clevel,
118+ filter=blosc_filter,
119+ codec=blosc2.Codec.ZSTD,
120+ )
121+
122+ # Combine all parts
123+ return header + coeffs_bytes + initial_bytes + compressed
124+
125+
126+def blosc2_markov_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
127+ import blosc2
128+ import struct
129+
130+ assert len(shape) == 1
131+
132+ # Extract header
133+ header_size = struct.calcsize("QQ")
134+ coeffs_len, initial_len = struct.unpack("QQ", x[:header_size])
135+
136+ # Extract coefficients and initial values
137+ pos = header_size
138+ coeffs = np.frombuffer(x[pos : pos + coeffs_len], dtype=np.float32)
139+ pos += coeffs_len
140+ initial = np.frombuffer(x[pos : pos + initial_len], dtype=dtype)
141+ pos += initial_len
142+
143+ # Decompress residuals
144+ decompressed = blosc2.decompress(x[pos:])
145+ assert isinstance(decompressed, (bytes, bytearray)) # Type assertion
146+ resid = np.frombuffer(decompressed, dtype=dtype)
147+
148+ # Reconstruct signal
149+ output = markov_reconstruct_cpp(coeffs, initial, resid)
150+ return output
151+
152+
153+def blosc2_markov_zrle_encode(x: np.ndarray, clevel: int) -> bytes:
154+ import blosc2
155+ import struct
156+
157+ assert x.ndim == 1
158+
159+ # Get run lengths for zero/non-zero sequences
160+ run_lengths = get_run_lengths(x)
161+
162+ # Determine run length dtype code
163+ if run_lengths.dtype == np.uint8:
164+ run_length_dtype_code = 0
165+ elif run_lengths.dtype == np.uint16:
166+ run_length_dtype_code = 1
167+ elif run_lengths.dtype == np.uint32:
168+ run_length_dtype_code = 2
169+ else:
170+ raise ValueError(f"Unsupported run length dtype: {run_lengths.dtype}")
171+
172+ # Extract non-zero data
173+ non_zero_arrays = []
174+ array_pos = 0
175+ i = 0
176+ while i < len(run_lengths):
177+ non_zero_len = int(run_lengths[i])
178+ if non_zero_len > 0:
179+ non_zero_arrays.append(x[array_pos : array_pos + non_zero_len])
180+ array_pos += non_zero_len
181+ i += 1
182+ if i < len(run_lengths):
183+ array_pos += int(run_lengths[i]) # Skip zeros
184+ i += 1
185+
186+ non_zero_data = np.concatenate(non_zero_arrays)
187+
188+ # Apply Markov prediction on non-zero data
189+ coeffs, initial, resid = markov_predict_cpp(
190+ non_zero_data, M=6, num_training_samples=10000
191+ )
192+
193+ # Convert data to bytes
194+ coeffs_bytes = coeffs.tobytes()
195+ initial_bytes = initial.tobytes()
196+ run_lengths_bytes = run_lengths.tobytes()
197+
198+ # Create header with lengths and dtype code
199+ header = struct.pack(
200+ "QQQQB",
201+ len(coeffs_bytes),
202+ len(initial_bytes),
203+ len(run_lengths_bytes),
204+ len(run_lengths),
205+ run_length_dtype_code,
206+ )
207+
208+ # Convert default filter int to proper enum
209+ blosc_filter = blosc2.Filter.BITSHUFFLE
210+
211+ # Get typesize from numpy array
212+ typesize = resid.dtype.itemsize
213+
214+ # Compress residuals
215+ compressed = blosc2.compress(
216+ resid.tobytes(),
217+ typesize=typesize,
218+ clevel=clevel,
219+ filter=blosc_filter,
220+ codec=blosc2.Codec.ZSTD,
221+ )
222+
223+ # Combine all parts
224+ return header + coeffs_bytes + initial_bytes + run_lengths_bytes + compressed
225+
226+
227+def blosc2_markov_zrle_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
228+ import blosc2
229+ import struct
230+
231+ assert len(shape) == 1
232+
233+ # Extract header
234+ header_size = struct.calcsize("QQQQB")
235+ coeffs_len, initial_len, run_lengths_len, num_run_lengths, run_length_dtype_code = (
236+ struct.unpack("QQQQB", x[:header_size])
237+ )
238+
239+ # Extract components
240+ pos = header_size
241+ coeffs = np.frombuffer(x[pos : pos + coeffs_len], dtype=np.float32)
242+ pos += coeffs_len
243+ initial = np.frombuffer(x[pos : pos + initial_len], dtype=dtype)
244+ pos += initial_len
245+
246+ # Get run lengths with proper dtype
247+ if run_length_dtype_code == 0:
248+ run_lengths = np.frombuffer(x[pos : pos + run_lengths_len], dtype=np.uint8)
249+ elif run_length_dtype_code == 1:
250+ run_lengths = np.frombuffer(x[pos : pos + run_lengths_len], dtype=np.uint16)
251+ elif run_length_dtype_code == 2:
252+ run_lengths = np.frombuffer(x[pos : pos + run_lengths_len], dtype=np.uint32)
253+ else:
254+ raise ValueError(f"Unsupported run length dtype code: {run_length_dtype_code}")
255+ pos += run_lengths_len
256+
257+ assert len(run_lengths) == num_run_lengths
258+
259+ # Decompress residuals
260+ decompressed = blosc2.decompress(x[pos:])
261+ assert isinstance(decompressed, (bytes, bytearray)) # Type assertion
262+ resid = np.frombuffer(decompressed, dtype=dtype)
263+
264+ # Reconstruct non-zero data
265+ non_zero_data = markov_reconstruct_cpp(coeffs, initial, resid)
266+
267+ # Reconstruct full array using run lengths
268+ non_zero_pos = 0
269+ i = 0
270+ segments = []
271+ while i < len(run_lengths):
272+ non_zero_len = int(run_lengths[i])
273+ if non_zero_len > 0:
274+ segment = non_zero_data[non_zero_pos : non_zero_pos + non_zero_len]
275+ segments.append(segment)
276+ non_zero_pos += non_zero_len
277+ i += 1
278+ if i < len(run_lengths):
279+ segments.append(np.zeros(int(run_lengths[i]), dtype=non_zero_data.dtype))
280+ i += 1
281+
282+ return np.concatenate(segments)
283+
284+
285+algorithms = [
286+ {
287+ "name": "blosc2-1",
288+ "version": "1",
289+ "encode": lambda x: blosc2_encode(x, clevel=1),
290+ "decode": lambda x, dtype, shape: blosc2_decode(x, dtype, shape),
291+ "description": "Blosc2 compression at level 1 (fastest compression).",
292+ "tags": ["blosc2"],
293+ "source_file": SOURCE_FILE,
294+ "long_description": LONG_DESCRIPTION,
295+ },
296+ {
297+ "name": "blosc2-5",
298+ "version": "1",
299+ "encode": lambda x: blosc2_encode(x, clevel=5),
300+ "decode": lambda x, dtype, shape: blosc2_decode(x, dtype, shape),
301+ "description": "Blosc2 compression at level 5 (balanced speed/compression).",
302+ "tags": ["blosc2"],
303+ "source_file": SOURCE_FILE,
304+ "long_description": LONG_DESCRIPTION,
305+ },
306+ {
307+ "name": "blosc2-9",
308+ "version": "1",
309+ "encode": lambda x: blosc2_encode(x, clevel=9),
310+ "decode": lambda x, dtype, shape: blosc2_decode(x, dtype, shape),
311+ "description": "Blosc2 compression at level 9 (maximum compression).",
312+ "tags": ["blosc2"],
313+ "source_file": SOURCE_FILE,
314+ "long_description": LONG_DESCRIPTION,
315+ },
316+ {
317+ "name": "blosc2-9-delta",
318+ "version": "1",
319+ "encode": lambda x: blosc2_delta_encode(x, clevel=9),
320+ "decode": lambda x, dtype, shape: blosc2_delta_decode(x, dtype, shape),
321+ "description": "Blosc2 compression at level 9 with delta encoding for improved compression of sequential data.",
322+ "tags": ["blosc2", "delta_encoding", "1d"],
323+ "source_file": SOURCE_FILE,
324+ "long_description": LONG_DESCRIPTION,
325+ },
326+ {
327+ "name": "blosc2-9-markov",
328+ "version": "1",
329+ "encode": lambda x: blosc2_markov_encode(x, clevel=9),
330+ "decode": lambda x, dtype, shape: blosc2_markov_decode(x, dtype, shape),
331+ "description": "Blosc2 compression at level 9 with Markov prediction for exploiting temporal correlations in the data.",
332+ "tags": ["blosc2", "markov_prediction", "1d"],
333+ "source_file": SOURCE_FILE,
334+ "long_description": LONG_DESCRIPTION,
335+ },
336+ {
337+ "name": "blosc2-9-markov-zrle",
338+ "version": "1",
339+ "encode": lambda x: blosc2_markov_zrle_encode(x, clevel=9),
340+ "decode": lambda x, dtype, shape: blosc2_markov_zrle_decode(x, dtype, shape),
341+ "description": "Blosc2 compression at level 9 with Markov prediction and zero run-length encoding for sparse data.",
342+ "tags": ["blosc2", "markov_prediction", "zero_rle", "1d"],
343+ "source_file": SOURCE_FILE,
344+ "long_description": LONG_DESCRIPTION,
345+ },
346+]
benchcompress/src/benchcompress/algorithms/blosc2/blosc2.mdadded+27−0View file
@@ -0,0 +1,27 @@
1+# Blosc2 Algorithm
2+
3+Blosc2 is a modern, fast data compression library that builds upon the original Blosc library. It is designed for efficient compression of binary data, particularly optimized for in-memory compression of numerical arrays. Blosc2 uses block-oriented compression with support for multithreading and SIMD instructions.
4+
5+## Features
6+
7+- Fast compression and decompression speeds
8+- Block-oriented compression for better cache usage
9+- Support for various shuffling filters to improve compression ratios
10+- Built-in support for delta filtering
11+- Uses ZSTD compression codec
12+
13+## Variants
14+
15+### Standard Compression
16+Different compression levels trading off speed vs compression ratio:
17+- blosc2-1: Fastest compression with level 1
18+- blosc2-5: Balanced speed/compression with level 5
19+- blosc2-9: Maximum compression with level 9
20+
21+### Advanced Variants
22+
23+#### Delta Encoding (blosc2-9-delta)
24+Uses Blosc2's delta filter along with maximum compression. The delta filter stores differences between consecutive values, which is particularly effective for time series data where adjacent values are similar. This variant combines:
25+- Delta filtering for temporal correlation
26+- Bit shuffling for improved compression
27+- Level 9 compression for maximum compression ratio