add lz4
4 changed files+106−2
benchcompress/pyproject.tomlmodified+2−1View file
@@ -22,7 +22,8 @@ dependencies = [
2222 "click",
2323 "numba",
2424 "pybind11>=2.11.1",
25- "segyio"
25+ "segyio",
26+ "lz4"
2627 ]
2728
2829 [tool.scikit-build]
benchcompress/src/benchcompress/algorithms/__init__.pymodified+2−0View file
@@ -4,6 +4,7 @@ from .zstd import algorithms as zstd_algorithms
44 from .ans import algorithms as ans_algorithms
55 from .lzma import algorithms as lzma_algorithms
66 from .brotli import algorithms as brotli_algorithms
7+from .lz4 import algorithms as lz4_algorithms
78
89 algorithms = (
910 bzip2_algorithms
@@ -12,4 +13,5 @@ algorithms = (
1213 + ans_algorithms
1314 + lzma_algorithms
1415 + brotli_algorithms
16+ + lz4_algorithms
1517 )
benchcompress/src/benchcompress/algorithms/lz4/__init__.pyadded+99−0View file
@@ -0,0 +1,99 @@
1+import numpy as np
2+
3+
4+SOURCE_FILE = "lz4/__init__.py"
5+
6+
7+def lz4_encode(x: np.ndarray, level: int) -> bytes:
8+ import lz4.frame
9+
10+ buf = x.tobytes()
11+ compressed = lz4.frame.compress(buf, compression_level=level)
12+ return compressed
13+
14+
15+def lz4_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
16+ import lz4.frame
17+
18+ buf = lz4.frame.decompress(x)
19+ y = np.frombuffer(buf, dtype=dtype)
20+ return y.reshape(shape)
21+
22+
23+def lz4_delta_encode(x: np.ndarray, level: int) -> bytes:
24+ import lz4.frame
25+
26+ assert x.ndim == 1
27+ y = np.diff(x)
28+ y = np.insert(y, 0, x[0])
29+ buf = y.tobytes()
30+ compressed = lz4.frame.compress(buf, compression_level=level)
31+ return compressed
32+
33+
34+def lz4_delta_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
35+ import lz4.frame
36+
37+ assert len(shape) == 1
38+
39+ buf = lz4.frame.decompress(x)
40+ y = np.frombuffer(buf, dtype=dtype)
41+ return np.cumsum(y)
42+
43+
44+algorithms = [
45+ {
46+ "name": "lz4-1",
47+ "version": "1",
48+ "encode": lambda x: lz4_encode(x, level=1),
49+ "decode": lambda x, dtype, shape: lz4_decode(x, dtype, shape),
50+ "description": "LZ4 compression at level 1 (fastest).",
51+ "tags": ["lz4"],
52+ "source_file": SOURCE_FILE,
53+ },
54+ {
55+ "name": "lz4-3",
56+ "version": "1",
57+ "encode": lambda x: lz4_encode(x, level=3),
58+ "decode": lambda x, dtype, shape: lz4_decode(x, dtype, shape),
59+ "description": "LZ4 compression at level 3.",
60+ "tags": ["lz4"],
61+ "source_file": SOURCE_FILE,
62+ },
63+ {
64+ "name": "lz4-6",
65+ "version": "1",
66+ "encode": lambda x: lz4_encode(x, level=6),
67+ "decode": lambda x, dtype, shape: lz4_decode(x, dtype, shape),
68+ "description": "LZ4 compression at level 6.",
69+ "tags": ["lz4"],
70+ "source_file": SOURCE_FILE,
71+ },
72+ {
73+ "name": "lz4-9",
74+ "version": "1",
75+ "encode": lambda x: lz4_encode(x, level=9),
76+ "decode": lambda x, dtype, shape: lz4_decode(x, dtype, shape),
77+ "description": "LZ4 compression at level 9.",
78+ "tags": ["lz4"],
79+ "source_file": SOURCE_FILE,
80+ },
81+ {
82+ "name": "lz4-12",
83+ "version": "1",
84+ "encode": lambda x: lz4_encode(x, level=12),
85+ "decode": lambda x, dtype, shape: lz4_decode(x, dtype, shape),
86+ "description": "LZ4 compression at maximum level 12.",
87+ "tags": ["lz4"],
88+ "source_file": SOURCE_FILE,
89+ },
90+ {
91+ "name": "lz4-12-delta",
92+ "version": "1",
93+ "encode": lambda x: lz4_delta_encode(x, level=12),
94+ "decode": lambda x, dtype, shape: lz4_delta_decode(x, dtype, shape),
95+ "description": "LZ4 compression at level 12 with delta encoding.",
96+ "tags": ["lz4", "delta_encoding", "1d"],
97+ "source_file": SOURCE_FILE,
98+ },
99+]
benchcompress/src/benchcompress/run_benchmarks.pymodified+3−1View file
@@ -31,12 +31,13 @@ def is_compatible(algorithm_tags: List[str], dataset_tags: List[str]) -> bool:
3131 Returns:
3232 True if the algorithm should be applied to the dataset
3333 """
34- # If algorithm has delta_encoding or markov_prediction, dataset must have continuous, timeseries, 1d
34+ # If algorithm has delta_encoding or markov_prediction, dataset must have continuous, timeseries, 1d, integer
3535 if "delta_encoding" in algorithm_tags or "markov_prediction" in algorithm_tags:
3636 if (
3737 "continuous" not in dataset_tags
3838 or "timeseries" not in dataset_tags
3939 or "1d" not in dataset_tags
40+ or "integer" not in dataset_tags
4041 ):
4142 return False
4243
@@ -371,6 +372,7 @@ def run_benchmarks(
371372 for j in range(len(data)):
372373 if data[j] != decoded[j]:
373374 print(f"Error at index {j}: {data[j]} != {decoded[j]}")
375+ break
374376 raise ValueError(
375377 f"Decompression verification failed for {alg_name} on {dataset['name']}"
376378 )