delta encoding
6 changed files+117−23
README.mdmodified+32−2View file
@@ -1,5 +1,35 @@
11 # zia
22
3-Benchmarking compression of integer arrays
3+A benchmarking framework for evaluating compression algorithms on integer array datasets, with a focus on scientific data.
44
5-Benchmark results: https://magland.github.io/zia/
5+## Overview
6+
7+Zia provides systematic benchmarking of compression methods for integer arrays, measuring:
8+- Compression ratio
9+- Encoding throughput (MB/s)
10+- Decoding throughput (MB/s)
11+
12+## Repository Structure
13+
14+Key components are located in the `zia_benchmark/src/zia_benchmark/` directory:
15+
16+- Algorithms: `/algorithms/`
17+ - LZMA: `/algorithms/lzma/`
18+ - Zstandard: `/algorithms/zstd/`
19+ - ZLIB: `/algorithms/zlib/`
20+ - Simple ANS: `/algorithms/simple_ans/`
21+ - etc.
22+
23+- Datasets: `/datasets/`
24+ - Synthetic data generators:
25+ - `/datasets/bernoulli/`: Binary random data
26+ - `/datasets/gaussian/`: Normal distribution samples
27+ - Real data: `/datasets/real/`
28+ - etc.
29+
30+- Core functionality:
31+ - `run_benchmarks.py`: Main benchmarking engine
32+
33+## Results
34+
35+Latest benchmark results: https://magland.github.io/zia/
zia_benchmark/src/zia_benchmark/algorithms/zlib/__init__.pymodified+32−5View file
@@ -14,35 +14,62 @@ def zlib_decode(x: bytes, dtype: str) -> np.ndarray:
1414 y = np.frombuffer(buf, dtype=dtype)
1515 return y
1616
17+def zlib_delta_encode(x: np.ndarray, level: int) -> bytes:
18+ import zlib
19+ assert x.ndim == 1
20+ y = np.diff(x)
21+ y = np.insert(y, 0, x[0])
22+ buf = y.tobytes()
23+ compressed = zlib.compress(buf, level=level)
24+ return compressed
25+
26+def zlib_delta_decode(x: bytes, dtype: str) -> np.ndarray:
27+ import zlib
28+ buf = zlib.decompress(x)
29+ y = np.frombuffer(buf, dtype=dtype)
30+ return np.cumsum(y)
31+
1732 algorithms = [
1833 {
1934 'name': 'zlib-1',
2035 'version': '1',
2136 'encode': lambda x: zlib_encode(x, level=1),
22- 'decode': lambda x, dtype: zlib_decode(x, dtype)
37+ 'decode': lambda x, dtype: zlib_decode(x, dtype),
38+ 'tags': []
2339 },
2440 {
2541 'name': 'zlib-3',
2642 'version': '1',
2743 'encode': lambda x: zlib_encode(x, level=3),
28- 'decode': lambda x, dtype: zlib_decode(x, dtype)
44+ 'decode': lambda x, dtype: zlib_decode(x, dtype),
45+ 'tags': []
2946 },
3047 {
3148 'name': 'zlib-5',
3249 'version': '1',
3350 'encode': lambda x: zlib_encode(x, level=5),
34- 'decode': lambda x, dtype: zlib_decode(x, dtype)
51+ 'decode': lambda x, dtype: zlib_decode(x, dtype),
52+ 'tags': []
3553 },
3654 {
3755 'name': 'zlib-7',
3856 'version': '1',
3957 'encode': lambda x: zlib_encode(x, level=7),
40- 'decode': lambda x, dtype: zlib_decode(x, dtype)
58+ 'decode': lambda x, dtype: zlib_decode(x, dtype),
59+ 'tags': []
4160 },
4261 {
4362 'name': 'zlib-9',
4463 'version': '1',
4564 'encode': lambda x: zlib_encode(x, level=9),
46- 'decode': lambda x, dtype: zlib_decode(x, dtype)
65+ 'decode': lambda x, dtype: zlib_decode(x, dtype),
66+ 'tags': []
67+ },
68+ {
69+ 'name': 'zlib-9-delta',
70+ 'version': '1',
71+ 'encode': lambda x: zlib_delta_encode(x, level=9),
72+ 'decode': lambda x, dtype: zlib_delta_decode(x, dtype),
73+ 'tags': ['delta_encoding']
4774 }
4875 ]
zia_benchmark/src/zia_benchmark/datasets/bernoulli/__init__.pymodified+10−5View file
@@ -10,26 +10,31 @@ datasets = [
1010 {
1111 'name': 'bernoulli-0.1',
1212 'version': '1',
13- 'create': lambda: create_bernoulli(n_samples=1_000_000, p=0.1, seed=0)
13+ 'create': lambda: create_bernoulli(n_samples=1_000_000, p=0.1, seed=0),
14+ 'tags': ['binary']
1415 },
1516 {
1617 'name': 'bernoulli-0.2',
1718 'version': '1',
18- 'create': lambda: create_bernoulli(n_samples=1_000_000, p=0.2, seed=0)
19+ 'create': lambda: create_bernoulli(n_samples=1_000_000, p=0.2, seed=0),
20+ 'tags': ['binary']
1921 },
2022 {
2123 'name': 'bernoulli-0.3',
2224 'version': '1',
23- 'create': lambda: create_bernoulli(n_samples=1_000_000, p=0.3, seed=0)
25+ 'create': lambda: create_bernoulli(n_samples=1_000_000, p=0.3, seed=0),
26+ 'tags': ['binary']
2427 },
2528 {
2629 'name': 'bernoulli-0.4',
2730 'version': '1',
28- 'create': lambda: create_bernoulli(n_samples=1_000_000, p=0.4, seed=0)
31+ 'create': lambda: create_bernoulli(n_samples=1_000_000, p=0.4, seed=0),
32+ 'tags': ['binary']
2933 },
3034 {
3135 'name': 'bernoulli-0.5',
3236 'version': '1',
33- 'create': lambda: create_bernoulli(n_samples=1_000_000, p=0.5, seed=0)
37+ 'create': lambda: create_bernoulli(n_samples=1_000_000, p=0.5, seed=0),
38+ 'tags': ['binary']
3439 }
3540 ]
zia_benchmark/src/zia_benchmark/datasets/gaussian/__init__.pymodified+10−5View file
@@ -10,26 +10,31 @@ datasets = [
1010 {
1111 'name': 'gaussian-1',
1212 'version': '1',
13- 'create': lambda: create_gaussian(n_samples=1_000_000, stddev=1, seed=0)
13+ 'create': lambda: create_gaussian(n_samples=1_000_000, stddev=1, seed=0),
14+ 'tags': []
1415 },
1516 {
1617 'name': 'gaussian-2',
1718 'version': '1',
18- 'create': lambda: create_gaussian(n_samples=1_000_000, stddev=2, seed=0)
19+ 'create': lambda: create_gaussian(n_samples=1_000_000, stddev=2, seed=0),
20+ 'tags': []
1921 },
2022 {
2123 'name': 'gaussian-3',
2224 'version': '1',
23- 'create': lambda: create_gaussian(n_samples=1_000_000, stddev=3, seed=0)
25+ 'create': lambda: create_gaussian(n_samples=1_000_000, stddev=3, seed=0),
26+ 'tags': []
2427 },
2528 {
2629 'name': 'gaussian-5',
2730 'version': '1',
28- 'create': lambda: create_gaussian(n_samples=1_000_000, stddev=5, seed=0)
31+ 'create': lambda: create_gaussian(n_samples=1_000_000, stddev=5, seed=0),
32+ 'tags': []
2933 },
3034 {
3135 'name': 'gaussian-8',
3236 'version': '1',
33- 'create': lambda: create_gaussian(n_samples=1_000_000, stddev=8, seed=0)
37+ 'create': lambda: create_gaussian(n_samples=1_000_000, stddev=8, seed=0),
38+ 'tags': []
3439 }
3540 ]
zia_benchmark/src/zia_benchmark/datasets/real/__init__.pymodified+6−3View file
@@ -65,18 +65,21 @@ datasets = [
6565 'name': 'real-000876-ch45',
6666 'version': '1',
6767 'description': 'Real neurophysiology data from DANDI:000876, channel 45',
68- 'create': lambda: _load_real_000876(num_samples=500_000, num_channels=1, start_channel=45).flatten()
68+ 'create': lambda: _load_real_000876(num_samples=500_000, num_channels=1, start_channel=45).flatten(),
69+ 'tags': ['continuous', 'neurophysiology']
6970 },
7071 {
7172 'name': 'real-000409-ch101',
7273 'version': '1',
7374 'description': 'Real neurophysiology data from DANDI:000409, channel 101',
74- 'create': lambda: _load_real_000409(num_samples=500_000, num_channels=1, start_channel=101).flatten()
75+ 'create': lambda: _load_real_000409(num_samples=500_000, num_channels=1, start_channel=101).flatten(),
76+ 'tags': ['continuous', 'neurophysiology']
7577 },
7678 {
7779 'name': 'real-001290-ch0',
7880 'version': '1',
7981 'description': 'Real neurophysiology data from DANDI:001290, channel 0',
80- 'create': lambda: _load_real_001290(num_samples=500_000, num_channels=1, start_channel=0).flatten()
82+ 'create': lambda: _load_real_001290(num_samples=500_000, num_channels=1, start_channel=0).flatten(),
83+ 'tags': ['continuous', 'neurophysiology']
8184 }
8285 ]
zia_benchmark/src/zia_benchmark/run_benchmarks.pymodified+27−3View file
@@ -1,7 +1,7 @@
11 import time
22 import json
33 import os
4-from typing import Dict, Any, Tuple
4+from typing import Dict, Any, Tuple, List
55 import numpy as np
66 from statistics import median
77 from .algorithms import algorithms
@@ -11,6 +11,21 @@ from ._memobin import construct_memobin_url, upload_to_memobin, download_from_me
1111
1212 system_version = 'v4'
1313
14+def is_compatible(algorithm_tags: List[str], dataset_tags: List[str]) -> bool:
15+ """Check if an algorithm is compatible with a dataset based on their tags.
16+
17+ Args:
18+ algorithm_tags: List of tags for the algorithm
19+ dataset_tags: List of tags for the dataset
20+
21+ Returns:
22+ True if the algorithm should be applied to the dataset
23+ """
24+ # If algorithm has delta_encoding tag, dataset must have continuous tag
25+ if 'delta_encoding' in algorithm_tags and 'continuous' not in dataset_tags:
26+ return False
27+ return True
28+
1429 def run_benchmarks(cache_dir: str = '.benchmark_cache', verbose: bool = True) -> Dict[str, Any]:
1530 """Run all benchmarks, with caching based on algorithm and dataset versions.
1631
@@ -37,7 +52,8 @@ def run_benchmarks(cache_dir: str = '.benchmark_cache', verbose: bool = True) ->
3752
3853 # Run benchmarks for each dataset and algorithm combination
3954 for dataset in datasets:
40- print(f"\n--- Dataset: {dataset['name']} ---")
55+ dataset_tags = dataset.get('tags', [])
56+ print(f"\n--- Dataset: {dataset['name']} (tags: {dataset_tags}) ---")
4157 # Create dataset once for all algorithms
4258 data = dataset['create']()
4359 dtype = str(data.dtype)
@@ -47,7 +63,15 @@ def run_benchmarks(cache_dir: str = '.benchmark_cache', verbose: bool = True) ->
4763
4864 for algorithm in algorithms:
4965 alg_name = algorithm['name']
50- print(f"\nTesting algorithm: {alg_name}")
66+ alg_tags = algorithm.get('tags', [])
67+
68+ # Skip if algorithm and dataset are not compatible based on tags
69+ if not is_compatible(alg_tags, dataset_tags):
70+ if verbose:
71+ print(f"\nSkipping algorithm {alg_name} (tags: {alg_tags}) - incompatible with dataset tags")
72+ continue
73+
74+ print(f"\nTesting algorithm: {alg_name} (tags: {alg_tags})")
5175
5276 # Check if we can use cached result
5377 test_dir = os.path.join(cache_dir, dataset['name'], alg_name)