delta encoding for other algs
5 changed files+153−8
web-ui/src/components/benchmark/charts/BenchmarkCharts.tsxmodified+26−7View file
@@ -1,4 +1,5 @@
11 import Plot from 'react-plotly.js';
2+import { useState } from 'react';
23
34 interface ChartData {
45 algorithm: string;
@@ -12,18 +13,35 @@ interface BenchmarkChartsProps {
1213 }
1314
1415 export function BenchmarkCharts({ chartData }: BenchmarkChartsProps) {
16+ const [sortByRatio, setSortByRatio] = useState(true);
17+
1518 if (!chartData.length) return null;
1619
20+ const sortedData = sortByRatio
21+ ? [...chartData].sort((a, b) => a.compression_ratio - b.compression_ratio)
22+ : chartData;
23+
1724 return (
18- <div style={{ marginBottom: '30px' }}>
25+ <div>
26+ <div style={{ marginBottom: '10px' }}>
27+ <label style={{ display: 'flex', alignItems: 'center', gap: '8px' }}>
28+ <input
29+ type="checkbox"
30+ checked={sortByRatio}
31+ onChange={(e) => setSortByRatio(e.target.checked)}
32+ />
33+ Sort by compression ratio
34+ </label>
35+ </div>
36+ <div style={{ marginBottom: '30px' }}>
1937 <div style={{ marginBottom: '20px' }}>
2038 <h3 style={{ marginBottom: '10px' }}>Compression Ratio</h3>
2139 <Plot
2240 data={[{
2341 type: 'bar',
2442 orientation: 'h',
25- y: chartData.map(d => d.algorithm),
26- x: chartData.map(d => d.compression_ratio),
43+ y: sortedData.map(d => d.algorithm),
44+ x: sortedData.map(d => d.compression_ratio),
2745 marker: { color: '#8884d8' }
2846 }]}
2947 layout={{
@@ -42,8 +60,8 @@ export function BenchmarkCharts({ chartData }: BenchmarkChartsProps) {
4260 data={[{
4361 type: 'bar',
4462 orientation: 'h',
45- y: chartData.map(d => d.algorithm),
46- x: chartData.map(d => d.encode_speed),
63+ y: sortedData.map(d => d.algorithm),
64+ x: sortedData.map(d => d.encode_speed),
4765 marker: { color: '#82ca9d' }
4866 }]}
4967 layout={{
@@ -62,8 +80,8 @@ export function BenchmarkCharts({ chartData }: BenchmarkChartsProps) {
6280 data={[{
6381 type: 'bar',
6482 orientation: 'h',
65- y: chartData.map(d => d.algorithm),
66- x: chartData.map(d => d.decode_speed),
83+ y: sortedData.map(d => d.algorithm),
84+ x: sortedData.map(d => d.decode_speed),
6785 marker: { color: '#ff7300' }
6886 }]}
6987 layout={{
@@ -75,6 +93,7 @@ export function BenchmarkCharts({ chartData }: BenchmarkChartsProps) {
7593 config={{ displayModeBar: false }}
7694 />
7795 </div>
96+ </div>
7897 </div>
7998 );
8099 }
web-ui/src/components/benchmark/table/BenchmarkTable.tsxmodified+1−1View file
@@ -81,7 +81,7 @@ export function BenchmarkTable() {
8181 <div className="table-container">
8282 <div style={{ marginBottom: '20px', display: 'flex', alignItems: 'center', gap: '10px', justifyContent: 'space-between' }}>
8383 <div style={{ display: 'flex', alignItems: 'center', gap: '10px' }}>
84- <label htmlFor="dataset-select">Filter by Dataset:</label>
84+ <label htmlFor="dataset-select">Dataset:</label>
8585 <select
8686 id="dataset-select"
8787 value={selectedDataset}
zia_benchmark/src/zia_benchmark/algorithms/lzma/__init__.pymodified+23−0View file
@@ -1,6 +1,22 @@
11 import numpy as np
22
33
4+def lzma_delta_encode(x: np.ndarray, preset: int) -> bytes:
5+ import lzma
6+ assert x.ndim == 1
7+ y = np.diff(x)
8+ y = np.insert(y, 0, x[0])
9+ buf = y.tobytes()
10+ compressed = lzma.compress(buf, preset=preset)
11+ return compressed
12+
13+def lzma_delta_decode(x: bytes, dtype: str) -> np.ndarray:
14+ import lzma
15+ buf = lzma.decompress(x)
16+ y = np.frombuffer(buf, dtype=dtype)
17+ return np.cumsum(y)
18+
19+
420 def lzma_encode(x: np.ndarray, preset: int) -> bytes:
521 import lzma
622 assert x.ndim == 1
@@ -20,5 +36,12 @@ algorithms = [
2036 'version': '1',
2137 'encode': lambda x: lzma_encode(x, preset=9),
2238 'decode': lambda x, dtype: lzma_decode(x, dtype)
39+ },
40+ {
41+ 'name': 'lzma-9-delta',
42+ 'version': '1',
43+ 'encode': lambda x: lzma_delta_encode(x, preset=9),
44+ 'decode': lambda x, dtype: lzma_delta_decode(x, dtype),
45+ 'tags': ['delta_encoding']
2346 }
2447 ]
zia_benchmark/src/zia_benchmark/algorithms/simple_ans/__init__.pymodified+78−0View file
@@ -1,6 +1,77 @@
11 import numpy as np
22
33
4+def simple_ans_delta_encode(x: np.ndarray) -> bytes:
5+ from simple_ans import ans_encode
6+ assert x.ndim == 1
7+ # Calculate differences without inserting x[0]
8+ y = np.diff(x)
9+ # Encode just the differences
10+ encoded = ans_encode(y)
11+ if x.dtype == np.uint8:
12+ dtype_code = 0
13+ elif x.dtype == np.uint16:
14+ dtype_code = 1
15+ elif x.dtype == np.uint32:
16+ dtype_code = 2
17+ elif x.dtype == np.int16:
18+ dtype_code = 3
19+ elif x.dtype == np.int32:
20+ dtype_code = 4
21+ else:
22+ raise ValueError(f"Unsupported dtype: {x.dtype}")
23+ # Include x[0] in the header
24+ header = [
25+ dtype_code,
26+ encoded.num_bits,
27+ encoded.signal_length,
28+ encoded.state,
29+ len(encoded.symbol_counts),
30+ x[0] # Store first value in header
31+ ] + [c for c in encoded.symbol_counts] + [v for v in encoded.symbol_values]
32+ header_bytes = np.array(header, dtype=np.int64).tobytes()
33+ header_size = np.uint32(len(header_bytes))
34+ return header_size.tobytes() + header_bytes + encoded.bitstream
35+
36+def simple_ans_delta_decode(x: bytes, dtype: str) -> np.ndarray:
37+ from simple_ans import ans_decode, EncodedSignal
38+ header_size = np.frombuffer(x[:4], dtype=np.uint32)[0]
39+ header = np.frombuffer(x[4:4 + header_size], dtype=np.int64)
40+ dtype_code, num_bits, signal_length, state, num_symbols, x0 = header[:6] # Extract x0 from header
41+ symbol_counts = header[6:6 + num_symbols]
42+ symbol_values = header[6 + num_symbols:]
43+ bitstream = x[4 + header_size:]
44+ if dtype_code == 0:
45+ assert dtype == 'uint8'
46+ elif dtype_code == 1:
47+ assert dtype == 'uint16'
48+ elif dtype_code == 2:
49+ assert dtype == 'uint32'
50+ elif dtype_code == 3:
51+ assert dtype == 'int16'
52+ elif dtype_code == 4:
53+ assert dtype == 'int32'
54+ else:
55+ raise ValueError(f"Unsupported dtype code: {dtype_code}")
56+
57+ encoded = EncodedSignal(
58+ num_bits=int(num_bits),
59+ signal_length=int(signal_length),
60+ state=int(state),
61+ symbol_counts=symbol_counts.astype(np.uint32),
62+ symbol_values=symbol_values.astype(dtype),
63+ bitstream=bitstream
64+ )
65+ # Decode the differences
66+ diffs = ans_decode(encoded)
67+ # Create output array starting with x0
68+ result = np.empty(len(diffs) + 1, dtype=diffs.dtype)
69+ result[0] = x0
70+ # Compute cumulative sum of differences
71+ np.cumsum(diffs, out=result[1:])
72+ return result
73+
74+
475 def simple_ans_encode(x: np.ndarray) -> bytes:
576 from simple_ans import ans_encode
677 assert x.ndim == 1
@@ -66,5 +137,12 @@ algorithms = [
66137 'version': '1',
67138 'encode': lambda x: simple_ans_encode(x),
68139 'decode': lambda x, dtype: simple_ans_decode(x, dtype)
140+ },
141+ {
142+ 'name': 'simple-ans-delta',
143+ 'version': '1',
144+ 'encode': lambda x: simple_ans_delta_encode(x),
145+ 'decode': lambda x, dtype: simple_ans_delta_decode(x, dtype),
146+ 'tags': ['delta_encoding']
69147 }
70148 ]
zia_benchmark/src/zia_benchmark/algorithms/zstd/__init__.pymodified+25−0View file
@@ -1,6 +1,24 @@
11 import numpy as np
22
33
4+def zstd_delta_encode(x: np.ndarray, level: int) -> bytes:
5+ import zstandard as zstd
6+ assert x.ndim == 1
7+ y = np.diff(x)
8+ y = np.insert(y, 0, x[0])
9+ buf = y.tobytes()
10+ compressor = zstd.ZstdCompressor(level=level)
11+ compressed = compressor.compress(buf)
12+ return compressed
13+
14+def zstd_delta_decode(x: bytes, dtype: str) -> np.ndarray:
15+ import zstandard as zstd
16+ decompressor = zstd.ZstdDecompressor()
17+ buf = decompressor.decompress(x)
18+ y = np.frombuffer(buf, dtype=dtype)
19+ return np.cumsum(y)
20+
21+
422 def zstd_encode(x: np.ndarray, level: int) -> bytes:
523 import zstandard as zstd
624 assert x.ndim == 1
@@ -58,5 +76,12 @@ algorithms = [
5876 'version': '1',
5977 'encode': lambda x: zstd_encode(x, level=22),
6078 'decode': lambda x, dtype: zstd_decode(x, dtype)
79+ },
80+ {
81+ 'name': 'zstd-22-delta',
82+ 'version': '1',
83+ 'encode': lambda x: zstd_delta_encode(x, level=22),
84+ 'decode': lambda x, dtype: zstd_delta_decode(x, dtype),
85+ 'tags': ['delta_encoding']
6186 }
6287 ]