move files
12 changed files+0−183
test1.pydeleted+0−176View file
@@ -1,176 +0,0 @@
1-# %%
2-import numpy as np
3-from zia._filters import bandpass_filter, highpass_filter
4-from zia._data_loaders import load_real_000876, load_real_000409, load_real_001290
5-from zia._compress_ints_lossless import compress_ints_lossless
6-from zia._analysis import linear_fit, compute_entropy_per_sample, estimate_noise_level
7-import matplotlib.pyplot as plt
8-
9-# %%
10-N = 500_000
11-
12-channel_number = 101
13-X = load_real_000409(num_samples=N, num_channels=1, start_channel=channel_number).flatten()
14-
15-# X = load_real_001290(num_samples=N, num_channels=1, start_channel=0).flatten()
16-# X = load_real_000876(num_samples=N, num_channels=1, start_channel=45).flatten()
17-
18-X = X.astype(np.int16)
19-
20-# %%
21-plt.figure(figsize=(12, 4))
22-plt.plot(X[:2400])
23-# %%
24-def print_ideal_compression_ratio(X):
25- ee = compute_entropy_per_sample(X)
26- print(f'Ideal compression ratio: {X.itemsize * 8 / ee:.2f} ({ee:.2f} bits per sample)')
27-
28-def print_actual_compression_ratios(X):
29- buf_zstd = compress_ints_lossless(X, method='zstd')
30- buf_zlib = compress_ints_lossless(X, method='zlib')
31- buf_lzma = compress_ints_lossless(X, method='lzma')
32- buf_ans = compress_ints_lossless(X, method='simple_ans')
33- print(f'Zstd compression ratio: {len(X) * X.itemsize / len(buf_zstd):.2f}')
34- print(f'Zlib compression ratio: {len(X) * X.itemsize / len(buf_zlib):.2f}')
35- print(f'Lzma compression ratio: {len(X) * X.itemsize / len(buf_lzma):.2f}')
36- print(f'simple_ans compression ratio: {len(X) * X.itemsize / len(buf_ans):.2f}')
37-
38-def get_marcovian_prediction_residual(X, M):
39- sequences = np.array([X[i:i+M] for i in range(len(X) - 2 * M + 1)])
40- predictors = sequences[:, :M - 1]
41- target = sequences[:, M - 1]
42-
43- coeffs, predict = linear_fit(predictors, target)
44- predictions = predict(predictors)
45- predictions = np.round(predictions)
46- residuals = target - predictions
47- residuals = residuals.astype(np.int16)
48- return residuals
49-
50-# %%
51-print('RAW')
52-print_ideal_compression_ratio(X)
53-
54-# %%
55-print('RAW DELTA ENCODING')
56-print_ideal_compression_ratio(np.diff(X))
57-
58-# %%
59-print('RAW DELTA ENCODING - actual compression ratios')
60-print_actual_compression_ratios(np.diff(X))
61-print_ideal_compression_ratio(np.diff(X))
62-
63-# %%
64-X_mr = get_marcovian_prediction_residual(X, 20)
65-print('RAW MARCOVIAN')
66-print_ideal_compression_ratio(X_mr)
67-
68-# %%
69-v = 0.25 # step size for quantization
70-lowcut = 300
71-highcut = 6000
72-X_filt = bandpass_filter(X - np.median(X), sampling_frequency=30000, lowcut=lowcut, highcut=highcut)
73-noise_level = estimate_noise_level(X_filt, sampling_frequency=30000)
74-X_filt_normalized = X_filt / noise_level
75-X2b = X_filt_normalized / v
76-X2 = np.round(X2b).astype(np.int16)
77-
78-# %%
79-plt.figure(figsize=(12, 4))
80-plt.plot(X[:2400])
81-
82-# %%
83-print('FILTERED (and quantized)')
84-print_ideal_compression_ratio(X2)
85-
86-# %%
87-print('FILTERED DELTA ENCODING')
88-print_ideal_compression_ratio(np.diff(X2))
89-
90-# %%
91-residuals2 = get_marcovian_prediction_residual(X2, 20)
92-print('FILTERED MARCOVIAN')
93-print_ideal_compression_ratio(residuals2)
94-
95-# %%
96-import matplotlib.pyplot as plt
97-plt.figure(figsize=(12, 4))
98-plt.plot(X[:2400])
99-plt.title('RAW')
100-
101-plt.figure(figsize=(12, 4))
102-plt.plot(X2[:2400])
103-plt.title('FILTERED')
104-
105-plt.figure(figsize=(12, 4))
106-plt.plot(residuals2[:2400])
107-plt.title('FILTERED MARCOVIAN')
108-
109-# %%
110-def sliding_max(x, delta):
111- y = np.zeros_like(x)
112- for i in range(len(x)):
113- y[i] = np.max(x[max(0, i - delta):min(len(x), i + delta + 1)])
114- return y
115-
116-def smoothed(x, delta):
117- y = np.zeros_like(x)
118- for i in range(len(x)):
119- y[i] = np.mean(x[max(0, i - delta):min(len(x), i + delta + 1)])
120- return y
121-
122-cc = [3, 6]
123-Y = sliding_max(np.abs(X_filt_normalized), 50)
124-Y = smoothed(Y, 20)
125-Y = np.minimum(1, np.maximum(0, (Y - cc[0]) / (cc[1] - cc[0])))
126-# Y = highpass_filter(Y, sampling_frequency=30000, lowcut=3)
127-Y_scaled = X_filt_normalized * Y
128-X3b = Y_scaled / v
129-X3 = np.round(X3b).astype(np.int16)
130-
131-plt.figure(figsize=(12, 4))
132-plt.plot(X2[:2400], color='lightgray')
133-# plt.plot(Y[4000:5000])
134-plt.plot(X3b[:2400])
135-# %%
136-print('FILTERED (and quantized) with suppression')
137-print_ideal_compression_ratio(X3)
138-print('')
139-print('FILTERED DELTA ENCODING with suppression')
140-print_ideal_compression_ratio(np.diff(X3))
141-print('')
142-print('FILTERED MARCOVIAN with suppression')
143-residuals3 = get_marcovian_prediction_residual(X3, 20)
144-print_ideal_compression_ratio(residuals3)
145-print('ACTUAL FILTERED MARCOVIAN with suppression')
146-print_actual_compression_ratios(residuals3)
147-# %%
148-def get_run_lengths(x):
149- runs = []
150- i = 0
151- current_nonzero_run_length = 0
152- while i < len(x):
153- if np.all(x[i:i+10] == 0):
154- runs.append(current_nonzero_run_length)
155- current_nonzero_run_length = 0
156- j = i
157- while j < len(x) and x[j] == 0:
158- j += 1
159- runs.append(j - i)
160- i = j
161- else:
162- current_nonzero_run_length += 1
163- i += 1
164- if np.max(runs) < 256:
165- return np.array(runs, dtype=np.uint8)
166- if np.max(runs) < 2 ** 16:
167- return np.array(runs, dtype=np.uint16)
168- return np.array(runs, dtype=np.uint32)
169-
170-AA = residuals3[residuals3 != 0]
171-run_lengths = get_run_lengths(residuals3)
172-print(run_lengths, run_lengths.itemsize)
173-ee = compute_entropy_per_sample(AA)
174-theoretical_size = (len(AA) * ee / 8) + run_lengths.nbytes
175-theoretical_compression_ratio = len(residuals3) * X.itemsize / theoretical_size
176-print(f'Theoretical compression ratio: {theoretical_compression_ratio:.2f}')
tests/conftest.pydeleted+0−7View file
@@ -1,7 +0,0 @@
1-"""Configuration for pytest."""
2-
3-import os
4-import sys
5-
6-# Add the src directory to the Python path
7-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../src")))
tests/test_basic.pydeleted+0−0View file
No changes to the file's content.
.gitignore →zia_benchmark/.gitignorerenamed+0−0View file
No changes to the file's content.
LICENSE →zia_benchmark/LICENSErenamed+0−0View file
No changes to the file's content.
pyproject.toml →zia_benchmark/pyproject.tomlrenamed+0−0View file
No changes to the file's content.
setup.py →zia_benchmark/setup.pyrenamed+0−0View file
No changes to the file's content.
src/zia/__init__.py →zia_benchmark/src/zia/__init__.pyrenamed+0−0View file
No changes to the file's content.
src/zia/_analysis.py →zia_benchmark/src/zia/_analysis.pyrenamed+0−0View file
No changes to the file's content.
src/zia/_compress_ints_lossless.py →zia_benchmark/src/zia/_compress_ints_lossless.pyrenamed+0−0View file
No changes to the file's content.
src/zia/_data_loaders.py →zia_benchmark/src/zia/_data_loaders.pyrenamed+0−0View file
No changes to the file's content.
src/zia/_filters.py →zia_benchmark/src/zia/_filters.pyrenamed+0−0View file
No changes to the file's content.