initial
39 changed files+2691−0
.github/workflows/benchmark.ymladded+75−0View file
@@ -0,0 +1,75 @@
1+name: Run Benchmarks
2+
3+# Add concurrency to cancel in-progress jobs
4+concurrency:
5+ group: ${{ github.workflow }}-${{ github.ref }}
6+ cancel-in-progress: true
7+
8+on:
9+ workflow_dispatch: # Manual trigger
10+ push:
11+ branches: [ main ] # Run on main branch pushes
12+ paths:
13+ - 'python/**' # Run only if benchmarks are updated
14+
15+permissions:
16+ contents: write
17+
18+jobs:
19+ benchmark:
20+ name: Run Benchmarks
21+ runs-on: ubuntu-latest
22+
23+ steps:
24+ - uses: actions/checkout@v4
25+
26+ - name: Set up Python
27+ uses: actions/setup-python@v4
28+ with:
29+ python-version: '3.12'
30+
31+ - name: Install package and dependencies
32+ run: |
33+ cd python
34+ pip install -e .
35+
36+ - name: Install simple_ans from source for now
37+ run: |
38+ git clone https://github.com/flatironinstitute/simple_ans.git
39+ cd simple_ans
40+ pip install -e .
41+
42+ - name: Run benchmarks
43+ env:
44+ MEMOBIN_API_KEY: ${{ secrets.MEMOBIN_API_KEY }}
45+ UPLOAD_TO_MEMOBIN: '1'
46+ run: |
47+ python scripts/run_benchmarks.py
48+
49+ - name: Upload benchmark results as artifacts
50+ uses: actions/upload-artifact@v4
51+ with:
52+ name: benchmark-results
53+ path: |
54+ python/benchmark_results/results.json
55+
56+ - name: Configure Git
57+ if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
58+ run: |
59+ git config user.name "GitHub Actions Bot"
60+ git config user.email "actions@github.com"
61+
62+ - name: Create fresh results branch
63+ if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
64+ run: |
65+ git checkout --orphan benchmark-results
66+
67+ - name: Commit benchmark results
68+ if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
69+ env:
70+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
71+ run: |
72+ # need to force add because it's in the .gitignore
73+ git add -f benchmark_results/
74+ git commit -m "Update benchmark results from $(date +'%Y-%m-%d %H:%M:%S') [skip ci]"
75+ git push -f https://${GITHUB_TOKEN}@github.com/${{ github.repository }} benchmark-results
.github/workflows/deploy-gh-pages.ymladded+67−0View file
@@ -0,0 +1,67 @@
1+name: Deploy to GitHub Pages
2+
3+on:
4+ push:
5+ branches: ['main']
6+ paths:
7+ - 'web-ui/**'
8+ - 'paper/**'
9+ workflow_dispatch:
10+
11+# Sets the GITHUB_TOKEN permissions to allow deployment to GitHub Pages
12+permissions:
13+ contents: read
14+ pages: write
15+ id-token: write
16+
17+# Allow one concurrent deployment
18+concurrency:
19+ group: 'pages'
20+ cancel-in-progress: true
21+
22+jobs:
23+ deploy:
24+ environment:
25+ name: github-pages
26+ url: ${{ steps.deployment.outputs.page_url }}
27+ runs-on: ubuntu-latest
28+ steps:
29+ - name: Checkout
30+ uses: actions/checkout@v4
31+
32+ - name: Install pandoc and texlive
33+ run: |
34+ sudo apt-get update
35+ sudo apt-get install -y pandoc texlive-latex-recommended
36+
37+ - name: Setup Node.js
38+ uses: actions/setup-node@v3
39+ with:
40+ node-version: '20'
41+ cache: 'npm'
42+ cache-dependency-path: web-ui/package-lock.json
43+
44+ - name: Build paper
45+ run: |
46+ chmod +x devel/copy_paper_to_web.sh
47+ ./devel/copy_paper_to_web.sh
48+
49+ - name: Install dependencies
50+ working-directory: web-ui
51+ run: npm ci
52+
53+ - name: Build website
54+ working-directory: web-ui
55+ run: npm run build
56+
57+ - name: Setup Pages
58+ uses: actions/configure-pages@v4
59+
60+ - name: Upload artifact
61+ uses: actions/upload-pages-artifact@v3
62+ with:
63+ path: ./web-ui/dist
64+
65+ - name: Deploy to GitHub Pages
66+ id: deployment
67+ uses: actions/deploy-pages@v4
.gitignoreadded+1−0View file
@@ -0,0 +1 @@
1+.benchmark_cache
python/.gitignoreadded+41−0View file
@@ -0,0 +1,41 @@
1+# Python
2+__pycache__/
3+*.py[cod]
4+*$py.class
5+*.so
6+.Python
7+build/
8+develop-eggs/
9+dist/
10+downloads/
11+eggs/
12+.eggs/
13+lib/
14+lib64/
15+parts/
16+sdist/
17+var/
18+wheels/
19+*.egg-info/
20+.installed.cfg
21+*.egg
22+
23+# Virtual Environment
24+venv/
25+env/
26+ENV/
27+
28+# IDE
29+.idea/
30+.vscode/
31+*.swp
32+*.swo
33+
34+# Testing
35+.coverage
36+htmlcov/
37+.pytest_cache/
38+.mypy_cache/
39+
40+# Misc
41+.DS_Store
python/LICENSEadded+201−0View file
@@ -0,0 +1,201 @@
1+ Apache License
2+ Version 2.0, January 2004
3+ http://www.apache.org/licenses/
4+
5+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6+
7+ 1. Definitions.
8+
9+ "License" shall mean the terms and conditions for use, reproduction,
10+ and distribution as defined by Sections 1 through 9 of this document.
11+
12+ "Licensor" shall mean the copyright owner or entity authorized by
13+ the copyright owner that is granting the License.
14+
15+ "Legal Entity" shall mean the union of the acting entity and all
16+ other entities that control, are controlled by, or are under common
17+ control with that entity. For the purposes of this definition,
18+ "control" means (i) the power, direct or indirect, to cause the
19+ direction or management of such entity, whether by contract or
20+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21+ outstanding shares, or (iii) beneficial ownership of such entity.
22+
23+ "You" (or "Your") shall mean an individual or Legal Entity
24+ exercising permissions granted by this License.
25+
26+ "Source" form shall mean the preferred form for making modifications,
27+ including but not limited to software source code, documentation
28+ source, and configuration files.
29+
30+ "Object" form shall mean any form resulting from mechanical
31+ transformation or translation of a Source form, including but
32+ not limited to compiled object code, generated documentation,
33+ and conversions to other media types.
34+
35+ "Work" shall mean the work of authorship, whether in Source or
36+ Object form, made available under the License, as indicated by a
37+ copyright notice that is included in or attached to the work
38+ (an example is provided in the Appendix below).
39+
40+ "Derivative Works" shall mean any work, whether in Source or Object
41+ form, that is based on (or derived from) the Work and for which the
42+ editorial revisions, annotations, elaborations, or other modifications
43+ represent, as a whole, an original work of authorship. For the purposes
44+ of this License, Derivative Works shall not include works that remain
45+ separable from, or merely link (or bind by name) to the interfaces of,
46+ the Work and Derivative Works thereof.
47+
48+ "Contribution" shall mean any work of authorship, including
49+ the original version of the Work and any modifications or additions
50+ to that Work or Derivative Works thereof, that is intentionally
51+ submitted to Licensor for inclusion in the Work by the copyright owner
52+ or by an individual or Legal Entity authorized to submit on behalf of
53+ the copyright owner. For the purposes of this definition, "submitted"
54+ means any form of electronic, verbal, or written communication sent
55+ to the Licensor or its representatives, including but not limited to
56+ communication on electronic mailing lists, source code control systems,
57+ and issue tracking systems that are managed by, or on behalf of, the
58+ Licensor for the purpose of discussing and improving the Work, but
59+ excluding communication that is conspicuously marked or otherwise
60+ designated in writing by the copyright owner as "Not a Contribution."
61+
62+ "Contributor" shall mean Licensor and any individual or Legal Entity
63+ on behalf of whom a Contribution has been received by Licensor and
64+ subsequently incorporated within the Work.
65+
66+ 2. Grant of Copyright License. Subject to the terms and conditions of
67+ this License, each Contributor hereby grants to You a perpetual,
68+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69+ copyright license to reproduce, prepare Derivative Works of,
70+ publicly display, publicly perform, sublicense, and distribute the
71+ Work and such Derivative Works in Source or Object form.
72+
73+ 3. Grant of Patent License. Subject to the terms and conditions of
74+ this License, each Contributor hereby grants to You a perpetual,
75+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76+ (except as stated in this section) patent license to make, have made,
77+ use, offer to sell, sell, import, and otherwise transfer the Work,
78+ where such license applies only to those patent claims licensable
79+ by such Contributor that are necessarily infringed by their
80+ Contribution(s) alone or by combination of their Contribution(s)
81+ with the Work to which such Contribution(s) was submitted. If You
82+ institute patent litigation against any entity (including a
83+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84+ or a Contribution incorporated within the Work constitutes direct
85+ or contributory patent infringement, then any patent licenses
86+ granted to You under this License for that Work shall terminate
87+ as of the date such litigation is filed.
88+
89+ 4. Redistribution. You may reproduce and distribute copies of the
90+ Work or Derivative Works thereof in any medium, with or without
91+ modifications, and in Source or Object form, provided that You
92+ meet the following conditions:
93+
94+ (a) You must give any other recipients of the Work or
95+ Derivative Works a copy of this License; and
96+
97+ (b) You must cause any modified files to carry prominent notices
98+ stating that You changed the files; and
99+
100+ (c) You must retain, in the Source form of any Derivative Works
101+ that You distribute, all copyright, patent, trademark, and
102+ attribution notices from the Source form of the Work,
103+ excluding those notices that do not pertain to any part of
104+ the Derivative Works; and
105+
106+ (d) If the Work includes a "NOTICE" text file as part of its
107+ distribution, then any Derivative Works that You distribute must
108+ include a readable copy of the attribution notices contained
109+ within such NOTICE file, excluding those notices that do not
110+ pertain to any part of the Derivative Works, in at least one
111+ of the following places: within a NOTICE text file distributed
112+ as part of the Derivative Works; within the Source form or
113+ documentation, if provided along with the Derivative Works; or,
114+ within a display generated by the Derivative Works, if and
115+ wherever such third-party notices normally appear. The contents
116+ of the NOTICE file are for informational purposes only and
117+ do not modify the License. You may add Your own attribution
118+ notices within Derivative Works that You distribute, alongside
119+ or as an addendum to the NOTICE text from the Work, provided
120+ that such additional attribution notices cannot be construed
121+ as modifying the License.
122+
123+ You may add Your own copyright statement to Your modifications and
124+ may provide additional or different license terms and conditions
125+ for use, reproduction, or distribution of Your modifications, or
126+ for any such Derivative Works as a whole, provided Your use,
127+ reproduction, and distribution of the Work otherwise complies with
128+ the conditions stated in this License.
129+
130+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131+ any Contribution intentionally submitted for inclusion in the Work
132+ by You to the Licensor shall be under the terms and conditions of
133+ this License, without any additional terms or conditions.
134+ Notwithstanding the above, nothing herein shall supersede or modify
135+ the terms of any separate license agreement you may have executed
136+ with Licensor regarding such Contributions.
137+
138+ 6. Trademarks. This License does not grant permission to use the trade
139+ names, trademarks, service marks, or product names of the Licensor,
140+ except as required for reasonable and customary use in describing the
141+ origin of the Work and reproducing the content of the NOTICE file.
142+
143+ 7. Disclaimer of Warranty. Unless required by applicable law or
144+ agreed to in writing, Licensor provides the Work (and each
145+ Contributor provides its Contributions) on an "AS IS" BASIS,
146+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147+ implied, including, without limitation, any warranties or conditions
148+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149+ PARTICULAR PURPOSE. You are solely responsible for determining the
150+ appropriateness of using or redistributing the Work and assume any
151+ risks associated with Your exercise of permissions under this License.
152+
153+ 8. Limitation of Liability. In no event and under no legal theory,
154+ whether in tort (including negligence), contract, or otherwise,
155+ unless required by applicable law (such as deliberate and grossly
156+ negligent acts) or agreed to in writing, shall any Contributor be
157+ liable to You for damages, including any direct, indirect, special,
158+ incidental, or consequential damages of any character arising as a
159+ result of this License or out of the use or inability to use the
160+ Work (including but not limited to damages for loss of goodwill,
161+ work stoppage, computer failure or malfunction, or any and all
162+ other commercial damages or losses), even if such Contributor
163+ has been advised of the possibility of such damages.
164+
165+ 9. Accepting Warranty or Additional Liability. While redistributing
166+ the Work or Derivative Works thereof, You may choose to offer,
167+ and charge a fee for, acceptance of support, warranty, indemnity,
168+ or other liability obligations and/or rights consistent with this
169+ License. However, in accepting such obligations, You may act only
170+ on Your own behalf and on Your sole responsibility, not on behalf
171+ of any other Contributor, and only if You agree to indemnify,
172+ defend, and hold each Contributor harmless for any liability
173+ incurred by, or claims asserted against, such Contributor by reason
174+ of your accepting any such warranty or additional liability.
175+
176+ END OF TERMS AND CONDITIONS
177+
178+ APPENDIX: How to apply the Apache License to your work.
179+
180+ To apply the Apache License to your work, attach the following
181+ boilerplate notice, with the fields enclosed by brackets "[]"
182+ replaced with your own identifying information. (Don't include
183+ the brackets!) The text should be enclosed in the appropriate
184+ comment syntax for the file format. We also recommend that a
185+ file or class name and description of purpose be included on the
186+ same "printed page" as the copyright notice for easier
187+ identification within third-party archives.
188+
189+ Copyright [yyyy] [name of copyright owner]
190+
191+ Licensed under the Apache License, Version 2.0 (the "License");
192+ you may not use this file except in compliance with the License.
193+ You may obtain a copy of the License at
194+
195+ http://www.apache.org/licenses/LICENSE-2.0
196+
197+ Unless required by applicable law or agreed to in writing, software
198+ distributed under the License is distributed on an "AS IS" BASIS,
199+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200+ See the License for the specific language governing permissions and
201+ limitations under the License.
python/MANIFEST.inadded+1−0View file
@@ -0,0 +1 @@
1+recursive-include ephys_compression_tests *.md
python/README.mdadded+1−0View file
@@ -0,0 +1 @@
1+# ephys_compression_tests
python/ephys_compression_tests/__init__.pyadded+0−0View file
No changes to the file's content.
python/ephys_compression_tests/_filters.pyadded+62−0View file
@@ -0,0 +1,62 @@
1+from typing import cast
2+import numpy as np
3+from scipy.signal import butter, lfilter
4+
5+
6+def bandpass_filter(
7+ array: np.ndarray, *, sampling_frequency: float, lowcut: float, highcut: float
8+) -> np.ndarray:
9+ """Apply a bandpass filter to the input array.
10+
11+ Args:
12+ array: Input signal array
13+ sampling_frequency: Sampling frequency in Hz
14+ lowcut: Lower cutoff frequency in Hz
15+ highcut: Higher cutoff frequency in Hz
16+
17+ Returns:
18+ Filtered signal array
19+ """
20+ nyquist = 0.5 * sampling_frequency
21+ low = lowcut / nyquist
22+ high = highcut / nyquist
23+ b, a = butter(5, [low, high], btype="band")
24+ return cast(np.ndarray, lfilter(b, a, array, axis=0))
25+
26+
27+def lowpass_filter(
28+ array: np.ndarray, *, sampling_frequency: float, highcut: float
29+) -> np.ndarray:
30+ """Apply a lowpass filter to the input array.
31+
32+ Args:
33+ array: Input signal array
34+ sampling_frequency: Sampling frequency in Hz
35+ highcut: Cutoff frequency in Hz
36+
37+ Returns:
38+ Filtered signal array
39+ """
40+ nyquist = 0.5 * sampling_frequency
41+ high = highcut / nyquist
42+ b, a = butter(5, high, btype="low")
43+ return cast(np.ndarray, lfilter(b, a, array, axis=0))
44+
45+
46+def highpass_filter(
47+ array: np.ndarray, *, sampling_frequency: float, lowcut: float
48+) -> np.ndarray:
49+ """Apply a highpass filter to the input array.
50+
51+ Args:
52+ array: Input signal array
53+ sampling_frequency: Sampling frequency in Hz
54+ lowcut: Cutoff frequency in Hz
55+
56+ Returns:
57+ Filtered signal array
58+ """
59+ nyquist = 0.5 * sampling_frequency
60+ low = lowcut / nyquist
61+ b, a = butter(5, low, btype="high")
62+ return cast(np.ndarray, lfilter(b, a, array, axis=0))
python/ephys_compression_tests/algorithms/__init__.pyadded+14−0View file
@@ -0,0 +1,14 @@
1+from .blosc2 import algorithms as blosc2_algorithms
2+from .ans import algorithms as ans_algorithms
3+from .wavpack import algorithms as wavpack_algorithms
4+from .lzma import algorithms as lzma_algorithms
5+from .zlib import algorithms as zlib_algorithms
6+from ..types import Algorithm
7+
8+algorithms: list[Algorithm] = (
9+ blosc2_algorithms
10+ + ans_algorithms
11+ + wavpack_algorithms
12+ + lzma_algorithms
13+ + zlib_algorithms
14+)
python/ephys_compression_tests/algorithms/ans/__init__.pyadded+261−0View file
@@ -0,0 +1,261 @@
1+import numpy as np
2+import os
3+from .ar import encode_ar, decode_ar
4+from ...types import Algorithm
5+
6+SOURCE_FILE = "ans/__init__.py"
7+
8+
9+def _load_long_description():
10+ current_dir = os.path.dirname(os.path.abspath(__file__))
11+ md_path = os.path.join(current_dir, "ans.md")
12+ with open(md_path, "r", encoding="utf-8") as f:
13+ return f.read()
14+
15+
16+LONG_DESCRIPTION = _load_long_description()
17+
18+def create_ans_header(
19+ dtype_code: int,
20+ num_words: int,
21+ signal_length: int,
22+ state: np.uint64,
23+ symbol_counts: np.ndarray,
24+ symbol_values: np.ndarray
25+) -> bytes:
26+ section1 = np.array([dtype_code, num_words, signal_length, len(symbol_counts)], dtype=np.uint32)
27+ section2 = np.array([state], dtype=np.uint64)
28+ symbol_counts_bytes = symbol_counts.astype(np.uint32).tobytes()
29+ symbol_values_bytes = symbol_values.tobytes()
30+
31+ return section1.tobytes() + section2.tobytes() + symbol_counts_bytes + symbol_values_bytes
32+
33+def unpack_ans_header(header_bytes: bytes) -> dict:
34+ # read section 1
35+ section1_size = 4 * 4 # 4 uint32
36+ section1 = np.frombuffer(header_bytes[:section1_size], dtype=np.uint32)
37+ dtype_code = int(section1[0])
38+ num_words = int(section1[1])
39+ signal_length = int(section1[2])
40+ num_symbols = int(section1[3])
41+ # read section 2
42+ section2_size = 8 # 1 uint64
43+ section2 = np.frombuffer(header_bytes[section1_size : section1_size + section2_size], dtype=np.uint64)
44+ state = np.uint64(section2[0])
45+ # read symbol counts and values
46+ remaining_bytes = header_bytes[section1_size + section2_size :]
47+ symbol_counts = np.frombuffer(remaining_bytes[: num_symbols * 4], dtype=np.uint32)
48+
49+ symbol_values_dtype = {0: np.uint8, 1: np.uint16, 2: np.uint32, 3: np.int16, 4: np.int32}.get(dtype_code)
50+ num_bytes_per_value = np.dtype(symbol_values_dtype).itemsize
51+ if symbol_values_dtype is None:
52+ raise ValueError(f"Unsupported dtype code: {dtype_code}")
53+ symbol_values = np.frombuffer(remaining_bytes[num_symbols * 4 : num_symbols * 4 + num_symbols * num_bytes_per_value], dtype=symbol_values_dtype)
54+
55+ if len(symbol_counts) != len(symbol_values):
56+ raise ValueError("Mismatch between number of symbol counts and symbol values")
57+
58+ return {
59+ "dtype_code": dtype_code,
60+ "num_words": num_words,
61+ "signal_length": signal_length,
62+ "state": state,
63+ "symbol_counts": symbol_counts,
64+ "symbol_values": symbol_values,
65+ }
66+
67+
68+def ans_encode_0(x: np.ndarray) -> bytes:
69+ from simple_ans import ans_encode
70+
71+ encoded = ans_encode(x)
72+ if x.dtype == np.uint8:
73+ dtype_code = 0
74+ elif x.dtype == np.uint16:
75+ dtype_code = 1
76+ elif x.dtype == np.uint32:
77+ dtype_code = 2
78+ elif x.dtype == np.int16:
79+ dtype_code = 3
80+ elif x.dtype == np.int32:
81+ dtype_code = 4
82+ else:
83+ raise ValueError(f"Unsupported dtype: {x.dtype}")
84+
85+ # Use the new header utilities
86+ header_bytes = create_ans_header(
87+ dtype_code=dtype_code,
88+ num_words=len(encoded.words),
89+ signal_length=encoded.signal_length,
90+ state=encoded.state,
91+ symbol_counts=encoded.symbol_counts,
92+ symbol_values=encoded.symbol_values,
93+ )
94+
95+ header_size = np.array([len(header_bytes)], dtype="uint32")
96+
97+ return header_size.tobytes() + header_bytes + encoded.words.tobytes()
98+
99+
100+
101+def ans_decode_0(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
102+ from simple_ans import ans_decode, EncodedSignal
103+
104+ header_size = np.frombuffer(x[:4], dtype=np.uint32)[0]
105+
106+ # Use the new header utilities
107+ header_dict = unpack_ans_header(x[4 : 4 + header_size])
108+
109+ dtype_code = header_dict["dtype_code"]
110+ num_words = header_dict["num_words"]
111+ signal_length = header_dict["signal_length"]
112+ state = header_dict["state"]
113+ symbol_counts = header_dict["symbol_counts"]
114+ symbol_values = header_dict["symbol_values"]
115+
116+ words_bytes = x[4 + header_size :]
117+
118+ if dtype_code == 0:
119+ assert dtype == "uint8"
120+ elif dtype_code == 1:
121+ assert dtype == "uint16"
122+ elif dtype_code == 2:
123+ assert dtype == "uint32"
124+ elif dtype_code == 3:
125+ assert dtype == "int16"
126+ elif dtype_code == 4:
127+ assert dtype == "int32"
128+ else:
129+ raise ValueError(f"Unsupported dtype code: {dtype_code}")
130+
131+ encoded = EncodedSignal(
132+ signal_length=int(signal_length),
133+ state=np.uint64(state),
134+ symbol_counts=symbol_counts.astype(np.uint32),
135+ symbol_values=symbol_values.astype(dtype),
136+ words=np.frombuffer(words_bytes, dtype=np.uint32, count=num_words),
137+ )
138+ return ans_decode(encoded).reshape(shape)
139+
140+algorithm_dicts_base = [
141+ {
142+ "name": "ans",
143+ "version": "1",
144+ "encode": lambda x: ans_encode_0(x),
145+ "decode": lambda x, dtype, shape: ans_decode_0(x, dtype, shape),
146+ "description": "ANS",
147+ "tags": ["ans"],
148+ "source_file": SOURCE_FILE,
149+ "long_description": LONG_DESCRIPTION,
150+ }
151+]
152+
153+algorithm_dicts = []
154+for a in algorithm_dicts_base:
155+ algorithm_dicts.append(a)
156+
157+# add delta encoding
158+for a in algorithm_dicts_base:
159+ def encode0(x: np.ndarray, a=a) -> bytes:
160+ x_diff = np.diff(x)
161+ x0 = x[0:1]
162+ encoded_diff = a["encode"](x_diff)
163+ # Store the first value at the start
164+ first_value_bytes = x0.tobytes()
165+ return first_value_bytes + encoded_diff
166+ def decode0(x: bytes, dtype: str, shape: tuple, a=a) -> np.ndarray:
167+ dtype_np = np.dtype(dtype)
168+ num_bytes_first_value = dtype_np.itemsize
169+ first_value_bytes = x[:num_bytes_first_value]
170+ x0 = np.frombuffer(first_value_bytes, dtype=dtype_np)
171+ encoded_diff = x[num_bytes_first_value:]
172+ x_diff = a["decode"](encoded_diff, dtype, (shape[0]-1,))
173+ x_reconstructed = np.empty(shape, dtype=dtype_np)
174+ x_reconstructed[0] = x0
175+ x_reconstructed[1:] = x0 + np.cumsum(x_diff)
176+ return x_reconstructed
177+ algorithm_dicts.append({
178+ "name": a["name"] + "-delta",
179+ "version": a["version"],
180+ "encode": encode0,
181+ "decode": decode0,
182+ "description": a["description"] + " with delta encoding",
183+ "tags": a["tags"] + ["delta"],
184+ "source_file": a["source_file"],
185+ "long_description": a["long_description"]
186+ })
187+
188+# add delta2 encoding
189+for a in algorithm_dicts_base:
190+ def encode0(x: np.ndarray, a=a) -> bytes:
191+ x_diff = np.diff(np.diff(x))
192+ x0 = x[0:1]
193+ encoded_diff = a["encode"](x_diff)
194+ # Store the first value at the start
195+ first_value_bytes = x0.tobytes()
196+ second_value_bytes = x[1:2].tobytes()
197+ return first_value_bytes + second_value_bytes + encoded_diff
198+ def decode0(x: bytes, dtype: str, shape: tuple, a=a) -> np.ndarray:
199+ dtype_np = np.dtype(dtype)
200+ num_bytes_first_value = dtype_np.itemsize
201+ first_value_bytes = x[:num_bytes_first_value]
202+ second_value_bytes = x[num_bytes_first_value:2*num_bytes_first_value]
203+ x0 = np.frombuffer(first_value_bytes, dtype=dtype_np)
204+ x1 = np.frombuffer(second_value_bytes, dtype=dtype_np)
205+ encoded_diff2 = x[2*num_bytes_first_value:]
206+ x_diff2 = a["decode"](encoded_diff2, dtype, (shape[0]-2,))
207+ x_recon1 = np.empty((shape[0]-1,), dtype=dtype_np)
208+ x_recon1[0] = x1 - x0
209+ x_recon1[1:] = x_recon1[0] + np.cumsum(x_diff2)
210+ x_reconstructed = np.empty(shape, dtype=dtype_np)
211+ x_reconstructed[0] = x0
212+ x_reconstructed[1:] = x0 + np.cumsum(x_recon1)
213+ return x_reconstructed
214+ algorithm_dicts.append({
215+ "name": a["name"] + "-delta2",
216+ "version": a["version"],
217+ "encode": encode0,
218+ "decode": decode0,
219+ "description": a["description"] + " with delta2 encoding",
220+ "tags": a["tags"] + ["delta2"],
221+ "source_file": a["source_file"],
222+ "long_description": a["long_description"]
223+ })
224+
225+# Add auto-regressive prediction encoding
226+for a in algorithm_dicts_base:
227+ for order in [2, 8]:
228+ def encode0(x: np.ndarray, a=a, order=order) -> bytes:
229+ coeffs, residuals, initial_values = encode_ar(x, order=order)
230+ encoded_residuals = a["encode"](residuals)
231+ coeffs_bytes = coeffs.astype(np.float32).tobytes()
232+ initial_values_bytes = initial_values.astype(np.int16).tobytes()
233+ return coeffs_bytes + initial_values_bytes + encoded_residuals
234+ def decode0(x: bytes, dtype: str, shape: tuple, a=a, order=order) -> np.ndarray:
235+ dtype_np = np.dtype(dtype)
236+ num_bytes_coeffs = order * np.dtype(np.float32).itemsize
237+ coeffs_bytes = x[:num_bytes_coeffs]
238+ coeffs = np.frombuffer(coeffs_bytes, dtype=np.float32)
239+ num_initial_values = len(coeffs)
240+ num_bytes_initial_values = num_initial_values * dtype_np.itemsize
241+ initial_values_bytes = x[num_bytes_coeffs : num_bytes_coeffs + num_bytes_initial_values]
242+ initial_values = np.frombuffer(initial_values_bytes, dtype=dtype_np)
243+ encoded_residuals = x[num_bytes_coeffs + num_bytes_initial_values :]
244+ residuals = a["decode"](encoded_residuals, dtype, (shape[0]-num_initial_values,))
245+ reconstructed = decode_ar(coeffs, residuals, initial_values)
246+ return reconstructed.reshape(shape)
247+ algorithm_dicts.append({
248+ "name": a["name"] + f"-ar{order}",
249+ "version": a["version"],
250+ "encode": encode0,
251+ "decode": decode0,
252+ "description": a["description"] + f" with auto-regressive prediction encoding of order {order}",
253+ "tags": a["tags"] + [f"ar{order}"],
254+ "source_file": a["source_file"],
255+ "long_description": a["long_description"]
256+ })
257+
258+algorithms = [
259+ Algorithm(**a)
260+ for a in algorithm_dicts
261+]
python/ephys_compression_tests/algorithms/ans/ans.mdadded+1−0View file
@@ -0,0 +1 @@
1+# ANS
python/ephys_compression_tests/algorithms/ans/ar.pyadded+173−0View file
@@ -0,0 +1,173 @@
1+"""Auto-regressive model utilities for ANS compression."""
2+
3+import numpy as np
4+from typing import Tuple
5+from numba import njit
6+
7+
8+def _warmup_numba_functions():
9+ """Warmup numba JIT compilation with small test data."""
10+ print("Warming up numba functions for AR model...")
11+ # Create small test data
12+ test_data = np.array([1, 2, 3, 4, 5, 6, 7, 8], dtype=np.int16)
13+ test_coeffs = np.array([0.5, 0.3], dtype=np.float32)
14+ test_residuals = np.array([1, 2, 3, 4], dtype=np.int16)
15+ test_initial = np.array([1, 2], dtype=np.int16)
16+
17+ # Warmup each numba function
18+ _create_design_matrix(test_data, 2)
19+ _apply_ar_residuals_kernel(test_data, test_coeffs)
20+ _decode_ar_kernel(test_coeffs, test_residuals, test_initial)
21+
22+
23+@njit
24+def _create_design_matrix(data: np.ndarray, order: int) -> Tuple[np.ndarray, np.ndarray]:
25+ """Numba-optimized design matrix creation for AR model."""
26+ n = len(data)
27+ X_design = np.zeros((n - order, order))
28+ y_target = data[order:]
29+
30+ for i in range(n - order):
31+ for j in range(order):
32+ X_design[i, j] = data[i + order - j - 1]
33+
34+ return X_design, y_target
35+
36+
37+def fit_ar_model(data: np.ndarray, order: int) -> np.ndarray:
38+ """
39+ Fit an autoregressive model of given order using least squares.
40+
41+ Args:
42+ data: Input data array
43+ order: AR model order
44+
45+ Returns:
46+ AR coefficients as numpy array
47+ """
48+ n = len(data)
49+ if order >= n:
50+ raise ValueError(f"AR order {order} must be less than data length {n}")
51+
52+ # Create design matrix using numba-optimized function
53+ X_design, y_target = _create_design_matrix(data, order)
54+
55+ # Use faster solve via normal equations: (X^T X) coeffs = X^T y
56+ # This is faster than lstsq for overdetermined systems
57+ XtX = X_design.T @ X_design
58+ Xty = X_design.T @ y_target
59+ coeffs = np.linalg.solve(XtX, Xty)
60+
61+ return coeffs
62+
63+
64+@njit
65+def _apply_ar_residuals_kernel(data: np.ndarray, coeffs: np.ndarray) -> np.ndarray:
66+ """Numba-optimized kernel for computing AR residuals."""
67+ order = len(coeffs)
68+ n = len(data)
69+ residuals = np.empty(n - order, dtype=data.dtype)
70+
71+ for i in range(order, n):
72+ # Predict using previous 'order' samples
73+ # Use float32 accumulation
74+ prediction = np.float32(0.0)
75+ for j in range(order):
76+ prediction += coeffs[j] * np.float32(data[i - j - 1])
77+
78+ # Round to nearest integer using numpy's round (banker's rounding)
79+ prediction_int = np.int16(np.round(prediction))
80+ residual = data[i] - prediction_int
81+ residuals[i - order] = residual
82+
83+ return residuals
84+
85+
86+def apply_ar_residuals(data: np.ndarray, coeffs: np.ndarray) -> np.ndarray:
87+ """
88+ Apply AR model with given coefficients and return residuals.
89+
90+ Args:
91+ data: Input data array
92+ coeffs: AR coefficients
93+
94+ Returns:
95+ Residuals array
96+ """
97+ # Ensure coeffs is float32
98+ coeffs = np.array(coeffs, dtype=np.float32)
99+
100+ return _apply_ar_residuals_kernel(data, coeffs)
101+
102+
103+def encode_ar(data: np.ndarray, order: int) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
104+ """
105+ Encode data using AR model - returns coefficients, residuals, and initial values.
106+
107+ Args:
108+ data: Input data array (int16)
109+ order: AR model order
110+
111+ Returns:
112+ Tuple of (coefficients, residuals, initial_values)
113+ """
114+ # Fit AR model
115+ coeffs = fit_ar_model(data, order)
116+
117+ # Convert coefficients to float32 to match what will be deserialized
118+ coeffs = coeffs.astype(np.float32)
119+
120+ # Compute residuals using float32 coefficients
121+ residuals = apply_ar_residuals(data, coeffs)
122+
123+ # Store initial values
124+ initial_values = data[:order]
125+
126+ return coeffs, residuals, initial_values
127+
128+
129+@njit
130+def _decode_ar_kernel(coeffs: np.ndarray, residuals: np.ndarray, initial_values: np.ndarray) -> np.ndarray:
131+ """Numba-optimized kernel for AR decoding."""
132+ order = len(coeffs)
133+ n = len(residuals) + order
134+ reconstructed = np.empty(n, dtype=np.int16)
135+ reconstructed[:order] = initial_values
136+
137+ for i in range(order, n):
138+ # Predict using AR model
139+ # Use float32 accumulation
140+ prediction = np.float32(0.0)
141+ for j in range(order):
142+ prediction += coeffs[j] * np.float32(reconstructed[i - j - 1])
143+
144+ # Round to nearest integer using numpy's round (banker's rounding)
145+ prediction_int = np.int16(np.round(prediction))
146+
147+ # Add residual
148+ reconstructed[i] = prediction_int + residuals[i - order]
149+
150+ return reconstructed
151+
152+
153+def decode_ar(coeffs: np.ndarray, residuals: np.ndarray, initial_values: np.ndarray) -> np.ndarray:
154+ """
155+ Decode AR encoded data.
156+
157+ Args:
158+ coeffs: AR coefficients (float32)
159+ residuals: Residuals array
160+ initial_values: Initial values (first 'order' samples)
161+
162+ Returns:
163+ Reconstructed data array
164+ """
165+ # Ensure coeffs is float32
166+ coeffs = np.array(coeffs, dtype=np.float32)
167+
168+ return _decode_ar_kernel(coeffs, residuals, initial_values)
169+
170+
171+# Warmup numba functions on module import
172+_warmup_numba_functions()
173+
python/ephys_compression_tests/algorithms/blosc2/__init__.pyadded+91−0View file
@@ -0,0 +1,91 @@
1+import numpy as np
2+import os
3+import blosc2
4+from ...types import Algorithm
5+
6+SOURCE_FILE = "blosc2/__init__.py"
7+
8+
9+def _load_long_description():
10+ current_dir = os.path.dirname(os.path.abspath(__file__))
11+ md_path = os.path.join(current_dir, "blosc2.md")
12+ with open(md_path, "r", encoding="utf-8") as f:
13+ return f.read()
14+
15+
16+LONG_DESCRIPTION = _load_long_description()
17+
18+
19+def blosc2_encode(x: np.ndarray, clevel: int, codec, filter: int = 2) -> bytes:
20+ import blosc2
21+
22+ # Convert filter int to proper enum
23+ if filter == 2:
24+ blosc_filter = blosc2.Filter.BITSHUFFLE
25+ elif filter == 1:
26+ blosc_filter = blosc2.Filter.SHUFFLE
27+ else:
28+ blosc_filter = blosc2.Filter.NOFILTER
29+
30+ # Get typesize from numpy array
31+ typesize = x.dtype.itemsize
32+
33+ # Compress data
34+ compressed = blosc2.compress(
35+ x, # numpy arrays support buffer interface
36+ typesize=typesize,
37+ clevel=clevel,
38+ filter=blosc_filter,
39+ codec=codec,
40+ )
41+ assert isinstance(compressed, bytes) # Type assertion
42+ return compressed
43+
44+
45+def blosc2_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
46+ import blosc2
47+
48+ decompressed = blosc2.decompress(x)
49+ assert isinstance(decompressed, (bytes, bytearray)) # Type assertion
50+ arr = np.frombuffer(decompressed, dtype=np.dtype(dtype))
51+ return arr.reshape(shape)
52+
53+zstd_codec = blosc2.Codec.ZSTD
54+
55+algorithms_dicts = [
56+ {
57+ "name": "blosc2-zstd-1",
58+ "version": "1b",
59+ "encode": lambda x: blosc2_encode(x, clevel=1, codec=zstd_codec),
60+ "decode": lambda x, dtype, shape: blosc2_decode(x, dtype, shape),
61+ "description": "Blosc2 compression at level 1 (fastest compression).",
62+ "tags": ["blosc2"],
63+ "source_file": SOURCE_FILE,
64+ "long_description": LONG_DESCRIPTION,
65+ },
66+ {
67+ "name": "blosc2-zstd-5",
68+ "version": "1",
69+ "encode": lambda x: blosc2_encode(x, clevel=5, codec=zstd_codec),
70+ "decode": lambda x, dtype, shape: blosc2_decode(x, dtype, shape),
71+ "description": "Blosc2 compression at level 5 (balanced speed/compression).",
72+ "tags": ["blosc2"],
73+ "source_file": SOURCE_FILE,
74+ "long_description": LONG_DESCRIPTION,
75+ },
76+ {
77+ "name": "blosc2-zstd-9",
78+ "version": "1",
79+ "encode": lambda x: blosc2_encode(x, clevel=9, codec=zstd_codec),
80+ "decode": lambda x, dtype, shape: blosc2_decode(x, dtype, shape),
81+ "description": "Blosc2 compression at level 9 (maximum compression).",
82+ "tags": ["blosc2"],
83+ "source_file": SOURCE_FILE,
84+ "long_description": LONG_DESCRIPTION,
85+ }
86+]
87+
88+algorithms = [
89+ Algorithm(**a)
90+ for a in algorithms_dicts
91+]
python/ephys_compression_tests/algorithms/blosc2/blosc2.mdadded+3−0View file
@@ -0,0 +1,3 @@
1+# Blosc2 Algorithm
2+
3+Blosc2 is a modern, fast data compression library that builds upon the original Blosc library. It is designed for efficient compression of binary data, particularly optimized for in-memory compression of numerical arrays. Blosc2 uses block-oriented compression with support for multithreading and SIMD instructions.
python/ephys_compression_tests/algorithms/lzma/__init__.pyadded+129−0View file
@@ -0,0 +1,129 @@
1+import numpy as np
2+import os
3+import lzma
4+from ...types import Algorithm
5+
6+SOURCE_FILE = "lzma/__init__.py"
7+
8+
9+def _load_long_description():
10+ current_dir = os.path.dirname(os.path.abspath(__file__))
11+ md_path = os.path.join(current_dir, "lzma.md")
12+ with open(md_path, "r", encoding="utf-8") as f:
13+ return f.read()
14+
15+
16+LONG_DESCRIPTION = _load_long_description()
17+
18+
19+def lzma_encode(x: np.ndarray, preset: int = 9) -> bytes:
20+ """Encode numpy array using LZMA compression.
21+
22+ Args:
23+ x: Input numpy array
24+ preset: Compression level (0-9, default 9 for maximum compression)
25+
26+ Returns:
27+ Compressed bytes
28+ """
29+ # Store dtype and shape information
30+ dtype_str = str(x.dtype)
31+ shape_bytes = np.array(x.shape, dtype=np.int64).tobytes()
32+ dtype_bytes = dtype_str.encode('utf-8')
33+ dtype_len = np.array([len(dtype_bytes)], dtype=np.uint32).tobytes()
34+
35+ # Compress the array data
36+ data_bytes = x.tobytes()
37+ compressed_data = lzma.compress(data_bytes, preset=preset)
38+
39+ # Combine metadata and compressed data
40+ return dtype_len + dtype_bytes + shape_bytes + compressed_data
41+
42+
43+def lzma_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
44+ """Decode LZMA compressed bytes back to numpy array.
45+
46+ Args:
47+ x: Compressed bytes
48+ dtype: Expected numpy dtype
49+ shape: Expected array shape
50+
51+ Returns:
52+ Decompressed numpy array
53+ """
54+ # Read dtype length
55+ dtype_len = np.frombuffer(x[:4], dtype=np.uint32)[0]
56+ offset = 4
57+
58+ # Read dtype string (not used but stored for completeness)
59+ # dtype_str = x[offset:offset + dtype_len].decode('utf-8')
60+ offset += dtype_len
61+
62+ # Read shape (not used but stored for completeness)
63+ # Determine number of dimensions from shape parameter
64+ num_dims = len(shape)
65+ shape_size = num_dims * 8 # int64
66+ # stored_shape = np.frombuffer(x[offset:offset + shape_size], dtype=np.int64)
67+ offset += shape_size
68+
69+ # Decompress the data
70+ compressed_data = x[offset:]
71+ decompressed_data = lzma.decompress(compressed_data)
72+
73+ # Reconstruct array
74+ arr = np.frombuffer(decompressed_data, dtype=np.dtype(dtype))
75+ return arr.reshape(shape)
76+
77+
78+algorithm_dicts_base = [
79+ {
80+ "name": "lzma",
81+ "version": "1",
82+ "encode": lambda x: lzma_encode(x, preset=9),
83+ "decode": lambda x, dtype, shape: lzma_decode(x, dtype, shape),
84+ "description": "LZMA compression at level 9 (maximum compression)",
85+ "tags": ["lzma"],
86+ "source_file": SOURCE_FILE,
87+ "long_description": LONG_DESCRIPTION,
88+ }
89+]
90+
91+algorithm_dicts = []
92+for a in algorithm_dicts_base:
93+ algorithm_dicts.append(a)
94+
95+# Add delta encoding
96+for a in algorithm_dicts_base:
97+ def encode0(x: np.ndarray, a=a) -> bytes:
98+ x_diff = np.diff(x)
99+ x0 = x[0:1]
100+ encoded_diff = a["encode"](x_diff)
101+ # Store the first value at the start
102+ first_value_bytes = x0.tobytes()
103+ return first_value_bytes + encoded_diff
104+ def decode0(x: bytes, dtype: str, shape: tuple, a=a) -> np.ndarray:
105+ dtype_np = np.dtype(dtype)
106+ num_bytes_first_value = dtype_np.itemsize
107+ first_value_bytes = x[:num_bytes_first_value]
108+ x0 = np.frombuffer(first_value_bytes, dtype=dtype_np)
109+ encoded_diff = x[num_bytes_first_value:]
110+ x_diff = a["decode"](encoded_diff, dtype, (shape[0]-1,))
111+ x_reconstructed = np.empty(shape, dtype=dtype_np)
112+ x_reconstructed[0] = x0
113+ x_reconstructed[1:] = x0 + np.cumsum(x_diff)
114+ return x_reconstructed
115+ algorithm_dicts.append({
116+ "name": a["name"] + "-delta",
117+ "version": a["version"],
118+ "encode": encode0,
119+ "decode": decode0,
120+ "description": a["description"] + " with delta encoding",
121+ "tags": a["tags"] + ["delta"],
122+ "source_file": a["source_file"],
123+ "long_description": a["long_description"]
124+ })
125+
126+algorithms = [
127+ Algorithm(**a)
128+ for a in algorithm_dicts
129+]
python/ephys_compression_tests/algorithms/lzma/lzma.mdadded+5−0View file
@@ -0,0 +1,5 @@
1+# LZMA Algorithm
2+
3+LZMA (Lempel-Ziv-Markov chain Algorithm) is a lossless data compression algorithm that provides a high compression ratio. It is the default and general compression method of 7z format in the 7-Zip program. LZMA uses a dictionary compression scheme and features a high compression ratio with variable dictionary size, while still maintaining fast decompression speed.
4+
5+The LZMA algorithm is particularly effective for compressing large files and is widely used in various applications including file archivers, software distribution, and embedded systems. Python's built-in `lzma` module provides access to this compression algorithm.
python/ephys_compression_tests/algorithms/wavpack/__init__.pyadded+83−0View file
@@ -0,0 +1,83 @@
1+import numpy as np
2+import os
3+from ...types import Algorithm
4+
5+SOURCE_FILE = "wavpack/__init__.py"
6+
7+
8+def _load_long_description():
9+ current_dir = os.path.dirname(os.path.abspath(__file__))
10+ md_path = os.path.join(current_dir, "wavpack.md")
11+ with open(md_path, "r", encoding="utf-8") as f:
12+ return f.read()
13+
14+
15+LONG_DESCRIPTION = _load_long_description()
16+
17+
18+def wavpack_encode(x: np.ndarray) -> bytes:
19+ from wavpack_numcodecs import WavPack
20+ codec = WavPack()
21+ encoded = codec.encode(x)
22+ assert isinstance(encoded, bytes)
23+ return encoded
24+
25+def wavpack_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
26+ from wavpack_numcodecs import WavPack
27+ codec = WavPack()
28+ decoded = codec.decode(x)
29+ arr = np.frombuffer(decoded, dtype=np.dtype(dtype))
30+ return arr.reshape(shape)
31+
32+algorithm_dicts_base = [
33+ {
34+ "name": "wavpack",
35+ "version": "1",
36+ "encode": lambda x: wavpack_encode(x),
37+ "decode": lambda x, dtype, shape: wavpack_decode(x, dtype, shape),
38+ "description": "WavPack",
39+ "tags": ["wavpack"],
40+ "source_file": SOURCE_FILE,
41+ "long_description": LONG_DESCRIPTION,
42+ }
43+]
44+
45+algorithm_dicts = []
46+for a in algorithm_dicts_base:
47+ algorithm_dicts.append(a)
48+
49+# add delta encoding
50+for a in algorithm_dicts_base:
51+ def encode0(x: np.ndarray, a=a) -> bytes:
52+ x_diff = np.diff(x)
53+ x0 = x[0:1]
54+ encoded_diff = a["encode"](x_diff)
55+ # Store the first value at the start
56+ first_value_bytes = x0.tobytes()
57+ return first_value_bytes + encoded_diff
58+ def decode0(x: bytes, dtype: str, shape: tuple, a=a) -> np.ndarray:
59+ dtype_np = np.dtype(dtype)
60+ num_bytes_first_value = dtype_np.itemsize
61+ first_value_bytes = x[:num_bytes_first_value]
62+ x0 = np.frombuffer(first_value_bytes, dtype=dtype_np)
63+ encoded_diff = x[num_bytes_first_value:]
64+ x_diff = a["decode"](encoded_diff, dtype, (shape[0]-1,))
65+ x_reconstructed = np.empty(shape, dtype=dtype_np)
66+ x_reconstructed[0] = x0
67+ x_reconstructed[1:] = x0 + np.cumsum(x_diff)
68+ return x_reconstructed
69+ algorithm_dicts.append({
70+ "name": a["name"] + "-delta",
71+ "version": a["version"],
72+ "encode": encode0,
73+ "decode": decode0,
74+ "description": a["description"] + " with delta encoding",
75+ "tags": a["tags"] + ["delta"],
76+ "source_file": a["source_file"],
77+ "long_description": a["long_description"]
78+ })
79+
80+algorithms = [
81+ Algorithm(**a)
82+ for a in algorithm_dicts
83+]
python/ephys_compression_tests/algorithms/wavpack/wavpack.mdadded+1−0View file
@@ -0,0 +1 @@
1+# wavpack
python/ephys_compression_tests/algorithms/zlib/__init__.pyadded+129−0View file
@@ -0,0 +1,129 @@
1+import numpy as np
2+import os
3+import zlib
4+from ...types import Algorithm
5+
6+SOURCE_FILE = "zlib/__init__.py"
7+
8+
9+def _load_long_description():
10+ current_dir = os.path.dirname(os.path.abspath(__file__))
11+ md_path = os.path.join(current_dir, "zlib.md")
12+ with open(md_path, "r", encoding="utf-8") as f:
13+ return f.read()
14+
15+
16+LONG_DESCRIPTION = _load_long_description()
17+
18+
19+def zlib_encode(x: np.ndarray, level: int = 9) -> bytes:
20+ """Encode numpy array using zlib compression.
21+
22+ Args:
23+ x: Input numpy array
24+ level: Compression level (0-9, default 9 for maximum compression)
25+
26+ Returns:
27+ Compressed bytes
28+ """
29+ # Store dtype and shape information
30+ dtype_str = str(x.dtype)
31+ shape_bytes = np.array(x.shape, dtype=np.int64).tobytes()
32+ dtype_bytes = dtype_str.encode('utf-8')
33+ dtype_len = np.array([len(dtype_bytes)], dtype=np.uint32).tobytes()
34+
35+ # Compress the array data
36+ data_bytes = x.tobytes()
37+ compressed_data = zlib.compress(data_bytes, level=level)
38+
39+ # Combine metadata and compressed data
40+ return dtype_len + dtype_bytes + shape_bytes + compressed_data
41+
42+
43+def zlib_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
44+ """Decode zlib compressed bytes back to numpy array.
45+
46+ Args:
47+ x: Compressed bytes
48+ dtype: Expected numpy dtype
49+ shape: Expected array shape
50+
51+ Returns:
52+ Decompressed numpy array
53+ """
54+ # Read dtype length
55+ dtype_len = np.frombuffer(x[:4], dtype=np.uint32)[0]
56+ offset = 4
57+
58+ # Read dtype string (not used but stored for completeness)
59+ # dtype_str = x[offset:offset + dtype_len].decode('utf-8')
60+ offset += dtype_len
61+
62+ # Read shape (not used but stored for completeness)
63+ # Determine number of dimensions from shape parameter
64+ num_dims = len(shape)
65+ shape_size = num_dims * 8 # int64
66+ # stored_shape = np.frombuffer(x[offset:offset + shape_size], dtype=np.int64)
67+ offset += shape_size
68+
69+ # Decompress the data
70+ compressed_data = x[offset:]
71+ decompressed_data = zlib.decompress(compressed_data)
72+
73+ # Reconstruct array
74+ arr = np.frombuffer(decompressed_data, dtype=np.dtype(dtype))
75+ return arr.reshape(shape)
76+
77+
78+algorithm_dicts_base = [
79+ {
80+ "name": "zlib",
81+ "version": "1",
82+ "encode": lambda x: zlib_encode(x, level=9),
83+ "decode": lambda x, dtype, shape: zlib_decode(x, dtype, shape),
84+ "description": "zlib compression at level 9 (maximum compression)",
85+ "tags": ["zlib"],
86+ "source_file": SOURCE_FILE,
87+ "long_description": LONG_DESCRIPTION,
88+ }
89+]
90+
91+algorithm_dicts = []
92+for a in algorithm_dicts_base:
93+ algorithm_dicts.append(a)
94+
95+# Add delta encoding
96+for a in algorithm_dicts_base:
97+ def encode0(x: np.ndarray, a=a) -> bytes:
98+ x_diff = np.diff(x)
99+ x0 = x[0:1]
100+ encoded_diff = a["encode"](x_diff)
101+ # Store the first value at the start
102+ first_value_bytes = x0.tobytes()
103+ return first_value_bytes + encoded_diff
104+ def decode0(x: bytes, dtype: str, shape: tuple, a=a) -> np.ndarray:
105+ dtype_np = np.dtype(dtype)
106+ num_bytes_first_value = dtype_np.itemsize
107+ first_value_bytes = x[:num_bytes_first_value]
108+ x0 = np.frombuffer(first_value_bytes, dtype=dtype_np)
109+ encoded_diff = x[num_bytes_first_value:]
110+ x_diff = a["decode"](encoded_diff, dtype, (shape[0]-1,))
111+ x_reconstructed = np.empty(shape, dtype=dtype_np)
112+ x_reconstructed[0] = x0
113+ x_reconstructed[1:] = x0 + np.cumsum(x_diff)
114+ return x_reconstructed
115+ algorithm_dicts.append({
116+ "name": a["name"] + "-delta",
117+ "version": a["version"],
118+ "encode": encode0,
119+ "decode": decode0,
120+ "description": a["description"] + " with delta encoding",
121+ "tags": a["tags"] + ["delta"],
122+ "source_file": a["source_file"],
123+ "long_description": a["long_description"]
124+ })
125+
126+algorithms = [
127+ Algorithm(**a)
128+ for a in algorithm_dicts
129+]
python/ephys_compression_tests/algorithms/zlib/zlib.mdadded+5−0View file
@@ -0,0 +1,5 @@
1+# zlib Algorithm
2+
3+zlib is a widely-used, general-purpose data compression library that implements the DEFLATE compression algorithm. DEFLATE is a combination of LZ77 (Lempel-Ziv 1977) and Huffman coding. The zlib library is extremely portable, efficient, and free for use in any application.
4+
5+zlib compression is commonly used in many applications including PNG image compression, HTTP compression, and git repositories. It provides a good balance between compression ratio and speed, making it suitable for a wide range of compression tasks. Python includes zlib as a built-in module, making it readily available without external dependencies.
python/ephys_compression_tests/cli.pyadded+143−0View file
@@ -0,0 +1,143 @@
1+#!/usr/bin/env python3
2+
3+import click
4+from typing import List, Optional
5+from .run_benchmarks.run_benchmarks import run_benchmarks
6+from .algorithms import algorithms
7+from .datasets import datasets
8+
9+
10+def get_available_algorithms() -> List[str]:
11+ """Get list of available algorithm names"""
12+ return [alg.name for alg in algorithms]
13+
14+
15+def get_available_datasets() -> List[str]:
16+ """Get list of available dataset names"""
17+ return [ds.name for ds in datasets]
18+
19+
20+def filter_algorithms(selected: Optional[List[str]] = None) -> List[dict]:
21+ """Filter algorithms based on selected names"""
22+ if not selected:
23+ return algorithms
24+ return [alg for alg in algorithms if alg.name in selected]
25+
26+
27+def filter_datasets(selected: Optional[List[str]] = None) -> List[dict]:
28+ """Filter datasets based on selected names"""
29+ if not selected:
30+ return datasets
31+ return [ds for ds in datasets if ds.name in selected]
32+
33+
34+def validate_algorithms(ctx, param, value):
35+ if not value:
36+ return None
37+ available = get_available_algorithms()
38+ invalid = [alg for alg in value if alg not in available]
39+ if invalid:
40+ raise click.BadParameter(
41+ f"Invalid algorithm(s): {', '.join(invalid)}. "
42+ f"Available algorithms: {', '.join(available)}"
43+ )
44+ return value
45+
46+
47+def validate_datasets(ctx, param, value):
48+ if not value:
49+ return None
50+ available = get_available_datasets()
51+ invalid = [ds for ds in value if ds not in available]
52+ if invalid:
53+ raise click.BadParameter(
54+ f"Invalid dataset(s): {', '.join(invalid)}. "
55+ f"Available datasets: {', '.join(available)}"
56+ )
57+ return value
58+
59+
60+@click.group()
61+def cli():
62+ """Benchmark compression algorithms for electrophysiology data"""
63+ pass
64+
65+
66+@cli.command()
67+def list():
68+ """List available algorithms and datasets"""
69+ click.echo("\nAvailable Algorithms:")
70+ for alg in algorithms:
71+ desc = alg.description if alg.description else "No description"
72+ click.echo(f" {alg.name:<20} - {desc}")
73+
74+ click.echo("\nAvailable Datasets:")
75+ for ds in datasets:
76+ desc = ds.description if ds.description else "No description"
77+ click.echo(f" {ds.name:<20} - {desc}")
78+
79+
80+@cli.command()
81+@click.option(
82+ "--algorithm",
83+ "-a",
84+ multiple=True,
85+ callback=validate_algorithms,
86+ help="Algorithm(s) to benchmark (can be specified multiple times)",
87+)
88+@click.option(
89+ "--dataset",
90+ "-d",
91+ multiple=True,
92+ callback=validate_datasets,
93+ help="Dataset(s) to benchmark (can be specified multiple times)",
94+)
95+@click.option(
96+ "--cache-dir",
97+ default=".benchmark_cache",
98+ help="Directory to store cached results",
99+ type=click.Path(),
100+)
101+@click.option("--quiet", "-q", is_flag=True, help="Reduce output verbosity")
102+@click.option("--force", "-f", is_flag=True, help="Force re-run without using cache")
103+def run(algorithm, dataset, cache_dir, quiet, force):
104+ """Run benchmarks with specified options"""
105+ # Filter algorithms and datasets
106+ filtered_algorithms = filter_algorithms(algorithm)
107+ filtered_datasets = filter_datasets(dataset)
108+
109+ if not filtered_algorithms:
110+ click.echo("Error: No matching algorithms found", err=True)
111+ ctx = click.get_current_context()
112+ ctx.exit(1)
113+ if not filtered_datasets:
114+ click.echo("Error: No matching datasets found", err=True)
115+ ctx = click.get_current_context()
116+ ctx.exit(1)
117+
118+ # Run benchmarks with filtered options
119+ results = run_benchmarks(
120+ cache_dir=cache_dir,
121+ verbose=not quiet,
122+ selected_algorithms=filtered_algorithms,
123+ selected_datasets=filtered_datasets,
124+ force=force,
125+ )
126+
127+ # Print summary
128+ click.echo("\nBenchmark Summary:")
129+ for result in results["results"]:
130+ click.echo(
131+ f"\n{result['dataset']} + {result['algorithm']}:"
132+ f"\n Compression ratio: {result['compression_ratio']:.2f}x"
133+ f"\n Encode speed: {result['encode_mb_per_sec']:.2f} MB/s"
134+ f"\n Decode speed: {result['decode_mb_per_sec']:.2f} MB/s"
135+ )
136+
137+
138+def main():
139+ cli()
140+
141+
142+if __name__ == "__main__":
143+ main()
python/ephys_compression_tests/datasets/__init__.pyadded+12−0View file
@@ -0,0 +1,12 @@
1+from .vyom import datasets as vyom_datasets
2+from .aind import datasets as aind_datasets
3+from ..types import Dataset
4+
5+datasets_list = [
6+ vyom_datasets,
7+ aind_datasets,
8+]
9+
10+datasets: list[Dataset] = []
11+for d in datasets_list:
12+ datasets.extend(d)
python/ephys_compression_tests/datasets/aind/__init__.pyadded+77−0View file
@@ -0,0 +1,77 @@
1+import numpy as np
2+import os
3+import requests
4+import io
5+from ...types import Dataset
6+
7+from ..._filters import bandpass_filter
8+
9+
10+SOURCE_FILE = "aind/__init__.py"
11+
12+
13+def _load_long_description():
14+ current_dir = os.path.dirname(os.path.abspath(__file__))
15+ md_path = os.path.join(current_dir, "aind.md")
16+ with open(md_path, "r", encoding="utf-8") as f:
17+ return f.read()
18+
19+
20+LONG_DESCRIPTION = _load_long_description()
21+
22+tags = ["real", "ecephys", "timeseries", "1d", "integer", "correlated"]
23+
24+
25+def load_aind_ch101() -> np.ndarray:
26+ """Load AIND CH101 dataset from external URL.
27+
28+ Returns:
29+ Array containing the loaded data
30+ """
31+ url = "https://tempory.net/ephys-compression-tests/aind_CH101.raw.npy"
32+ print(f'Loading AIND dataset from {url}...')
33+ response = requests.get(url)
34+ response.raise_for_status()
35+ data = np.load(io.BytesIO(response.content)).flatten()
36+ return data
37+
38+
39+
40+dataset_dicts_base = [
41+ {
42+ "name": "aind-ch101",
43+ "version": "1",
44+ "description": "AIND CH101 dataset",
45+ "create": load_aind_ch101,
46+ "tags": tags,
47+ "source_file": SOURCE_FILE,
48+ "long_description": LONG_DESCRIPTION,
49+ }
50+]
51+
52+dataset_dicts = []
53+for d in dataset_dicts_base:
54+ dataset_dicts.append(d)
55+
56+# Add filtered versions
57+for d in dataset_dicts_base:
58+ def create0(d=d) -> np.ndarray:
59+ data = d["create"]()
60+ filtered = bandpass_filter(data, sampling_frequency=30000, lowcut=300, highcut=4000)
61+ filtered = filtered.astype(data.dtype)
62+ return filtered
63+
64+ dataset_dicts.append(
65+ {
66+ "name": f'{d["name"]}-bandpass',
67+ "version": "1",
68+ "description": f'{d["description"]} (bandpass filtered 300-4000 Hz)',
69+ "create": create0,
70+ "tags": d["tags"] + ["filtered", "bandpass"],
71+ "source_file": SOURCE_FILE,
72+ "long_description": LONG_DESCRIPTION,
73+ }
74+ )
75+
76+
77+datasets = [Dataset(**a) for a in dataset_dicts]
python/ephys_compression_tests/datasets/aind/aind.mdadded+1−0View file
@@ -0,0 +1 @@
1+Examples from AIND
python/ephys_compression_tests/datasets/vyom/__init__.pyadded+77−0View file
@@ -0,0 +1,77 @@
1+import numpy as np
2+import os
3+import requests
4+import io
5+from ...types import Dataset
6+
7+from ..._filters import bandpass_filter
8+
9+
10+SOURCE_FILE = "vyom/__init__.py"
11+
12+
13+def _load_long_description():
14+ current_dir = os.path.dirname(os.path.abspath(__file__))
15+ md_path = os.path.join(current_dir, "vyom.md")
16+ with open(md_path, "r", encoding="utf-8") as f:
17+ return f.read()
18+
19+
20+LONG_DESCRIPTION = _load_long_description()
21+
22+tags = ["real", "ecephys", "timeseries", "1d", "integer", "correlated"]
23+
24+
25+def load_vyom_example_ch0_seg2_6() -> np.ndarray:
26+ """Load Vyom example dataset from external URL.
27+
28+ Returns:
29+ Array containing the loaded data
30+ """
31+ url = "https://tempory.net/ephys-compression-tests/vyom_example_ch0_seg2-6.npy"
32+ print(f'Loading Vyom example dataset from {url}...')
33+ response = requests.get(url)
34+ response.raise_for_status()
35+ data = np.load(io.BytesIO(response.content))
36+ return data
37+
38+
39+
40+dataset_dicts_base = [
41+ {
42+ "name": "vyom-example-ch0-seg2-6",
43+ "version": "1",
44+ "description": "Vyom example dataset",
45+ "create": load_vyom_example_ch0_seg2_6,
46+ "tags": tags,
47+ "source_file": SOURCE_FILE,
48+ "long_description": LONG_DESCRIPTION,
49+ }
50+]
51+
52+dataset_dicts = []
53+for d in dataset_dicts_base:
54+ dataset_dicts.append(d)
55+
56+# Add filtered versions
57+for d in dataset_dicts_base:
58+ def create0(d=d) -> np.ndarray:
59+ data = d["create"]()
60+ filtered = bandpass_filter(data, sampling_frequency=20000, lowcut=300, highcut=4000)
61+ filtered = filtered.astype(data.dtype)
62+ return filtered
63+
64+ dataset_dicts.append(
65+ {
66+ "name": f'{d["name"]}-bandpass',
67+ "version": "1",
68+ "description": f'{d["description"]} (bandpass filtered 300-4000 Hz)',
69+ "create": create0,
70+ "tags": d["tags"] + ["filtered", "bandpass"],
71+ "source_file": SOURCE_FILE,
72+ "long_description": LONG_DESCRIPTION,
73+ }
74+ )
75+
76+
77+datasets = [Dataset(**a) for a in dataset_dicts]
python/ephys_compression_tests/datasets/vyom/vyom.mdadded+1−0View file
@@ -0,0 +1 @@
1+Examples from Vyom Raval
python/ephys_compression_tests/run_benchmarks/__init__.pyadded+1−0View file
@@ -0,0 +1 @@
1+from .run_benchmarks import run_benchmarks
python/ephys_compression_tests/run_benchmarks/_memobin.pyadded+228−0View file
@@ -0,0 +1,228 @@
1+import json
2+import requests
3+import time
4+from typing import Optional, TypeVar, Callable
5+
6+T = TypeVar("T")
7+
8+
9+def _retry_with_backoff(
10+ func: Callable[..., T], num_retries: int = 4, *args, **kwargs
11+) -> T:
12+ """Execute a function with exponential backoff retry logic.
13+
14+ Args:
15+ func: Function to execute
16+ num_retries: Maximum number of retries
17+ args: Positional arguments for the function
18+ kwargs: Keyword arguments for the function
19+
20+ Returns:
21+ The function's return value
22+
23+ Raises:
24+ The last exception encountered after all retries are exhausted
25+ """
26+ last_exception = None
27+ for attempt in range(num_retries):
28+ try:
29+ return func(*args, **kwargs)
30+ except Exception as e:
31+ last_exception = e
32+ if attempt < num_retries - 1:
33+ sleep_time = 2**attempt # 1, 2, 4, 8 seconds
34+ time.sleep(sleep_time)
35+ else:
36+ raise last_exception
37+ raise RuntimeError("Unexpected: retry loop completed without return or raise")
38+
39+
40+def create_signed_upload_url(
41+ url: str, size: int, user_id: str, memobin_api_key: str, num_retries: int = 4
42+) -> str:
43+ """Create a signed upload URL for memobin.
44+
45+ Args:
46+ url: The target URL for the file
47+ size: Size of the file in bytes
48+ user_id: User ID for memobin
49+ memobin_api_key: API key for memobin authentication
50+
51+ Returns:
52+ The signed upload URL
53+
54+ Raises:
55+ ValueError: If the URL prefix is invalid
56+ requests.RequestException: If the API request fails
57+ """
58+
59+ def _create_url() -> str:
60+ prefix = "https://tempory.net/f/memobin/"
61+ if not url.startswith(prefix):
62+ raise ValueError("Invalid url. Does not have proper prefix")
63+
64+ file_path = url[len(prefix) :]
65+ tempory_api_url = "https://hub.tempory.net/api/uploadFile"
66+
67+ response = requests.post(
68+ tempory_api_url,
69+ headers={
70+ "Content-Type": "application/json",
71+ "Authorization": f"Bearer {memobin_api_key}",
72+ },
73+ json={
74+ "appName": "memobin",
75+ "filePath": file_path,
76+ "size": size,
77+ "userId": user_id,
78+ },
79+ )
80+
81+ if not response.ok:
82+ raise requests.RequestException("Failed to get signed url")
83+
84+ result = response.json()
85+ upload_url = result["uploadUrl"]
86+ download_url = result["downloadUrl"]
87+
88+ if download_url != url:
89+ raise ValueError("Mismatch between download url and url")
90+
91+ return upload_url
92+
93+ return _retry_with_backoff(_create_url, num_retries)
94+
95+
96+def construct_memobin_url(
97+ alg_name: str,
98+ dataset_name: str,
99+ alg_version: str,
100+ dataset_version: str,
101+ system_version: str,
102+ file_type: str = "metadata.json",
103+) -> str:
104+ """Construct the memobin URL for a specific benchmark result or dataset.
105+
106+ Args:
107+ alg_name: Name of the algorithm
108+ dataset_name: Name of the dataset
109+ alg_version: Version of the algorithm
110+ dataset_version: Version of the dataset
111+ system_version: Version of the system
112+ file_type: Type of file (metadata.json or data.bin)
113+
114+ Returns:
115+ The constructed memobin URL
116+ """
117+ path = f"{alg_name}/{dataset_name}/{alg_version}/{dataset_version}/{system_version}/{file_type}"
118+ return f"https://tempory.net/f/memobin/{path}"
119+
120+
121+def construct_dataset_url(
122+ dataset_name: str, dataset_version: str, format: str = "dat"
123+) -> str:
124+ """Construct the memobin URL for a dataset.
125+
126+ Args:
127+ dataset_name: Name of the dataset
128+ dataset_version: Version of the dataset
129+ format: File format ("dat", "npy", or "json")
130+
131+ Returns:
132+ The constructed memobin URL for the dataset
133+ """
134+ path = f"datasets/{dataset_name}/{dataset_version}/{dataset_name}-{dataset_version}.{format}"
135+ return f"https://tempory.net/f/memobin/{path}"
136+
137+
138+def upload_to_memobin(
139+ data: dict | bytes,
140+ url: str,
141+ memobin_api_key: str,
142+ content_type: str = "application/json",
143+ num_retries: int = 4,
144+) -> None:
145+ """Upload data to memobin.
146+
147+ Args:
148+ data: The data to upload (dict for JSON or bytes for binary)
149+ url: The target URL for the file
150+ memobin_api_key: API key for memobin authentication
151+ content_type: Content type of the data
152+
153+ Raises:
154+ requests.RequestException: If the upload fails
155+ """
156+ if isinstance(data, dict):
157+ data_bytes = json.dumps(data).encode("utf-8")
158+ else:
159+ data_bytes = data
160+ size = len(data_bytes)
161+
162+ def _do_upload() -> None:
163+ upload_url = create_signed_upload_url(
164+ url, size, "ephys_compression_tests", memobin_api_key, num_retries
165+ )
166+
167+ response = requests.put(
168+ upload_url, data=data_bytes, headers={"Content-Type": content_type}
169+ )
170+
171+ if not response.ok:
172+ raise requests.RequestException("Failed to upload data to memobin")
173+
174+ _retry_with_backoff(_do_upload, num_retries)
175+
176+
177+def exists_in_memobin(url: str, num_retries: int = 4) -> bool:
178+ """Check if a file exists in memobin using a HEAD request.
179+
180+ Args:
181+ url: The URL to check
182+
183+ Returns:
184+ True if the file exists, False otherwise
185+ """
186+
187+ def _check_exists() -> bool:
188+ try:
189+ response = requests.head(url)
190+ return (
191+ 200 <= response.status_code < 300
192+ ) # Any 2xx status code indicates success
193+ except requests.RequestException:
194+ return False
195+
196+ return _retry_with_backoff(_check_exists, num_retries)
197+
198+
199+def download_from_memobin(
200+ url: str, as_json: bool = True, num_retries: int = 4
201+) -> Optional[dict | bytes]:
202+ """Download data from memobin.
203+
204+ Args:
205+ url: The URL to download from
206+ as_json: Whether to parse the response as JSON
207+
208+ Returns:
209+ The downloaded data as a dictionary or bytes, or None if not found
210+
211+ Raises:
212+ requests.RequestException: If the download fails for a reason other than 404
213+ """
214+
215+ def _do_download() -> Optional[dict | bytes]:
216+ response = None
217+ try:
218+ response = requests.get(url)
219+ if response.status_code == 404:
220+ return None
221+ response.raise_for_status()
222+ return response.json() if as_json else response.content
223+ except requests.RequestException as e:
224+ if response and response.status_code == 404:
225+ return None
226+ raise e
227+
228+ return _retry_with_backoff(_do_download, num_retries)
python/ephys_compression_tests/run_benchmarks/benchmark_timing.pyadded+122−0View file
@@ -0,0 +1,122 @@
1+from typing import Any, Tuple, Callable, Dict
2+from statistics import median
3+import time
4+import numpy as np
5+
6+
7+def run_timed_trials(
8+ data: np.ndarray, operation: Callable, *args
9+) -> Tuple[float, float, Any]:
10+ """Run multiple trials of an operation until total time exceeds 1 second.
11+
12+ Args:
13+ data: Input numpy array for calculating throughput
14+ operation: Function to benchmark
15+ *args: Arguments to pass to the operation
16+
17+ Returns:
18+ Tuple containing:
19+ - median_time: Median execution time across trials
20+ - mb_per_sec: Throughput in MB/s
21+ - result: Result from the last trial execution
22+ """
23+ times = []
24+ total_time = 0
25+ array_size_mb = data.nbytes / (1024 * 1024) # Convert to MB
26+
27+ operation(
28+ *args
29+ ) # execute once prior to timing in case there's any initial overhead
30+
31+ ret = None
32+ while total_time < 1.0:
33+ start_time = time.perf_counter()
34+ ret = operation(*args) # Execute operation
35+ trial_time = time.perf_counter() - start_time
36+ times.append(trial_time)
37+ total_time += trial_time
38+
39+ median_time = median(times)
40+ mb_per_sec = array_size_mb / median_time
41+ return median_time, mb_per_sec, ret
42+
43+
44+def run_compression_benchmark(
45+ data: np.ndarray,
46+ algorithm_name: str,
47+ encode_fn: Callable,
48+ decode_fn: Callable,
49+ verbose: bool = True,
50+) -> Tuple[Dict[str, Any], bytes]:
51+ """Run compression and decompression benchmarks for an algorithm.
52+
53+ Args:
54+ data: Input numpy array to compress
55+ algorithm_name: Name of the algorithm being benchmarked
56+ encode_fn: Compression function
57+ decode_fn: Decompression function
58+ verbose: Whether to print progress messages
59+
60+ Returns:
61+ Tuple containing:
62+ - result: Dictionary with benchmark metrics
63+ - encoded: Compressed data bytes
64+ """
65+ original_size = len(data.tobytes())
66+ dtype = str(data.dtype)
67+
68+ if verbose:
69+ print(" Encoding...")
70+ encode_time, encode_mb_per_sec, encoded = run_timed_trials(data, encode_fn, data)
71+ compressed_size = len(encoded)
72+ compression_ratio = original_size / compressed_size
73+
74+ if verbose:
75+ print(" Compression complete:")
76+ print(f" Compressed size: {compressed_size:,} bytes")
77+ print(f" Compression ratio: {compression_ratio:.2f}x")
78+ print(f" Encode time: {encode_time*1000:.2f}ms")
79+ print(f" Encode throughput: {encode_mb_per_sec:.2f} MB/s")
80+ print(" Decoding...")
81+
82+ decode_time, decode_mb_per_sec, decoded = run_timed_trials(
83+ data, decode_fn, encoded, dtype, data.shape
84+ )
85+
86+ if verbose:
87+ print(f" Decode time: {decode_time*1000:.2f}ms")
88+ print(f" Decode throughput: {decode_mb_per_sec:.2f} MB/s")
89+
90+ # Verify correctness
91+ if len(data) != len(decoded):
92+ raise ValueError(
93+ f"Decompression failed: decoded length {len(decoded)} != original length {len(data)}"
94+ )
95+
96+ if not np.array_equal(data, decoded):
97+ print(data[:100])
98+ print(decoded[:100])
99+ for j in range(len(data)):
100+ if data[j] != decoded[j]:
101+ print(f"Error at index {j}: {data[j]} != {decoded[j]}")
102+ break
103+ raise ValueError(f"Decompression verification failed for {algorithm_name}")
104+
105+ if verbose:
106+ print(" Verification successful!")
107+
108+ result = {
109+ "compression_ratio": compression_ratio,
110+ "encode_time": encode_time,
111+ "decode_time": decode_time,
112+ "encode_mb_per_sec": encode_mb_per_sec,
113+ "decode_mb_per_sec": decode_mb_per_sec,
114+ "original_size": original_size,
115+ "compressed_size": compressed_size,
116+ "array_shape": data.shape,
117+ "array_dtype": dtype,
118+ "timestamp": time.time(),
119+ "cache_status": "new",
120+ }
121+
122+ return result, encoded
python/ephys_compression_tests/run_benchmarks/cache_management.pyadded+118−0View file
@@ -0,0 +1,118 @@
1+import os
2+import json
3+from typing import Optional, Dict, Any
4+from ._memobin import (
5+ construct_memobin_url,
6+ download_from_memobin,
7+)
8+
9+
10+def check_cached_result(
11+ cache_dir: str,
12+ dataset_name: str,
13+ algorithm_name: str,
14+ algorithm_version: str,
15+ dataset_version: str,
16+ system_version: str,
17+ force: bool = False,
18+ verbose: bool = True,
19+) -> Optional[Dict[str, Any]]:
20+ """Check for cached benchmark results locally and in memobin.
21+
22+ Args:
23+ cache_dir: Directory containing cached results
24+ dataset_name: Name of the dataset
25+ algorithm_name: Name of the algorithm
26+ algorithm_version: Version of the algorithm
27+ dataset_version: Version of the dataset
28+ system_version: Version of the system
29+ force: If True, ignore cached results
30+ verbose: Whether to print progress messages
31+
32+ Returns:
33+ Cached result dictionary if found and valid, None otherwise
34+ """
35+ test_dir = os.path.join(cache_dir, dataset_name, algorithm_name)
36+ metadata_file = os.path.join(test_dir, "metadata.json")
37+
38+ # First try local cache (unless force flag is set)
39+ cached_data = None
40+ if not force and os.path.exists(metadata_file):
41+ with open(metadata_file, "r") as f:
42+ cached_data = json.load(f)
43+ # if versions do not match, then set to None
44+ if isinstance(cached_data, dict) and "result" in cached_data:
45+ result = cached_data["result"]
46+ if (
47+ result["algorithm_version"] != algorithm_version
48+ or result["dataset_version"] != dataset_version
49+ or result.get("system_version", "") != system_version
50+ ):
51+ cached_data = None
52+
53+ # If not in local cache, try memobin (unless force flag is set)
54+ if cached_data is None and not force:
55+ memobin_url = construct_memobin_url(
56+ algorithm_name,
57+ dataset_name,
58+ algorithm_version,
59+ dataset_version,
60+ system_version,
61+ "metadata.json",
62+ )
63+ if verbose:
64+ print(" Looking for cached result in memobin...")
65+ cached_data = download_from_memobin(memobin_url)
66+ if cached_data is not None:
67+ if verbose:
68+ print(" Found result in memobin, saving locally...")
69+ # Save to local cache
70+ os.makedirs(test_dir, exist_ok=True)
71+ with open(metadata_file, "w") as f:
72+ json.dump(cached_data, f, indent=2)
73+
74+ if (
75+ cached_data is not None
76+ and isinstance(cached_data, dict)
77+ and "result" in cached_data
78+ ):
79+ result = cached_data["result"]
80+ if (
81+ isinstance(result, dict)
82+ and result.get("algorithm_version") == algorithm_version
83+ and result.get("dataset_version") == dataset_version
84+ and result.get("system_version", "") == system_version
85+ ):
86+ result["cache_status"] = "cached"
87+ return result
88+
89+ return None
90+
91+
92+def save_result_to_cache(
93+ result: Dict[str, Any],
94+ encoded_data: bytes,
95+ cache_dir: str,
96+ dataset_name: str,
97+ algorithm_name: str,
98+) -> None:
99+ """Save benchmark result and compressed data to cache.
100+
101+ Args:
102+ result: Benchmark result dictionary
103+ encoded_data: Compressed data bytes
104+ cache_dir: Directory to store cached results
105+ dataset_name: Name of the dataset
106+ algorithm_name: Name of the algorithm
107+ """
108+ test_dir = os.path.join(cache_dir, dataset_name, algorithm_name)
109+ metadata_file = os.path.join(test_dir, "metadata.json")
110+ compressed_file = os.path.join(test_dir, "compressed.dat")
111+
112+ os.makedirs(test_dir, exist_ok=True)
113+ cache_data = {"result": result}
114+
115+ with open(metadata_file, "w") as f:
116+ json.dump(cache_data, f, indent=2)
117+ with open(compressed_file, "wb") as f:
118+ f.write(encoded_data)
python/ephys_compression_tests/run_benchmarks/collect_info.pyadded+63−0View file
@@ -0,0 +1,63 @@
1+from typing import List, Dict, Any
2+from ._memobin import construct_dataset_url
3+from ..types import Algorithm
4+
5+GITHUB_ALGORITHMS_PREFIX = "https://github.com/magland/ephys_compression_tests/blob/main/python/ephys_compression_tests/algorithms/"
6+GITHUB_DATASETS_PREFIX = "https://github.com/magland/ephys_compression_tests/blob/main/python/ephys_compression_tests/datasets/"
7+
8+
9+def collect_algorithm_info(algorithms: List[Dict[str, Algorithm]]) -> List[Dict[str, Any]]:
10+ """Collect information about compression algorithms.
11+
12+ Args:
13+ algorithms: List of algorithm dictionaries
14+
15+ Returns:
16+ List of algorithm information dictionaries
17+ """
18+ algorithm_info = []
19+ for algorithm in algorithms:
20+ info = {
21+ "name": algorithm.name,
22+ "description": algorithm.description if algorithm.description else "",
23+ "long_description": algorithm.long_description if algorithm.long_description else "",
24+ "version": algorithm.version,
25+ "tags": algorithm.tags if algorithm.tags else [],
26+ }
27+ if algorithm.source_file:
28+ info["source_file"] = GITHUB_ALGORITHMS_PREFIX + algorithm.source_file
29+ algorithm_info.append(info)
30+ return algorithm_info
31+
32+
33+def collect_dataset_info(datasets: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
34+ """Collect information about benchmark datasets.
35+
36+ Args:
37+ datasets: List of dataset dictionaries
38+
39+ Returns:
40+ List of dataset information dictionaries
41+ """
42+ dataset_info = []
43+ for dataset in datasets:
44+ info = {
45+ "name": dataset.name,
46+ "description": dataset.description if dataset.description else "",
47+ "long_description": dataset.long_description if dataset.long_description else "",
48+ "version": dataset.version,
49+ "tags": dataset.tags if dataset.tags else [],
50+ "data_url_raw": construct_dataset_url(
51+ dataset.name, dataset.version, "dat"
52+ ),
53+ "data_url_npy": construct_dataset_url(
54+ dataset.name, dataset.version, "npy"
55+ ),
56+ "data_url_json": construct_dataset_url(
57+ dataset.name, dataset.version, "json"
58+ ),
59+ }
60+ if dataset.source_file:
61+ info["source_file"] = GITHUB_DATASETS_PREFIX + dataset.source_file
62+ dataset_info.append(info)
63+ return dataset_info
python/ephys_compression_tests/run_benchmarks/is_compatible.pyadded+43−0View file
@@ -0,0 +1,43 @@
1+from typing import List
2+
3+
4+def is_compatible(algorithm_tags: List[str], dataset_tags: List[str]) -> bool:
5+ """Check if an algorithm is compatible with a dataset based on their tags.
6+
7+ Args:
8+ algorithm_tags: List of tags for the algorithm
9+ dataset_tags: List of tags for the dataset
10+
11+ Returns:
12+ True if the algorithm should be applied to the dataset
13+ """
14+ # If algorithm has delta_encoding or lpc_prediction, dataset must have continuous, timeseries, 1d, integer
15+ if "delta_encoding" in algorithm_tags or "lpc_prediction" in algorithm_tags:
16+ if (
17+ "correlated" not in dataset_tags
18+ or "timeseries" not in dataset_tags
19+ or "1d" not in dataset_tags
20+ or "integer" not in dataset_tags
21+ ):
22+ return False
23+
24+ # If algorithm has zero_rle, dataset must have sparse, timeseries, 1d
25+ if "zero_rle" in algorithm_tags:
26+ if (
27+ "sparse" not in dataset_tags
28+ or "timeseries" not in dataset_tags
29+ or "1d" not in dataset_tags
30+ ):
31+ return False
32+
33+ # If algorithm has integer, dataset must have integer
34+ if "integer" in algorithm_tags:
35+ if "integer" not in dataset_tags:
36+ return False
37+
38+ # If algorithm has "no_bernoulli", dataset must not have "bernoulli"
39+ if "no_bernoulli" in algorithm_tags:
40+ if "bernoulli" in dataset_tags:
41+ return False
42+
43+ return True
python/ephys_compression_tests/run_benchmarks/run_benchmarks.pyadded+226−0View file
@@ -0,0 +1,226 @@
1+import os
2+import time
3+from typing import Dict, Any, List, Optional
4+import numpy as np
5+
6+from ..algorithms import algorithms
7+from ..datasets import datasets
8+from ._memobin import construct_memobin_url, upload_to_memobin
9+from .upload_dataset import upload_dataset_to_memobin
10+from .cache_management import check_cached_result, save_result_to_cache
11+from .benchmark_timing import run_compression_benchmark
12+from .collect_info import collect_algorithm_info, collect_dataset_info
13+from .is_compatible import is_compatible
14+from .upload_benchmark_status import upload_benchmark_status
15+from ..types import Algorithm, Dataset
16+
17+system_version = "v6"
18+
19+
20+def run_benchmarks(
21+ cache_dir: str = ".benchmark_cache",
22+ verbose: bool = True,
23+ selected_algorithms: Optional[List[Algorithm]] = None,
24+ selected_datasets: Optional[List[Dataset]] = None,
25+ force: bool = False,
26+) -> Dict[str, Any]:
27+ """Run all benchmarks, with caching based on algorithm and dataset versions.
28+
29+ Results are stored in separate directories for each dataset/algorithm combination:
30+ cache_dir/
31+ dataset_name/
32+ algorithm_name/
33+ metadata.json # Contains algorithm version, dataset version, and results
34+ compressed.dat # The actual compressed data
35+
36+ Args:
37+ cache_dir: Directory to store cached results
38+ verbose: Whether to print progress messages
39+ selected_algorithms: Optional list of specific algorithms to run
40+ selected_datasets: Optional list of specific datasets to run
41+ force: If True, ignore cached results
42+
43+ Returns:
44+ Dictionary containing benchmark results and metadata
45+ """
46+ print("\n=== Starting Benchmark Run ===")
47+ print(f"Cache directory: {cache_dir}")
48+
49+ os.makedirs(cache_dir, exist_ok=True)
50+
51+ start_time = time.time()
52+ last_status_upload = 0 # Track last status upload time
53+ results = []
54+ print("\nRunning benchmarks for all dataset-algorithm combinations...")
55+
56+ # Use selected datasets/algorithms or fall back to all
57+ datasets_to_run = selected_datasets if selected_datasets is not None else datasets
58+ algorithms_to_run = (
59+ selected_algorithms if selected_algorithms is not None else algorithms
60+ )
61+
62+ # Calculate total number of benchmarks
63+ total_benchmarks = sum(
64+ 1
65+ for dataset in datasets_to_run
66+ for algorithm in algorithms_to_run
67+ if is_compatible(algorithm.tags, dataset.tags)
68+ )
69+
70+ # Run benchmarks for each dataset and algorithm combination
71+ memobin_api_key = os.environ.get("MEMOBIN_API_KEY")
72+ upload_enabled = os.environ.get("UPLOAD_TO_MEMOBIN") == "1"
73+
74+ for dataset in datasets_to_run:
75+ dataset_tags = dataset.tags
76+ print(f"\n*** Dataset: {dataset.name} (tags: {dataset_tags}) ***")
77+
78+ # only create the dataset if it is needed
79+ data = None
80+
81+ for algorithm in algorithms_to_run:
82+ alg_name = algorithm.name
83+ alg_tags = algorithm.tags
84+
85+ # Skip if algorithm and dataset are not compatible based on tags
86+ if not is_compatible(alg_tags, dataset_tags):
87+ if verbose:
88+ print(
89+ f"\nSkipping algorithm {alg_name} (tags: {alg_tags}) - incompatible with dataset tags"
90+ )
91+ continue
92+
93+ print(f"\nTesting algorithm: {alg_name} on dataset: {dataset.name}")
94+
95+ # Upload current status to memobin if enabled (once per minute)
96+ current_time = time.time()
97+ if (
98+ memobin_api_key
99+ and upload_enabled
100+ and (current_time - last_status_upload >= 60)
101+ ): # Check if 60 seconds have passed
102+ try:
103+ upload_benchmark_status(
104+ memobin_api_key,
105+ dataset.name,
106+ alg_name,
107+ results,
108+ total_benchmarks,
109+ start_time,
110+ )
111+ last_status_upload = current_time # Update last upload time
112+ except Exception as e:
113+ print(f" Warning: Failed to upload status to memobin: {str(e)}")
114+
115+ # Check if we can use cached result
116+ cached_result = check_cached_result(
117+ cache_dir,
118+ dataset.name,
119+ alg_name,
120+ algorithm.version,
121+ dataset.version,
122+ system_version,
123+ force,
124+ verbose,
125+ )
126+
127+ if cached_result is not None:
128+ print(" Using cached result")
129+ results.append(cached_result)
130+ continue
131+
132+ print(f" Running benchmark for {alg_name} on {dataset.name}...")
133+ if data is None:
134+ data = dataset.create()
135+ print(f"Created dataset: shape={data.shape}, dtype={data.dtype}")
136+ else:
137+ print("Dataset already created")
138+
139+ # Upload dataset to memobin if enabled
140+ if memobin_api_key and upload_enabled:
141+ try:
142+ upload_dataset_to_memobin(
143+ data,
144+ dataset.name,
145+ dataset.version,
146+ memobin_api_key,
147+ cache_dir,
148+ verbose,
149+ )
150+ except Exception as e:
151+ print(f" Warning: Failed to upload dataset to memobin: {str(e)}")
152+
153+ # Run the benchmark
154+ result, encoded = run_compression_benchmark(
155+ data,
156+ alg_name,
157+ algorithm.encode,
158+ algorithm.decode,
159+ verbose,
160+ )
161+
162+ # Add metadata to result
163+ result.update(
164+ {
165+ "dataset": dataset.name,
166+ "algorithm": alg_name,
167+ "algorithm_version": algorithm.version,
168+ "dataset_version": dataset.version,
169+ "system_version": system_version,
170+ }
171+ )
172+ results.append(result)
173+
174+ # Save result and compressed data
175+ save_result_to_cache(
176+ result,
177+ encoded,
178+ cache_dir,
179+ dataset.name,
180+ alg_name,
181+ )
182+ print(
183+ f" Results saved to: {os.path.join(cache_dir, dataset.name, alg_name)}"
184+ )
185+
186+ # Upload to memobin if enabled
187+ if memobin_api_key and upload_enabled:
188+ try:
189+ memobin_url = construct_memobin_url(
190+ alg_name,
191+ dataset.name,
192+ algorithm.version,
193+ dataset.version,
194+ system_version,
195+ )
196+ upload_to_memobin(
197+ {"result": result},
198+ memobin_url,
199+ memobin_api_key,
200+ )
201+ if verbose:
202+ print(" Successfully uploaded to memobin")
203+ except Exception as e:
204+ print(f" Warning: Failed to upload to memobin: {str(e)}")
205+
206+ print("\n=== Benchmark Run Complete ===\n")
207+
208+ # Collect algorithm and dataset information
209+ algorithm_info = collect_algorithm_info(algorithms)
210+ dataset_info = collect_dataset_info(datasets)
211+
212+ # Upload final benchmark status
213+ if memobin_api_key and upload_enabled:
214+ try:
215+ upload_benchmark_status(
216+ memobin_api_key,
217+ "All datasets",
218+ "All algorithms",
219+ results,
220+ total_benchmarks,
221+ start_time,
222+ )
223+ except Exception as e:
224+ print(f" Warning: Failed to upload final status to memobin: {str(e)}")
225+
226+ return {"results": results, "algorithms": algorithm_info, "datasets": dataset_info}
python/ephys_compression_tests/run_benchmarks/upload_benchmark_status.pyadded+39−0View file
@@ -0,0 +1,39 @@
1+from typing import Any, Dict, List
2+import time
3+from datetime import datetime
4+from ._memobin import (
5+ upload_to_memobin,
6+)
7+
8+
9+def upload_benchmark_status(
10+ memobin_api_key: str,
11+ current_dataset: str,
12+ current_algorithm: str,
13+ completed_benchmarks: List[Dict[str, Any]],
14+ total_benchmarks: int,
15+ start_time: float,
16+) -> None:
17+ """Upload current benchmark status to memobin.
18+
19+ Args:
20+ memobin_api_key: API key for memobin authentication
21+ current_dataset: Name of the current dataset being processed
22+ current_algorithm: Name of the current algorithm being tested
23+ completed_benchmarks: List of completed benchmark results
24+ total_benchmarks: Total number of benchmarks to run
25+ start_time: Timestamp when the benchmark run started
26+ """
27+ status = {
28+ "current_dataset": current_dataset,
29+ "current_algorithm": current_algorithm,
30+ "completed_count": len(completed_benchmarks),
31+ "total_count": total_benchmarks,
32+ "progress_percentage": (len(completed_benchmarks) / total_benchmarks) * 100,
33+ "elapsed_time": time.time() - start_time,
34+ "last_update": datetime.now().isoformat(),
35+ "completed_benchmarks": completed_benchmarks,
36+ }
37+
38+ status_url = "https://tempory.net/f/memobin/benchmark_status/current.json"
39+ upload_to_memobin(status, status_url, memobin_api_key)
python/ephys_compression_tests/run_benchmarks/upload_dataset.pyadded+79−0View file
@@ -0,0 +1,79 @@
1+import os
2+import numpy as np
3+from ._memobin import (
4+ construct_dataset_url,
5+ exists_in_memobin,
6+ upload_to_memobin,
7+)
8+
9+
10+def upload_dataset_to_memobin(
11+ data: np.ndarray,
12+ dataset_name: str,
13+ dataset_version: str,
14+ memobin_api_key: str,
15+ cache_dir: str,
16+ verbose: bool = True,
17+) -> None:
18+ """Upload dataset to memobin in multiple formats.
19+
20+ Args:
21+ data: The numpy array dataset to upload
22+ dataset_name: Name of the dataset
23+ dataset_version: Version of the dataset
24+ memobin_api_key: API key for memobin
25+ cache_dir: Directory for temporary files
26+ verbose: Whether to print progress messages
27+ """
28+ try:
29+ # Upload array metadata as JSON
30+ dataset_url_json = construct_dataset_url(dataset_name, dataset_version, "json")
31+ if not exists_in_memobin(dataset_url_json):
32+ if verbose:
33+ print(" Uploading dataset metadata to memobin...")
34+ metadata = {"dtype": str(data.dtype), "shape": data.shape}
35+ upload_to_memobin(
36+ metadata,
37+ dataset_url_json,
38+ memobin_api_key,
39+ content_type="application/json",
40+ )
41+ if verbose:
42+ print(" Successfully uploaded metadata")
43+
44+ # Upload raw .dat format
45+ dataset_url_raw = construct_dataset_url(dataset_name, dataset_version, "dat")
46+ if not exists_in_memobin(dataset_url_raw):
47+ if verbose:
48+ print(" Uploading dataset (raw) to memobin...")
49+ upload_to_memobin(
50+ data.tobytes(),
51+ dataset_url_raw,
52+ memobin_api_key,
53+ content_type="application/octet-stream",
54+ )
55+ if verbose:
56+ print(" Successfully uploaded raw dataset")
57+
58+ # Upload .npy format
59+ dataset_url_npy = construct_dataset_url(dataset_name, dataset_version, "npy")
60+ if not exists_in_memobin(dataset_url_npy):
61+ if verbose:
62+ print(" Uploading dataset (npy) to memobin...")
63+ # Save array to a temporary .npy file
64+ temp_npy = os.path.join(cache_dir, "temp.npy")
65+ np.save(temp_npy, data)
66+ with open(temp_npy, "rb") as f:
67+ npy_bytes = f.read()
68+ os.remove(temp_npy) # Clean up temp file
69+
70+ upload_to_memobin(
71+ npy_bytes,
72+ dataset_url_npy,
73+ memobin_api_key,
74+ content_type="application/octet-stream",
75+ )
76+ if verbose:
77+ print(" Successfully uploaded npy dataset")
78+ except Exception as e:
79+ print(f" Warning: Failed to upload dataset to memobin: {str(e)}")
python/ephys_compression_tests/types.pyadded+42−0View file
@@ -0,0 +1,42 @@
1+from typing import Callable
2+import numpy as np
3+
4+class Algorithm:
5+ def __init__(self, *,
6+ name: str,
7+ version: str,
8+ encode: Callable[[np.ndarray], bytes],
9+ decode: Callable[[bytes, np.dtype, tuple], np.ndarray],
10+ description: str,
11+ tags: list[str],
12+ source_file: str,
13+ long_description: str
14+ ):
15+ self.name = name
16+ self.version = version
17+ self.encode = encode
18+ self.decode = decode
19+ self.description = description
20+ self.tags = tags
21+ self.source_file = source_file
22+ self.long_description = long_description
23+
24+class Dataset:
25+ def __init__(self, *,
26+ name: str,
27+ version: str,
28+ create: Callable[[], np.ndarray],
29+ description: str,
30+ tags: list[str],
31+ source_file: str,
32+ long_description: str,
33+ ideal_compression_ratio: float = 0
34+ ):
35+ self.name = name
36+ self.version = version
37+ self.create = create
38+ self.description = description
39+ self.tags = tags
40+ self.source_file = source_file
41+ self.long_description = long_description
42+ self.ideal_compression_ratio = ideal_compression_ratio
python/pyproject.tomladded+37−0View file
@@ -0,0 +1,37 @@
1+[build-system]
2+requires = ["setuptools>=61.0", "wheel"]
3+build-backend = "setuptools.build_meta"
4+
5+[project]
6+name = "ephys_compression_tests"
7+version = "0.1.0"
8+description = "Benchmarking compression methods for electrophysiology data"
9+readme = "README.md"
10+requires-python = ">=3.8"
11+authors = [
12+ { name = "Jeremy Magland" }
13+]
14+dependencies = [
15+ "numpy",
16+ "scipy",
17+ "zstandard",
18+ "simple_ans",
19+ "requests",
20+ "lindi",
21+ "brotli",
22+ "click",
23+ "numba",
24+ "segyio",
25+ "lz4",
26+ "pyedflib",
27+ "nibabel",
28+ "blosc2",
29+ "wavpack-numcodecs"
30+]
31+
32+[tool.setuptools.packages.find]
33+where = ["."]
34+include = ["ephys_compression_tests*"]
35+
36+[project.scripts]
37+ephys_compression_tests = "ephys_compression_tests.cli:main"
scripts/run_benchmarks.pyadded+38−0View file
@@ -0,0 +1,38 @@
1+#!/usr/bin/env python3
2+
3+import json
4+import os
5+from pathlib import Path
6+from ephys_compression_tests import run_benchmarks
7+from ephys_compression_tests.run_benchmarks._memobin import upload_to_memobin, construct_memobin_url
8+
9+def main():
10+ # Run benchmarks
11+ print("Running benchmarks...")
12+ results = run_benchmarks()
13+
14+ # Save detailed results to JSON
15+ output_dir = Path("benchmark_results")
16+ output_dir.mkdir(exist_ok=True)
17+ output_file = output_dir / "results.json"
18+
19+ with open(output_file, "w") as f:
20+ json.dump(results, f, indent=2)
21+
22+ print(f"\nDetailed results saved to {output_file}")
23+
24+ # Upload results to memobin if enabled
25+ memobin_api_key = os.environ.get("MEMOBIN_API_KEY")
26+ upload_enabled = os.environ.get("UPLOAD_TO_MEMOBIN") == "1"
27+
28+ if memobin_api_key and upload_enabled:
29+ try:
30+ # Construct URL for the global results file
31+ url = "https://tempory.net/f/memobin/ephys_compression_tests/global/results.json"
32+ upload_to_memobin(results, url, memobin_api_key)
33+ print("Successfully uploaded results to memobin")
34+ except Exception as e:
35+ print(f"Warning: Failed to upload results to memobin: {str(e)}")
36+
37+if __name__ == "__main__":
38+ main()