Update benchmark results from 2025-07-24 16:44:35 [skip ci]
158 changed files+47216−0
.github/workflows/benchmark.ymladded+75−0View file
@@ -0,0 +1,75 @@
1+name: Run Benchmarks
2+
3+# Add concurrency to cancel in-progress jobs
4+concurrency:
5+ group: ${{ github.workflow }}-${{ github.ref }}
6+ cancel-in-progress: true
7+
8+on:
9+ workflow_dispatch: # Manual trigger
10+ push:
11+ branches: [ main ] # Run on main branch pushes
12+ paths:
13+ - 'benchcompress/**' # Run only if benchmarks are updated
14+
15+permissions:
16+ contents: write
17+
18+jobs:
19+ benchmark:
20+ name: Run Benchmarks
21+ runs-on: ubuntu-latest
22+
23+ steps:
24+ - uses: actions/checkout@v4
25+
26+ - name: Set up Python
27+ uses: actions/setup-python@v4
28+ with:
29+ python-version: '3.12'
30+
31+ - name: Install package and dependencies
32+ run: |
33+ cd benchcompress
34+ pip install -e .
35+
36+ - name: Install simple_ans from source for now
37+ run: |
38+ git clone https://github.com/flatironinstitute/simple_ans.git
39+ cd simple_ans
40+ pip install -e .
41+
42+ - name: Run benchmarks
43+ env:
44+ MEMOBIN_API_KEY: ${{ secrets.MEMOBIN_API_KEY }}
45+ UPLOAD_TO_MEMOBIN: '1'
46+ run: |
47+ python scripts/run_benchmarks.py
48+
49+ - name: Upload benchmark results as artifacts
50+ uses: actions/upload-artifact@v4
51+ with:
52+ name: benchmark-results
53+ path: |
54+ benchcompress/benchmark_results/results.json
55+
56+ - name: Configure Git
57+ if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
58+ run: |
59+ git config user.name "GitHub Actions Bot"
60+ git config user.email "actions@github.com"
61+
62+ - name: Create fresh results branch
63+ if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
64+ run: |
65+ git checkout --orphan benchmark-results
66+
67+ - name: Commit benchmark results
68+ if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
69+ env:
70+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
71+ run: |
72+ # need to force add because it's in the .gitignore
73+ git add -f benchmark_results/
74+ git commit -m "Update benchmark results from $(date +'%Y-%m-%d %H:%M:%S') [skip ci]"
75+ git push -f https://${GITHUB_TOKEN}@github.com/${{ github.repository }} benchmark-results
.github/workflows/deploy-gh-pages.ymladded+67−0View file
@@ -0,0 +1,67 @@
1+name: Deploy to GitHub Pages
2+
3+on:
4+ push:
5+ branches: ['main']
6+ paths:
7+ - 'web-ui/**'
8+ - 'paper/**'
9+ workflow_dispatch:
10+
11+# Sets the GITHUB_TOKEN permissions to allow deployment to GitHub Pages
12+permissions:
13+ contents: read
14+ pages: write
15+ id-token: write
16+
17+# Allow one concurrent deployment
18+concurrency:
19+ group: 'pages'
20+ cancel-in-progress: true
21+
22+jobs:
23+ deploy:
24+ environment:
25+ name: github-pages
26+ url: ${{ steps.deployment.outputs.page_url }}
27+ runs-on: ubuntu-latest
28+ steps:
29+ - name: Checkout
30+ uses: actions/checkout@v4
31+
32+ - name: Install pandoc and texlive
33+ run: |
34+ sudo apt-get update
35+ sudo apt-get install -y pandoc texlive-latex-recommended
36+
37+ - name: Setup Node.js
38+ uses: actions/setup-node@v3
39+ with:
40+ node-version: '20'
41+ cache: 'npm'
42+ cache-dependency-path: web-ui/package-lock.json
43+
44+ - name: Build paper
45+ run: |
46+ chmod +x devel/copy_paper_to_web.sh
47+ ./devel/copy_paper_to_web.sh
48+
49+ - name: Install dependencies
50+ working-directory: web-ui
51+ run: npm ci
52+
53+ - name: Build website
54+ working-directory: web-ui
55+ run: npm run build
56+
57+ - name: Setup Pages
58+ uses: actions/configure-pages@v4
59+
60+ - name: Upload artifact
61+ uses: actions/upload-pages-artifact@v3
62+ with:
63+ path: ./web-ui/dist
64+
65+ - name: Deploy to GitHub Pages
66+ id: deployment
67+ uses: actions/deploy-pages@v4
.gitignoreadded+11−0View file
@@ -0,0 +1,11 @@
1+*.out
2+benchmark_results
3+.benchmark_cache
4+__pycache__
5+*.dat
6+*.npy
7+*.segy
8+*.mat
9+*.edf
10+*.zip
11+data
.pre-commit-config.yamladded+8−0View file
@@ -0,0 +1,8 @@
1+repos:
2+ - repo: local
3+ hooks:
4+ - id: check-format
5+ name: Check Code Formatting
6+ entry: devel/check_format.sh
7+ language: script
8+ pass_filenames: false
.vscode/tasks.jsonadded+55−0View file
@@ -0,0 +1,55 @@
1+{
2+ "version": "2.0.0",
3+ "tasks": [
4+ {
5+ "label": "Compile Paper",
6+ "type": "shell",
7+ "command": "./devel/compile_paper.sh",
8+ "group": "build",
9+ "presentation": {
10+ "reveal": "always",
11+ "panel": "new"
12+ },
13+ "problemMatcher": []
14+ },
15+ {
16+ "label": "Generate Posts Index",
17+ "type": "shell",
18+ "command": "./devel/generate_posts_index.sh",
19+ "group": "build",
20+ "presentation": {
21+ "reveal": "silent",
22+ "panel": "shared"
23+ },
24+ "problemMatcher": []
25+ },
26+ {
27+ "label": "Format Code",
28+ "type": "shell",
29+ "command": "./devel/format_code.sh",
30+ "group": {
31+ "kind": "build",
32+ "isDefault": true
33+ },
34+ "presentation": {
35+ "reveal": "always",
36+ "panel": "shared"
37+ },
38+ "problemMatcher": []
39+ },
40+ {
41+ "label": "Web UI Dev Server",
42+ "type": "shell",
43+ "command": "cd web-ui && npm run dev",
44+ "group": {
45+ "kind": "build",
46+ "isDefault": true
47+ },
48+ "presentation": {
49+ "reveal": "always",
50+ "panel": "new"
51+ },
52+ "problemMatcher": []
53+ }
54+ ]
55+}
LICENSEadded+201−0View file
@@ -0,0 +1,201 @@
1+ Apache License
2+ Version 2.0, January 2004
3+ http://www.apache.org/licenses/
4+
5+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6+
7+ 1. Definitions.
8+
9+ "License" shall mean the terms and conditions for use, reproduction,
10+ and distribution as defined by Sections 1 through 9 of this document.
11+
12+ "Licensor" shall mean the copyright owner or entity authorized by
13+ the copyright owner that is granting the License.
14+
15+ "Legal Entity" shall mean the union of the acting entity and all
16+ other entities that control, are controlled by, or are under common
17+ control with that entity. For the purposes of this definition,
18+ "control" means (i) the power, direct or indirect, to cause the
19+ direction or management of such entity, whether by contract or
20+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21+ outstanding shares, or (iii) beneficial ownership of such entity.
22+
23+ "You" (or "Your") shall mean an individual or Legal Entity
24+ exercising permissions granted by this License.
25+
26+ "Source" form shall mean the preferred form for making modifications,
27+ including but not limited to software source code, documentation
28+ source, and configuration files.
29+
30+ "Object" form shall mean any form resulting from mechanical
31+ transformation or translation of a Source form, including but
32+ not limited to compiled object code, generated documentation,
33+ and conversions to other media types.
34+
35+ "Work" shall mean the work of authorship, whether in Source or
36+ Object form, made available under the License, as indicated by a
37+ copyright notice that is included in or attached to the work
38+ (an example is provided in the Appendix below).
39+
40+ "Derivative Works" shall mean any work, whether in Source or Object
41+ form, that is based on (or derived from) the Work and for which the
42+ editorial revisions, annotations, elaborations, or other modifications
43+ represent, as a whole, an original work of authorship. For the purposes
44+ of this License, Derivative Works shall not include works that remain
45+ separable from, or merely link (or bind by name) to the interfaces of,
46+ the Work and Derivative Works thereof.
47+
48+ "Contribution" shall mean any work of authorship, including
49+ the original version of the Work and any modifications or additions
50+ to that Work or Derivative Works thereof, that is intentionally
51+ submitted to Licensor for inclusion in the Work by the copyright owner
52+ or by an individual or Legal Entity authorized to submit on behalf of
53+ the copyright owner. For the purposes of this definition, "submitted"
54+ means any form of electronic, verbal, or written communication sent
55+ to the Licensor or its representatives, including but not limited to
56+ communication on electronic mailing lists, source code control systems,
57+ and issue tracking systems that are managed by, or on behalf of, the
58+ Licensor for the purpose of discussing and improving the Work, but
59+ excluding communication that is conspicuously marked or otherwise
60+ designated in writing by the copyright owner as "Not a Contribution."
61+
62+ "Contributor" shall mean Licensor and any individual or Legal Entity
63+ on behalf of whom a Contribution has been received by Licensor and
64+ subsequently incorporated within the Work.
65+
66+ 2. Grant of Copyright License. Subject to the terms and conditions of
67+ this License, each Contributor hereby grants to You a perpetual,
68+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69+ copyright license to reproduce, prepare Derivative Works of,
70+ publicly display, publicly perform, sublicense, and distribute the
71+ Work and such Derivative Works in Source or Object form.
72+
73+ 3. Grant of Patent License. Subject to the terms and conditions of
74+ this License, each Contributor hereby grants to You a perpetual,
75+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76+ (except as stated in this section) patent license to make, have made,
77+ use, offer to sell, sell, import, and otherwise transfer the Work,
78+ where such license applies only to those patent claims licensable
79+ by such Contributor that are necessarily infringed by their
80+ Contribution(s) alone or by combination of their Contribution(s)
81+ with the Work to which such Contribution(s) was submitted. If You
82+ institute patent litigation against any entity (including a
83+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84+ or a Contribution incorporated within the Work constitutes direct
85+ or contributory patent infringement, then any patent licenses
86+ granted to You under this License for that Work shall terminate
87+ as of the date such litigation is filed.
88+
89+ 4. Redistribution. You may reproduce and distribute copies of the
90+ Work or Derivative Works thereof in any medium, with or without
91+ modifications, and in Source or Object form, provided that You
92+ meet the following conditions:
93+
94+ (a) You must give any other recipients of the Work or
95+ Derivative Works a copy of this License; and
96+
97+ (b) You must cause any modified files to carry prominent notices
98+ stating that You changed the files; and
99+
100+ (c) You must retain, in the Source form of any Derivative Works
101+ that You distribute, all copyright, patent, trademark, and
102+ attribution notices from the Source form of the Work,
103+ excluding those notices that do not pertain to any part of
104+ the Derivative Works; and
105+
106+ (d) If the Work includes a "NOTICE" text file as part of its
107+ distribution, then any Derivative Works that You distribute must
108+ include a readable copy of the attribution notices contained
109+ within such NOTICE file, excluding those notices that do not
110+ pertain to any part of the Derivative Works, in at least one
111+ of the following places: within a NOTICE text file distributed
112+ as part of the Derivative Works; within the Source form or
113+ documentation, if provided along with the Derivative Works; or,
114+ within a display generated by the Derivative Works, if and
115+ wherever such third-party notices normally appear. The contents
116+ of the NOTICE file are for informational purposes only and
117+ do not modify the License. You may add Your own attribution
118+ notices within Derivative Works that You distribute, alongside
119+ or as an addendum to the NOTICE text from the Work, provided
120+ that such additional attribution notices cannot be construed
121+ as modifying the License.
122+
123+ You may add Your own copyright statement to Your modifications and
124+ may provide additional or different license terms and conditions
125+ for use, reproduction, or distribution of Your modifications, or
126+ for any such Derivative Works as a whole, provided Your use,
127+ reproduction, and distribution of the Work otherwise complies with
128+ the conditions stated in this License.
129+
130+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131+ any Contribution intentionally submitted for inclusion in the Work
132+ by You to the Licensor shall be under the terms and conditions of
133+ this License, without any additional terms or conditions.
134+ Notwithstanding the above, nothing herein shall supersede or modify
135+ the terms of any separate license agreement you may have executed
136+ with Licensor regarding such Contributions.
137+
138+ 6. Trademarks. This License does not grant permission to use the trade
139+ names, trademarks, service marks, or product names of the Licensor,
140+ except as required for reasonable and customary use in describing the
141+ origin of the Work and reproducing the content of the NOTICE file.
142+
143+ 7. Disclaimer of Warranty. Unless required by applicable law or
144+ agreed to in writing, Licensor provides the Work (and each
145+ Contributor provides its Contributions) on an "AS IS" BASIS,
146+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147+ implied, including, without limitation, any warranties or conditions
148+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149+ PARTICULAR PURPOSE. You are solely responsible for determining the
150+ appropriateness of using or redistributing the Work and assume any
151+ risks associated with Your exercise of permissions under this License.
152+
153+ 8. Limitation of Liability. In no event and under no legal theory,
154+ whether in tort (including negligence), contract, or otherwise,
155+ unless required by applicable law (such as deliberate and grossly
156+ negligent acts) or agreed to in writing, shall any Contributor be
157+ liable to You for damages, including any direct, indirect, special,
158+ incidental, or consequential damages of any character arising as a
159+ result of this License or out of the use or inability to use the
160+ Work (including but not limited to damages for loss of goodwill,
161+ work stoppage, computer failure or malfunction, or any and all
162+ other commercial damages or losses), even if such Contributor
163+ has been advised of the possibility of such damages.
164+
165+ 9. Accepting Warranty or Additional Liability. While redistributing
166+ the Work or Derivative Works thereof, You may choose to offer,
167+ and charge a fee for, acceptance of support, warranty, indemnity,
168+ or other liability obligations and/or rights consistent with this
169+ License. However, in accepting such obligations, You may act only
170+ on Your own behalf and on Your sole responsibility, not on behalf
171+ of any other Contributor, and only if You agree to indemnify,
172+ defend, and hold each Contributor harmless for any liability
173+ incurred by, or claims asserted against, such Contributor by reason
174+ of your accepting any such warranty or additional liability.
175+
176+ END OF TERMS AND CONDITIONS
177+
178+ APPENDIX: How to apply the Apache License to your work.
179+
180+ To apply the Apache License to your work, attach the following
181+ boilerplate notice, with the fields enclosed by brackets "[]"
182+ replaced with your own identifying information. (Don't include
183+ the brackets!) The text should be enclosed in the appropriate
184+ comment syntax for the file format. We also recommend that a
185+ file or class name and description of purpose be included on the
186+ same "printed page" as the copyright notice for easier
187+ identification within third-party archives.
188+
189+ Copyright [yyyy] [name of copyright owner]
190+
191+ Licensed under the Apache License, Version 2.0 (the "License");
192+ you may not use this file except in compliance with the License.
193+ You may obtain a copy of the License at
194+
195+ http://www.apache.org/licenses/LICENSE-2.0
196+
197+ Unless required by applicable law or agreed to in writing, software
198+ distributed under the License is distributed on an "AS IS" BASIS,
199+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200+ See the License for the specific language governing permissions and
201+ limitations under the License.
README.mdadded+92−0View file
@@ -0,0 +1,92 @@
1+# benchcompress
2+
3+A benchmarking framework for evaluating compression algorithms on scientific data arrays.
4+
5+Latest benchmark results: https://magland.github.io/benchcompress/
6+
7+Paper (WIP): https://magland.github.io/benchcompress/paper
8+
9+## Overview
10+
11+Benchcompress is a comprehensive benchmarking framework for evaluating compression algorithms on scientific data arrays. The system follows an automated workflow:
12+
13+1. **Defining Components**
14+ - Algorithms are implemented in `benchcompress/src/benchcompress/algorithms/`
15+ - Datasets are defined in `benchcompress/src/benchcompress/datasets/`
16+ - Each component specifies metadata like version, tags, and compatibility requirements
17+
18+2. **Automated Benchmarking**
19+ - Benchmarks run automatically via GitHub Actions on pushes to main branch
20+ - For each compatible algorithm-dataset pair, measures:
21+ - Compression ratio
22+ - Encoding throughput (MB/s)
23+ - Decoding throughput (MB/s)
24+ - Results are verified by decompressing and comparing with original data
25+
26+3. **Result Storage**
27+ - Results are committed to a dedicated `benchmark-results` branch
28+ - Local and remote caching system prevents redundant rerunning of benchmarks (only modified or added components are re-benchmarked)
29+ - Caching is based on algorithm and dataset versions
30+
31+4. **Web Interface**
32+ - Interactive visualization at https://magland.github.io/benchcompress/
33+ - Filter and sort results by dataset or algorithm
34+ - Visual charts for comparing performance metrics
35+ - Export results to CSV for further analysis
36+
37+## For developers
38+
39+The project consists of two main components:
40+
41+- `benchcompress/`: Python package containing the core benchmarking framework, algorithms, and datasets
42+- `web-ui/`: React-based web interface for visualizing benchmark results
43+
44+### Local Development Setup
45+
46+1. Install Python dependencies:
47+```bash
48+# You may need to first install wavpack
49+# apt-get install libwavpack-dev
50+
51+cd benchcompress
52+pip install -e .
53+
54+benchcompress --help
55+benchcompress list
56+benchcompress run --help
57+```
58+
59+2. Install web UI dependencies:
60+```bash
61+cd web-ui
62+npm install
63+```
64+
65+3. Run web UI locally:
66+```bash
67+cd web-ui
68+npm run dev
69+```
70+
71+### Code Formatting
72+
73+This project uses pre-commit hooks to automatically check format code before each commit. The formatting includes:
74+- Python code formatting using black
75+- TypeScript/JavaScript code formatting using npm scripts
76+- C++ code formatting using clang-format
77+
78+To set up the pre-commit hooks after cloning the repository:
79+
80+1. Install pre-commit:
81+```bash
82+pip install pre-commit
83+```
84+
85+2. Install the git hook scripts:
86+```bash
87+pre-commit install
88+```
89+
90+After this setup, code will be automatically checked for formatting when you make a commit.
91+
92+Running `./devel/format_code.sh` which will format all code in the repository.
benchcompress/.gitignoreadded+41−0View file
@@ -0,0 +1,41 @@
1+# Python
2+__pycache__/
3+*.py[cod]
4+*$py.class
5+*.so
6+.Python
7+build/
8+develop-eggs/
9+dist/
10+downloads/
11+eggs/
12+.eggs/
13+lib/
14+lib64/
15+parts/
16+sdist/
17+var/
18+wheels/
19+*.egg-info/
20+.installed.cfg
21+*.egg
22+
23+# Virtual Environment
24+venv/
25+env/
26+ENV/
27+
28+# IDE
29+.idea/
30+.vscode/
31+*.swp
32+*.swo
33+
34+# Testing
35+.coverage
36+htmlcov/
37+.pytest_cache/
38+.mypy_cache/
39+
40+# Misc
41+.DS_Store
benchcompress/LICENSEadded+201−0View file
@@ -0,0 +1,201 @@
1+ Apache License
2+ Version 2.0, January 2004
3+ http://www.apache.org/licenses/
4+
5+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6+
7+ 1. Definitions.
8+
9+ "License" shall mean the terms and conditions for use, reproduction,
10+ and distribution as defined by Sections 1 through 9 of this document.
11+
12+ "Licensor" shall mean the copyright owner or entity authorized by
13+ the copyright owner that is granting the License.
14+
15+ "Legal Entity" shall mean the union of the acting entity and all
16+ other entities that control, are controlled by, or are under common
17+ control with that entity. For the purposes of this definition,
18+ "control" means (i) the power, direct or indirect, to cause the
19+ direction or management of such entity, whether by contract or
20+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21+ outstanding shares, or (iii) beneficial ownership of such entity.
22+
23+ "You" (or "Your") shall mean an individual or Legal Entity
24+ exercising permissions granted by this License.
25+
26+ "Source" form shall mean the preferred form for making modifications,
27+ including but not limited to software source code, documentation
28+ source, and configuration files.
29+
30+ "Object" form shall mean any form resulting from mechanical
31+ transformation or translation of a Source form, including but
32+ not limited to compiled object code, generated documentation,
33+ and conversions to other media types.
34+
35+ "Work" shall mean the work of authorship, whether in Source or
36+ Object form, made available under the License, as indicated by a
37+ copyright notice that is included in or attached to the work
38+ (an example is provided in the Appendix below).
39+
40+ "Derivative Works" shall mean any work, whether in Source or Object
41+ form, that is based on (or derived from) the Work and for which the
42+ editorial revisions, annotations, elaborations, or other modifications
43+ represent, as a whole, an original work of authorship. For the purposes
44+ of this License, Derivative Works shall not include works that remain
45+ separable from, or merely link (or bind by name) to the interfaces of,
46+ the Work and Derivative Works thereof.
47+
48+ "Contribution" shall mean any work of authorship, including
49+ the original version of the Work and any modifications or additions
50+ to that Work or Derivative Works thereof, that is intentionally
51+ submitted to Licensor for inclusion in the Work by the copyright owner
52+ or by an individual or Legal Entity authorized to submit on behalf of
53+ the copyright owner. For the purposes of this definition, "submitted"
54+ means any form of electronic, verbal, or written communication sent
55+ to the Licensor or its representatives, including but not limited to
56+ communication on electronic mailing lists, source code control systems,
57+ and issue tracking systems that are managed by, or on behalf of, the
58+ Licensor for the purpose of discussing and improving the Work, but
59+ excluding communication that is conspicuously marked or otherwise
60+ designated in writing by the copyright owner as "Not a Contribution."
61+
62+ "Contributor" shall mean Licensor and any individual or Legal Entity
63+ on behalf of whom a Contribution has been received by Licensor and
64+ subsequently incorporated within the Work.
65+
66+ 2. Grant of Copyright License. Subject to the terms and conditions of
67+ this License, each Contributor hereby grants to You a perpetual,
68+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69+ copyright license to reproduce, prepare Derivative Works of,
70+ publicly display, publicly perform, sublicense, and distribute the
71+ Work and such Derivative Works in Source or Object form.
72+
73+ 3. Grant of Patent License. Subject to the terms and conditions of
74+ this License, each Contributor hereby grants to You a perpetual,
75+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76+ (except as stated in this section) patent license to make, have made,
77+ use, offer to sell, sell, import, and otherwise transfer the Work,
78+ where such license applies only to those patent claims licensable
79+ by such Contributor that are necessarily infringed by their
80+ Contribution(s) alone or by combination of their Contribution(s)
81+ with the Work to which such Contribution(s) was submitted. If You
82+ institute patent litigation against any entity (including a
83+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84+ or a Contribution incorporated within the Work constitutes direct
85+ or contributory patent infringement, then any patent licenses
86+ granted to You under this License for that Work shall terminate
87+ as of the date such litigation is filed.
88+
89+ 4. Redistribution. You may reproduce and distribute copies of the
90+ Work or Derivative Works thereof in any medium, with or without
91+ modifications, and in Source or Object form, provided that You
92+ meet the following conditions:
93+
94+ (a) You must give any other recipients of the Work or
95+ Derivative Works a copy of this License; and
96+
97+ (b) You must cause any modified files to carry prominent notices
98+ stating that You changed the files; and
99+
100+ (c) You must retain, in the Source form of any Derivative Works
101+ that You distribute, all copyright, patent, trademark, and
102+ attribution notices from the Source form of the Work,
103+ excluding those notices that do not pertain to any part of
104+ the Derivative Works; and
105+
106+ (d) If the Work includes a "NOTICE" text file as part of its
107+ distribution, then any Derivative Works that You distribute must
108+ include a readable copy of the attribution notices contained
109+ within such NOTICE file, excluding those notices that do not
110+ pertain to any part of the Derivative Works, in at least one
111+ of the following places: within a NOTICE text file distributed
112+ as part of the Derivative Works; within the Source form or
113+ documentation, if provided along with the Derivative Works; or,
114+ within a display generated by the Derivative Works, if and
115+ wherever such third-party notices normally appear. The contents
116+ of the NOTICE file are for informational purposes only and
117+ do not modify the License. You may add Your own attribution
118+ notices within Derivative Works that You distribute, alongside
119+ or as an addendum to the NOTICE text from the Work, provided
120+ that such additional attribution notices cannot be construed
121+ as modifying the License.
122+
123+ You may add Your own copyright statement to Your modifications and
124+ may provide additional or different license terms and conditions
125+ for use, reproduction, or distribution of Your modifications, or
126+ for any such Derivative Works as a whole, provided Your use,
127+ reproduction, and distribution of the Work otherwise complies with
128+ the conditions stated in this License.
129+
130+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131+ any Contribution intentionally submitted for inclusion in the Work
132+ by You to the Licensor shall be under the terms and conditions of
133+ this License, without any additional terms or conditions.
134+ Notwithstanding the above, nothing herein shall supersede or modify
135+ the terms of any separate license agreement you may have executed
136+ with Licensor regarding such Contributions.
137+
138+ 6. Trademarks. This License does not grant permission to use the trade
139+ names, trademarks, service marks, or product names of the Licensor,
140+ except as required for reasonable and customary use in describing the
141+ origin of the Work and reproducing the content of the NOTICE file.
142+
143+ 7. Disclaimer of Warranty. Unless required by applicable law or
144+ agreed to in writing, Licensor provides the Work (and each
145+ Contributor provides its Contributions) on an "AS IS" BASIS,
146+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147+ implied, including, without limitation, any warranties or conditions
148+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149+ PARTICULAR PURPOSE. You are solely responsible for determining the
150+ appropriateness of using or redistributing the Work and assume any
151+ risks associated with Your exercise of permissions under this License.
152+
153+ 8. Limitation of Liability. In no event and under no legal theory,
154+ whether in tort (including negligence), contract, or otherwise,
155+ unless required by applicable law (such as deliberate and grossly
156+ negligent acts) or agreed to in writing, shall any Contributor be
157+ liable to You for damages, including any direct, indirect, special,
158+ incidental, or consequential damages of any character arising as a
159+ result of this License or out of the use or inability to use the
160+ Work (including but not limited to damages for loss of goodwill,
161+ work stoppage, computer failure or malfunction, or any and all
162+ other commercial damages or losses), even if such Contributor
163+ has been advised of the possibility of such damages.
164+
165+ 9. Accepting Warranty or Additional Liability. While redistributing
166+ the Work or Derivative Works thereof, You may choose to offer,
167+ and charge a fee for, acceptance of support, warranty, indemnity,
168+ or other liability obligations and/or rights consistent with this
169+ License. However, in accepting such obligations, You may act only
170+ on Your own behalf and on Your sole responsibility, not on behalf
171+ of any other Contributor, and only if You agree to indemnify,
172+ defend, and hold each Contributor harmless for any liability
173+ incurred by, or claims asserted against, such Contributor by reason
174+ of your accepting any such warranty or additional liability.
175+
176+ END OF TERMS AND CONDITIONS
177+
178+ APPENDIX: How to apply the Apache License to your work.
179+
180+ To apply the Apache License to your work, attach the following
181+ boilerplate notice, with the fields enclosed by brackets "[]"
182+ replaced with your own identifying information. (Don't include
183+ the brackets!) The text should be enclosed in the appropriate
184+ comment syntax for the file format. We also recommend that a
185+ file or class name and description of purpose be included on the
186+ same "printed page" as the copyright notice for easier
187+ identification within third-party archives.
188+
189+ Copyright [yyyy] [name of copyright owner]
190+
191+ Licensed under the Apache License, Version 2.0 (the "License");
192+ you may not use this file except in compliance with the License.
193+ You may obtain a copy of the License at
194+
195+ http://www.apache.org/licenses/LICENSE-2.0
196+
197+ Unless required by applicable law or agreed to in writing, software
198+ distributed under the License is distributed on an "AS IS" BASIS,
199+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200+ See the License for the specific language governing permissions and
201+ limitations under the License.
benchcompress/MANIFEST.inadded+1−0View file
@@ -0,0 +1 @@
1+recursive-include src/benchcompress *.md
benchcompress/README.mdadded+1−0View file
@@ -0,0 +1 @@
1+# benchcompress
benchcompress/pyproject.tomladded+40−0View file
@@ -0,0 +1,40 @@
1+[build-system]
2+requires = ["scikit-build-core>=0.5.0", "pybind11>=2.11.1"]
3+build-backend = "scikit_build_core.build"
4+
5+[project]
6+name = "benchcompress"
7+version = "0.1.0"
8+description = "Benchmarking compression methods for integer timeseries data"
9+readme = "README.md"
10+requires-python = ">=3.8"
11+authors = [
12+ { name = "Jeremy Magland" }
13+]
14+dependencies = [
15+ "numpy",
16+ "scipy",
17+ "zstandard",
18+ "simple_ans",
19+ "requests",
20+ "lindi",
21+ "brotli",
22+ "click",
23+ "numba",
24+ "pybind11>=2.11.1",
25+ "segyio",
26+ "lz4",
27+ "pyedflib",
28+ "nibabel",
29+ "blosc2",
30+ "wavpack-numcodecs"
31+]
32+
33+[tool.scikit-build]
34+cmake.minimum-version = "3.15"
35+cmake.source-dir = "src/benchcompress/algorithms/ans"
36+cmake.build-type = "Release"
37+wheel.packages = ["src/benchcompress"]
38+
39+[project.scripts]
40+benchcompress = "benchcompress.cli:main"
benchcompress/setup.pyadded+66−0View file
@@ -0,0 +1,66 @@
1+from setuptools import setup, find_packages
2+from setuptools.command.build_ext import build_ext
3+from setuptools import Extension
4+import os
5+import sys
6+import subprocess
7+
8+class CMakeExtension(Extension):
9+ def __init__(self, name, sourcedir=""):
10+ Extension.__init__(self, name, sources=[])
11+ self.sourcedir = os.path.abspath(sourcedir)
12+
13+class CMakeBuild(build_ext):
14+ def build_extension(self, ext):
15+ extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name)))
16+
17+ cmake_args = [
18+ f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}",
19+ f"-DPYTHON_EXECUTABLE={sys.executable}"
20+ ]
21+
22+ build_args = []
23+
24+ if not os.path.exists(self.build_temp):
25+ os.makedirs(self.build_temp)
26+
27+ subprocess.check_call(["cmake", ext.sourcedir] + cmake_args, cwd=self.build_temp)
28+ subprocess.check_call(["cmake", "--build", "."] + build_args, cwd=self.build_temp)
29+
30+setup(
31+ name="benchcompress",
32+ version="0.1.0",
33+ packages=find_packages(where="src"),
34+ package_dir={"": "src"},
35+ ext_modules=[
36+ CMakeExtension("benchcompress.algorithms.ans.lpc_reconstruct_cpp_ext",
37+ sourcedir="src/benchcompress/algorithms/ans"),
38+ CMakeExtension("benchcompress.algorithms.ans.lpc_predict_cpp_ext",
39+ sourcedir="src/benchcompress/algorithms/ans"),
40+ CMakeExtension("benchcompress.algorithms.ans.get_run_lengths_cpp_ext",
41+ sourcedir="src/benchcompress/algorithms/ans")
42+ ],
43+ cmdclass={
44+ "build_ext": CMakeBuild,
45+ },
46+ python_requires=">=3.8",
47+ entry_points={
48+ "console_scripts": [
49+ "benchcompress=benchcompress.cli:main",
50+ ],
51+ },
52+ author="Jeremy Magland",
53+ description="Benchmarking compression methods for scientific data arrays",
54+ long_description=open("README.md").read(),
55+ long_description_content_type="text/markdown",
56+ keywords="compression, signal processing",
57+ classifiers=[
58+ "Development Status :: 3 - Alpha",
59+ "Intended Audience :: Science/Research",
60+ "Programming Language :: Python :: 3",
61+ "Programming Language :: Python :: 3.8",
62+ "Programming Language :: Python :: 3.9",
63+ "Programming Language :: Python :: 3.10",
64+ "Programming Language :: Python :: 3.11"
65+ ],
66+)
benchcompress/src/benchcompress/__init__.pyadded+5−0View file
@@ -0,0 +1,5 @@
1+from .algorithms import algorithms
2+from .datasets import datasets
3+from .run_benchmarks import run_benchmarks
4+
5+__all__ = ["algorithms", "datasets", "run_benchmarks"]
benchcompress/src/benchcompress/_analysis.pyadded+65−0View file
@@ -0,0 +1,65 @@
1+import numpy as np
2+from ._filters import highpass_filter
3+
4+
5+def estimate_noise_level(array: np.ndarray, *, sampling_frequency: float) -> float:
6+ """Estimate the noise level of a signal using the median absolute deviation.
7+
8+ Args:
9+ array: Input signal array
10+ sampling_frequency: Sampling frequency in Hz
11+
12+ Returns:
13+ Estimated noise level
14+ """
15+ array_filtered = highpass_filter(
16+ array, sampling_frequency=sampling_frequency, lowcut=300
17+ )
18+ MAD = float(
19+ np.median(np.abs(array_filtered.ravel() - np.median(array_filtered.ravel())))
20+ / 0.6745
21+ )
22+ return MAD
23+
24+
25+def compute_entropy_per_sample(array: np.ndarray) -> float:
26+ """Compute the entropy per sample of a signal.
27+
28+ Args:
29+ array: Input signal array
30+
31+ Returns:
32+ Entropy per sample in bits
33+ """
34+ _, counts = np.unique(array, return_counts=True)
35+ p = counts / len(array)
36+ return float(-np.sum(p * np.log2(p)))
37+
38+
39+from typing import Callable
40+
41+
42+def linear_fit(
43+ x: np.ndarray, y: np.ndarray
44+) -> tuple[np.ndarray, Callable[[np.ndarray], np.ndarray]]:
45+ """Perform linear fit with constant term.
46+
47+ Args:
48+ x: Input array of shape (N, M-1) containing M-1 predictors for N samples
49+ y: Target array of shape (N,) containing values to predict
50+
51+ Returns:
52+ Tuple containing:
53+ - coefficients array of shape (M,)
54+ - prediction function that takes x_new and returns predictions
55+ """
56+ from numpy.linalg import lstsq
57+
58+ X = np.column_stack([x, np.ones(len(x))])
59+ coeffs = lstsq(X, y, rcond=None)[0]
60+
61+ def predict(x_new: np.ndarray) -> np.ndarray:
62+ X_new = np.column_stack([x_new, np.ones(len(x_new))])
63+ return np.dot(X_new, coeffs)
64+
65+ return coeffs, predict
benchcompress/src/benchcompress/_compress_ints_lossless.pyadded+38−0View file
@@ -0,0 +1,38 @@
1+def compress_ints_lossless(x, *, method: str = "zstd") -> bytes:
2+ """Compress integer data using various compression methods.
3+
4+ Args:
5+ x: NumPy array of integers to compress
6+ method: Compression method to use. One of:
7+ - "zstd": Zstandard compression (default)
8+ - "zlib": zlib compression
9+ - "lzma": LZMA compression
10+ - "ANS": Asymmetric Numeral Systems
11+
12+ Returns:
13+ Compressed bytes
14+ """
15+ if method == "zstd":
16+ import zstandard as zstd
17+
18+ cctx = zstd.ZstdCompressor(level=22)
19+ return cctx.compress(x.tobytes())
20+ elif method == "zlib":
21+ import zlib
22+
23+ return zlib.compress(x.tobytes(), level=9)
24+ elif method == "lzma":
25+ import lzma
26+
27+ return lzma.compress(x.tobytes(), preset=9)
28+ elif method == "ANS":
29+ from simple_ans import ans_encode
30+
31+ encoding = ans_encode(x)
32+ return (
33+ encoding.bitstream
34+ + encoding.symbol_counts.tobytes()
35+ + encoding.symbol_values.tobytes()
36+ )
37+ else:
38+ raise ValueError(f"Unknown method: {method}")
benchcompress/src/benchcompress/_data_loaders.pyadded+84−0View file
@@ -0,0 +1,84 @@
1+"""Data loading utilities."""
2+
3+import numpy as np
4+import lindi
5+from typing import cast
6+import warnings
7+
8+
9+def load_real_000876(
10+ *, num_samples: int, num_channels: int, start_channel: int
11+) -> np.ndarray:
12+ """Load data from DANDI dataset 000876.
13+
14+ Args:
15+ num_samples: Number of samples to load
16+ num_channels: Number of channels to load
17+ start_channel: Starting channel index
18+
19+ Returns:
20+ Array of shape (num_samples, num_channels) containing the loaded data
21+ """
22+ warnings.warn(
23+ "This function is deprecated. Use datasets['real-000876-ch45']['create']() instead.",
24+ DeprecationWarning,
25+ stacklevel=2,
26+ )
27+ nwb_url = "https://api.dandiarchive.org/api/assets/7e1de06d-d478-40e2-9b64-9dd04eafaa4c/download/"
28+ h5f = lindi.LindiH5pyFile.from_hdf5_file(nwb_url)
29+ ds = h5f["/acquisition/ElectricalSeriesAP/data"]
30+ assert isinstance(ds, lindi.LindiH5pyDataset)
31+ ret = ds[:num_samples, start_channel : start_channel + num_channels]
32+ return cast(np.ndarray, ret)
33+
34+
35+def load_real_000409(
36+ *, num_samples: int, num_channels: int, start_channel: int
37+) -> np.ndarray:
38+ """Load data from DANDI dataset 000409.
39+
40+ Args:
41+ num_samples: Number of samples to load
42+ num_channels: Number of channels to load
43+ start_channel: Starting channel index
44+
45+ Returns:
46+ Array of shape (num_samples, num_channels) containing the loaded data
47+ """
48+ warnings.warn(
49+ "This function is deprecated. Use datasets['real-000409-ch101']['create']() instead.",
50+ DeprecationWarning,
51+ stacklevel=2,
52+ )
53+ nwb_url = "https://api.dandiarchive.org/api/assets/c04f6b30-82bf-40e1-9210-34f0bcd8be24/download/"
54+ h5f = lindi.LindiH5pyFile.from_hdf5_file(nwb_url)
55+ ds = h5f["/acquisition/ElectricalSeriesAp/data"]
56+ assert isinstance(ds, lindi.LindiH5pyDataset)
57+ ret = ds[:num_samples, start_channel : start_channel + num_channels]
58+ return cast(np.ndarray, ret)
59+
60+
61+def load_real_001290(
62+ *, num_samples: int, num_channels: int, start_channel: int
63+) -> np.ndarray:
64+ """Load data from DANDI dataset 001290.
65+
66+ Args:
67+ num_samples: Number of samples to load
68+ num_channels: Number of channels to load
69+ start_channel: Starting channel index
70+
71+ Returns:
72+ Array of shape (num_samples, num_channels) containing the loaded data
73+ """
74+ warnings.warn(
75+ "This function is deprecated. Use datasets['real-001290-ch0']['create']() instead.",
76+ DeprecationWarning,
77+ stacklevel=2,
78+ )
79+ nwb_url = "https://api.dandiarchive.org/api/assets/78c99d23-da88-4ecd-9086-c488a126eac5/download/"
80+ h5f = lindi.LindiH5pyFile.from_hdf5_file(nwb_url)
81+ ds = h5f["/acquisition/ElectricalSeriesAPImec/data"]
82+ assert isinstance(ds, lindi.LindiH5pyDataset)
83+ ret = ds[:num_samples, start_channel : start_channel + num_channels]
84+ return cast(np.ndarray, ret)
benchcompress/src/benchcompress/_filters.pyadded+62−0View file
@@ -0,0 +1,62 @@
1+from typing import cast
2+import numpy as np
3+from scipy.signal import butter, lfilter
4+
5+
6+def bandpass_filter(
7+ array: np.ndarray, *, sampling_frequency: float, lowcut: float, highcut: float
8+) -> np.ndarray:
9+ """Apply a bandpass filter to the input array.
10+
11+ Args:
12+ array: Input signal array
13+ sampling_frequency: Sampling frequency in Hz
14+ lowcut: Lower cutoff frequency in Hz
15+ highcut: Higher cutoff frequency in Hz
16+
17+ Returns:
18+ Filtered signal array
19+ """
20+ nyquist = 0.5 * sampling_frequency
21+ low = lowcut / nyquist
22+ high = highcut / nyquist
23+ b, a = butter(5, [low, high], btype="band")
24+ return cast(np.ndarray, lfilter(b, a, array, axis=0))
25+
26+
27+def lowpass_filter(
28+ array: np.ndarray, *, sampling_frequency: float, highcut: float
29+) -> np.ndarray:
30+ """Apply a lowpass filter to the input array.
31+
32+ Args:
33+ array: Input signal array
34+ sampling_frequency: Sampling frequency in Hz
35+ highcut: Cutoff frequency in Hz
36+
37+ Returns:
38+ Filtered signal array
39+ """
40+ nyquist = 0.5 * sampling_frequency
41+ high = highcut / nyquist
42+ b, a = butter(5, high, btype="low")
43+ return cast(np.ndarray, lfilter(b, a, array, axis=0))
44+
45+
46+def highpass_filter(
47+ array: np.ndarray, *, sampling_frequency: float, lowcut: float
48+) -> np.ndarray:
49+ """Apply a highpass filter to the input array.
50+
51+ Args:
52+ array: Input signal array
53+ sampling_frequency: Sampling frequency in Hz
54+ lowcut: Cutoff frequency in Hz
55+
56+ Returns:
57+ Filtered signal array
58+ """
59+ nyquist = 0.5 * sampling_frequency
60+ low = lowcut / nyquist
61+ b, a = butter(5, low, btype="high")
62+ return cast(np.ndarray, lfilter(b, a, array, axis=0))
benchcompress/src/benchcompress/algorithms/__init__.pyadded+21−0View file
@@ -0,0 +1,21 @@
1+from .bzip2 import algorithms as bzip2_algorithms
2+from .zlib import algorithms as zlib_algorithms
3+from .zstd import algorithms as zstd_algorithms
4+from .ans import algorithms as ans_algorithms
5+from .lzma import algorithms as lzma_algorithms
6+from .brotli import algorithms as brotli_algorithms
7+from .lz4 import algorithms as lz4_algorithms
8+from .blosc2 import algorithms as blosc2_algorithms
9+from .wavpack import algorithms as wavpack_algorithms
10+
11+algorithms = (
12+ bzip2_algorithms
13+ + zlib_algorithms
14+ + zstd_algorithms
15+ + ans_algorithms
16+ + lzma_algorithms
17+ + brotli_algorithms
18+ + lz4_algorithms
19+ + blosc2_algorithms
20+ + wavpack_algorithms
21+)
benchcompress/src/benchcompress/algorithms/ans/CMakeLists.txtadded+39−0View file
@@ -0,0 +1,39 @@
1+cmake_minimum_required(VERSION 3.15)
2+project(lpc_cpp)
3+
4+set(CMAKE_CXX_STANDARD 14)
5+set(CMAKE_CXX_STANDARD_REQUIRED ON)
6+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
7+
8+find_package(Python REQUIRED COMPONENTS Interpreter Development.Module)
9+
10+# Fetch and include pybind11
11+include(FetchContent)
12+FetchContent_Declare(
13+ pybind11
14+ GIT_REPOSITORY https://github.com/pybind/pybind11.git
15+ GIT_TAG v2.11.1
16+)
17+
18+# Fetch and include Eigen
19+FetchContent_Declare(
20+ eigen
21+ GIT_REPOSITORY https://gitlab.com/libeigen/eigen.git
22+ GIT_TAG 3.4.0
23+)
24+
25+FetchContent_MakeAvailable(pybind11 eigen)
26+
27+# Build lpc_reconstruct module
28+pybind11_add_module(lpc_reconstruct_cpp_ext lpc_reconstruct.cpp)
29+
30+# Build lpc_predict module
31+pybind11_add_module(lpc_predict_cpp_ext lpc_predict.cpp)
32+target_link_libraries(lpc_predict_cpp_ext PRIVATE Eigen3::Eigen)
33+
34+# Build get_run_lengths module
35+pybind11_add_module(get_run_lengths_cpp_ext get_run_lengths.cpp)
36+
37+# Install all modules
38+install(TARGETS lpc_reconstruct_cpp_ext lpc_predict_cpp_ext get_run_lengths_cpp_ext
39+ DESTINATION benchcompress/algorithms/ans)
benchcompress/src/benchcompress/algorithms/ans/__init__.pyadded+488−0View file
@@ -0,0 +1,488 @@
1+import numpy as np
2+import os
3+from .lpc_reconstruct import (
4+ lpc_reconstruct as lpc_reconstruct_cpp,
5+)
6+from .lpc_predict import (
7+ lpc_predict as lpc_predict_cpp,
8+)
9+from .get_run_lengths import get_run_lengths
10+from .header_utils import (
11+ create_header_with_schema,
12+ unpack_header_with_schema,
13+ create_ans_header,
14+ create_lpc_header,
15+ unpack_lpc_header,
16+ create_lpc_sparse_header,
17+ unpack_lpc_sparse_header,
18+)
19+
20+
21+SOURCE_FILE = "ans/__init__.py"
22+
23+
24+def _load_long_description():
25+ current_dir = os.path.dirname(os.path.abspath(__file__))
26+ md_path = os.path.join(current_dir, "ans.md")
27+ with open(md_path, "r", encoding="utf-8") as f:
28+ return f.read()
29+
30+
31+LONG_DESCRIPTION = _load_long_description()
32+
33+
34+def ans_encode(x: np.ndarray) -> bytes:
35+ from simple_ans import ans_encode as ans_encode_0
36+
37+ encoded = ans_encode_0(x)
38+ if x.dtype == np.uint8:
39+ dtype_code = 0
40+ elif x.dtype == np.uint16:
41+ dtype_code = 1
42+ elif x.dtype == np.uint32:
43+ dtype_code = 2
44+ elif x.dtype == np.int16:
45+ dtype_code = 3
46+ elif x.dtype == np.int32:
47+ dtype_code = 4
48+ else:
49+ raise ValueError(f"Unsupported dtype: {x.dtype}")
50+
51+ # Use the new header utilities
52+ header_bytes, schema = create_ans_header(
53+ dtype_code=dtype_code,
54+ num_words=len(encoded.words),
55+ signal_length=encoded.signal_length,
56+ state=encoded.state,
57+ symbol_counts=encoded.symbol_counts,
58+ symbol_values=encoded.symbol_values,
59+ )
60+
61+ # Create self-describing header
62+ complete_header = create_header_with_schema(header_bytes, schema)
63+ header_size = np.uint32(len(complete_header))
64+
65+ return header_size.tobytes() + complete_header + encoded.words.tobytes()
66+
67+
68+def ans_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
69+ from simple_ans import ans_decode as ans_decode_0, EncodedSignal
70+
71+ header_size = np.frombuffer(x[:4], dtype=np.uint32)[0]
72+
73+ # Use the new header utilities
74+ header_dict, total_header_size = unpack_header_with_schema(x[4 : 4 + header_size])
75+
76+ dtype_code = header_dict["dtype_code"]
77+ num_words = header_dict["num_words"]
78+ signal_length = header_dict["signal_length"]
79+ state = header_dict["state"]
80+ symbol_counts = header_dict["symbol_counts"]
81+ symbol_values = header_dict["symbol_values"]
82+
83+ words_bytes = x[4 + header_size :]
84+
85+ if dtype_code == 0:
86+ assert dtype == "uint8"
87+ elif dtype_code == 1:
88+ assert dtype == "uint16"
89+ elif dtype_code == 2:
90+ assert dtype == "uint32"
91+ elif dtype_code == 3:
92+ assert dtype == "int16"
93+ elif dtype_code == 4:
94+ assert dtype == "int32"
95+ else:
96+ raise ValueError(f"Unsupported dtype code: {dtype_code}")
97+
98+ encoded = EncodedSignal(
99+ signal_length=int(signal_length),
100+ state=np.uint64(state),
101+ symbol_counts=symbol_counts.astype(np.uint32),
102+ symbol_values=symbol_values.astype(dtype),
103+ words=np.frombuffer(words_bytes, dtype=np.uint32, count=num_words),
104+ )
105+ return ans_decode_0(encoded).reshape(shape)
106+
107+
108+def ans_delta_encode(x: np.ndarray) -> bytes:
109+ from simple_ans import ans_encode
110+
111+ assert x.ndim == 1
112+
113+ y = np.diff(x)
114+ # Encode just the differences
115+ encoded = ans_encode(y)
116+ if x.dtype == np.uint8:
117+ dtype_code = 0
118+ elif x.dtype == np.uint16:
119+ dtype_code = 1
120+ elif x.dtype == np.uint32:
121+ dtype_code = 2
122+ elif x.dtype == np.int16:
123+ dtype_code = 3
124+ elif x.dtype == np.int32:
125+ dtype_code = 4
126+ else:
127+ raise ValueError(f"Unsupported dtype: {x.dtype}")
128+
129+ # Use the new header utilities with extra value for x[0]
130+ extra_values = [(x[0], x.dtype.name)]
131+ header_bytes, schema = create_ans_header(
132+ dtype_code=dtype_code,
133+ num_words=len(encoded.words),
134+ signal_length=encoded.signal_length,
135+ state=encoded.state,
136+ symbol_counts=encoded.symbol_counts,
137+ symbol_values=encoded.symbol_values,
138+ extra_values=extra_values,
139+ )
140+
141+ # Create self-describing header
142+ complete_header = create_header_with_schema(header_bytes, schema)
143+ header_size = np.uint32(len(complete_header))
144+
145+ return header_size.tobytes() + complete_header + encoded.words.tobytes()
146+
147+
148+def ans_delta_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
149+ from simple_ans import ans_decode, EncodedSignal
150+
151+ assert len(shape) == 1
152+
153+ header_size = np.frombuffer(x[:4], dtype=np.uint32)[0]
154+
155+ # Use the new header utilities
156+ header_dict, total_header_size = unpack_header_with_schema(x[4 : 4 + header_size])
157+
158+ dtype_code = header_dict["dtype_code"]
159+ num_words = header_dict["num_words"]
160+ signal_length = header_dict["signal_length"]
161+ state = header_dict["state"]
162+ symbol_counts = header_dict["symbol_counts"]
163+ symbol_values = header_dict["symbol_values"]
164+ x0 = header_dict["extra_values"][0] # First extra value is x[0]
165+
166+ words_bytes = x[4 + header_size :]
167+
168+ if dtype_code == 0:
169+ assert dtype == "uint8"
170+ elif dtype_code == 1:
171+ assert dtype == "uint16"
172+ elif dtype_code == 2:
173+ assert dtype == "uint32"
174+ elif dtype_code == 3:
175+ assert dtype == "int16"
176+ elif dtype_code == 4:
177+ assert dtype == "int32"
178+ else:
179+ raise ValueError(f"Unsupported dtype code: {dtype_code}")
180+
181+ encoded = EncodedSignal(
182+ signal_length=int(signal_length),
183+ state=state,
184+ symbol_counts=symbol_counts.astype(np.uint32),
185+ symbol_values=symbol_values.astype(dtype),
186+ words=np.frombuffer(words_bytes, dtype=np.uint32, count=num_words),
187+ )
188+ # Decode the differences
189+ diffs = ans_decode(encoded)
190+ # Insert x0 at the beginning and cumulatively sum the differences
191+ result = np.cumsum(np.insert(diffs, 0, x0))
192+ # Cast back to the original dtype
193+ return result.astype(dtype)
194+
195+
196+def ans_lpc_encode(x: np.ndarray) -> bytes:
197+ from simple_ans import ans_encode
198+
199+ assert x.ndim == 1
200+
201+ coeffs, initial, resid = lpc_predict_cpp(x, M=6, num_training_samples=10000)
202+ # Encode just the differences
203+ encoded = ans_encode(resid)
204+ if x.dtype == np.uint8:
205+ dtype_code = 0
206+ elif x.dtype == np.uint16:
207+ dtype_code = 1
208+ elif x.dtype == np.uint32:
209+ dtype_code = 2
210+ elif x.dtype == np.int16:
211+ dtype_code = 3
212+ elif x.dtype == np.int32:
213+ dtype_code = 4
214+ else:
215+ raise ValueError(f"Unsupported dtype: {x.dtype}")
216+
217+ # Use the new LPC header utilities
218+ header_bytes, schema = create_lpc_header(
219+ dtype_code=dtype_code,
220+ num_words=len(encoded.words),
221+ signal_length=encoded.signal_length,
222+ state=encoded.state,
223+ symbol_counts=encoded.symbol_counts,
224+ symbol_values=encoded.symbol_values,
225+ coeffs=coeffs,
226+ initial=initial,
227+ )
228+
229+ # Create self-describing header
230+ complete_header = create_header_with_schema(header_bytes, schema)
231+ header_size = np.uint32(len(complete_header))
232+
233+ return header_size.tobytes() + complete_header + encoded.words.tobytes()
234+
235+
236+def ans_lpc_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
237+ from simple_ans import ans_decode, EncodedSignal
238+
239+ assert len(shape) == 1
240+
241+ header_size = np.frombuffer(x[:4], dtype=np.uint32)[0]
242+
243+ # Use the new header utilities
244+ header_dict, total_header_size = unpack_header_with_schema(x[4 : 4 + header_size])
245+
246+ dtype_code = header_dict["dtype_code"]
247+ num_words = header_dict["num_words"]
248+ signal_length = header_dict["signal_length"]
249+ state = header_dict["state"]
250+ symbol_counts = header_dict["symbol_counts"]
251+ symbol_values = header_dict["symbol_values"]
252+ coeffs = header_dict["coeffs"]
253+ initial = header_dict["initial"]
254+
255+ bitstream = x[4 + header_size :]
256+
257+ if dtype_code == 0:
258+ assert dtype == "uint8"
259+ elif dtype_code == 1:
260+ assert dtype == "uint16"
261+ elif dtype_code == 2:
262+ assert dtype == "uint32"
263+ elif dtype_code == 3:
264+ assert dtype == "int16"
265+ elif dtype_code == 4:
266+ assert dtype == "int32"
267+ else:
268+ raise ValueError(f"Unsupported dtype code: {dtype_code}")
269+
270+ encoded = EncodedSignal(
271+ signal_length=int(signal_length),
272+ state=state,
273+ symbol_counts=symbol_counts.astype(np.uint32),
274+ symbol_values=symbol_values.astype(dtype),
275+ words=np.frombuffer(bitstream, dtype=np.uint32, count=num_words),
276+ )
277+
278+ resid = ans_decode(encoded)
279+ output = lpc_reconstruct_cpp(coeffs, initial, resid)
280+ return output
281+
282+
283+def ans_lpc_sparse_encode(x: np.ndarray) -> bytes:
284+ from simple_ans import ans_encode
285+
286+ assert x.ndim == 1
287+
288+ run_lengths = get_run_lengths(x)
289+
290+ non_zero_arrays = []
291+ array_pos = 0
292+ i = 0
293+ while i < len(run_lengths):
294+ non_zero_len = int(run_lengths[i])
295+ if non_zero_len > 0:
296+ non_zero_arrays.append(x[array_pos : array_pos + non_zero_len])
297+ array_pos += non_zero_len
298+ i += 1
299+ if i < len(run_lengths):
300+ zero_len = int(run_lengths[i])
301+ array_pos += zero_len
302+ i += 1
303+
304+ non_zero_data = np.concatenate(non_zero_arrays)
305+ assert len(non_zero_data) == np.sum(run_lengths[::2])
306+ coeffs, initial, resid = lpc_predict_cpp(
307+ non_zero_data, M=6, num_training_samples=10000
308+ )
309+ encoded = ans_encode(resid)
310+
311+ if x.dtype == np.uint8:
312+ dtype_code = 0
313+ elif x.dtype == np.uint16:
314+ dtype_code = 1
315+ elif x.dtype == np.uint32:
316+ dtype_code = 2
317+ elif x.dtype == np.int16:
318+ dtype_code = 3
319+ elif x.dtype == np.int32:
320+ dtype_code = 4
321+ else:
322+ raise ValueError(f"Unsupported dtype: {x.dtype}")
323+
324+ if run_lengths.dtype == np.uint8:
325+ run_length_dtype_code = 0
326+ elif run_lengths.dtype == np.uint16:
327+ run_length_dtype_code = 1
328+ elif run_lengths.dtype == np.uint32:
329+ run_length_dtype_code = 2
330+ else:
331+ raise ValueError(f"Unsupported run length dtype: {run_lengths.dtype}")
332+
333+ # Use the new LPC sparse header utilities
334+ header_bytes, schema = create_lpc_sparse_header(
335+ dtype_code=dtype_code,
336+ num_words=len(encoded.words),
337+ signal_length=encoded.signal_length,
338+ state=encoded.state,
339+ symbol_counts=encoded.symbol_counts,
340+ symbol_values=encoded.symbol_values,
341+ coeffs=coeffs,
342+ initial=initial,
343+ run_length_dtype_code=run_length_dtype_code,
344+ num_run_lengths=len(run_lengths),
345+ )
346+
347+ # Create self-describing header
348+ complete_header = create_header_with_schema(header_bytes, schema)
349+ header_size = np.uint32(len(complete_header))
350+
351+ return (
352+ header_size.tobytes()
353+ + complete_header
354+ + encoded.words.tobytes()
355+ + run_lengths.tobytes()
356+ )
357+
358+
359+def ans_lpc_sparse_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
360+ from simple_ans import ans_decode, EncodedSignal
361+
362+ assert len(shape) == 1
363+
364+ header_size = np.frombuffer(x[:4], dtype=np.uint32)[0]
365+
366+ # Use the new header utilities
367+ header_dict, total_header_size = unpack_header_with_schema(x[4 : 4 + header_size])
368+
369+ dtype_code = header_dict["dtype_code"]
370+ num_words = header_dict["num_words"]
371+ signal_length = header_dict["signal_length"]
372+ state = header_dict["state"]
373+ symbol_counts = header_dict["symbol_counts"]
374+ symbol_values = header_dict["symbol_values"]
375+ coeffs = header_dict["coeffs"]
376+ initial = header_dict["initial"]
377+ run_length_dtype_code = header_dict["run_length_dtype_code"]
378+ num_run_lengths = header_dict["num_run_lengths"]
379+
380+ words_end = 4 + header_size + num_words * 4
381+ words_bytes = x[4 + header_size : words_end]
382+
383+ # Get run lengths from the remaining bytes
384+ if run_length_dtype_code == 0:
385+ run_lengths = np.frombuffer(x[words_end:], dtype=np.uint8)
386+ elif run_length_dtype_code == 1:
387+ run_lengths = np.frombuffer(x[words_end:], dtype=np.uint16)
388+ elif run_length_dtype_code == 2:
389+ run_lengths = np.frombuffer(x[words_end:], dtype=np.uint32)
390+ else:
391+ raise ValueError(f"Unsupported run length dtype code: {run_length_dtype_code}")
392+
393+ if len(run_lengths) != num_run_lengths:
394+ raise ValueError(
395+ f"Expected {num_run_lengths} run lengths, got {len(run_lengths)}"
396+ )
397+
398+ if dtype_code == 0:
399+ assert dtype == "uint8"
400+ elif dtype_code == 1:
401+ assert dtype == "uint16"
402+ elif dtype_code == 2:
403+ assert dtype == "uint32"
404+ elif dtype_code == 3:
405+ assert dtype == "int16"
406+ elif dtype_code == 4:
407+ assert dtype == "int32"
408+ else:
409+ raise ValueError(f"Unsupported dtype code: {dtype_code}")
410+
411+ encoded = EncodedSignal(
412+ signal_length=int(signal_length),
413+ state=state,
414+ symbol_counts=symbol_counts.astype(np.uint32),
415+ symbol_values=symbol_values.astype(dtype),
416+ words=np.frombuffer(words_bytes, dtype=np.uint32, count=num_words),
417+ )
418+
419+ # Decode residuals and reconstruct non-zero data
420+ resid = ans_decode(encoded)
421+ non_zero_data = lpc_reconstruct_cpp(coeffs, initial, resid)
422+
423+ assert len(non_zero_data) == np.sum(run_lengths[::2])
424+
425+ # Reconstruct full array using run lengths
426+ non_zero_pos = 0 # Position in non_zero_data
427+ i = 0
428+ segments = []
429+ while i < len(run_lengths):
430+ non_zero_len = int(run_lengths[i])
431+ if non_zero_len > 0:
432+ # print(f'--- b non_zero_len={non_zero_len} non_zero_pos={non_zero_pos} pos={pos} len(non_zero_data)={len(non_zero_data)} len(output)={len(output)}')
433+ # print('---- y')
434+ segment = non_zero_data[non_zero_pos : non_zero_pos + non_zero_len]
435+ assert len(segment) == non_zero_len
436+ segments.append(segment)
437+ non_zero_pos += non_zero_len
438+ i += 1
439+ if i < len(run_lengths):
440+ segments.append(np.zeros(int(run_lengths[i]), dtype=non_zero_data.dtype))
441+ i += 1
442+ output = np.concatenate(segments)
443+
444+ return output
445+
446+
447+algorithms = [
448+ {
449+ "name": "ANS",
450+ "version": "11",
451+ "encode": lambda x: ans_encode(x),
452+ "decode": lambda x, dtype, shape: ans_decode(x, dtype, shape),
453+ "description": "ANS compression via simple_ans for efficient data compression.",
454+ "tags": ["ANS", "integer"],
455+ "source_file": SOURCE_FILE,
456+ "long_description": LONG_DESCRIPTION,
457+ },
458+ {
459+ "name": "ANS-delta",
460+ "version": "11",
461+ "encode": lambda x: ans_delta_encode(x),
462+ "decode": lambda x, dtype, shape: ans_delta_decode(x, dtype, shape),
463+ "description": "ANS compression via simple_ans with delta encoding for improved compression of sequential data.",
464+ "tags": ["ANS", "integer", "delta_encoding", "1d"],
465+ "source_file": SOURCE_FILE,
466+ "long_description": LONG_DESCRIPTION,
467+ },
468+ {
469+ "name": "ANS-lpc",
470+ "version": "11",
471+ "encode": lambda x: ans_lpc_encode(x),
472+ "decode": lambda x, dtype, shape: ans_lpc_decode(x, dtype, shape),
473+ "description": "ANS compression via simple_ans with linear predictive codingn for exploiting temporal correlations in the data.",
474+ "tags": ["ANS", "integer", "lpc_prediction", "1d"],
475+ "source_file": SOURCE_FILE,
476+ "long_description": LONG_DESCRIPTION,
477+ },
478+ {
479+ "name": "ANS-lpc-zrle",
480+ "version": "11",
481+ "encode": lambda x: ans_lpc_sparse_encode(x),
482+ "decode": lambda x, dtype, shape: ans_lpc_sparse_decode(x, dtype, shape),
483+ "description": "ANS compression via simple_ans with linear predictive codingn and zero run-length encoding for sparse data.",
484+ "tags": ["ANS", "integer", "lpc_prediction", "zero_rle", "1d"],
485+ "source_file": SOURCE_FILE,
486+ "long_description": LONG_DESCRIPTION,
487+ },
488+]
benchcompress/src/benchcompress/algorithms/ans/ans.mdadded+26−0View file
@@ -0,0 +1,26 @@
1+# ANS (Asymmetric Numeral Systems) Algorithm
2+
3+ANS is a modern entropy coding method that achieves a compression ratio near arithmetic coding with relatively efficient encoding and decoding. It achieves this by maintaining a state that represents the encoded data in a way that can be efficiently updated. The implementation uses the [simple_ans](https://github.com/flatironinstitute/simple_ans) Python package.
4+
5+## Variants
6+
7+### Basic ANS
8+- ANS: Standard implementation using the simple_ans library
9+- Efficiently encodes integer data by modeling symbol frequencies
10+
11+### Delta Encoding
12+- ANS-delta: ANS compression with delta encoding
13+- Stores differences between consecutive values
14+- Effective for sequences where adjacent values are similar
15+
16+### Linear Predictive Coding (LPC)
17+- ANS-lpc: ANS with linear predictive codingn
18+- Uses a 6th-order Markov model to predict values based on previous samples
19+- Compresses the prediction residuals
20+- Particularly effective for data with temporal correlations
21+
22+### Linear Predictive Coding with Zero RLE
23+- ANS-lpc-zrle: Combines linear predictive coding with zero run-length encoding
24+- Identifies runs of zero values and encodes their lengths
25+- Applies Markov prediction to the non-zero regions
26+- Ideal for sparse data with many zeros interspersed with correlated non-zero values
benchcompress/src/benchcompress/algorithms/ans/get_run_lengths.cppadded+21−0View file
@@ -0,0 +1,21 @@
1+#include "get_run_lengths.hpp"
2+
3+namespace py = pybind11;
4+
5+// Explicit instantiation for int16_t
6+py::array get_run_lengths_int16(py::array_t<int16_t> x) {
7+ return get_run_lengths_impl<int16_t>(x);
8+}
9+
10+// Explicit instantiation for int32_t
11+py::array get_run_lengths_int32(py::array_t<int32_t> x) {
12+ return get_run_lengths_impl<int32_t>(x);
13+}
14+
15+PYBIND11_MODULE(get_run_lengths_cpp_ext, m) {
16+ m.doc() = "C++ implementation of get_run_lengths using pybind11";
17+ m.def("get_run_lengths_int16", &get_run_lengths_int16,
18+ "Calculate run lengths of zeros and non-zeros in a signal (int16)");
19+ m.def("get_run_lengths_int32", &get_run_lengths_int32,
20+ "Calculate run lengths of zeros and non-zeros in a signal (int32)");
21+}
benchcompress/src/benchcompress/algorithms/ans/get_run_lengths.hppadded+88−0View file
@@ -0,0 +1,88 @@
1+#pragma once
2+
3+#include <cstdint>
4+#include <pybind11/numpy.h>
5+#include <pybind11/pybind11.h>
6+#include <vector>
7+
8+namespace py = pybind11;
9+
10+template <typename T> py::array get_run_lengths_impl(py::array_t<T> x) {
11+ auto x_buf = x.request();
12+ T *x_ptr = static_cast<T *>(x_buf.ptr);
13+ size_t N = x_buf.shape[0];
14+
15+ std::vector<uint32_t> runs;
16+ size_t i = 0;
17+ uint32_t current_nonzero_run_length = 0;
18+
19+ while (i < N) {
20+ // Check for a sequence of at least 10 zeros
21+ bool has_zeros = true;
22+ for (size_t j = 0; j < 10 && i + j < N; j++) {
23+ if (x_ptr[i + j] != 0) {
24+ has_zeros = false;
25+ break;
26+ }
27+ }
28+
29+ if (has_zeros) {
30+ // Add current non-zero run if any
31+ runs.push_back(current_nonzero_run_length);
32+ current_nonzero_run_length = 0;
33+
34+ // Count consecutive zeros
35+ size_t j = i;
36+ while (j < N && x_ptr[j] == 0) {
37+ j++;
38+ }
39+ runs.push_back(j - i);
40+ i = j;
41+ } else {
42+ current_nonzero_run_length++;
43+ i++;
44+ }
45+ }
46+
47+ // Add final non-zero run if any
48+ if (current_nonzero_run_length > 0) {
49+ runs.push_back(current_nonzero_run_length);
50+ }
51+
52+ // Determine appropriate dtype based on max run length
53+ uint32_t max_run = 0;
54+ for (const auto &run : runs) {
55+ if (run > max_run) {
56+ max_run = run;
57+ }
58+ }
59+
60+ // Create numpy array with appropriate dtype
61+ std::vector<ssize_t> shape = {static_cast<ssize_t>(runs.size())};
62+
63+ if (max_run < 256) {
64+ py::array_t<uint8_t> result(shape);
65+ auto result_buf = result.request();
66+ uint8_t *result_ptr = static_cast<uint8_t *>(result_buf.ptr);
67+ for (size_t i = 0; i < runs.size(); i++) {
68+ result_ptr[i] = static_cast<uint8_t>(runs[i]);
69+ }
70+ return result;
71+ } else if (max_run < 65536) {
72+ py::array_t<uint16_t> result(shape);
73+ auto result_buf = result.request();
74+ uint16_t *result_ptr = static_cast<uint16_t *>(result_buf.ptr);
75+ for (size_t i = 0; i < runs.size(); i++) {
76+ result_ptr[i] = static_cast<uint16_t>(runs[i]);
77+ }
78+ return result;
79+ } else {
80+ py::array_t<uint32_t> result(shape);
81+ auto result_buf = result.request();
82+ uint32_t *result_ptr = static_cast<uint32_t *>(result_buf.ptr);
83+ for (size_t i = 0; i < runs.size(); i++) {
84+ result_ptr[i] = runs[i];
85+ }
86+ return result;
87+ }
88+}
benchcompress/src/benchcompress/algorithms/ans/get_run_lengths.pyadded+24−0View file
@@ -0,0 +1,24 @@
1+import numpy as np
2+from .get_run_lengths_cpp_ext import get_run_lengths_int16, get_run_lengths_int32
3+
4+
5+def get_run_lengths(x: np.ndarray) -> np.ndarray:
6+ """Calculate run lengths of zeros and non-zeros in a signal using C++ implementation.
7+
8+ Args:
9+ x: Input signal (must be int16 or int32)
10+
11+ Returns:
12+ np.ndarray: Array of run lengths alternating between non-zero and zero runs.
13+ The dtype will be uint8, uint16, or uint32 depending on the maximum run length.
14+
15+ Raises:
16+ ValueError: If input array is not int16 or int32
17+ """
18+ # Check input dtype and call appropriate implementation
19+ if x.dtype == np.int16:
20+ return get_run_lengths_int16(x)
21+ elif x.dtype == np.int32:
22+ return get_run_lengths_int32(x)
23+ else:
24+ raise ValueError(f"Input array must be int16 or int32, got {x.dtype}")
benchcompress/src/benchcompress/algorithms/ans/header_utils.pyadded+485−0View file
@@ -0,0 +1,485 @@
1+"""
2+Utilities for serializing and deserializing mixed-type headers in ANS compression.
3+
4+This module provides functions to pack and unpack headers containing different
5+data types while preserving their original types and precision.
6+"""
7+
8+import numpy as np
9+from typing import List, Tuple, Any, Union
10+
11+
12+def pack_header_values(values: List[Tuple[Any, str]]) -> bytes:
13+ """
14+ Pack a list of (value, dtype) tuples into a binary format.
15+
16+ Args:
17+ values: List of (value, dtype_string) tuples where dtype_string is like 'int32', 'float32', etc.
18+
19+ Returns:
20+ bytes: Packed binary data
21+ """
22+ packed_data = []
23+
24+ for value, dtype_str in values:
25+ if isinstance(value, (list, np.ndarray)):
26+ # Handle arrays
27+ arr = np.array(value, dtype=dtype_str)
28+ packed_data.append(arr.tobytes())
29+ else:
30+ # Handle scalars
31+ scalar = np.array([value], dtype=dtype_str)
32+ packed_data.append(scalar.tobytes())
33+
34+ return b"".join(packed_data)
35+
36+
37+def unpack_header_values(data: bytes, schema: List[Tuple[str, int]]) -> List[Any]:
38+ """
39+ Unpack binary data according to a schema.
40+
41+ Args:
42+ data: Binary data to unpack
43+ schema: List of (dtype_string, count) tuples where count is number of elements
44+ Use count=1 for scalars, count>1 for arrays
45+
46+ Returns:
47+ List of unpacked values
48+ """
49+ values = []
50+ offset = 0
51+
52+ for dtype_str, count in schema:
53+ dtype = np.dtype(dtype_str)
54+ size = dtype.itemsize * count
55+
56+ if offset + size > len(data):
57+ raise ValueError(f"Not enough data to unpack {count} {dtype_str} values")
58+
59+ chunk = data[offset : offset + size]
60+ arr = np.frombuffer(chunk, dtype=dtype)
61+
62+ values.append(arr.flatten()) # Always return flattened array to ensure 1D
63+
64+ offset += size
65+
66+ return values
67+
68+
69+def create_ans_header(
70+ dtype_code: int,
71+ num_words: int,
72+ signal_length: int,
73+ state: np.uint64,
74+ symbol_counts: np.ndarray,
75+ symbol_values: np.ndarray,
76+ extra_values: Union[List[Tuple[Any, str]], None] = None,
77+) -> Tuple[bytes, List[Tuple[str, int]]]:
78+ """
79+ Create a header for ANS compression with proper type preservation.
80+
81+ Args:
82+ dtype_code: Integer code for the data type
83+ num_words: Number of words in the encoded data
84+ signal_length: Length of the original signal
85+ state: ANS state
86+ symbol_counts: Array of symbol counts (will be stored as uint32)
87+ symbol_values: Array of symbol values (will be stored with original dtype)
88+ extra_values: Optional list of (value, dtype_string) tuples for additional data
89+
90+ Returns:
91+ Tuple of (header_bytes, schema) where schema describes the unpacking format
92+ """
93+ # Build the values list with their types
94+ values: List[Tuple[Any, str]] = [
95+ (dtype_code, "int32"),
96+ (num_words, "int32"),
97+ (signal_length, "int32"),
98+ (state, "uint64"),
99+ (len(symbol_counts), "int32"),
100+ ]
101+
102+ # Add symbol counts as uint32
103+ values.append((symbol_counts, "uint32"))
104+
105+ # Add symbol values with their original dtype
106+ symbol_dtype = symbol_values.dtype.name
107+ values.append((symbol_values, symbol_dtype))
108+
109+ # Add any extra values
110+ if extra_values:
111+ values.extend(extra_values)
112+
113+ # Create schema for unpacking
114+ schema = [
115+ ("int32", 1), # dtype_code
116+ ("int32", 1), # num_words
117+ ("int32", 1), # signal_length
118+ ("uint64", 1), # state
119+ ("int32", 1), # num_symbols
120+ ("uint32", len(symbol_counts)), # symbol_counts
121+ (symbol_dtype, len(symbol_values)), # symbol_values
122+ ]
123+
124+ # Add schema entries for extra values
125+ if extra_values:
126+ for value, dtype_str in extra_values:
127+ if isinstance(value, (list, np.ndarray)):
128+ count = len(value)
129+ else:
130+ count = 1
131+ schema.append((dtype_str, count))
132+
133+ header_bytes = pack_header_values(values)
134+ return header_bytes, schema
135+
136+
137+def unpack_ans_header(data: bytes, schema: List[Tuple[str, int]]) -> dict:
138+ """
139+ Unpack an ANS header according to the provided schema.
140+
141+ Args:
142+ data: Binary header data
143+ schema: Schema describing the data layout
144+
145+ Returns:
146+ Dictionary with unpacked header values
147+ """
148+ values = unpack_header_values(data, schema)
149+
150+ result = {
151+ "dtype_code": values[0][0], # Extract scalar from 1-element array
152+ "num_words": values[1][0],
153+ "signal_length": values[2][0],
154+ "state": values[3][0],
155+ "num_symbols": values[4][0],
156+ "symbol_counts": values[5], # Keep as array
157+ "symbol_values": values[6], # Keep as array
158+ }
159+
160+ # Add any extra values
161+ if len(values) > 7:
162+ extra_values = []
163+ for val in values[7:]:
164+ if len(val) == 1:
165+ extra_values.append(val[0]) # Extract scalar
166+ else:
167+ extra_values.append(val) # Keep as array
168+ result["extra_values"] = extra_values
169+
170+ return result
171+
172+
173+def create_lpc_sparse_header(
174+ dtype_code: int,
175+ num_words: int,
176+ signal_length: int,
177+ state: np.uint64,
178+ symbol_counts: np.ndarray,
179+ symbol_values: np.ndarray,
180+ coeffs: np.ndarray,
181+ initial: np.ndarray,
182+ run_length_dtype_code: int,
183+ num_run_lengths: int,
184+) -> Tuple[bytes, List[Tuple[str, int]]]:
185+ """
186+ Create a header for ANS LPC sparse compression with proper type preservation.
187+
188+ Args:
189+ dtype_code: Integer code for the data type
190+ num_words: Number of words in the encoded data
191+ signal_length: Length of the original signal
192+ state: ANS state
193+ symbol_counts: Array of symbol counts (will be stored as uint32)
194+ symbol_values: Array of symbol values (will be stored with original dtype)
195+ coeffs: LPC coefficients (will be stored as float32)
196+ initial: Initial values (will be stored with original dtype)
197+ run_length_dtype_code: Integer code for run length data type
198+ num_run_lengths: Number of run length values
199+
200+ Returns:
201+ Tuple of (header_bytes, schema) where schema describes the unpacking format
202+ """
203+ # Build the values list with their types
204+ values: List[Tuple[Any, str]] = [
205+ (dtype_code, "int32"),
206+ (num_words, "int32"),
207+ (signal_length, "int32"),
208+ (state, "uint64"), # Use uint64 for state as it can be very large
209+ (len(symbol_counts), "int32"),
210+ (len(coeffs), "int32"),
211+ (len(initial), "int32"),
212+ (run_length_dtype_code, "int32"),
213+ (num_run_lengths, "int32"),
214+ ]
215+
216+ # Add symbol counts as uint32
217+ values.append((symbol_counts, "uint32"))
218+
219+ # Add symbol values with their original dtype
220+ symbol_dtype = symbol_values.dtype.name
221+ values.append((symbol_values, symbol_dtype))
222+
223+ # Add coefficients as float32
224+ values.append((coeffs, "float32"))
225+
226+ # Add initial values with their original dtype
227+ initial_dtype = initial.dtype.name
228+ values.append((initial, initial_dtype))
229+
230+ # Create schema for unpacking
231+ schema = [
232+ ("int32", 1), # dtype_code
233+ ("int32", 1), # num_words
234+ ("int32", 1), # signal_length
235+ ("uint64", 1), # state
236+ ("int32", 1), # num_symbols
237+ ("int32", 1), # num_coeffs
238+ ("int32", 1), # num_initial
239+ ("int32", 1), # run_length_dtype_code
240+ ("int32", 1), # num_run_lengths
241+ ("uint32", len(symbol_counts)), # symbol_counts
242+ (symbol_dtype, len(symbol_values)), # symbol_values
243+ ("float32", len(coeffs)), # coeffs
244+ (initial_dtype, len(initial)), # initial
245+ ]
246+
247+ header_bytes = pack_header_values(values)
248+ return header_bytes, schema
249+
250+
251+def unpack_lpc_sparse_header(data: bytes, schema: List[Tuple[str, int]]) -> dict:
252+ """
253+ Unpack an ANS LPC sparse header according to the provided schema.
254+
255+ Args:
256+ data: Binary header data
257+ schema: Schema describing the data layout
258+
259+ Returns:
260+ Dictionary with unpacked header values
261+ """
262+ values = unpack_header_values(data, schema)
263+
264+ result = {
265+ "dtype_code": values[0][0], # Extract scalar from 1-element array
266+ "num_words": values[1][0],
267+ "signal_length": values[2][0],
268+ "state": values[3][0],
269+ "num_symbols": values[4][0],
270+ "num_coeffs": values[5][0],
271+ "num_initial": values[6][0],
272+ "run_length_dtype_code": values[7][0],
273+ "num_run_lengths": values[8][0],
274+ "symbol_counts": values[9], # Keep as array
275+ "symbol_values": values[10], # Keep as array
276+ "coeffs": values[11], # Keep as array
277+ "initial": values[12], # Keep as array
278+ }
279+
280+ return result
281+
282+
283+def create_header_with_schema(
284+ header_bytes: bytes, schema: List[Tuple[str, int]]
285+) -> bytes:
286+ """
287+ Create a complete header that includes the schema information for self-describing headers.
288+
289+ Args:
290+ header_bytes: The actual header data
291+ schema: Schema describing the header layout
292+
293+ Returns:
294+ Complete header with embedded schema
295+ """
296+ # Serialize the schema
297+ schema_data = []
298+ for dtype_str, count in schema:
299+ # Store dtype string length, dtype string, and count
300+ dtype_bytes = dtype_str.encode("utf-8")
301+ schema_data.append(np.array([len(dtype_bytes)], dtype="uint8").tobytes())
302+ schema_data.append(dtype_bytes)
303+ schema_data.append(np.array([count], dtype="int32").tobytes())
304+
305+ schema_bytes = b"".join(schema_data)
306+ schema_length = np.array([len(schema_bytes)], dtype="uint32").tobytes()
307+
308+ # Combine: schema_length + schema + header_data
309+ return schema_length + schema_bytes + header_bytes
310+
311+
312+def unpack_header_with_schema(data: bytes) -> Tuple[dict, int]:
313+ """
314+ Unpack a self-describing header that includes schema information.
315+
316+ Args:
317+ data: Complete header data with embedded schema
318+
319+ Returns:
320+ Tuple of (unpacked_header_dict, total_header_size)
321+ """
322+ # Read schema length
323+ schema_length = np.frombuffer(data[:4], dtype="uint32")[0]
324+
325+ # Read and parse schema
326+ schema_data = data[4 : 4 + schema_length]
327+ schema = []
328+ offset = 0
329+
330+ while offset < len(schema_data):
331+ # Read dtype string length
332+ dtype_len = np.frombuffer(schema_data[offset : offset + 1], dtype="uint8")[0]
333+ offset += 1
334+
335+ # Read dtype string
336+ dtype_str = schema_data[offset : offset + dtype_len].decode("utf-8")
337+ offset += dtype_len
338+
339+ # Read count
340+ count = np.frombuffer(schema_data[offset : offset + 4], dtype="int32")[0]
341+ offset += 4
342+
343+ schema.append((dtype_str, count))
344+
345+ # Unpack the actual header
346+ header_data = data[4 + schema_length :]
347+
348+ # Determine header type based on schema structure
349+ if len(schema) >= 13 and schema[7][0] == "int32" and schema[8][0] == "int32":
350+ # This looks like an LPC sparse header (has run_length_dtype_code and num_run_lengths)
351+ header_dict = unpack_lpc_sparse_header(header_data, schema)
352+ elif len(schema) >= 11 and schema[5][0] == "int32" and schema[6][0] == "int32":
353+ # This looks like an LPC header
354+ header_dict = unpack_lpc_header(header_data, schema)
355+ else:
356+ # This is a basic ANS header
357+ header_dict = unpack_ans_header(header_data, schema)
358+
359+ total_size = 4 + schema_length + len(header_data)
360+ return header_dict, total_size
361+
362+
363+def create_lpc_header(
364+ dtype_code: int,
365+ num_words: int,
366+ signal_length: int,
367+ state: np.uint64,
368+ symbol_counts: np.ndarray,
369+ symbol_values: np.ndarray,
370+ coeffs: np.ndarray,
371+ initial: np.ndarray,
372+ extra_values: Union[List[Tuple[Any, str]], None] = None,
373+) -> Tuple[bytes, List[Tuple[str, int]]]:
374+ """
375+ Create a header for ANS LPC compression with proper type preservation.
376+
377+ Args:
378+ dtype_code: Integer code for the data type
379+ num_words: Number of words in the encoded data
380+ signal_length: Length of the original signal
381+ state: ANS state
382+ symbol_counts: Array of symbol counts (will be stored as uint32)
383+ symbol_values: Array of symbol values (will be stored with original dtype)
384+ coeffs: LPC coefficients (will be stored as float32)
385+ initial: Initial values (will be stored with original dtype)
386+ extra_values: Optional list of (value, dtype_string) tuples for additional data
387+
388+ Returns:
389+ Tuple of (header_bytes, schema) where schema describes the unpacking format
390+ """
391+ # Build the values list with their types
392+ values: List[Tuple[Any, str]] = [
393+ (dtype_code, "int32"),
394+ (num_words, "int32"),
395+ (signal_length, "int32"),
396+ (state, "uint64"), # Use uint64 for state as it can be very large
397+ (len(symbol_counts), "int32"),
398+ (len(coeffs), "int32"),
399+ (len(initial), "int32"),
400+ ]
401+
402+ # Add symbol counts as uint32
403+ values.append((symbol_counts, "uint32"))
404+
405+ # Add symbol values with their original dtype
406+ symbol_dtype = symbol_values.dtype.name
407+ values.append((symbol_values, symbol_dtype))
408+
409+ # Add coefficients as float32
410+ values.append((coeffs, "float32"))
411+
412+ # Add initial values with their original dtype
413+ initial_dtype = initial.dtype.name
414+ values.append((initial, initial_dtype))
415+
416+ # Add any extra values
417+ if extra_values:
418+ values.extend(extra_values)
419+
420+ # Create schema for unpacking
421+ schema = [
422+ ("int32", 1), # dtype_code
423+ ("int32", 1), # num_words
424+ ("int32", 1), # signal_length
425+ ("uint64", 1), # state
426+ ("int32", 1), # num_symbols
427+ ("int32", 1), # num_coeffs
428+ ("int32", 1), # num_initial
429+ ("uint32", len(symbol_counts)), # symbol_counts
430+ (symbol_dtype, len(symbol_values)), # symbol_values
431+ ("float32", len(coeffs)), # coeffs
432+ (initial_dtype, len(initial)), # initial
433+ ]
434+
435+ # Add schema entries for extra values
436+ if extra_values:
437+ for value, dtype_str in extra_values:
438+ if isinstance(value, (list, np.ndarray)):
439+ count = len(value)
440+ else:
441+ count = 1
442+ schema.append((dtype_str, count))
443+
444+ header_bytes = pack_header_values(values)
445+ return header_bytes, schema
446+
447+
448+def unpack_lpc_header(data: bytes, schema: List[Tuple[str, int]]) -> dict:
449+ """
450+ Unpack an ANS LPC header according to the provided schema.
451+
452+ Args:
453+ data: Binary header data
454+ schema: Schema describing the data layout
455+
456+ Returns:
457+ Dictionary with unpacked header values
458+ """
459+ values = unpack_header_values(data, schema)
460+
461+ result = {
462+ "dtype_code": values[0][0], # Extract scalar from 1-element array
463+ "num_words": values[1][0],
464+ "signal_length": values[2][0],
465+ "state": values[3][0],
466+ "num_symbols": values[4][0],
467+ "num_coeffs": values[5][0],
468+ "num_initial": values[6][0],
469+ "symbol_counts": values[7], # Keep as array
470+ "symbol_values": values[8], # Keep as array
471+ "coeffs": values[9], # Keep as array
472+ "initial": values[10], # Keep as array
473+ }
474+
475+ # Add any extra values
476+ if len(values) > 11:
477+ extra_values = []
478+ for val in values[11:]:
479+ if len(val) == 1:
480+ extra_values.append(val[0]) # Extract scalar
481+ else:
482+ extra_values.append(val) # Keep as array
483+ result["extra_values"] = extra_values
484+
485+ return result
benchcompress/src/benchcompress/algorithms/ans/lpc_predict.cppadded+29−0View file
@@ -0,0 +1,29 @@
1+#include "lpc_predict.hpp"
2+
3+namespace py = pybind11;
4+
5+// Explicit instantiation for int16_t
6+std::tuple<py::array_t<float>, py::array_t<int16_t>, py::array_t<int16_t>>
7+lpc_predict_int16(py::array_t<int16_t> x, size_t M,
8+ size_t num_training_samples) {
9+ return lpc_predict_impl<int16_t>(x, M, num_training_samples);
10+}
11+
12+// Explicit instantiation for int32_t
13+std::tuple<py::array_t<float>, py::array_t<int32_t>, py::array_t<int32_t>>
14+lpc_predict_int32(py::array_t<int32_t> x, size_t M,
15+ size_t num_training_samples) {
16+ return lpc_predict_impl<int32_t>(x, M, num_training_samples);
17+}
18+
19+PYBIND11_MODULE(lpc_predict_cpp_ext, m) {
20+ m.doc() = "C++ implementation of lpc_predict using pybind11";
21+ m.def("lpc_predict_int16", &lpc_predict_int16,
22+ "Predict signal using Markov model and return coefficients, initial "
23+ "values and residuals (int16)",
24+ py::arg("x"), py::arg("M"), py::arg("num_training_samples") = 10000);
25+ m.def("lpc_predict_int32", &lpc_predict_int32,
26+ "Predict signal using Markov model and return coefficients, initial "
27+ "values and residuals (int32)",
28+ py::arg("x"), py::arg("M"), py::arg("num_training_samples") = 10000);
29+}
benchcompress/src/benchcompress/algorithms/ans/lpc_predict.hppadded+89−0View file
@@ -0,0 +1,89 @@
1+#pragma once
2+
3+#include <Eigen/Dense>
4+#include <cmath>
5+#include <iostream>
6+#include <pybind11/eigen.h>
7+#include <pybind11/numpy.h>
8+#include <pybind11/pybind11.h>
9+
10+namespace py = pybind11;
11+
12+template <typename T>
13+std::tuple<py::array_t<float>, py::array_t<T>, py::array_t<T>>
14+lpc_predict_impl(py::array_t<T> x, size_t M, size_t num_training_samples) {
15+ // Get array buffer
16+ auto x_buf = x.request();
17+ T *x_ptr = static_cast<T *>(x_buf.ptr);
18+ size_t N = x_buf.shape[0];
19+
20+ // Keep initial values for reconstruction
21+ size_t initial_size = M - 1;
22+ std::vector<ssize_t> initial_shape = {static_cast<ssize_t>(initial_size)};
23+ py::array_t<T> initial(initial_shape);
24+ py::buffer_info initial_buf = initial.request(true);
25+ T *initial_ptr = static_cast<T *>(initial_buf.ptr);
26+
27+ // Copy initial values
28+ for (size_t i = 0; i < initial_size; i++) {
29+ initial_ptr[i] = x_ptr[i];
30+ }
31+
32+ // Create sequences matrix for linear regression
33+ size_t resid_size = N - M + 1;
34+ // Use only num_training_samples sequences for model fitting
35+ size_t num_samples_for_fit = std::min(resid_size, num_training_samples);
36+ // Take evenly spaced samples for training
37+ size_t stride =
38+ resid_size > num_training_samples ? resid_size / num_training_samples : 1;
39+
40+ Eigen::MatrixXf predictors(num_samples_for_fit, M - 1);
41+ Eigen::VectorXf target(num_samples_for_fit);
42+
43+ // Fill predictors matrix and target vector with strided training samples
44+ for (size_t i = 0; i < num_samples_for_fit; i++) {
45+ size_t idx = i * stride;
46+ for (size_t j = 0; j < M - 1; j++) {
47+ predictors(i, j) = static_cast<float>(x_ptr[idx + j]);
48+ }
49+ target(i) = static_cast<float>(x_ptr[idx + M - 1]);
50+ }
51+
52+ // Add constant term column (ones) to predictors
53+ Eigen::MatrixXf X(predictors.rows(), predictors.cols() + 1);
54+ X << predictors, Eigen::VectorXf::Ones(predictors.rows());
55+
56+ // Solve least squares problem: X * coeffs = target
57+ Eigen::VectorXf coeffs = X.colPivHouseholderQr().solve(target);
58+
59+ // Create coefficients array
60+ std::vector<ssize_t> coeffs_shape = {static_cast<ssize_t>(M)};
61+ py::array_t<float> coeffs_array(coeffs_shape);
62+ py::buffer_info coeffs_buf = coeffs_array.request(true);
63+ float *coeffs_ptr = static_cast<float *>(coeffs_buf.ptr);
64+
65+ // Copy coefficients
66+ for (size_t i = 0; i < M - 1; i++) {
67+ coeffs_ptr[i] = coeffs(i);
68+ }
69+ coeffs_ptr[M - 1] = coeffs(M - 1); // bias term
70+
71+ // Calculate residuals
72+ std::vector<ssize_t> resid_shape = {static_cast<ssize_t>(resid_size)};
73+ py::array_t<T> residuals(resid_shape);
74+ py::buffer_info resid_buf = residuals.request(true);
75+ T *resid_ptr = static_cast<T *>(resid_buf.ptr);
76+
77+ for (size_t i = 0; i < resid_size; i++) {
78+ float prediction = 0.0f;
79+ for (size_t j = 0; j < M - 1; j++) {
80+ float term = coeffs_ptr[j] * static_cast<float>(x_ptr[i + j]);
81+ prediction += term;
82+ }
83+ prediction += coeffs_ptr[M - 1]; // bias term
84+ float rounded_prediction = std::round(prediction);
85+ resid_ptr[i] = x_ptr[i + M - 1] - static_cast<T>(rounded_prediction);
86+ }
87+
88+ return std::make_tuple(coeffs_array, initial, residuals);
89+}
benchcompress/src/benchcompress/algorithms/ans/lpc_predict.pyadded+26−0View file
@@ -0,0 +1,26 @@
1+import numpy as np
2+from .lpc_predict_cpp_ext import lpc_predict_int16, lpc_predict_int32
3+
4+
5+def lpc_predict(x: np.ndarray, M: int, num_training_samples: int = 10000) -> tuple:
6+ """Predict signal using Markov model and return coefficients, initial values and residuals using C++ implementation.
7+
8+ Args:
9+ x: Input signal (must be int16 or int32)
10+ M: Number of previous samples to use for prediction
11+ num_training_samples: Maximum number of samples to use for fitting the model coefficients (default: 10000).
12+ Using fewer samples speeds up model fitting on large inputs while maintaining accuracy.
13+
14+ Returns:
15+ tuple: (coefficients (float32), initial_values (same dtype as input), residuals (same dtype as input))
16+
17+ Raises:
18+ ValueError: If input array is not int16 or int32
19+ """
20+ # Check input dtype and call appropriate implementation
21+ if x.dtype == np.int16:
22+ return lpc_predict_int16(x, M, num_training_samples)
23+ elif x.dtype == np.int32:
24+ return lpc_predict_int32(x, M, num_training_samples)
25+ else:
26+ raise ValueError(f"Input array must be int16 or int32, got {x.dtype}")
benchcompress/src/benchcompress/algorithms/ans/lpc_reconstruct.cppadded+27−0View file
@@ -0,0 +1,27 @@
1+#include "lpc_reconstruct.hpp"
2+
3+namespace py = pybind11;
4+
5+// Explicit instantiation for int16_t
6+py::array_t<int16_t> lpc_reconstruct_int16(py::array_t<float> coeffs,
7+ py::array_t<int16_t> initial,
8+ py::array_t<int16_t> resid) {
9+ return lpc_reconstruct_impl<int16_t>(coeffs, initial, resid);
10+}
11+
12+// Explicit instantiation for int32_t
13+py::array_t<int32_t> lpc_reconstruct_int32(py::array_t<float> coeffs,
14+ py::array_t<int32_t> initial,
15+ py::array_t<int32_t> resid) {
16+ return lpc_reconstruct_impl<int32_t>(coeffs, initial, resid);
17+}
18+
19+PYBIND11_MODULE(lpc_reconstruct_cpp_ext, m) {
20+ m.doc() = "C++ implementation of lpc_reconstruct using pybind11";
21+ m.def("lpc_reconstruct_int16", &lpc_reconstruct_int16,
22+ "Reconstruct signal from Markov model parameters and residuals (int16)",
23+ py::arg("coeffs"), py::arg("initial"), py::arg("resid"));
24+ m.def("lpc_reconstruct_int32", &lpc_reconstruct_int32,
25+ "Reconstruct signal from Markov model parameters and residuals (int32)",
26+ py::arg("coeffs"), py::arg("initial"), py::arg("resid"));
27+}
benchcompress/src/benchcompress/algorithms/ans/lpc_reconstruct.hppadded+64−0View file
@@ -0,0 +1,64 @@
1+#pragma once
2+
3+#include <cmath>
4+#include <iostream>
5+#include <pybind11/numpy.h>
6+#include <pybind11/pybind11.h>
7+
8+namespace py = pybind11;
9+
10+template <typename T>
11+py::array_t<T> lpc_reconstruct_impl(py::array_t<float> coeffs,
12+ py::array_t<T> initial,
13+ py::array_t<T> resid) {
14+ // Get array buffers
15+ auto coeffs_buf = coeffs.request();
16+ auto initial_buf = initial.request();
17+ auto resid_buf = resid.request();
18+
19+ // Get raw pointers to data
20+ float *coeffs_ptr = static_cast<float *>(coeffs_buf.ptr);
21+ T *initial_ptr = static_cast<T *>(initial_buf.ptr);
22+ T *resid_ptr = static_cast<T *>(resid_buf.ptr);
23+
24+ // Calculate dimensions
25+ size_t M = initial_buf.shape[0] + 1; // Number of samples used in prediction
26+ size_t output_size = resid_buf.shape[0] + initial_buf.shape[0];
27+
28+ // Create output array with explicit shape and memory ownership
29+ std::vector<ssize_t> shape = {static_cast<ssize_t>(output_size)};
30+ py::array_t<T> output(shape);
31+ py::buffer_info output_buf = output.request(true); // Request writable buffer
32+ T *output_ptr = static_cast<T *>(output_buf.ptr);
33+
34+ // Copy initial values with bounds check
35+ for (size_t i = 0; i < initial_buf.shape[0] && i < output_size; i++) {
36+ output_ptr[i] = initial_ptr[i];
37+ }
38+
39+ size_t resid_size = resid_buf.shape[0];
40+
41+ // Reconstruct signal iteratively
42+ for (size_t i = 0; i < resid_size; i++) {
43+ float prediction = 0.0f;
44+
45+ // Calculate prediction using coefficients (excluding bias term)
46+ for (size_t j = 0; j < M - 1; j++) {
47+ float term = coeffs_ptr[j] * static_cast<float>(output_ptr[i + j]);
48+ prediction += term;
49+ }
50+
51+ // Add bias term separately
52+ prediction += coeffs_ptr[coeffs_buf.shape[0] - 1];
53+
54+ // Round prediction to nearest integer
55+ float rounded_prediction = std::round(prediction);
56+
57+ // Add residual and store result
58+ T final_value =
59+ static_cast<T>(rounded_prediction + static_cast<float>(resid_ptr[i]));
60+ output_ptr[i + M - 1] = final_value;
61+ }
62+
63+ return output;
64+}
benchcompress/src/benchcompress/algorithms/ans/lpc_reconstruct.pyadded+39−0View file
@@ -0,0 +1,39 @@
1+import numpy as np
2+from .lpc_reconstruct_cpp_ext import (
3+ lpc_reconstruct_int16,
4+ lpc_reconstruct_int32,
5+)
6+
7+
8+def lpc_reconstruct(coeffs, initial, resid):
9+ """Reconstruct signal from Markov model parameters and residuals using C++ implementation.
10+
11+ Args:
12+ coeffs: Model coefficients from linear regression (float32)
13+ initial: Initial values needed for prediction (int16 or int32)
14+ resid: Prediction residuals (must match initial dtype)
15+
16+ Returns:
17+ np.ndarray: Reconstructed signal (same dtype as input)
18+
19+ Raises:
20+ ValueError: If initial/resid arrays are not int16 or int32, or if their dtypes don't match
21+ """
22+ # Convert coeffs to float32 if needed
23+ coeffs = coeffs.astype(np.float32)
24+
25+ # Check input dtypes
26+ if initial.dtype != resid.dtype:
27+ raise ValueError(
28+ f"Initial and residual arrays must have same dtype, got {initial.dtype} and {resid.dtype}"
29+ )
30+
31+ # Call appropriate implementation based on dtype
32+ if initial.dtype == np.int16:
33+ return lpc_reconstruct_int16(coeffs, initial, resid)
34+ elif initial.dtype == np.int32:
35+ return lpc_reconstruct_int32(coeffs, initial, resid)
36+ else:
37+ raise ValueError(
38+ f"Initial/residual arrays must be int16 or int32, got {initial.dtype}"
39+ )
benchcompress/src/benchcompress/algorithms/blosc2/__init__.pyadded+366−0View file
@@ -0,0 +1,366 @@
1+import numpy as np
2+import os
3+from ..ans.lpc_reconstruct import lpc_reconstruct as lpc_reconstruct_cpp
4+from ..ans.lpc_predict import lpc_predict as lpc_predict_cpp
5+from ..ans.get_run_lengths import get_run_lengths
6+
7+SOURCE_FILE = "blosc2/__init__.py"
8+
9+
10+def _load_long_description():
11+ current_dir = os.path.dirname(os.path.abspath(__file__))
12+ md_path = os.path.join(current_dir, "blosc2.md")
13+ with open(md_path, "r", encoding="utf-8") as f:
14+ return f.read()
15+
16+
17+LONG_DESCRIPTION = _load_long_description()
18+
19+
20+def blosc2_encode(x: np.ndarray, clevel: int, filter: int = 2) -> bytes:
21+ import blosc2
22+
23+ # Convert filter int to proper enum
24+ if filter == 2:
25+ blosc_filter = blosc2.Filter.BITSHUFFLE
26+ elif filter == 1:
27+ blosc_filter = blosc2.Filter.SHUFFLE
28+ else:
29+ blosc_filter = blosc2.Filter.NOFILTER
30+
31+ # Get typesize from numpy array
32+ typesize = x.dtype.itemsize
33+
34+ # Compress data
35+ compressed = blosc2.compress(
36+ x, # numpy arrays support buffer interface
37+ typesize=typesize,
38+ clevel=clevel,
39+ filter=blosc_filter,
40+ codec=blosc2.Codec.ZSTD,
41+ )
42+ assert isinstance(compressed, bytes) # Type assertion
43+ return compressed
44+
45+
46+def blosc2_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
47+ import blosc2
48+
49+ decompressed = blosc2.decompress(x)
50+ assert isinstance(decompressed, (bytes, bytearray)) # Type assertion
51+ arr = np.frombuffer(decompressed, dtype=np.dtype(dtype))
52+ return arr.reshape(shape)
53+
54+
55+def blosc2_delta_encode(x: np.ndarray, clevel: int) -> bytes:
56+ import blosc2
57+
58+ assert x.ndim == 1
59+
60+ y = np.diff(x)
61+ y = np.insert(y, 0, x[0])
62+ buf = y.tobytes()
63+
64+ # Convert default filter int to proper enum
65+ blosc_filter = blosc2.Filter.BITSHUFFLE
66+
67+ # Get typesize from numpy array
68+ typesize = y.dtype.itemsize
69+
70+ # Compress data
71+ compressed = blosc2.compress(
72+ buf,
73+ typesize=typesize,
74+ clevel=clevel,
75+ filter=blosc_filter,
76+ codec=blosc2.Codec.ZSTD,
77+ )
78+ assert isinstance(compressed, bytes) # Type assertion
79+ return compressed
80+
81+
82+def blosc2_delta_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
83+ import blosc2
84+
85+ assert len(shape) == 1
86+
87+ decompressed = blosc2.decompress(x)
88+ assert isinstance(decompressed, (bytes, bytearray)) # Type assertion
89+ y = np.frombuffer(decompressed, dtype=np.dtype(dtype))
90+ return np.cumsum(y)
91+
92+
93+def blosc2_lpc_encode(x: np.ndarray, clevel: int) -> bytes:
94+ import blosc2
95+ import struct
96+
97+ assert x.ndim == 1
98+ coeffs, initial, resid = lpc_predict_cpp(x, M=6, num_training_samples=10000)
99+
100+ # Convert coeffs and initial to bytes
101+ coeffs_bytes = coeffs.tobytes()
102+ initial_bytes = initial.tobytes()
103+
104+ # Create header with lengths
105+ header = struct.pack("QQ", len(coeffs_bytes), len(initial_bytes))
106+
107+ # Convert default filter int to proper enum
108+ blosc_filter = blosc2.Filter.BITSHUFFLE
109+
110+ # Get typesize from numpy array
111+ typesize = resid.dtype.itemsize
112+
113+ # Compress data
114+ compressed = blosc2.compress(
115+ resid.tobytes(),
116+ typesize=typesize,
117+ clevel=clevel,
118+ filter=blosc_filter,
119+ codec=blosc2.Codec.ZSTD,
120+ )
121+
122+ # Combine all parts
123+ return header + coeffs_bytes + initial_bytes + compressed
124+
125+
126+def blosc2_lpc_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
127+ import blosc2
128+ import struct
129+
130+ assert len(shape) == 1
131+
132+ # Extract header
133+ header_size = struct.calcsize("QQ")
134+ coeffs_len, initial_len = struct.unpack("QQ", x[:header_size])
135+
136+ # Extract coefficients and initial values
137+ pos = header_size
138+ coeffs = np.frombuffer(x[pos : pos + coeffs_len], dtype=np.float32)
139+ pos += coeffs_len
140+ initial = np.frombuffer(x[pos : pos + initial_len], dtype=dtype)
141+ pos += initial_len
142+
143+ # Decompress residuals
144+ decompressed = blosc2.decompress(x[pos:])
145+ assert isinstance(decompressed, (bytes, bytearray)) # Type assertion
146+ resid = np.frombuffer(decompressed, dtype=dtype)
147+
148+ # Reconstruct signal
149+ output = lpc_reconstruct_cpp(coeffs, initial, resid)
150+ return output
151+
152+
153+def blosc2_lpc_zrle_encode(x: np.ndarray, clevel: int) -> bytes:
154+ import blosc2
155+ import struct
156+
157+ assert x.ndim == 1
158+
159+ # Get run lengths for zero/non-zero sequences
160+ run_lengths = get_run_lengths(x)
161+
162+ # Determine run length dtype code
163+ if run_lengths.dtype == np.uint8:
164+ run_length_dtype_code = 0
165+ elif run_lengths.dtype == np.uint16:
166+ run_length_dtype_code = 1
167+ elif run_lengths.dtype == np.uint32:
168+ run_length_dtype_code = 2
169+ else:
170+ raise ValueError(f"Unsupported run length dtype: {run_lengths.dtype}")
171+
172+ # Extract non-zero data
173+ non_zero_arrays = []
174+ array_pos = 0
175+ i = 0
176+ while i < len(run_lengths):
177+ non_zero_len = int(run_lengths[i])
178+ if non_zero_len > 0:
179+ non_zero_arrays.append(x[array_pos : array_pos + non_zero_len])
180+ array_pos += non_zero_len
181+ i += 1
182+ if i < len(run_lengths):
183+ array_pos += int(run_lengths[i]) # Skip zeros
184+ i += 1
185+
186+ non_zero_data = np.concatenate(non_zero_arrays)
187+
188+ # Apply linear predictive coding on non-zero data
189+ coeffs, initial, resid = lpc_predict_cpp(
190+ non_zero_data, M=6, num_training_samples=10000
191+ )
192+
193+ # Convert data to bytes
194+ coeffs_bytes = coeffs.tobytes()
195+ initial_bytes = initial.tobytes()
196+ run_lengths_bytes = run_lengths.tobytes()
197+
198+ # Create header with lengths and dtype code
199+ header = struct.pack(
200+ "QQQQB",
201+ len(coeffs_bytes),
202+ len(initial_bytes),
203+ len(run_lengths_bytes),
204+ len(run_lengths),
205+ run_length_dtype_code,
206+ )
207+
208+ # Convert default filter int to proper enum
209+ blosc_filter = blosc2.Filter.BITSHUFFLE
210+
211+ # Get typesize from numpy array
212+ typesize = resid.dtype.itemsize
213+
214+ # Compress residuals
215+ compressed = blosc2.compress(
216+ resid.tobytes(),
217+ typesize=typesize,
218+ clevel=clevel,
219+ filter=blosc_filter,
220+ codec=blosc2.Codec.ZSTD,
221+ )
222+
223+ # Combine all parts
224+ return header + coeffs_bytes + initial_bytes + run_lengths_bytes + compressed
225+
226+
227+def blosc2_lpc_zrle_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
228+ import blosc2
229+ import struct
230+
231+ assert len(shape) == 1
232+
233+ # Extract header
234+ header_size = struct.calcsize("QQQQB")
235+ coeffs_len, initial_len, run_lengths_len, num_run_lengths, run_length_dtype_code = (
236+ struct.unpack("QQQQB", x[:header_size])
237+ )
238+
239+ # Extract components
240+ pos = header_size
241+ coeffs = np.frombuffer(x[pos : pos + coeffs_len], dtype=np.float32)
242+ pos += coeffs_len
243+ initial = np.frombuffer(x[pos : pos + initial_len], dtype=dtype)
244+ pos += initial_len
245+
246+ # Get run lengths with proper dtype
247+ if run_length_dtype_code == 0:
248+ run_lengths = np.frombuffer(x[pos : pos + run_lengths_len], dtype=np.uint8)
249+ elif run_length_dtype_code == 1:
250+ run_lengths = np.frombuffer(x[pos : pos + run_lengths_len], dtype=np.uint16)
251+ elif run_length_dtype_code == 2:
252+ run_lengths = np.frombuffer(x[pos : pos + run_lengths_len], dtype=np.uint32)
253+ else:
254+ raise ValueError(f"Unsupported run length dtype code: {run_length_dtype_code}")
255+ pos += run_lengths_len
256+
257+ assert len(run_lengths) == num_run_lengths
258+
259+ # Decompress residuals
260+ decompressed = blosc2.decompress(x[pos:])
261+ assert isinstance(decompressed, (bytes, bytearray)) # Type assertion
262+ resid = np.frombuffer(decompressed, dtype=dtype)
263+
264+ # Reconstruct non-zero data
265+ non_zero_data = lpc_reconstruct_cpp(coeffs, initial, resid)
266+
267+ # Reconstruct full array using run lengths
268+ non_zero_pos = 0
269+ i = 0
270+ segments = []
271+ while i < len(run_lengths):
272+ non_zero_len = int(run_lengths[i])
273+ if non_zero_len > 0:
274+ segment = non_zero_data[non_zero_pos : non_zero_pos + non_zero_len]
275+ segments.append(segment)
276+ non_zero_pos += non_zero_len
277+ i += 1
278+ if i < len(run_lengths):
279+ segments.append(np.zeros(int(run_lengths[i]), dtype=non_zero_data.dtype))
280+ i += 1
281+
282+ return np.concatenate(segments)
283+
284+
285+algorithms = [
286+ {
287+ "name": "blosc2-1",
288+ "version": "1b",
289+ "encode": lambda x: blosc2_encode(x, clevel=1),
290+ "decode": lambda x, dtype, shape: blosc2_decode(x, dtype, shape),
291+ "description": "Blosc2 compression at level 1 (fastest compression).",
292+ "tags": ["blosc2"],
293+ "source_file": SOURCE_FILE,
294+ "long_description": LONG_DESCRIPTION,
295+ },
296+ {
297+ "name": "blosc2-5",
298+ "version": "1",
299+ "encode": lambda x: blosc2_encode(x, clevel=5),
300+ "decode": lambda x, dtype, shape: blosc2_decode(x, dtype, shape),
301+ "description": "Blosc2 compression at level 5 (balanced speed/compression).",
302+ "tags": ["blosc2"],
303+ "source_file": SOURCE_FILE,
304+ "long_description": LONG_DESCRIPTION,
305+ },
306+ {
307+ "name": "blosc2-9",
308+ "version": "1",
309+ "encode": lambda x: blosc2_encode(x, clevel=9),
310+ "decode": lambda x, dtype, shape: blosc2_decode(x, dtype, shape),
311+ "description": "Blosc2 compression at level 9 (maximum compression).",
312+ "tags": ["blosc2"],
313+ "source_file": SOURCE_FILE,
314+ "long_description": LONG_DESCRIPTION,
315+ },
316+ {
317+ "name": "blosc2-9-delta",
318+ "version": "1",
319+ "encode": lambda x: blosc2_delta_encode(x, clevel=9),
320+ "decode": lambda x, dtype, shape: blosc2_delta_decode(x, dtype, shape),
321+ "description": "Blosc2 compression at level 9 with delta encoding for improved compression of sequential data.",
322+ "tags": ["blosc2", "delta_encoding", "1d"],
323+ "source_file": SOURCE_FILE,
324+ "long_description": LONG_DESCRIPTION,
325+ },
326+ {
327+ "name": "blosc2-1-lpc",
328+ "version": "1",
329+ "encode": lambda x: blosc2_lpc_encode(x, clevel=1),
330+ "decode": lambda x, dtype, shape: blosc2_lpc_decode(x, dtype, shape),
331+ "description": "Blosc2 compression at level 1 with linear predictive codingn for exploiting temporal correlations in the data.",
332+ "tags": ["blosc2", "lpc_prediction", "1d"],
333+ "source_file": SOURCE_FILE,
334+ "long_description": LONG_DESCRIPTION,
335+ },
336+ {
337+ "name": "blosc2-5-lpc",
338+ "version": "1",
339+ "encode": lambda x: blosc2_lpc_encode(x, clevel=1),
340+ "decode": lambda x, dtype, shape: blosc2_lpc_decode(x, dtype, shape),
341+ "description": "Blosc2 compression at level 5 with linear predictive codingn for exploiting temporal correlations in the data.",
342+ "tags": ["blosc2", "lpc_prediction", "1d"],
343+ "source_file": SOURCE_FILE,
344+ "long_description": LONG_DESCRIPTION,
345+ },
346+ {
347+ "name": "blosc2-9-lpc",
348+ "version": "1",
349+ "encode": lambda x: blosc2_lpc_encode(x, clevel=9),
350+ "decode": lambda x, dtype, shape: blosc2_lpc_decode(x, dtype, shape),
351+ "description": "Blosc2 compression at level 9 with linear predictive codingn for exploiting temporal correlations in the data.",
352+ "tags": ["blosc2", "lpc_prediction", "1d"],
353+ "source_file": SOURCE_FILE,
354+ "long_description": LONG_DESCRIPTION,
355+ },
356+ {
357+ "name": "blosc2-9-lpc-zrle",
358+ "version": "1",
359+ "encode": lambda x: blosc2_lpc_zrle_encode(x, clevel=9),
360+ "decode": lambda x, dtype, shape: blosc2_lpc_zrle_decode(x, dtype, shape),
361+ "description": "Blosc2 compression at level 9 with linear predictive codingn and zero run-length encoding for sparse data.",
362+ "tags": ["blosc2", "lpc_prediction", "zero_rle", "1d"],
363+ "source_file": SOURCE_FILE,
364+ "long_description": LONG_DESCRIPTION,
365+ },
366+]
benchcompress/src/benchcompress/algorithms/blosc2/blosc2.mdadded+27−0View file
@@ -0,0 +1,27 @@
1+# Blosc2 Algorithm
2+
3+Blosc2 is a modern, fast data compression library that builds upon the original Blosc library. It is designed for efficient compression of binary data, particularly optimized for in-memory compression of numerical arrays. Blosc2 uses block-oriented compression with support for multithreading and SIMD instructions.
4+
5+## Features
6+
7+- Fast compression and decompression speeds
8+- Block-oriented compression for better cache usage
9+- Support for various shuffling filters to improve compression ratios
10+- Built-in support for delta filtering
11+- Uses ZSTD compression codec
12+
13+## Variants
14+
15+### Standard Compression
16+Different compression levels trading off speed vs compression ratio:
17+- blosc2-1: Fastest compression with level 1
18+- blosc2-5: Balanced speed/compression with level 5
19+- blosc2-9: Maximum compression with level 9
20+
21+### Advanced Variants
22+
23+#### Delta Encoding (blosc2-9-delta)
24+Uses Blosc2's delta filter along with maximum compression. The delta filter stores differences between consecutive values, which is particularly effective for time series data where adjacent values are similar. This variant combines:
25+- Delta filtering for temporal correlation
26+- Bit shuffling for improved compression
27+- Level 9 compression for maximum compression ratio
benchcompress/src/benchcompress/algorithms/brotli/__init__.pyadded+98−0View file
@@ -0,0 +1,98 @@
1+import numpy as np
2+import brotli
3+import os
4+
5+
6+SOURCE_FILE = "brotli/__init__.py"
7+
8+
9+def _load_long_description():
10+ current_dir = os.path.dirname(os.path.abspath(__file__))
11+ md_path = os.path.join(current_dir, "brotli.md")
12+ with open(md_path, "r", encoding="utf-8") as f:
13+ return f.read()
14+
15+
16+LONG_DESCRIPTION = _load_long_description()
17+
18+
19+def brotli_delta_encode(x: np.ndarray, level: int) -> bytes:
20+ assert x.ndim == 1
21+ y = np.diff(x)
22+ y = np.insert(y, 0, x[0])
23+ buf = y.tobytes()
24+ compressed = brotli.compress(buf, quality=level)
25+ return compressed
26+
27+
28+def brotli_delta_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
29+ assert len(shape) == 1
30+ buf = brotli.decompress(x)
31+ y = np.frombuffer(buf, dtype=dtype)
32+ return np.cumsum(y)
33+
34+
35+def brotli_encode(x: np.ndarray, level: int) -> bytes:
36+ buf = x.tobytes()
37+ compressed = brotli.compress(buf, quality=level)
38+ return compressed
39+
40+
41+def brotli_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
42+ buf = brotli.decompress(x)
43+ y = np.frombuffer(buf, dtype=dtype)
44+ return y.reshape(shape)
45+
46+
47+algorithms = [
48+ {
49+ "name": "brotli-4",
50+ "version": "1",
51+ "encode": lambda x: brotli_encode(x, level=4),
52+ "decode": lambda x, dtype, shape: brotli_decode(x, dtype, shape),
53+ "description": "Brotli compression at level 4 (faster).",
54+ "tags": ["brotli"],
55+ "source_file": SOURCE_FILE,
56+ "long_description": LONG_DESCRIPTION,
57+ },
58+ {
59+ "name": "brotli-6",
60+ "version": "1",
61+ "encode": lambda x: brotli_encode(x, level=6),
62+ "decode": lambda x, dtype, shape: brotli_decode(x, dtype, shape),
63+ "description": "Brotli compression at level 6 (balanced).",
64+ "tags": ["brotli"],
65+ "source_file": SOURCE_FILE,
66+ "long_description": LONG_DESCRIPTION,
67+ },
68+ {
69+ "name": "brotli-8",
70+ "version": "1",
71+ "encode": lambda x: brotli_encode(x, level=8),
72+ "decode": lambda x, dtype, shape: brotli_decode(x, dtype, shape),
73+ "description": "Brotli compression at level 8 (better compression).",
74+ "tags": ["brotli"],
75+ "source_file": SOURCE_FILE,
76+ "long_description": LONG_DESCRIPTION,
77+ },
78+ {
79+ "name": "brotli-11",
80+ "version": "1",
81+ "encode": lambda x: brotli_encode(x, level=11),
82+ "decode": lambda x, dtype, shape: brotli_decode(x, dtype, shape),
83+ "description": "Brotli compression at maximum level 11.",
84+ "tags": ["brotli"],
85+ "source_file": SOURCE_FILE,
86+ "long_description": LONG_DESCRIPTION,
87+ },
88+ {
89+ "name": "brotli-11-delta",
90+ "version": "1",
91+ "encode": lambda x: brotli_delta_encode(x, level=11),
92+ "decode": lambda x, dtype, shape: brotli_delta_decode(x, dtype, shape),
93+ "description": "Brotli compression at level 11 with delta encoding.",
94+ "tags": ["brotli", "delta_encoding", "1d"],
95+ "source_file": SOURCE_FILE,
96+ "long_description": LONG_DESCRIPTION,
97+ },
98+]
benchcompress/src/benchcompress/algorithms/brotli/brotli.mdadded+18−0View file
@@ -0,0 +1,18 @@
1+# Brotli Algorithm
2+
3+Brotli is a general-purpose data compression algorithm developed by Google. It uses a combination of modern compression techniques including LZ77, Huffman coding, and context modeling.
4+
5+## Variants
6+
7+We provide several variants with different compression levels:
8+
9+### Standard Compression
10+- brotli-4: Fast compression with moderate ratio
11+- brotli-6: Balanced between speed and compression
12+- brotli-8: Better compression at cost of speed
13+- brotli-11: Maximum compression level
14+
15+### Delta-Encoded Compression
16+- brotli-11-delta: Maximum compression with delta encoding
17+
18+Delta encoding stores differences between consecutive values rather than the values themselves. This can improve compression for sequences where adjacent values are often similar, like time series data.
benchcompress/src/benchcompress/algorithms/bzip2/__init__.pyadded+116−0View file
@@ -0,0 +1,116 @@
1+import numpy as np
2+import os
3+
4+
5+SOURCE_FILE = "bzip2/__init__.py"
6+
7+
8+def _load_long_description():
9+ current_dir = os.path.dirname(os.path.abspath(__file__))
10+ md_path = os.path.join(current_dir, "bzip2.md")
11+ with open(md_path, "r", encoding="utf-8") as f:
12+ return f.read()
13+
14+
15+LONG_DESCRIPTION = _load_long_description()
16+
17+
18+def bzip2_encode(x: np.ndarray, level: int) -> bytes:
19+ import bz2
20+
21+ buf = x.tobytes()
22+ compressed = bz2.compress(buf, compresslevel=level)
23+ return compressed
24+
25+
26+def bzip2_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
27+ import bz2
28+
29+ buf = bz2.decompress(x)
30+ y = np.frombuffer(buf, dtype=dtype)
31+ return y.reshape(shape)
32+
33+
34+def bzip2_delta_encode(x: np.ndarray, level: int) -> bytes:
35+ import bz2
36+
37+ assert x.ndim == 1
38+ y = np.diff(x)
39+ y = np.insert(y, 0, x[0])
40+ buf = y.tobytes()
41+ compressed = bz2.compress(buf, compresslevel=level)
42+ return compressed
43+
44+
45+def bzip2_delta_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
46+ import bz2
47+
48+ assert len(shape) == 1
49+
50+ buf = bz2.decompress(x)
51+ y = np.frombuffer(buf, dtype=dtype)
52+ return np.cumsum(y)
53+
54+
55+algorithms = [
56+ {
57+ "name": "bzip2-1",
58+ "version": "1",
59+ "encode": lambda x: bzip2_encode(x, level=1),
60+ "decode": lambda x, dtype, shape: bzip2_decode(x, dtype, shape),
61+ "description": "Bzip2 compression at level 1 (fastest).",
62+ "tags": ["bzip2"],
63+ "source_file": SOURCE_FILE,
64+ "long_description": LONG_DESCRIPTION,
65+ },
66+ {
67+ "name": "bzip2-3",
68+ "version": "1",
69+ "encode": lambda x: bzip2_encode(x, level=3),
70+ "decode": lambda x, dtype, shape: bzip2_decode(x, dtype, shape),
71+ "description": "Bzip2 compression at level 3.",
72+ "tags": ["bzip2"],
73+ "source_file": SOURCE_FILE,
74+ "long_description": LONG_DESCRIPTION,
75+ },
76+ {
77+ "name": "bzip2-5",
78+ "version": "1",
79+ "encode": lambda x: bzip2_encode(x, level=5),
80+ "decode": lambda x, dtype, shape: bzip2_decode(x, dtype, shape),
81+ "description": "Bzip2 compression at level 5 (medium).",
82+ "tags": ["bzip2"],
83+ "source_file": SOURCE_FILE,
84+ "long_description": LONG_DESCRIPTION,
85+ },
86+ {
87+ "name": "bzip2-7",
88+ "version": "1",
89+ "encode": lambda x: bzip2_encode(x, level=7),
90+ "decode": lambda x, dtype, shape: bzip2_decode(x, dtype, shape),
91+ "description": "Bzip2 compression at level 7.",
92+ "tags": ["bzip2"],
93+ "source_file": SOURCE_FILE,
94+ "long_description": LONG_DESCRIPTION,
95+ },
96+ {
97+ "name": "bzip2-9",
98+ "version": "1",
99+ "encode": lambda x: bzip2_encode(x, level=9),
100+ "decode": lambda x, dtype, shape: bzip2_decode(x, dtype, shape),
101+ "description": "Bzip2 compression at maximum level 9.",
102+ "tags": ["bzip2"],
103+ "source_file": SOURCE_FILE,
104+ "long_description": LONG_DESCRIPTION,
105+ },
106+ {
107+ "name": "bzip2-9-delta",
108+ "version": "1",
109+ "encode": lambda x: bzip2_delta_encode(x, level=9),
110+ "decode": lambda x, dtype, shape: bzip2_delta_decode(x, dtype, shape),
111+ "description": "Bzip2 compression at level 9 with delta encoding.",
112+ "tags": ["bzip2", "delta_encoding", "1d"],
113+ "source_file": SOURCE_FILE,
114+ "long_description": LONG_DESCRIPTION,
115+ },
116+]
benchcompress/src/benchcompress/algorithms/bzip2/bzip2.mdadded+18−0View file
@@ -0,0 +1,18 @@
1+# Bzip2 Algorithm
2+
3+Bzip2 is a compression algorithm that uses the Burrows-Wheeler transform along with Huffman coding. It typically achieves better compression than traditional LZ77/LZ78-based algorithms but at the cost of speed.
4+
5+## Variants
6+
7+### Standard Compression
8+Different compression levels trading off speed vs compression ratio:
9+- bzip2-1: Fastest compression
10+- bzip2-3: Better compression than level 1
11+- bzip2-5: Medium compression
12+- bzip2-7: Better compression than level 5
13+- bzip2-9: Maximum compression
14+
15+### Delta Encoding
16+- bzip2-9-delta: Maximum compression with delta encoding
17+
18+Delta encoding stores differences between consecutive values rather than the values themselves. This can improve compression for sequences where adjacent values are often similar, such as time series data.
benchcompress/src/benchcompress/algorithms/lz4/__init__.pyadded+106−0View file
@@ -0,0 +1,106 @@
1+import numpy as np
2+import os
3+
4+
5+SOURCE_FILE = "lz4/__init__.py"
6+
7+
8+def _load_long_description():
9+ current_dir = os.path.dirname(os.path.abspath(__file__))
10+ md_path = os.path.join(current_dir, "lz4.md")
11+ with open(md_path, "r", encoding="utf-8") as f:
12+ return f.read()
13+
14+
15+LONG_DESCRIPTION = _load_long_description()
16+
17+
18+def lz4_encode(x: np.ndarray, level: int) -> bytes:
19+ import lz4.frame
20+
21+ buf = x.tobytes()
22+ compressed = lz4.frame.compress(buf, compression_level=level)
23+ return compressed
24+
25+
26+def lz4_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
27+ import lz4.frame
28+
29+ buf = lz4.frame.decompress(x)
30+ y = np.frombuffer(buf, dtype=dtype)
31+ return y.reshape(shape)
32+
33+
34+def lz4_delta_encode(x: np.ndarray, level: int) -> bytes:
35+ import lz4.frame
36+
37+ assert x.ndim == 1
38+ y = np.diff(x)
39+ y = np.insert(y, 0, x[0])
40+ buf = y.tobytes()
41+ compressed = lz4.frame.compress(buf, compression_level=level)
42+ return compressed
43+
44+
45+def lz4_delta_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
46+ import lz4.frame
47+
48+ assert len(shape) == 1
49+
50+ buf = lz4.frame.decompress(x)
51+ y = np.frombuffer(buf, dtype=dtype)
52+ return np.cumsum(y)
53+
54+
55+algorithms = [
56+ {
57+ "name": "lz4-0",
58+ "version": "1",
59+ "encode": lambda x: lz4_encode(x, level=0),
60+ "decode": lambda x, dtype, shape: lz4_decode(x, dtype, shape),
61+ "description": "LZ4 compression at level 0 (fastest).",
62+ "tags": ["lz4"],
63+ "source_file": SOURCE_FILE,
64+ "long_description": LONG_DESCRIPTION,
65+ },
66+ {
67+ "name": "lz4-3",
68+ "version": "1",
69+ "encode": lambda x: lz4_encode(x, level=3),
70+ "decode": lambda x, dtype, shape: lz4_decode(x, dtype, shape),
71+ "description": "LZ4 compression at level 3 (minimum high compression).",
72+ "tags": ["lz4"],
73+ "source_file": SOURCE_FILE,
74+ "long_description": LONG_DESCRIPTION,
75+ },
76+ {
77+ "name": "lz4-10",
78+ "version": "1",
79+ "encode": lambda x: lz4_encode(x, level=10),
80+ "decode": lambda x, dtype, shape: lz4_decode(x, dtype, shape),
81+ "description": "LZ4 compression at level 10.",
82+ "tags": ["lz4"],
83+ "source_file": SOURCE_FILE,
84+ "long_description": LONG_DESCRIPTION,
85+ },
86+ {
87+ "name": "lz4-16",
88+ "version": "1",
89+ "encode": lambda x: lz4_encode(x, level=16),
90+ "decode": lambda x, dtype, shape: lz4_decode(x, dtype, shape),
91+ "description": "LZ4 compression at level 16 (highest compression).",
92+ "tags": ["lz4"],
93+ "source_file": SOURCE_FILE,
94+ "long_description": LONG_DESCRIPTION,
95+ },
96+ {
97+ "name": "lz4-16-delta",
98+ "version": "1",
99+ "encode": lambda x: lz4_delta_encode(x, level=16),
100+ "decode": lambda x, dtype, shape: lz4_delta_decode(x, dtype, shape),
101+ "description": "LZ4 compression at level 16 with delta encoding.",
102+ "tags": ["lz4", "delta_encoding", "1d"],
103+ "source_file": SOURCE_FILE,
104+ "long_description": LONG_DESCRIPTION,
105+ },
106+]
benchcompress/src/benchcompress/algorithms/lz4/lz4.mdadded+17−0View file
@@ -0,0 +1,17 @@
1+# LZ4 Algorithm
2+
3+LZ4 is a lossless compression algorithm focused on compression and decompression speed. It belongs to the LZ77 family of byte-oriented compression schemes and is particularly well-suited for real-time compression scenarios.
4+
5+## Variants
6+
7+### Standard Compression
8+Different compression levels trading off speed vs compression ratio:
9+- lz4-0: Fastest compression mode
10+- lz4-3: Minimum high compression mode
11+- lz4-10: Higher compression
12+- lz4-16: Maximum compression
13+
14+### Delta Encoding
15+- lz4-16-delta: Maximum compression with delta encoding
16+
17+Delta encoding stores differences between consecutive values rather than the values themselves. This can improve compression for sequences where adjacent values are often similar, such as time series data.
benchcompress/src/benchcompress/algorithms/lzma/__init__.pyadded+76−0View file
@@ -0,0 +1,76 @@
1+import numpy as np
2+import os
3+
4+
5+SOURCE_FILE = "lzma/__init__.py"
6+
7+
8+def _load_long_description():
9+ current_dir = os.path.dirname(os.path.abspath(__file__))
10+ md_path = os.path.join(current_dir, "lzma.md")
11+ with open(md_path, "r", encoding="utf-8") as f:
12+ return f.read()
13+
14+
15+LONG_DESCRIPTION = _load_long_description()
16+
17+
18+def lzma_delta_encode(x: np.ndarray, preset: int) -> bytes:
19+ import lzma
20+
21+ assert x.ndim == 1
22+ y = np.diff(x)
23+ y = np.insert(y, 0, x[0])
24+ buf = y.tobytes()
25+ compressed = lzma.compress(buf, preset=preset)
26+ return compressed
27+
28+
29+def lzma_delta_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
30+ import lzma
31+
32+ assert len(shape) == 1
33+
34+ buf = lzma.decompress(x)
35+ y = np.frombuffer(buf, dtype=dtype)
36+ return np.cumsum(y)
37+
38+
39+def lzma_encode(x: np.ndarray, preset: int) -> bytes:
40+ import lzma
41+
42+ buf = x.tobytes()
43+ compressed = lzma.compress(buf, preset=preset)
44+ return compressed
45+
46+
47+def lzma_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
48+ import lzma
49+
50+ buf = lzma.decompress(x)
51+ y = np.frombuffer(buf, dtype=dtype)
52+ return y.reshape(shape)
53+
54+
55+algorithms = [
56+ {
57+ "name": "lzma-9",
58+ "version": "1",
59+ "encode": lambda x: lzma_encode(x, preset=9),
60+ "decode": lambda x, dtype, shape: lzma_decode(x, dtype, shape),
61+ "description": "LZMA compression at maximum preset 9 for highest compression ratio.",
62+ "tags": ["lzma"],
63+ "source_file": SOURCE_FILE,
64+ "long_description": LONG_DESCRIPTION,
65+ },
66+ {
67+ "name": "lzma-9-delta",
68+ "version": "1",
69+ "encode": lambda x: lzma_delta_encode(x, preset=9),
70+ "decode": lambda x, dtype, shape: lzma_delta_decode(x, dtype, shape),
71+ "description": "LZMA compression at preset 9 with delta encoding for improved compression of sequential data.",
72+ "tags": ["lzma", "delta_encoding", "1d"],
73+ "source_file": SOURCE_FILE,
74+ "long_description": LONG_DESCRIPTION,
75+ },
76+]
benchcompress/src/benchcompress/algorithms/lzma/lzma.mdadded+16−0View file
@@ -0,0 +1,16 @@
1+# LZMA Algorithm
2+
3+LZMA (Lempel-Ziv-Markov chain Algorithm) is a compression algorithm that provides high compression ratios. It uses a dictionary compression scheme similar to LZ77 but with sophisticated modeling of repeating patterns using Markov chains.
4+
5+## Variants
6+
7+### Standard Compression
8+- lzma-9: Maximum compression preset (level 9)
9+ - Focuses on achieving the highest possible compression ratio
10+ - Uses larger dictionary sizes and more complex modeling
11+ - Generally slower than other compression algorithms but produces smaller files
12+
13+### Delta Encoding
14+- lzma-9-delta: Maximum compression with delta encoding
15+
16+Delta encoding stores differences between consecutive values rather than the values themselves. This can improve compression for sequences where adjacent values are often similar, such as time series data.
benchcompress/src/benchcompress/algorithms/wavpack/__init__.pyadded+89−0View file
@@ -0,0 +1,89 @@
1+import numpy as np
2+import os
3+
4+
5+SOURCE_FILE = "wavpack/__init__.py"
6+
7+
8+def _load_long_description():
9+ current_dir = os.path.dirname(os.path.abspath(__file__))
10+ md_path = os.path.join(current_dir, "wavpack.md")
11+ with open(md_path, "r", encoding="utf-8") as f:
12+ return f.read()
13+
14+
15+LONG_DESCRIPTION = _load_long_description()
16+
17+
18+def wavpack_encode(x: np.ndarray, level: int) -> bytes:
19+ from wavpack_numcodecs import WavPack
20+
21+ # if uint8, convert to int8
22+ if x.dtype == np.uint8:
23+ x = x.astype(np.int8)
24+
25+ # if uint16, convert to int16
26+ elif x.dtype == np.uint16:
27+ x = x.astype(np.int16)
28+
29+ assert x.ndim == 1 or x.ndim == 2
30+ wv = WavPack(level=level)
31+ compressed = wv.encode(x)
32+ return compressed
33+
34+
35+def wavpack_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
36+ from wavpack_numcodecs import WavPack
37+
38+ wv = WavPack()
39+ buf = wv.decode(x)
40+ y = np.frombuffer(buf, dtype=dtype)
41+
42+ # make sure the dtype matches the original
43+ y = y.astype(dtype)
44+
45+ return y.reshape(shape)
46+
47+
48+algorithms = [
49+ {
50+ "name": "wavpack-1",
51+ "version": "1",
52+ "encode": lambda x: wavpack_encode(x, level=1),
53+ "decode": lambda x, dtype, shape: wavpack_decode(x, dtype, shape),
54+ "description": "WavPack compression at level 1, minimum compression level.",
55+ "tags": ["wavpack", "no_bernoulli"],
56+ "source_file": SOURCE_FILE,
57+ "long_description": LONG_DESCRIPTION,
58+ },
59+ {
60+ "name": "wavpack-2",
61+ "version": "1",
62+ "encode": lambda x: wavpack_encode(x, level=2),
63+ "decode": lambda x, dtype, shape: wavpack_decode(x, dtype, shape),
64+ "description": "WavPack compression at level 2, medium compression level.",
65+ "tags": ["wavpack", "no_bernoulli"],
66+ "source_file": SOURCE_FILE,
67+ "long_description": LONG_DESCRIPTION,
68+ },
69+ {
70+ "name": "wavpack-3",
71+ "version": "1",
72+ "encode": lambda x: wavpack_encode(x, level=3),
73+ "decode": lambda x, dtype, shape: wavpack_decode(x, dtype, shape),
74+ "description": "WavPack compression at level 3, high compression level.",
75+ "tags": ["wavpack", "no_bernoulli"],
76+ "source_file": SOURCE_FILE,
77+ "long_description": LONG_DESCRIPTION,
78+ },
79+ {
80+ "name": "wavpack-4",
81+ "version": "1",
82+ "encode": lambda x: wavpack_encode(x, level=4),
83+ "decode": lambda x, dtype, shape: wavpack_decode(x, dtype, shape),
84+ "description": "WavPack compression at level 4, highest compression level.",
85+ "tags": ["wavpack", "no_bernoulli"],
86+ "source_file": SOURCE_FILE,
87+ "long_description": LONG_DESCRIPTION,
88+ },
89+]
benchcompress/src/benchcompress/algorithms/wavpack/wavpack.mdadded+12−0View file
@@ -0,0 +1,12 @@
1+# WavPack
2+
3+[WavPack](https://www.wavpack.com/) is a completely open audio compression format providing lossless, high-quality lossy, and a unique hybrid compression mode. It is designed to offer a versatile and efficient solution for audio compression.
4+
5+## Variants
6+
7+WavPack supports different levels of lossless compression, which can be adjusted to balance between compression ratio and encoding/decoding speed. The following variants are available:
8+
9+- **Fast**: Prioritizes speed over compression ratio.
10+- **Normal**: Balanced between speed and compression ratio.
11+- **High**: Prioritizes compression ratio over speed.
12+- **Very High**: Maximum compression ratio with slower speed.
benchcompress/src/benchcompress/algorithms/zlib/__init__.pyadded+116−0View file
@@ -0,0 +1,116 @@
1+import numpy as np
2+import os
3+
4+
5+SOURCE_FILE = "zlib/__init__.py"
6+
7+
8+def _load_long_description():
9+ current_dir = os.path.dirname(os.path.abspath(__file__))
10+ md_path = os.path.join(current_dir, "zlib.md")
11+ with open(md_path, "r", encoding="utf-8") as f:
12+ return f.read()
13+
14+
15+LONG_DESCRIPTION = _load_long_description()
16+
17+
18+def zlib_encode(x: np.ndarray, level: int) -> bytes:
19+ import zlib
20+
21+ buf = x.tobytes()
22+ compressed = zlib.compress(buf, level=level)
23+ return compressed
24+
25+
26+def zlib_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
27+ import zlib
28+
29+ buf = zlib.decompress(x)
30+ y = np.frombuffer(buf, dtype=dtype)
31+ return y.reshape(shape)
32+
33+
34+def zlib_delta_encode(x: np.ndarray, level: int) -> bytes:
35+ import zlib
36+
37+ assert x.ndim == 1
38+ y = np.diff(x)
39+ y = np.insert(y, 0, x[0])
40+ buf = y.tobytes()
41+ compressed = zlib.compress(buf, level=level)
42+ return compressed
43+
44+
45+def zlib_delta_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
46+ import zlib
47+
48+ assert len(shape) == 1
49+
50+ buf = zlib.decompress(x)
51+ y = np.frombuffer(buf, dtype=dtype)
52+ return np.cumsum(y)
53+
54+
55+algorithms = [
56+ {
57+ "name": "zlib-1",
58+ "version": "1",
59+ "encode": lambda x: zlib_encode(x, level=1),
60+ "decode": lambda x, dtype, shape: zlib_decode(x, dtype, shape),
61+ "description": "Zlib DEFLATE compression at level 1 (fastest).",
62+ "tags": ["zlib"],
63+ "source_file": SOURCE_FILE,
64+ "long_description": LONG_DESCRIPTION,
65+ },
66+ {
67+ "name": "zlib-3",
68+ "version": "1",
69+ "encode": lambda x: zlib_encode(x, level=3),
70+ "decode": lambda x, dtype, shape: zlib_decode(x, dtype, shape),
71+ "description": "Zlib DEFLATE compression at level 3.",
72+ "tags": ["zlib"],
73+ "source_file": SOURCE_FILE,
74+ "long_description": LONG_DESCRIPTION,
75+ },
76+ {
77+ "name": "zlib-5",
78+ "version": "1",
79+ "encode": lambda x: zlib_encode(x, level=5),
80+ "decode": lambda x, dtype, shape: zlib_decode(x, dtype, shape),
81+ "description": "Zlib DEFLATE compression at level 5 (medium).",
82+ "tags": ["zlib"],
83+ "source_file": SOURCE_FILE,
84+ "long_description": LONG_DESCRIPTION,
85+ },
86+ {
87+ "name": "zlib-7",
88+ "version": "1",
89+ "encode": lambda x: zlib_encode(x, level=7),
90+ "decode": lambda x, dtype, shape: zlib_decode(x, dtype, shape),
91+ "description": "Zlib DEFLATE compression at level 7.",
92+ "tags": ["zlib"],
93+ "source_file": SOURCE_FILE,
94+ "long_description": LONG_DESCRIPTION,
95+ },
96+ {
97+ "name": "zlib-9",
98+ "version": "1",
99+ "encode": lambda x: zlib_encode(x, level=9),
100+ "decode": lambda x, dtype, shape: zlib_decode(x, dtype, shape),
101+ "description": "Zlib DEFLATE compression at maximum level 9.",
102+ "tags": ["zlib"],
103+ "source_file": SOURCE_FILE,
104+ "long_description": LONG_DESCRIPTION,
105+ },
106+ {
107+ "name": "zlib-9-delta",
108+ "version": "1",
109+ "encode": lambda x: zlib_delta_encode(x, level=9),
110+ "decode": lambda x, dtype, shape: zlib_delta_decode(x, dtype, shape),
111+ "description": "Zlib DEFLATE compression at level 9 with delta encoding.",
112+ "tags": ["zlib", "delta_encoding", "1d"],
113+ "source_file": SOURCE_FILE,
114+ "long_description": LONG_DESCRIPTION,
115+ },
116+]
benchcompress/src/benchcompress/algorithms/zlib/zlib.mdadded+18−0View file
@@ -0,0 +1,18 @@
1+# Zlib Algorithm
2+
3+Zlib is a data compression library that uses the DEFLATE algorithm, which combines LZ77 and Huffman coding. It provides a good balance between compression ratio and speed, making it widely used in many applications.
4+
5+## Variants
6+
7+### Standard Compression
8+Different compression levels trading off speed vs compression ratio:
9+- zlib-1: Fastest compression
10+- zlib-3: Better compression than level 1
11+- zlib-5: Medium compression
12+- zlib-7: Better compression than level 5
13+- zlib-9: Maximum compression
14+
15+### Delta Encoding
16+- zlib-9-delta: Maximum compression with delta encoding
17+
18+Delta encoding stores differences between consecutive values rather than the values themselves. This can improve compression for sequences where adjacent values are often similar, such as time series data.
benchcompress/src/benchcompress/algorithms/zstd/__init__.pyadded+336−0View file
@@ -0,0 +1,336 @@
1+import numpy as np
2+import os
3+from ..ans.lpc_reconstruct import lpc_reconstruct as lpc_reconstruct_cpp
4+from ..ans.lpc_predict import lpc_predict as lpc_predict_cpp
5+from ..ans.get_run_lengths import get_run_lengths
6+
7+
8+SOURCE_FILE = "zstd/__init__.py"
9+
10+
11+def _load_long_description():
12+ current_dir = os.path.dirname(os.path.abspath(__file__))
13+ md_path = os.path.join(current_dir, "zstd.md")
14+ with open(md_path, "r", encoding="utf-8") as f:
15+ return f.read()
16+
17+
18+LONG_DESCRIPTION = _load_long_description()
19+
20+
21+def zstd_delta_encode(x: np.ndarray, level: int) -> bytes:
22+ import zstandard as zstd
23+
24+ assert x.ndim == 1
25+
26+ y = np.diff(x)
27+ y = np.insert(y, 0, x[0])
28+ buf = y.tobytes()
29+ compressor = zstd.ZstdCompressor(level=level)
30+ compressed = compressor.compress(buf)
31+ return compressed
32+
33+
34+def zstd_delta_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
35+ import zstandard as zstd
36+
37+ assert len(shape) == 1
38+
39+ decompressor = zstd.ZstdDecompressor()
40+ buf = decompressor.decompress(x)
41+ y = np.frombuffer(buf, dtype=dtype)
42+ return np.cumsum(y)
43+
44+
45+def zstd_encode(x: np.ndarray, level: int) -> bytes:
46+ import zstandard as zstd
47+
48+ buf = x.tobytes()
49+ compressor = zstd.ZstdCompressor(level=level)
50+ compressed = compressor.compress(buf)
51+ return compressed
52+
53+
54+def zstd_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
55+ import zstandard as zstd
56+
57+ decompressor = zstd.ZstdDecompressor()
58+ buf = decompressor.decompress(x)
59+ y = np.frombuffer(buf, dtype=dtype)
60+ return y.reshape(shape)
61+
62+
63+def zstd_lpc_encode(x: np.ndarray, level: int) -> bytes:
64+ import zstandard as zstd
65+ import struct
66+
67+ assert x.ndim == 1
68+ coeffs, initial, resid = lpc_predict_cpp(x, M=6, num_training_samples=10000)
69+
70+ # Convert coeffs and initial to bytes
71+ coeffs_bytes = coeffs.tobytes()
72+ initial_bytes = initial.tobytes()
73+
74+ # Create header with lengths
75+ header = struct.pack("QQ", len(coeffs_bytes), len(initial_bytes))
76+
77+ # Compress residuals
78+ resid_bytes = resid.tobytes()
79+ compressor = zstd.ZstdCompressor(level=level)
80+ compressed_resid = compressor.compress(resid_bytes)
81+
82+ # Combine all parts
83+ return header + coeffs_bytes + initial_bytes + compressed_resid
84+
85+
86+def zstd_lpc_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
87+ import zstandard as zstd
88+ import struct
89+
90+ assert len(shape) == 1
91+
92+ # Extract header
93+ header_size = struct.calcsize("QQ")
94+ coeffs_len, initial_len = struct.unpack("QQ", x[:header_size])
95+
96+ # Extract coefficients and initial values
97+ pos = header_size
98+ coeffs = np.frombuffer(x[pos : pos + coeffs_len], dtype=np.float32)
99+ pos += coeffs_len
100+ initial = np.frombuffer(x[pos : pos + initial_len], dtype=dtype)
101+ pos += initial_len
102+
103+ # Decompress residuals
104+ decompressor = zstd.ZstdDecompressor()
105+ resid_buf = decompressor.decompress(x[pos:])
106+ resid = np.frombuffer(resid_buf, dtype=dtype)
107+
108+ # Reconstruct signal
109+ output = lpc_reconstruct_cpp(coeffs, initial, resid)
110+ return output
111+
112+
113+def zstd_lpc_zrle_encode(x: np.ndarray, level: int) -> bytes:
114+ import zstandard as zstd
115+ import struct
116+
117+ assert x.ndim == 1
118+
119+ # Get run lengths for zero/non-zero sequences
120+ run_lengths = get_run_lengths(x)
121+
122+ # Determine run length dtype code
123+ if run_lengths.dtype == np.uint8:
124+ run_length_dtype_code = 0
125+ elif run_lengths.dtype == np.uint16:
126+ run_length_dtype_code = 1
127+ elif run_lengths.dtype == np.uint32:
128+ run_length_dtype_code = 2
129+ else:
130+ raise ValueError(f"Unsupported run length dtype: {run_lengths.dtype}")
131+
132+ # Extract non-zero data
133+ non_zero_arrays = []
134+ array_pos = 0
135+ i = 0
136+ while i < len(run_lengths):
137+ non_zero_len = int(run_lengths[i])
138+ if non_zero_len > 0:
139+ non_zero_arrays.append(x[array_pos : array_pos + non_zero_len])
140+ array_pos += non_zero_len
141+ i += 1
142+ if i < len(run_lengths):
143+ array_pos += int(run_lengths[i]) # Skip zeros
144+ i += 1
145+
146+ non_zero_data = np.concatenate(non_zero_arrays)
147+
148+ # Apply linear predictive coding on non-zero data
149+ coeffs, initial, resid = lpc_predict_cpp(
150+ non_zero_data, M=6, num_training_samples=10000
151+ )
152+
153+ # Convert data to bytes
154+ coeffs_bytes = coeffs.tobytes()
155+ initial_bytes = initial.tobytes()
156+ run_lengths_bytes = run_lengths.tobytes()
157+
158+ # Create header with lengths and dtype code
159+ header = struct.pack(
160+ "QQQQB",
161+ len(coeffs_bytes),
162+ len(initial_bytes),
163+ len(run_lengths_bytes),
164+ len(run_lengths),
165+ run_length_dtype_code,
166+ )
167+
168+ # Compress residuals
169+ resid_bytes = resid.tobytes()
170+ compressor = zstd.ZstdCompressor(level=level)
171+ compressed_resid = compressor.compress(resid_bytes)
172+
173+ # Combine all parts
174+ return header + coeffs_bytes + initial_bytes + run_lengths_bytes + compressed_resid
175+
176+
177+def zstd_lpc_zrle_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
178+ import zstandard as zstd
179+ import struct
180+
181+ assert len(shape) == 1
182+
183+ # Extract header
184+ header_size = struct.calcsize("QQQQB")
185+ coeffs_len, initial_len, run_lengths_len, num_run_lengths, run_length_dtype_code = (
186+ struct.unpack("QQQQB", x[:header_size])
187+ )
188+
189+ # Extract components
190+ pos = header_size
191+ coeffs = np.frombuffer(x[pos : pos + coeffs_len], dtype=np.float32)
192+ pos += coeffs_len
193+ initial = np.frombuffer(x[pos : pos + initial_len], dtype=dtype)
194+ pos += initial_len
195+
196+ # Get run lengths with proper dtype
197+ if run_length_dtype_code == 0:
198+ run_lengths = np.frombuffer(x[pos : pos + run_lengths_len], dtype=np.uint8)
199+ elif run_length_dtype_code == 1:
200+ run_lengths = np.frombuffer(x[pos : pos + run_lengths_len], dtype=np.uint16)
201+ elif run_length_dtype_code == 2:
202+ run_lengths = np.frombuffer(x[pos : pos + run_lengths_len], dtype=np.uint32)
203+ else:
204+ raise ValueError(f"Unsupported run length dtype code: {run_length_dtype_code}")
205+ pos += run_lengths_len
206+
207+ assert len(run_lengths) == num_run_lengths
208+
209+ # Decompress residuals
210+ decompressor = zstd.ZstdDecompressor()
211+ resid_buf = decompressor.decompress(x[pos:])
212+ resid = np.frombuffer(resid_buf, dtype=dtype)
213+
214+ # Reconstruct non-zero data
215+ non_zero_data = lpc_reconstruct_cpp(coeffs, initial, resid)
216+
217+ # Reconstruct full array using run lengths
218+ non_zero_pos = 0
219+ i = 0
220+ segments = []
221+ while i < len(run_lengths):
222+ non_zero_len = int(run_lengths[i])
223+ if non_zero_len > 0:
224+ segment = non_zero_data[non_zero_pos : non_zero_pos + non_zero_len]
225+ segments.append(segment)
226+ non_zero_pos += non_zero_len
227+ i += 1
228+ if i < len(run_lengths):
229+ segments.append(np.zeros(int(run_lengths[i]), dtype=non_zero_data.dtype))
230+ i += 1
231+
232+ return np.concatenate(segments)
233+
234+
235+algorithms = [
236+ {
237+ "name": "zstd-4",
238+ "version": "1",
239+ "encode": lambda x: zstd_encode(x, level=4),
240+ "decode": lambda x, dtype, shape: zstd_decode(x, dtype, shape),
241+ "description": "Zstandard compression at level 4 (fast compression).",
242+ "tags": ["zstd"],
243+ "source_file": SOURCE_FILE,
244+ "long_description": LONG_DESCRIPTION,
245+ },
246+ {
247+ "name": "zstd-7",
248+ "version": "1",
249+ "encode": lambda x: zstd_encode(x, level=7),
250+ "decode": lambda x, dtype, shape: zstd_decode(x, dtype, shape),
251+ "description": "Zstandard compression at level 7 (balanced speed/compression).",
252+ "tags": ["zstd"],
253+ "source_file": SOURCE_FILE,
254+ "long_description": LONG_DESCRIPTION,
255+ },
256+ {
257+ "name": "zstd-10",
258+ "version": "1",
259+ "encode": lambda x: zstd_encode(x, level=10),
260+ "decode": lambda x, dtype, shape: zstd_decode(x, dtype, shape),
261+ "description": "Zstandard compression at level 10 (better compression).",
262+ "tags": ["zstd"],
263+ "source_file": SOURCE_FILE,
264+ "long_description": LONG_DESCRIPTION,
265+ },
266+ {
267+ "name": "zstd-13",
268+ "version": "1",
269+ "encode": lambda x: zstd_encode(x, level=13),
270+ "decode": lambda x, dtype, shape: zstd_decode(x, dtype, shape),
271+ "description": "Zstandard compression at level 13 (high compression).",
272+ "tags": ["zstd"],
273+ "source_file": SOURCE_FILE,
274+ "long_description": LONG_DESCRIPTION,
275+ },
276+ {
277+ "name": "zstd-16",
278+ "version": "1",
279+ "encode": lambda x: zstd_encode(x, level=16),
280+ "decode": lambda x, dtype, shape: zstd_decode(x, dtype, shape),
281+ "description": "Zstandard compression at level 16 (very high compression).",
282+ "tags": ["zstd"],
283+ "source_file": SOURCE_FILE,
284+ "long_description": LONG_DESCRIPTION,
285+ },
286+ {
287+ "name": "zstd-19",
288+ "version": "1",
289+ "encode": lambda x: zstd_encode(x, level=19),
290+ "decode": lambda x, dtype, shape: zstd_decode(x, dtype, shape),
291+ "description": "Zstandard compression at level 19 (ultra high compression).",
292+ "tags": ["zstd"],
293+ "source_file": SOURCE_FILE,
294+ "long_description": LONG_DESCRIPTION,
295+ },
296+ {
297+ "name": "zstd-22",
298+ "version": "1",
299+ "encode": lambda x: zstd_encode(x, level=22),
300+ "decode": lambda x, dtype, shape: zstd_decode(x, dtype, shape),
301+ "description": "Zstandard compression at maximum level 22 (highest compression).",
302+ "tags": ["zstd"],
303+ "source_file": SOURCE_FILE,
304+ "long_description": LONG_DESCRIPTION,
305+ },
306+ {
307+ "name": "zstd-22-delta",
308+ "version": "1",
309+ "encode": lambda x: zstd_delta_encode(x, level=22),
310+ "decode": lambda x, dtype, shape: zstd_delta_decode(x, dtype, shape),
311+ "description": "Zstandard compression at level 22 with delta encoding for improved compression of sequential data.",
312+ "tags": ["zstd", "delta_encoding", "1d"],
313+ "source_file": SOURCE_FILE,
314+ "long_description": LONG_DESCRIPTION,
315+ },
316+ {
317+ "name": "zstd-22-lpc",
318+ "version": "1",
319+ "encode": lambda x: zstd_lpc_encode(x, level=22),
320+ "decode": lambda x, dtype, shape: zstd_lpc_decode(x, dtype, shape),
321+ "description": "Zstandard compression at level 22 with linear predictive codingn for exploiting temporal correlations in the data.",
322+ "tags": ["zstd", "lpc_prediction", "1d"],
323+ "source_file": SOURCE_FILE,
324+ "long_description": LONG_DESCRIPTION,
325+ },
326+ {
327+ "name": "zstd-22-lpc-zrle",
328+ "version": "1",
329+ "encode": lambda x: zstd_lpc_zrle_encode(x, level=22),
330+ "decode": lambda x, dtype, shape: zstd_lpc_zrle_decode(x, dtype, shape),
331+ "description": "Zstandard compression at level 22 with linear predictive codingn and zero run-length encoding for sparse data.",
332+ "tags": ["zstd", "lpc_prediction", "zero_rle", "1d"],
333+ "source_file": SOURCE_FILE,
334+ "long_description": LONG_DESCRIPTION,
335+ },
336+]
benchcompress/src/benchcompress/algorithms/zstd/zstd.mdadded+26−0View file
@@ -0,0 +1,26 @@
1+# Zstandard Algorithm
2+
3+Zstandard (zstd) is a fast real-time compression algorithm developed by Facebook. It provides a wide range of compression levels and supports custom dictionaries for improved compression.
4+
5+## Variants
6+
7+### Standard Compression
8+Different compression levels trading off speed vs compression ratio:
9+- zstd-4: Fast compression
10+- zstd-7: Balanced speed/compression
11+- zstd-10: Better compression
12+- zstd-13: High compression
13+- zstd-16: Very high compression
14+- zstd-19: Ultra high compression
15+- zstd-22: Maximum compression
16+
17+### Advanced Variants
18+
19+#### Delta Encoding (zstd-22-delta)
20+Stores differences between consecutive values. Effective for sequences where adjacent values are similar, like time series data.
21+
22+#### Linear Predictive Coding (zstd-22-lpc)
23+Uses a Markov model to predict values based on previous samples. The prediction residuals are then compressed using zstd. This can significantly improve compression for data with temporal correlations.
24+
25+#### Linear Predictive Coding with Zero RLE (zstd-22-lpc-zrle)
26+Combines Markov prediction with zero run-length encoding. Particularly effective for sparse data where many values are zero, as it efficiently encodes runs of zeros while using Markov prediction for the non-zero regions.
benchcompress/src/benchcompress/cli.pyadded+143−0View file
@@ -0,0 +1,143 @@
1+#!/usr/bin/env python3
2+
3+import click
4+from typing import List, Optional
5+from .run_benchmarks.run_benchmarks import run_benchmarks
6+from .algorithms import algorithms
7+from .datasets import datasets
8+
9+
10+def get_available_algorithms() -> List[str]:
11+ """Get list of available algorithm names"""
12+ return [alg["name"] for alg in algorithms]
13+
14+
15+def get_available_datasets() -> List[str]:
16+ """Get list of available dataset names"""
17+ return [ds["name"] for ds in datasets]
18+
19+
20+def filter_algorithms(selected: Optional[List[str]] = None) -> List[dict]:
21+ """Filter algorithms based on selected names"""
22+ if not selected:
23+ return algorithms
24+ return [alg for alg in algorithms if alg["name"] in selected]
25+
26+
27+def filter_datasets(selected: Optional[List[str]] = None) -> List[dict]:
28+ """Filter datasets based on selected names"""
29+ if not selected:
30+ return datasets
31+ return [ds for ds in datasets if ds["name"] in selected]
32+
33+
34+def validate_algorithms(ctx, param, value):
35+ if not value:
36+ return None
37+ available = get_available_algorithms()
38+ invalid = [alg for alg in value if alg not in available]
39+ if invalid:
40+ raise click.BadParameter(
41+ f"Invalid algorithm(s): {', '.join(invalid)}. "
42+ f"Available algorithms: {', '.join(available)}"
43+ )
44+ return value
45+
46+
47+def validate_datasets(ctx, param, value):
48+ if not value:
49+ return None
50+ available = get_available_datasets()
51+ invalid = [ds for ds in value if ds not in available]
52+ if invalid:
53+ raise click.BadParameter(
54+ f"Invalid dataset(s): {', '.join(invalid)}. "
55+ f"Available datasets: {', '.join(available)}"
56+ )
57+ return value
58+
59+
60+@click.group()
61+def cli():
62+ """Benchmark compression algorithms for scientific data arrays"""
63+ pass
64+
65+
66+@cli.command()
67+def list():
68+ """List available algorithms and datasets"""
69+ click.echo("\nAvailable Algorithms:")
70+ for alg in algorithms:
71+ desc = alg.get("description", "No description")
72+ click.echo(f" {alg['name']:<20} - {desc}")
73+
74+ click.echo("\nAvailable Datasets:")
75+ for ds in datasets:
76+ desc = ds.get("description", "No description")
77+ click.echo(f" {ds['name']:<20} - {desc}")
78+
79+
80+@cli.command()
81+@click.option(
82+ "--algorithm",
83+ "-a",
84+ multiple=True,
85+ callback=validate_algorithms,
86+ help="Algorithm(s) to benchmark (can be specified multiple times)",
87+)
88+@click.option(
89+ "--dataset",
90+ "-d",
91+ multiple=True,
92+ callback=validate_datasets,
93+ help="Dataset(s) to benchmark (can be specified multiple times)",
94+)
95+@click.option(
96+ "--cache-dir",
97+ default=".benchmark_cache",
98+ help="Directory to store cached results",
99+ type=click.Path(),
100+)
101+@click.option("--quiet", "-q", is_flag=True, help="Reduce output verbosity")
102+@click.option("--force", "-f", is_flag=True, help="Force re-run without using cache")
103+def run(algorithm, dataset, cache_dir, quiet, force):
104+ """Run benchmarks with specified options"""
105+ # Filter algorithms and datasets
106+ filtered_algorithms = filter_algorithms(algorithm)
107+ filtered_datasets = filter_datasets(dataset)
108+
109+ if not filtered_algorithms:
110+ click.echo("Error: No matching algorithms found", err=True)
111+ ctx = click.get_current_context()
112+ ctx.exit(1)
113+ if not filtered_datasets:
114+ click.echo("Error: No matching datasets found", err=True)
115+ ctx = click.get_current_context()
116+ ctx.exit(1)
117+
118+ # Run benchmarks with filtered options
119+ results = run_benchmarks(
120+ cache_dir=cache_dir,
121+ verbose=not quiet,
122+ selected_algorithms=filtered_algorithms,
123+ selected_datasets=filtered_datasets,
124+ force=force,
125+ )
126+
127+ # Print summary
128+ click.echo("\nBenchmark Summary:")
129+ for result in results["results"]:
130+ click.echo(
131+ f"\n{result['dataset']} + {result['algorithm']}:"
132+ f"\n Compression ratio: {result['compression_ratio']:.2f}x"
133+ f"\n Encode speed: {result['encode_mb_per_sec']:.2f} MB/s"
134+ f"\n Decode speed: {result['decode_mb_per_sec']:.2f} MB/s"
135+ )
136+
137+
138+def main():
139+ cli()
140+
141+
142+if __name__ == "__main__":
143+ main()
benchcompress/src/benchcompress/datasets/__init__.pyadded+23−0View file
@@ -0,0 +1,23 @@
1+from .bernoulli import datasets as bernoulli_datasets
2+from .gaussian import datasets as gaussian_datasets
3+from .gaussian_ar1 import datasets as gaussian_ar1_datasets
4+from .gaussian_lowpass import datasets as gaussian_lowpass_datasets
5+from .ecephys import datasets as ecephys_datasets
6+from .seismic import datasets as seismic_datasets
7+from .ieeg import datasets as ieeg_datasets
8+from .fmri import datasets as fmri_datasets
9+
10+datasets_list = [
11+ bernoulli_datasets,
12+ gaussian_datasets,
13+ gaussian_ar1_datasets,
14+ gaussian_lowpass_datasets,
15+ ecephys_datasets,
16+ seismic_datasets,
17+ ieeg_datasets,
18+ fmri_datasets,
19+]
20+
21+datasets = []
22+for d in datasets_list:
23+ datasets.extend(d)
benchcompress/src/benchcompress/datasets/bernoulli/__init__.pyadded+72−0View file
@@ -0,0 +1,72 @@
1+import numpy as np
2+import os
3+
4+
5+SOURCE_FILE = "bernoulli/__init__.py"
6+
7+
8+def _load_long_description():
9+ current_dir = os.path.dirname(os.path.abspath(__file__))
10+ md_path = os.path.join(current_dir, "bernoulli.md")
11+ with open(md_path, "r", encoding="utf-8") as f:
12+ return f.read()
13+
14+
15+LONG_DESCRIPTION = _load_long_description()
16+
17+
18+def create_bernoulli(*, n_samples: int, p: float, seed: int) -> np.ndarray:
19+ rng = np.random.default_rng(seed)
20+ x = rng.binomial(1, p, n_samples).astype(np.uint8)
21+ return x
22+
23+
24+tags = ["bernoulli", "timeseries", "1d", "integer", "discrete", "synthetic", "i.i.d."]
25+
26+datasets = [
27+ {
28+ "name": "bernoulli-0.1",
29+ "version": "3c",
30+ "create": lambda: create_bernoulli(n_samples=1_000_000, p=0.1, seed=0),
31+ "description": "Binary sequence with 10% probability of ones.",
32+ "tags": tags,
33+ "source_file": SOURCE_FILE,
34+ "long_description": LONG_DESCRIPTION,
35+ },
36+ {
37+ "name": "bernoulli-0.2",
38+ "version": "3",
39+ "create": lambda: create_bernoulli(n_samples=1_000_000, p=0.2, seed=0),
40+ "description": "Binary sequence with 20% probability of ones.",
41+ "tags": tags,
42+ "source_file": SOURCE_FILE,
43+ "long_description": LONG_DESCRIPTION,
44+ },
45+ {
46+ "name": "bernoulli-0.3",
47+ "version": "3",
48+ "create": lambda: create_bernoulli(n_samples=1_000_000, p=0.3, seed=0),
49+ "description": "Binary sequence with 30% probability of ones.",
50+ "tags": tags,
51+ "source_file": SOURCE_FILE,
52+ "long_description": LONG_DESCRIPTION,
53+ },
54+ {
55+ "name": "bernoulli-0.4",
56+ "version": "3",
57+ "create": lambda: create_bernoulli(n_samples=1_000_000, p=0.4, seed=0),
58+ "description": "Binary sequence with 40% probability of ones.",
59+ "tags": tags,
60+ "source_file": SOURCE_FILE,
61+ "long_description": LONG_DESCRIPTION,
62+ },
63+ {
64+ "name": "bernoulli-0.5",
65+ "version": "3",
66+ "create": lambda: create_bernoulli(n_samples=1_000_000, p=0.5, seed=0),
67+ "description": "Binary sequence with 50% probability of ones and 50% probability of zeros.",
68+ "tags": tags,
69+ "source_file": SOURCE_FILE,
70+ "long_description": LONG_DESCRIPTION,
71+ },
72+]
benchcompress/src/benchcompress/datasets/bernoulli/bernoulli.mdadded+21−0View file
@@ -0,0 +1,21 @@
1+# Bernoulli Dataset
2+
3+A Bernoulli dataset is a sequence of binary values (0s and 1s) where each value is independently generated with a fixed probability `p` of being 1. These datasets are useful for testing compression algorithms because they have well-defined theoretical properties.
4+
5+## Variants
6+
7+We provide five variants with different probabilities of generating a 1:
8+- bernoulli-0.1 (10% ones)
9+- bernoulli-0.2 (20% ones)
10+- bernoulli-0.3 (30% ones)
11+- bernoulli-0.4 (40% ones)
12+- bernoulli-0.5 (50% ones)
13+
14+## Theoretical Properties
15+
16+The theoretical entropy H(p) provides a lower bound for compression:
17+```
18+H(p) = -p log₂(p) - (1-p) log₂(1-p) bits/symbol
19+```
20+
21+For example, at p = 0.1 the theoretical minimum is 0.469 bits/symbol, while at p = 0.5 it reaches the maximum of 1.000 bits/symbol.
benchcompress/src/benchcompress/datasets/ecephys/__init__.pyadded+349−0View file
@@ -0,0 +1,349 @@
1+import numpy as np
2+import lindi
3+import os
4+from typing import cast
5+from ..._filters import bandpass_filter
6+from ..._analysis import estimate_noise_level
7+
8+
9+SOURCE_FILE = "ecephys/__init__.py"
10+
11+
12+def _load_long_description():
13+ current_dir = os.path.dirname(os.path.abspath(__file__))
14+ md_path = os.path.join(current_dir, "ecephys.md")
15+ with open(md_path, "r", encoding="utf-8") as f:
16+ return f.read()
17+
18+
19+LONG_DESCRIPTION = _load_long_description()
20+
21+tags = ["real", "ecephys", "timeseries", "1d", "integer", "correlated"]
22+
23+
24+def _load_real_000876(
25+ *, num_samples: int, num_channels: int, start_channel: int
26+) -> np.ndarray:
27+ """Load data from DANDI dataset 000876.
28+
29+ Args:
30+ num_samples: Number of samples to load
31+ num_channels: Number of channels to load
32+ start_channel: Starting channel index
33+
34+ Returns:
35+ Array of shape (num_samples, num_channels) containing the loaded data
36+ """
37+ nwb_url = "https://api.dandiarchive.org/api/assets/7e1de06d-d478-40e2-9b64-9dd04eafaa4c/download/"
38+ h5f = lindi.LindiH5pyFile.from_hdf5_file(nwb_url)
39+ ds = h5f["/acquisition/ElectricalSeriesAP/data"]
40+ assert isinstance(ds, lindi.LindiH5pyDataset)
41+ ret = ds[:num_samples, start_channel : start_channel + num_channels]
42+ return cast(np.ndarray, ret)
43+
44+
45+def _load_real_000409(
46+ *, num_samples: int, num_channels: int, start_channel: int
47+) -> np.ndarray:
48+ """Load data from DANDI dataset 000409.
49+
50+ Args:
51+ num_samples: Number of samples to load
52+ num_channels: Number of channels to load
53+ start_channel: Starting channel index
54+
55+ Returns:
56+ Array of shape (num_samples, num_channels) containing the loaded data
57+ """
58+ nwb_url = "https://api.dandiarchive.org/api/assets/c04f6b30-82bf-40e1-9210-34f0bcd8be24/download/"
59+ h5f = lindi.LindiH5pyFile.from_hdf5_file(nwb_url)
60+ ds = h5f["/acquisition/ElectricalSeriesAp/data"]
61+ assert isinstance(ds, lindi.LindiH5pyDataset)
62+ ret = ds[:num_samples, start_channel : start_channel + num_channels]
63+ return cast(np.ndarray, ret)
64+
65+
66+def _load_real_001290(
67+ *, num_samples: int, num_channels: int, start_channel: int
68+) -> np.ndarray:
69+ """Load data from DANDI dataset 001290.
70+
71+ Args:
72+ num_samples: Number of samples to load
73+ num_channels: Number of channels to load
74+ start_channel: Starting channel index
75+
76+ Returns:
77+ Array of shape (num_samples, num_channels) containing the loaded data
78+ """
79+ nwb_url = "https://api.dandiarchive.org/api/assets/78c99d23-da88-4ecd-9086-c488a126eac5/download/"
80+ h5f = lindi.LindiH5pyFile.from_hdf5_file(nwb_url)
81+ ds = h5f["/acquisition/ElectricalSeriesAPImec/data"]
82+ assert isinstance(ds, lindi.LindiH5pyDataset)
83+ ret = ds[:num_samples, start_channel : start_channel + num_channels]
84+ return cast(np.ndarray, ret)
85+
86+
87+# embargoed
88+# def _load_real_001259(*, num_samples: int) -> np.ndarray:
89+# """Load data from DANDI dataset 001259.
90+
91+# Args:
92+# num_samples: Number of samples to load
93+
94+# Returns:
95+# Array of shape (num_samples, 1) containing the loaded data
96+# """
97+# # this one only has one channel
98+# # https://neurosift.app/?p=/nwb&url=https://api.dandiarchive.org/api/assets/6fa78218-f4d4-45f3-a627-5afe4aa88c3e/download/&dandisetId=001259&dandisetVersion=draft
99+# nwb_url = "https://api.dandiarchive.org/api/assets/6fa78218-f4d4-45f3-a627-5afe4aa88c3e/download/"
100+# h5f = lindi.LindiH5pyFile.from_hdf5_file(nwb_url)
101+# ds = h5f["/acquisition/ElectricalSeries"]
102+# assert isinstance(ds, lindi.LindiH5pyDataset)
103+# # this one only has one channel
104+# ret = ds[:num_samples, 0]
105+# return cast(np.ndarray, ret)
106+
107+
108+def _create_filtered_version(X: np.ndarray) -> np.ndarray:
109+ """Create filtered version of a dataset using bandpass filtering and quantization.
110+
111+ Args:
112+ X: Input signal array
113+
114+ Returns:
115+ Filtered and quantized signal array
116+ """
117+ v = 0.25 # step size for quantization
118+ lowcut = 300
119+ highcut = 6000
120+ sampling_frequency = 30000
121+
122+ # Bandpass filter
123+ X_filt = bandpass_filter(
124+ X - np.median(X),
125+ sampling_frequency=sampling_frequency,
126+ lowcut=lowcut,
127+ highcut=highcut,
128+ )
129+
130+ # Normalize by noise level
131+ noise_level = estimate_noise_level(X_filt, sampling_frequency=sampling_frequency)
132+ X_filt_normalized = X_filt / noise_level
133+
134+ # Quantize
135+ X2b = X_filt_normalized / v
136+ X2 = np.round(X2b).astype(np.int16)
137+
138+ return X2
139+
140+
141+def _create_sparse_version(X: np.ndarray) -> np.ndarray:
142+ """Create sparse version of a dataset using activity-based suppression.
143+
144+ Args:
145+ X: Input signal array
146+
147+ Returns:
148+ Sparse signal array with suppressed low-activity regions
149+ """
150+ v = 0.25 # step size for quantization
151+ lowcut = 300
152+ highcut = 6000
153+ sampling_frequency = 30000
154+
155+ # Bandpass filter
156+ X_filt = bandpass_filter(
157+ X - np.median(X),
158+ sampling_frequency=sampling_frequency,
159+ lowcut=lowcut,
160+ highcut=highcut,
161+ )
162+
163+ # Normalize by noise level
164+ noise_level = estimate_noise_level(X_filt, sampling_frequency=sampling_frequency)
165+ X_filt_normalized = X_filt / noise_level
166+
167+ # Activity detection
168+ def sliding_max(x: np.ndarray, delta: int) -> np.ndarray:
169+ y = np.zeros_like(x)
170+ for i in range(len(x)):
171+ y[i] = np.max(x[max(0, i - delta) : min(len(x), i + delta + 1)])
172+ return y
173+
174+ def smoothed(x: np.ndarray, delta: int) -> np.ndarray:
175+ y = np.zeros_like(x)
176+ for i in range(len(x)):
177+ y[i] = np.mean(x[max(0, i - delta) : min(len(x), i + delta + 1)])
178+ return y
179+
180+ # Activity detection parameters
181+ activity_threshold = [3, 6] # [min, max] thresholds for activity detection
182+
183+ # Detect activity regions
184+ Y = sliding_max(np.abs(X_filt_normalized), 50)
185+ Y = smoothed(Y, 20)
186+ Y = np.minimum(
187+ 1,
188+ np.maximum(
189+ 0,
190+ (Y - activity_threshold[0])
191+ / (activity_threshold[1] - activity_threshold[0]),
192+ ),
193+ )
194+
195+ # Apply suppression
196+ Y_scaled = X_filt_normalized * Y
197+ X3b = Y_scaled / v
198+ X3 = np.round(X3b).astype(np.int16)
199+
200+ return X3
201+
202+
203+datasets = [
204+ {
205+ "name": "ecephys-000876-ch45",
206+ "version": "1",
207+ "description": "Raw extracellular electrophysiology recording from DANDI:000876.",
208+ "create": lambda: _load_real_000876(
209+ num_samples=500_000, num_channels=1, start_channel=45
210+ ).flatten(),
211+ "tags": tags,
212+ "source_file": SOURCE_FILE,
213+ "long_description": LONG_DESCRIPTION,
214+ },
215+ {
216+ "name": "ecephys-000409-ch101",
217+ "version": "1",
218+ "description": "Raw extracellular electrophysiology recording from DANDI:000409.",
219+ "create": lambda: _load_real_000409(
220+ num_samples=500_000, num_channels=1, start_channel=101
221+ ).flatten(),
222+ "tags": tags,
223+ "source_file": SOURCE_FILE,
224+ "long_description": LONG_DESCRIPTION,
225+ },
226+ {
227+ "name": "ecephys-001290-ch0",
228+ "version": "1",
229+ "description": "Raw extracellular electrophysiology recording from DANDI:001290.",
230+ "create": lambda: _load_real_001290(
231+ num_samples=500_000, num_channels=1, start_channel=0
232+ ).flatten(),
233+ "tags": tags,
234+ "source_file": SOURCE_FILE,
235+ "long_description": LONG_DESCRIPTION,
236+ },
237+ # embargoed
238+ # {
239+ # "name": "ecephys-001259",
240+ # "version": "1",
241+ # "description": "Raw extracellular electrophysiology recording from DANDI:001259.",
242+ # "create": lambda: _load_real_001259(num_samples=500_000).flatten(),
243+ # "tags": tags,
244+ # "source_file": SOURCE_FILE,
245+ # "long_description": LONG_DESCRIPTION,
246+ # },
247+ {
248+ "name": "ecephys-000876-ch45-filtered",
249+ "version": "1",
250+ "description": "Preprocessed version of real-000876-ch45. Bandpass filtered (300-6000 Hz).",
251+ "create": lambda: _create_filtered_version(
252+ _load_real_000876(
253+ num_samples=500_000, num_channels=1, start_channel=45
254+ ).flatten()
255+ ),
256+ "tags": tags + ["filtered"],
257+ "source_file": SOURCE_FILE,
258+ "long_description": LONG_DESCRIPTION,
259+ },
260+ {
261+ "name": "ecephys-000409-ch101-filtered",
262+ "version": "1",
263+ "description": "Preprocessed version of real-000409-ch101. Bandpass filtered (300-6000 Hz).",
264+ "create": lambda: _create_filtered_version(
265+ _load_real_000409(
266+ num_samples=500_000, num_channels=1, start_channel=101
267+ ).flatten()
268+ ),
269+ "tags": tags + ["filtered"],
270+ "source_file": SOURCE_FILE,
271+ "long_description": LONG_DESCRIPTION,
272+ },
273+ {
274+ "name": "ecephys-001290-ch0-filtered",
275+ "version": "1",
276+ "description": "Preprocessed version of real-001290-ch0. Bandpass filtered (300-6000 Hz).",
277+ "create": lambda: _create_filtered_version(
278+ _load_real_001290(
279+ num_samples=500_000, num_channels=1, start_channel=0
280+ ).flatten()
281+ ),
282+ "tags": tags + ["filtered"],
283+ "source_file": SOURCE_FILE,
284+ "long_description": LONG_DESCRIPTION,
285+ },
286+ # embargoed
287+ # {
288+ # "name": "ecephys-001259-filtered",
289+ # "version": "1",
290+ # "description": "Preprocessed version of real-001259. Bandpass filtered (300-6000 Hz).",
291+ # "create": lambda: _create_filtered_version(
292+ # _load_real_001259(num_samples=500_000).flatten()
293+ # ),
294+ # "tags": tags + ["filtered"],
295+ # "source_file": SOURCE_FILE,
296+ # "long_description": LONG_DESCRIPTION,
297+ # },
298+ {
299+ "name": "ecephys-000876-ch45-sparse",
300+ "version": "1",
301+ "description": "Sparse version of real-000876-ch45. Activity-based suppression applied.",
302+ "create": lambda: _create_sparse_version(
303+ _load_real_000876(
304+ num_samples=500_000, num_channels=1, start_channel=45
305+ ).flatten()
306+ ),
307+ "tags": tags + ["filtered", "sparse"],
308+ "source_file": SOURCE_FILE,
309+ "long_description": LONG_DESCRIPTION,
310+ },
311+ {
312+ "name": "ecephys-000409-ch101-sparse",
313+ "version": "1",
314+ "description": "Sparse version of real-000409-ch101. Activity-based suppression applied.",
315+ "create": lambda: _create_sparse_version(
316+ _load_real_000409(
317+ num_samples=500_000, num_channels=1, start_channel=101
318+ ).flatten()
319+ ),
320+ "tags": tags + ["filtered", "sparse"],
321+ "source_file": SOURCE_FILE,
322+ "long_description": LONG_DESCRIPTION,
323+ },
324+ {
325+ "name": "ecephys-001290-ch0-sparse",
326+ "version": "1",
327+ "description": "Sparse version of real-001290-ch0. Activity-based suppression applied.",
328+ "create": lambda: _create_sparse_version(
329+ _load_real_001290(
330+ num_samples=500_000, num_channels=1, start_channel=0
331+ ).flatten()
332+ ),
333+ "tags": tags + ["filtered", "sparse"],
334+ "source_file": SOURCE_FILE,
335+ "long_description": LONG_DESCRIPTION,
336+ },
337+ # embargoed
338+ # {
339+ # "name": "ecephys-001259-sparse",
340+ # "version": "1",
341+ # "description": "Sparse version of real-001259. Activity-based suppression applied.",
342+ # "create": lambda: _create_sparse_version(
343+ # _load_real_001259(num_samples=500_000).flatten()
344+ # ),
345+ # "tags": tags + ["filtered", "sparse"],
346+ # "source_file": SOURCE_FILE,
347+ # "long_description": LONG_DESCRIPTION,
348+ # },
349+]
benchcompress/src/benchcompress/datasets/ecephys/ecephys.mdadded+44−0View file
@@ -0,0 +1,44 @@
1+# Extracellular Electrophysiology Dataset
2+
3+Extracellular Electrophysiology datasets contain recordings of electrical activity from neurons in the brain. These recordings are made using microelectrodes that detect the small voltage changes produced by neurons when they fire action potentials (spikes).
4+
5+## Data Sources
6+
7+The datasets are samples from single channels within larger electrophysiology recordings available on the DANDI Archive:
8+
9+- Channel 45 from a session in [Dandiset 000876](https://dandiarchive.org/dandiset/000876)
10+- Channel 101 from a session in [Dandiset 000409](https://dandiarchive.org/dandiset/000409)
11+- Channel 0 from a session in [Dandiset 001290](https://dandiarchive.org/dandiset/001290)
12+- Only channel from a session in [Dandiset 001259](https://dandiarchive.org/dandiset/001259)
13+
14+## Dataset Variants
15+
16+We provide three types of variants for each sampled channel:
17+
18+### 1. Raw Data
19+- Direct recordings from the electrodes
20+- Contains both neural signals and background noise
21+- Integer-valued samples representing voltage
22+- Sampling rate: 30 kHz
23+
24+### 2. Filtered Data
25+- Bandpass filtered between 300-6000 Hz to isolate spike activity
26+- Normalized by estimated noise level
27+- Quantized with step size 0.25
28+- Stored as 16-bit integers
29+
30+### 3. Sparse Data
31+- Applies activity-based suppression to focus on regions with neural firing
32+- Uses adaptive thresholding to detect active regions
33+- Suppresses low-activity regions while preserving spike waveforms
34+- Also quantized and stored as 16-bit integers
35+
36+## Compression Considerations
37+
38+These datasets present different challenges for compression algorithms:
39+
40+1. Raw data contains broadband noise and slow drifts
41+2. Filtered data emphasizes spike waveforms but maintains continuous values
42+3. Sparse data has many near-zero regions interspersed with spike events
43+
44+The sparse variants are particularly interesting as they represent a common preprocessing step in neuroscience analysis, where only time periods containing neural activity are retained.
benchcompress/src/benchcompress/datasets/fmri/__init__.pyadded+71−0View file
@@ -0,0 +1,71 @@
1+import numpy as np
2+import os
3+import nibabel as nib
4+from typing import cast, Optional, List
5+
6+SOURCE_FILE = "fmri/__init__.py"
7+
8+
9+def _load_long_description():
10+ current_dir = os.path.dirname(os.path.abspath(__file__))
11+ md_path = os.path.join(current_dir, "fmri.md")
12+ with open(md_path, "r", encoding="utf-8") as f:
13+ return f.read()
14+
15+
16+LONG_DESCRIPTION = _load_long_description()
17+
18+tags = ["real", "fmri", "timeseries", "1d", "integer", "bold", "correlated"]
19+
20+
21+def _load_bold_data(*, slice_indices: Optional[List[int]] = None) -> np.ndarray:
22+ """Load BOLD fMRI data from OpenNeuro dataset ds005880.
23+
24+ Args:
25+ slice_indices: Optional list of slice indices to load. If None, loads full volume.
26+ Use range(15, 30) for middle 15 slices.
27+
28+ Returns:
29+ Array of shape (X, Y, Z, T) where X,Y,Z are spatial dimensions and T is time points.
30+ Z dimension will be length 1 if a single slice index is provided.
31+ """
32+ url = "https://s3.amazonaws.com/openneuro.org/ds005880/sub-01/func/sub-01_task-rest_run-01_bold.nii.gz?versionId=0z5_YvqoLC4pXDVUVDt9Y1nrJBRxMqXb"
33+ # Download and load the data
34+ import requests
35+ from pathlib import Path
36+
37+ cache_dir = Path(os.path.expanduser("~/.cache/benchcompress/fmri"))
38+ cache_dir.mkdir(parents=True, exist_ok=True)
39+ local_file = cache_dir / "sub-01_task-rest_run-01_bold.nii.gz"
40+
41+ if not local_file.exists():
42+ response = requests.get(url)
43+ response.raise_for_status()
44+ local_file.write_bytes(response.content)
45+
46+ img = nib.load(str(local_file)) # type: ignore
47+ data = img.get_fdata() # type: ignore
48+
49+ # Convert to int16
50+ data = data.astype(np.int16)
51+
52+ if slice_indices is not None:
53+ data = data[:, :, slice_indices, :]
54+
55+ # Now we are going to convert to 1d array, and we make sure that the time dimension varies the fastest
56+ data = data.ravel()
57+
58+ return data
59+
60+
61+datasets = [
62+ {
63+ "name": "fmri-ds005880",
64+ "version": "2",
65+ "description": "Middle 15 slices from BOLD fMRI recording from ds005880 OpenNeuro dataset.",
66+ "create": lambda: _load_bold_data(slice_indices=list(range(15, 30))),
67+ "tags": tags,
68+ "source_file": SOURCE_FILE,
69+ "long_description": LONG_DESCRIPTION,
70+ }
71+]
benchcompress/src/benchcompress/datasets/fmri/fmri.mdadded+17−0View file
@@ -0,0 +1,17 @@
1+# Functional MRI Dataset
2+
3+This dataset contains functional MRI (fMRI) data from a study investigating neural activity elicited by the diminished seventh chord in listeners with refined music listening skills.
4+
5+## Data Source
6+
7+The data comes from OpenNeuro dataset [ds005880](https://openneuro.org/datasets/ds005880/versions/1.0.1) titled "BIDS Dataset for the Diminished Seventh Chord". The specific data used is a single functional run from subject 01.
8+
9+Data were acquired at Imaging Center for Integrated Body, Mind, and Culture Research at National Taiwan University using a 3T MR system (MAGNETOM Prisma, Siemens, Erlangen, Germany).
10+
11+## Dataset Details
12+
13+- Type: BOLD fMRI timeseries
14+- Format: 4D NIfTI file (3D volumes over time)
15+- Data type: 16-bit integers
16+- Content: Represents changes in blood oxygenation level dependent (BOLD) signal over time
17+- Structure: Each timepoint is a 3D brain volume showing neural activity patterns
benchcompress/src/benchcompress/datasets/gaussian/__init__.pyadded+108−0View file
@@ -0,0 +1,108 @@
1+import numpy as np
2+import os
3+
4+
5+SOURCE_FILE = "gaussian/__init__.py"
6+
7+
8+def _load_long_description():
9+ current_dir = os.path.dirname(os.path.abspath(__file__))
10+ md_path = os.path.join(current_dir, "gaussian.md")
11+ with open(md_path, "r", encoding="utf-8") as f:
12+ return f.read()
13+
14+
15+LONG_DESCRIPTION = _load_long_description()
16+
17+
18+def create_gaussian_quantized(
19+ *, n_samples: int, stddev: float, seed: int
20+) -> np.ndarray:
21+ rng = np.random.default_rng(seed)
22+ x = np.round(rng.normal(0, stddev, n_samples)).astype(np.int16)
23+ return x
24+
25+
26+def create_gaussian_float(*, n_samples: int, stddev: float, seed: int) -> np.ndarray:
27+ rng = np.random.default_rng(seed)
28+ x = rng.normal(0, stddev, n_samples).astype(np.float32)
29+ return x
30+
31+
32+tags_quantized = [
33+ "gaussian",
34+ "integer",
35+ "discrete",
36+ "timeseries",
37+ "1d",
38+ "synthetic",
39+ "i.i.d.",
40+]
41+tags_float = ["gaussian", "float", "timeseries", "1d", "synthetic", "i.i.d."]
42+
43+datasets = [
44+ {
45+ "name": "gaussian-q1",
46+ "version": "1",
47+ "create": lambda: create_gaussian_quantized(
48+ n_samples=1_000_000, stddev=1, seed=0
49+ ),
50+ "description": "Rounded Gaussian integers with σ=1.",
51+ "tags": tags_quantized,
52+ "source_file": SOURCE_FILE,
53+ "long_description": LONG_DESCRIPTION,
54+ },
55+ {
56+ "name": "gaussian-q2",
57+ "version": "1",
58+ "create": lambda: create_gaussian_quantized(
59+ n_samples=1_000_000, stddev=2, seed=0
60+ ),
61+ "description": "Rounded Gaussian integers with σ=2.",
62+ "tags": tags_quantized,
63+ "source_file": SOURCE_FILE,
64+ "long_description": LONG_DESCRIPTION,
65+ },
66+ {
67+ "name": "gaussian-q3",
68+ "version": "1",
69+ "create": lambda: create_gaussian_quantized(
70+ n_samples=1_000_000, stddev=3, seed=0
71+ ),
72+ "description": "Rounded Gaussian integers with σ=3.",
73+ "tags": tags_quantized,
74+ "source_file": SOURCE_FILE,
75+ "long_description": LONG_DESCRIPTION,
76+ },
77+ {
78+ "name": "gaussian-q5",
79+ "version": "1",
80+ "create": lambda: create_gaussian_quantized(
81+ n_samples=1_000_000, stddev=5, seed=0
82+ ),
83+ "description": "Rounded Gaussian integers with σ=5.",
84+ "tags": tags_quantized,
85+ "source_file": SOURCE_FILE,
86+ "long_description": LONG_DESCRIPTION,
87+ },
88+ {
89+ "name": "gaussian-q8",
90+ "version": "1",
91+ "create": lambda: create_gaussian_quantized(
92+ n_samples=1_000_000, stddev=8, seed=0
93+ ),
94+ "description": "Rounded Gaussian integers with σ=8.",
95+ "tags": tags_quantized,
96+ "source_file": SOURCE_FILE,
97+ "long_description": LONG_DESCRIPTION,
98+ },
99+ {
100+ "name": "gaussian-flt1",
101+ "version": "1",
102+ "create": lambda: create_gaussian_float(n_samples=1_000_000, stddev=8, seed=0),
103+ "description": "Floating point Gaussian numbers with σ=1.",
104+ "tags": tags_float,
105+ "source_file": SOURCE_FILE,
106+ "long_description": LONG_DESCRIPTION,
107+ },
108+]
benchcompress/src/benchcompress/datasets/gaussian/gaussian.mdadded+22−0View file
@@ -0,0 +1,22 @@
1+# Gaussian Dataset
2+
3+These datasets contain sequences of numbers drawn from Gaussian distributions with μ=0 and various values of σ. All values are independently and identically distributed (i.i.d.).
4+
5+## Variants
6+
7+We provide two types of Gaussian datasets:
8+
9+### Quantized Integer Variants
10+Numbers are drawn from a Gaussian distribution then rounded to the nearest integer:
11+- gaussian-q1 (σ=1)
12+- gaussian-q2 (σ=2)
13+- gaussian-q3 (σ=3)
14+- gaussian-q5 (σ=5)
15+- gaussian-q8 (σ=8)
16+
17+The quantization creates discrete integer values, with larger σ values producing a wider spread of integers.
18+
19+### Floating Point Variant
20+- gaussian-flt1 (σ=1)
21+
22+This variant preserves the continuous nature of the Gaussian distribution using 32-bit floating point numbers.
benchcompress/src/benchcompress/datasets/gaussian_ar1/__init__.pyadded+86−0View file
@@ -0,0 +1,86 @@
1+import numpy as np
2+import os
3+
4+SOURCE_FILE = "gaussian_ar1/__init__.py"
5+
6+
7+def _load_long_description():
8+ current_dir = os.path.dirname(os.path.abspath(__file__))
9+ md_path = os.path.join(current_dir, "gaussian_ar1.md")
10+ with open(md_path, "r", encoding="utf-8") as f:
11+ return f.read()
12+
13+
14+LONG_DESCRIPTION = _load_long_description()
15+
16+
17+def create_gaussian_ar1_quantized(
18+ *, n_samples: int, phi: float, stddev: float, seed: int
19+) -> np.ndarray:
20+ """Generate quantized Gaussian AR(1) process.
21+
22+ Args:
23+ n_samples: Number of samples to generate
24+ phi: AR(1) coefficient (-1 < phi < 1)
25+ stddev: Standard deviation of Gaussian innovations
26+ seed: Random seed
27+ """
28+ rng = np.random.default_rng(seed)
29+ x = np.zeros(n_samples)
30+ x[0] = rng.normal(0, stddev) # Initialize first sample
31+
32+ # Generate AR(1) process
33+ for t in range(1, n_samples):
34+ x[t] = phi * x[t - 1] + rng.normal(0, stddev)
35+
36+ # Quantize to integers
37+ x = np.round(x).astype(np.int16)
38+ return x
39+
40+
41+tags = [
42+ "gaussian",
43+ "integer",
44+ "discrete",
45+ "timeseries",
46+ "1d",
47+ "synthetic",
48+ "ar1",
49+ "correlated",
50+]
51+
52+datasets = [
53+ {
54+ "name": "gaussian-ar1-02",
55+ "version": "1",
56+ "create": lambda: create_gaussian_ar1_quantized(
57+ n_samples=1_000_000, phi=0.2, stddev=3, seed=0
58+ ),
59+ "description": "Quantized Gaussian AR(1) process with φ=0.2, σ=3.",
60+ "tags": tags,
61+ "source_file": SOURCE_FILE,
62+ "long_description": LONG_DESCRIPTION,
63+ },
64+ {
65+ "name": "gaussian-ar1-05",
66+ "version": "1",
67+ "create": lambda: create_gaussian_ar1_quantized(
68+ n_samples=1_000_000, phi=0.5, stddev=3, seed=0
69+ ),
70+ "description": "Quantized Gaussian AR(1) process with φ=0.5, σ=3.",
71+ "tags": tags,
72+ "source_file": SOURCE_FILE,
73+ "long_description": LONG_DESCRIPTION,
74+ },
75+ {
76+ "name": "gaussian-ar1-08",
77+ "version": "1",
78+ "create": lambda: create_gaussian_ar1_quantized(
79+ n_samples=1_000_000, phi=0.8, stddev=3, seed=0
80+ ),
81+ "description": "Quantized Gaussian AR(1) process with φ=0.8, σ=3.",
82+ "tags": tags,
83+ "source_file": SOURCE_FILE,
84+ "long_description": LONG_DESCRIPTION,
85+ },
86+]
benchcompress/src/benchcompress/datasets/gaussian_ar1/gaussian_ar1.mdadded+22−0View file
@@ -0,0 +1,22 @@
1+# Gaussian AR(1) Dataset
2+
3+These datasets contain sequences from autoregressive AR(1) processes with Gaussian innovations. The data is generated using the model:
4+
5+```
6+x[t] = φ * x[t-1] + ε[t]
7+```
8+
9+where:
10+- φ is the autoregressive parameter controlling temporal correlation
11+- ε[t] follows a Gaussian distribution with μ=0 and σ=3
12+- Values are quantized to integers after generation
13+
14+## Variants
15+
16+We provide three AR(1) variants with different temporal correlation strengths:
17+
18+- gaussian-ar1-02 (φ=0.2): Weak temporal correlation
19+- gaussian-ar1-05 (φ=0.5): Moderate temporal correlation
20+- gaussian-ar1-08 (φ=0.8): Strong temporal correlation
21+
22+Each variant has σ=3 for the Gaussian innovations and uses integer quantization after generation to maintain compatibility with integer-based compression algorithms.
benchcompress/src/benchcompress/datasets/gaussian_lowpass/__init__.pyadded+89−0View file
@@ -0,0 +1,89 @@
1+import numpy as np
2+import os
3+
4+SOURCE_FILE = "gaussian_lowpass/__init__.py"
5+
6+
7+def _load_long_description():
8+ current_dir = os.path.dirname(os.path.abspath(__file__))
9+ md_path = os.path.join(current_dir, "gaussian_lowpass.md")
10+ with open(md_path, "r", encoding="utf-8") as f:
11+ return f.read()
12+
13+
14+LONG_DESCRIPTION = _load_long_description()
15+
16+
17+def create_gaussian_lowpass_quantized(
18+ *, n_samples: int, window_size: int, stddev: float, seed: int
19+) -> np.ndarray:
20+ """Generate quantized Gaussian lowpass filtered sequence.
21+
22+ Args:
23+ n_samples: Number of samples to generate
24+ window_size: Size of moving average window
25+ stddev: Standard deviation of initial Gaussian noise
26+ seed: Random seed
27+ """
28+ rng = np.random.default_rng(seed)
29+
30+ # Generate Gaussian noise
31+ x = rng.normal(0, stddev, n_samples)
32+
33+ # Apply moving average filter
34+ # Use 'same' mode to maintain input length, and valid edges
35+ window = np.ones(window_size) / window_size
36+ x_filtered = np.convolve(x, window, mode="same")
37+
38+ # Quantize to integers
39+ x_quantized = np.round(x_filtered).astype(np.int16)
40+ return x_quantized
41+
42+
43+tags = [
44+ "gaussian",
45+ "integer",
46+ "discrete",
47+ "timeseries",
48+ "1d",
49+ "synthetic",
50+ "correlated",
51+ "lowpass",
52+ "filtered",
53+]
54+
55+datasets = [
56+ {
57+ "name": "gaussian-lp-03",
58+ "version": "1",
59+ "create": lambda: create_gaussian_lowpass_quantized(
60+ n_samples=1_000_000, window_size=3, stddev=5, seed=0
61+ ),
62+ "description": "Quantized Gaussian lowpass filtered with window size 3, σ=5.",
63+ "tags": tags,
64+ "source_file": SOURCE_FILE,
65+ "long_description": LONG_DESCRIPTION,
66+ },
67+ {
68+ "name": "gaussian-lp-07",
69+ "version": "1",
70+ "create": lambda: create_gaussian_lowpass_quantized(
71+ n_samples=1_000_000, window_size=7, stddev=5, seed=0
72+ ),
73+ "description": "Quantized Gaussian lowpass filtered with window size 7, σ=5.",
74+ "tags": tags,
75+ "source_file": SOURCE_FILE,
76+ "long_description": LONG_DESCRIPTION,
77+ },
78+ {
79+ "name": "gaussian-lp-15",
80+ "version": "1",
81+ "create": lambda: create_gaussian_lowpass_quantized(
82+ n_samples=1_000_000, window_size=15, stddev=5, seed=0
83+ ),
84+ "description": "Quantized Gaussian lowpass filtered with window size 15, σ=5.",
85+ "tags": tags,
86+ "source_file": SOURCE_FILE,
87+ "long_description": LONG_DESCRIPTION,
88+ },
89+]
benchcompress/src/benchcompress/datasets/gaussian_lowpass/gaussian_lowpass.mdadded+18−0View file
@@ -0,0 +1,18 @@
1+# Gaussian Lowpass Dataset
2+
3+These datasets contain sequences of quantized Gaussian noise that has been smoothed using a moving average (lowpass) filter. The data is generated by:
4+1. Drawing samples from a Gaussian distribution (σ=5)
5+2. Applying a moving average filter of varying window sizes
6+3. Quantizing the filtered result to integers
7+
8+The moving average filter acts as a lowpass filter, attenuating high-frequency components while preserving low-frequency trends. Larger window sizes result in smoother sequences with stronger low-frequency characteristics.
9+
10+## Variants
11+
12+We provide three variants with different window sizes:
13+
14+- gaussian-lp-03 (window=3): Light smoothing
15+- gaussian-lp-07 (window=7): Moderate smoothing
16+- gaussian-lp-15 (window=15): Strong smoothing
17+
18+Each variant uses σ=5 for the initial Gaussian noise and applies integer quantization after filtering to maintain compatibility with integer-based compression algorithms.
benchcompress/src/benchcompress/datasets/ieeg/__init__.pyadded+85−0View file
@@ -0,0 +1,85 @@
1+import numpy as np
2+import os
3+import requests
4+import pyedflib
5+
6+SOURCE_FILE = "ieeg/__init__.py"
7+
8+
9+def _load_long_description():
10+ current_dir = os.path.dirname(os.path.abspath(__file__))
11+ md_path = os.path.join(current_dir, "ieeg.md")
12+ with open(md_path, "r", encoding="utf-8") as f:
13+ return f.read()
14+
15+
16+LONG_DESCRIPTION = _load_long_description()
17+
18+tags = ["real", "ecephys", "ieeg", "timeseries", "1d", "correlated"]
19+tags_float = tags + ["float"]
20+tags_integer = tags + ["integer"]
21+
22+
23+def _load_ieeg_openneuro_005592() -> np.ndarray:
24+ """Load data from OpenNeuro dataset 005592.
25+
26+ Returns:
27+ Array containing concatenated data from first 10 channels
28+ """
29+ url = "https://s3.amazonaws.com/openneuro.org/ds006107/sub-01/ses-01/ieeg/sub-01_ses-01_task-sws_run-01_ieeg.edf?versionId=lF3VchWxQteKF.R8nwm5kss6b846EPek"
30+ edf_filename = "sub-01_ieeg.edf"
31+
32+ # Download if needed
33+ if not os.path.exists(edf_filename):
34+ print(f"Downloading {edf_filename} {url}...")
35+ response = requests.get(url)
36+ with open(edf_filename, "wb") as f:
37+ f.write(response.content)
38+
39+ # Load and concatenate channels
40+ f = pyedflib.EdfReader(edf_filename)
41+ max_channels = 10
42+ n_signals = min(f.signals_in_file, max_channels)
43+
44+ # Read and concatenate first n_signals
45+ signals = []
46+ for i in range(n_signals):
47+ signals.append(f.readSignal(i))
48+ concatenated = np.concatenate(signals).astype(np.float32)
49+
50+ f.close()
51+ return concatenated
52+
53+
54+def _load_quantized_ieeg_openneuro_005592() -> np.ndarray:
55+ """Load and quantize data from OpenNeuro dataset 005592.
56+
57+ Returns:
58+ Array containing quantized data from first 10 channels concatenated
59+ """
60+ X = _load_ieeg_openneuro_005592()
61+ step = 1
62+ X = np.round(X / step).astype(np.int32)
63+ return X
64+
65+
66+datasets = [
67+ {
68+ "name": "ieeg-005592",
69+ "version": "1",
70+ "description": "iEEG recording from OpenNeuro dataset ds005592, first 10 channels concatenated.",
71+ "create": lambda: _load_ieeg_openneuro_005592(),
72+ "tags": tags_float,
73+ "source_file": SOURCE_FILE,
74+ "long_description": LONG_DESCRIPTION,
75+ },
76+ {
77+ "name": "ieeg-005592-quantized",
78+ "version": "1",
79+ "description": "iEEG recording from OpenNeuro dataset ds005592, first 10 channels concatenated, quantized.",
80+ "create": lambda: _load_quantized_ieeg_openneuro_005592(),
81+ "tags": tags_integer,
82+ "source_file": SOURCE_FILE,
83+ "long_description": LONG_DESCRIPTION,
84+ },
85+]
benchcompress/src/benchcompress/datasets/ieeg/ieeg.mdadded+15−0View file
@@ -0,0 +1,15 @@
1+# iEEG Dataset
2+
3+This dataset contains intracranial EEG (iEEG) recordings from OpenNeuro dataset ds005592. The data consists of the first 10 channels concatenated into a single continuous time series.
4+
5+## Data Source
6+
7+OpenNeuro dataset ds005592:
8+- Dataset URL: https://openneuro.org/datasets/ds005592
9+- Specific file: sub-01/ses-01/ieeg/sub-01_ses-01_task-sws_run-01_ieeg.edf
10+
11+## Data Properties
12+
13+- Type: Intracranial EEG recordings
14+- Format: Concatenated time series from first 10 channels
15+- Values: 32-bit floating point numbers representing voltage measurements
benchcompress/src/benchcompress/datasets/seismic/__init__.pyadded+87−0View file
@@ -0,0 +1,87 @@
1+import numpy as np
2+import segyio
3+import os
4+import requests
5+
6+
7+SOURCE_FILE = "seismic/__init__.py"
8+
9+
10+def _load_long_description():
11+ current_dir = os.path.dirname(os.path.abspath(__file__))
12+ md_path = os.path.join(current_dir, "seismic.md")
13+ with open(md_path, "r", encoding="utf-8") as f:
14+ return f.read()
15+
16+
17+LONG_DESCRIPTION = _load_long_description()
18+
19+tags = ["real", "seismic", "correlated", "timeseries", "1d"]
20+tags_float = tags + ["float"]
21+tags_integer = tags + ["integer"]
22+
23+
24+def _load_04A_04B_seismic_data() -> np.ndarray:
25+ """Load seismic data from the SEG-Y file.
26+
27+ Returns:
28+ Array containing the loaded seismic data
29+ """
30+ file_path = "04A+04B.segy"
31+ if not os.path.exists(file_path):
32+ # Download the SEG-Y file
33+ url = "https://zenodo.org/records/8152964/files/04A+04B.segy?download=1"
34+ response = requests.get(url)
35+ with open(file_path, "wb") as f:
36+ f.write(response.content)
37+ print(f"Downloaded {file_path}")
38+ else:
39+ print(f"{file_path} already exists locally.")
40+
41+ # Open the SEG-Y file
42+ with segyio.open(file_path, "r", ignore_geometry=True) as f:
43+ # Read the seismic data
44+ data = f.trace.raw[:]
45+
46+ # Consider only the first 3000 traces, because the others have a bunch of zeros
47+ X = data[:3000]
48+ # The first part of each trace is zeros
49+ first_nonzero_indices = []
50+ for j in range(X.shape[0]):
51+ inds = np.where(X[j] != 0)[0]
52+ first_nonzero_indices.append(inds[0] if len(inds) > 0 else -1)
53+ # plt.figure(figsize=(10, 5))
54+ # plt.hist(first_nonzero_indices, bins=20)
55+ # print(np.max(first_nonzero_indices)) # 1607
56+ X = X[:, 1700:]
57+
58+ return X.ravel()
59+
60+
61+def _load_quantized_04A_04B_seismic_data():
62+ X = _load_04A_04B_seismic_data()
63+ step = 3000
64+ X = np.round(X / step).astype(np.int32)
65+ return X
66+
67+
68+datasets = [
69+ {
70+ "name": "seismic-04A-04B",
71+ "version": "1",
72+ "description": "Seismic data from Roger Revelle voyage RR1508.",
73+ "create": lambda: _load_04A_04B_seismic_data(),
74+ "tags": tags_float,
75+ "source_file": SOURCE_FILE,
76+ "long_description": LONG_DESCRIPTION,
77+ },
78+ {
79+ "name": "seismic-04A-04B-quantized",
80+ "version": "4",
81+ "description": "Seismic data from Roger Revelle voyage RR1508, quantized.",
82+ "create": lambda: _load_quantized_04A_04B_seismic_data(),
83+ "tags": tags_integer,
84+ "source_file": SOURCE_FILE,
85+ "long_description": LONG_DESCRIPTION,
86+ },
87+]
benchcompress/src/benchcompress/datasets/seismic/explore_seismic.pyadded+131−0View file
@@ -0,0 +1,131 @@
1+# %%
2+import numpy as np
3+import matplotlib.pyplot as plt
4+import os
5+import requests
6+import segyio
7+
8+file_path = "04A+04B.segy"
9+if not os.path.exists(file_path):
10+ # Download the SEG-Y file
11+ url = "https://zenodo.org/records/8152964/files/04A+04B.segy?download=1"
12+ response = requests.get(url)
13+ with open(file_path, "wb") as f:
14+ f.write(response.content)
15+ print(f"Downloaded {file_path}")
16+else:
17+ print(f"{file_path} already exists locally.")
18+
19+# Open the SEG-Y file
20+with segyio.open(file_path, "r", ignore_geometry=True) as f:
21+ # Read the seismic data
22+ data = f.trace.raw[:]
23+# %%
24+print(data.shape) # (4101, 3751)
25+print(data.dtype) # float32
26+# %%
27+# Consider only the first 3000 traces, because the others have a bunch of zeros
28+X = data[:3000]
29+# The first part of each trace is zeros
30+first_nonzero_indices = []
31+for j in range(X.shape[0]):
32+ inds = np.where(X[j] != 0)[0]
33+ first_nonzero_indices.append(inds[0] if len(inds) > 0 else -1)
34+# plt.figure(figsize=(10, 5))
35+# plt.hist(first_nonzero_indices, bins=20)
36+print(np.max(first_nonzero_indices)) # 1607
37+X = X[:, 1700:]
38+# %%
39+plt.figure(figsize=(10, 5))
40+plt.plot(X.ravel()[:12000])
41+# %%
42+print(X.shape) # (4101, 2051)
43+print(X[0, :5]) # [-2891.2651 -2550.8093 -2744.9373 -111.93254 6441.383 ]
44+# %%
45+plt.figure(figsize=(10, 5))
46+plt.plot(X[0])
47+plt.figure(figsize=(10, 5))
48+plt.plot(X[1])
49+plt.figure(figsize=(10, 5))
50+plt.plot(X[1000])
51+plt.figure(figsize=(10, 5))
52+plt.plot(X[1001])
53+plt.figure(figsize=(10, 5))
54+plt.plot(X[2000])
55+plt.figure(figsize=(10, 5))
56+plt.plot(X[-1])
57+# %%
58+plt.figure(figsize=(10, 5))
59+plt.plot(X[:, 0])
60+plt.figure(figsize=(10, 5))
61+plt.plot(X[:, 1])
62+plt.figure(figsize=(10, 5))
63+plt.plot(X[:, 1000])
64+# %%
65+# Extract the exponent part of the float32 array X as uint8
66+X_exponent = ((X.view(np.uint32) >> 23) & 0xFF).astype(np.uint8)
67+print(X_exponent)
68+
69+# %%
70+import simple_ans
71+
72+a = simple_ans.ans_encode(X_exponent.ravel())
73+# %%
74+vals, counts = np.unique(X.ravel(), return_counts=True)
75+print(len(vals))
76+print(len(X.ravel()))
77+# plt.figure(figsize=(10, 5))
78+# plt.plot(vals, '.')
79+
80+# %%
81+v = len(a.bitstream) + a.symbol_counts.nbytes + a.symbol_values.nbytes
82+compression_ratio = len(X_exponent.tobytes()) / v
83+print(compression_ratio)
84+
85+print(4 / (v / X_exponent.size + 3))
86+
87+steps = np.exp(np.arange(20))
88+tests = []
89+for step in steps:
90+ X_quantized = np.round(X / step).astype(np.int32)
91+ resid = X / step - X_quantized
92+ resid_delta = np.diff(np.diff(resid.ravel()))
93+ v = np.median(np.abs(resid_delta))
94+ tests.append(v)
95+
96+plt.figure(figsize=(10, 5))
97+plt.plot(steps, tests)
98+plt.semilogx()
99+
100+# %%
101+q_step = 10000
102+X_quantized = np.round(X / q_step).astype(np.int32)
103+
104+# %%
105+plt.figure(figsize=(10, 5))
106+plt.plot(X_quantized[0])
107+plt.figure(figsize=(10, 5))
108+plt.plot(X[0])
109+# %%
110+import simple_ans
111+
112+a = simple_ans.ans_encode(X_quantized.ravel())
113+v = len(a.bitstream) + a.symbol_counts.nbytes + a.symbol_values.nbytes
114+print(len(X.tobytes()) / v)
115+# %%
116+import zstandard as zstd
117+
118+cctx = zstd.ZstdCompressor(level=13)
119+compressed = cctx.compress(X_quantized.tobytes())
120+v_zstd = len(compressed)
121+print(len(X.tobytes()) / v_zstd)
122+# %%
123+X_quantized_diff = np.diff(X_quantized.ravel())
124+a = simple_ans.ans_encode(X_quantized_diff)
125+v = len(a.bitstream) + a.symbol_counts.nbytes + a.symbol_values.nbytes
126+print(len(X.tobytes()) / v)
127+# %%
128+compressed = cctx.compress(X_quantized_diff.tobytes())
129+v_zstd = len(compressed)
130+print(len(X.tobytes()) / v_zstd)
131+# %%
benchcompress/src/benchcompress/datasets/seismic/seismic.mdadded+22−0View file
@@ -0,0 +1,22 @@
1+# Seismic Dataset
2+
3+This dataset contains marine seismic reflection data. Processed seismic reflection data and associated files from Roger Revelle voyage RR1508: 16 May - 18 June 2015. The research voyage aimed to characterise the thermal regime of the gas hydrate systems on the southern Hikurangi margin east of New Zealand. Data were processed using the Globe Claritas processing software.
4+
5+## Data Source
6+
7+The data comes from a SEG-Y file available on Zenodo (https://zenodo.org/records/8152964). SEG-Y is a standard format for storing seismic data that includes both the recorded waveforms and metadata about the survey.
8+
9+### Revelle RR1508 Data (seismic-04A-04B)
10+- Original floating point values from the SEG-Y file
11+- Contains the natural amplitude variations of the seismic waves
12+- Stored as 32-bit floating point numbers
13+
14+### Revelle RR1508 Quantized Data (seismic-04A-04B-quantized)
15+- Values are scaled and rounded to integers
16+- Uses a quantization step of 10000
17+- Stored as 32-bit integers
18+
19+## Source Details
20+
21+### Roger Revelle RR1508
22+The data comes from a SEG-Y file available on Zenodo (https://zenodo.org/records/8152964). SEG-Y is a standard format for storing seismic data that includes both the recorded waveforms and metadata about the survey.
benchcompress/src/benchcompress/run_benchmarks/__init__.pyadded+1−0View file
@@ -0,0 +1 @@
1+from .run_benchmarks import run_benchmarks
benchcompress/src/benchcompress/run_benchmarks/_memobin.pyadded+228−0View file
@@ -0,0 +1,228 @@
1+import json
2+import requests
3+import time
4+from typing import Optional, TypeVar, Callable
5+
6+T = TypeVar("T")
7+
8+
9+def _retry_with_backoff(
10+ func: Callable[..., T], num_retries: int = 4, *args, **kwargs
11+) -> T:
12+ """Execute a function with exponential backoff retry logic.
13+
14+ Args:
15+ func: Function to execute
16+ num_retries: Maximum number of retries
17+ args: Positional arguments for the function
18+ kwargs: Keyword arguments for the function
19+
20+ Returns:
21+ The function's return value
22+
23+ Raises:
24+ The last exception encountered after all retries are exhausted
25+ """
26+ last_exception = None
27+ for attempt in range(num_retries):
28+ try:
29+ return func(*args, **kwargs)
30+ except Exception as e:
31+ last_exception = e
32+ if attempt < num_retries - 1:
33+ sleep_time = 2**attempt # 1, 2, 4, 8 seconds
34+ time.sleep(sleep_time)
35+ else:
36+ raise last_exception
37+ raise RuntimeError("Unexpected: retry loop completed without return or raise")
38+
39+
40+def create_signed_upload_url(
41+ url: str, size: int, user_id: str, memobin_api_key: str, num_retries: int = 4
42+) -> str:
43+ """Create a signed upload URL for memobin.
44+
45+ Args:
46+ url: The target URL for the file
47+ size: Size of the file in bytes
48+ user_id: User ID for memobin
49+ memobin_api_key: API key for memobin authentication
50+
51+ Returns:
52+ The signed upload URL
53+
54+ Raises:
55+ ValueError: If the URL prefix is invalid
56+ requests.RequestException: If the API request fails
57+ """
58+
59+ def _create_url() -> str:
60+ prefix = "https://tempory.net/f/memobin/"
61+ if not url.startswith(prefix):
62+ raise ValueError("Invalid url. Does not have proper prefix")
63+
64+ file_path = url[len(prefix) :]
65+ tempory_api_url = "https://hub.tempory.net/api/uploadFile"
66+
67+ response = requests.post(
68+ tempory_api_url,
69+ headers={
70+ "Content-Type": "application/json",
71+ "Authorization": f"Bearer {memobin_api_key}",
72+ },
73+ json={
74+ "appName": "memobin",
75+ "filePath": file_path,
76+ "size": size,
77+ "userId": user_id,
78+ },
79+ )
80+
81+ if not response.ok:
82+ raise requests.RequestException("Failed to get signed url")
83+
84+ result = response.json()
85+ upload_url = result["uploadUrl"]
86+ download_url = result["downloadUrl"]
87+
88+ if download_url != url:
89+ raise ValueError("Mismatch between download url and url")
90+
91+ return upload_url
92+
93+ return _retry_with_backoff(_create_url, num_retries)
94+
95+
96+def construct_memobin_url(
97+ alg_name: str,
98+ dataset_name: str,
99+ alg_version: str,
100+ dataset_version: str,
101+ system_version: str,
102+ file_type: str = "metadata.json",
103+) -> str:
104+ """Construct the memobin URL for a specific benchmark result or dataset.
105+
106+ Args:
107+ alg_name: Name of the algorithm
108+ dataset_name: Name of the dataset
109+ alg_version: Version of the algorithm
110+ dataset_version: Version of the dataset
111+ system_version: Version of the system
112+ file_type: Type of file (metadata.json or data.bin)
113+
114+ Returns:
115+ The constructed memobin URL
116+ """
117+ path = f"{alg_name}/{dataset_name}/{alg_version}/{dataset_version}/{system_version}/{file_type}"
118+ return f"https://tempory.net/f/memobin/{path}"
119+
120+
121+def construct_dataset_url(
122+ dataset_name: str, dataset_version: str, format: str = "dat"
123+) -> str:
124+ """Construct the memobin URL for a dataset.
125+
126+ Args:
127+ dataset_name: Name of the dataset
128+ dataset_version: Version of the dataset
129+ format: File format ("dat", "npy", or "json")
130+
131+ Returns:
132+ The constructed memobin URL for the dataset
133+ """
134+ path = f"datasets/{dataset_name}/{dataset_version}/{dataset_name}-{dataset_version}.{format}"
135+ return f"https://tempory.net/f/memobin/{path}"
136+
137+
138+def upload_to_memobin(
139+ data: dict | bytes,
140+ url: str,
141+ memobin_api_key: str,
142+ content_type: str = "application/json",
143+ num_retries: int = 4,
144+) -> None:
145+ """Upload data to memobin.
146+
147+ Args:
148+ data: The data to upload (dict for JSON or bytes for binary)
149+ url: The target URL for the file
150+ memobin_api_key: API key for memobin authentication
151+ content_type: Content type of the data
152+
153+ Raises:
154+ requests.RequestException: If the upload fails
155+ """
156+ if isinstance(data, dict):
157+ data_bytes = json.dumps(data).encode("utf-8")
158+ else:
159+ data_bytes = data
160+ size = len(data_bytes)
161+
162+ def _do_upload() -> None:
163+ upload_url = create_signed_upload_url(
164+ url, size, "benchcompress", memobin_api_key, num_retries
165+ )
166+
167+ response = requests.put(
168+ upload_url, data=data_bytes, headers={"Content-Type": content_type}
169+ )
170+
171+ if not response.ok:
172+ raise requests.RequestException("Failed to upload data to memobin")
173+
174+ _retry_with_backoff(_do_upload, num_retries)
175+
176+
177+def exists_in_memobin(url: str, num_retries: int = 4) -> bool:
178+ """Check if a file exists in memobin using a HEAD request.
179+
180+ Args:
181+ url: The URL to check
182+
183+ Returns:
184+ True if the file exists, False otherwise
185+ """
186+
187+ def _check_exists() -> bool:
188+ try:
189+ response = requests.head(url)
190+ return (
191+ 200 <= response.status_code < 300
192+ ) # Any 2xx status code indicates success
193+ except requests.RequestException:
194+ return False
195+
196+ return _retry_with_backoff(_check_exists, num_retries)
197+
198+
199+def download_from_memobin(
200+ url: str, as_json: bool = True, num_retries: int = 4
201+) -> Optional[dict | bytes]:
202+ """Download data from memobin.
203+
204+ Args:
205+ url: The URL to download from
206+ as_json: Whether to parse the response as JSON
207+
208+ Returns:
209+ The downloaded data as a dictionary or bytes, or None if not found
210+
211+ Raises:
212+ requests.RequestException: If the download fails for a reason other than 404
213+ """
214+
215+ def _do_download() -> Optional[dict | bytes]:
216+ response = None
217+ try:
218+ response = requests.get(url)
219+ if response.status_code == 404:
220+ return None
221+ response.raise_for_status()
222+ return response.json() if as_json else response.content
223+ except requests.RequestException as e:
224+ if response and response.status_code == 404:
225+ return None
226+ raise e
227+
228+ return _retry_with_backoff(_do_download, num_retries)
benchcompress/src/benchcompress/run_benchmarks/benchmark_timing.pyadded+122−0View file
@@ -0,0 +1,122 @@
1+from typing import Any, Tuple, Callable, Dict
2+from statistics import median
3+import time
4+import numpy as np
5+
6+
7+def run_timed_trials(
8+ data: np.ndarray, operation: Callable, *args
9+) -> Tuple[float, float, Any]:
10+ """Run multiple trials of an operation until total time exceeds 1 second.
11+
12+ Args:
13+ data: Input numpy array for calculating throughput
14+ operation: Function to benchmark
15+ *args: Arguments to pass to the operation
16+
17+ Returns:
18+ Tuple containing:
19+ - median_time: Median execution time across trials
20+ - mb_per_sec: Throughput in MB/s
21+ - result: Result from the last trial execution
22+ """
23+ times = []
24+ total_time = 0
25+ array_size_mb = data.nbytes / (1024 * 1024) # Convert to MB
26+
27+ operation(
28+ *args
29+ ) # execute once prior to timing in case there's any initial overhead
30+
31+ ret = None
32+ while total_time < 1.0:
33+ start_time = time.perf_counter()
34+ ret = operation(*args) # Execute operation
35+ trial_time = time.perf_counter() - start_time
36+ times.append(trial_time)
37+ total_time += trial_time
38+
39+ median_time = median(times)
40+ mb_per_sec = array_size_mb / median_time
41+ return median_time, mb_per_sec, ret
42+
43+
44+def run_compression_benchmark(
45+ data: np.ndarray,
46+ algorithm_name: str,
47+ encode_fn: Callable,
48+ decode_fn: Callable,
49+ verbose: bool = True,
50+) -> Tuple[Dict[str, Any], bytes]:
51+ """Run compression and decompression benchmarks for an algorithm.
52+
53+ Args:
54+ data: Input numpy array to compress
55+ algorithm_name: Name of the algorithm being benchmarked
56+ encode_fn: Compression function
57+ decode_fn: Decompression function
58+ verbose: Whether to print progress messages
59+
60+ Returns:
61+ Tuple containing:
62+ - result: Dictionary with benchmark metrics
63+ - encoded: Compressed data bytes
64+ """
65+ original_size = len(data.tobytes())
66+ dtype = str(data.dtype)
67+
68+ if verbose:
69+ print(" Encoding...")
70+ encode_time, encode_mb_per_sec, encoded = run_timed_trials(data, encode_fn, data)
71+ compressed_size = len(encoded)
72+ compression_ratio = original_size / compressed_size
73+
74+ if verbose:
75+ print(" Compression complete:")
76+ print(f" Compressed size: {compressed_size:,} bytes")
77+ print(f" Compression ratio: {compression_ratio:.2f}x")
78+ print(f" Encode time: {encode_time*1000:.2f}ms")
79+ print(f" Encode throughput: {encode_mb_per_sec:.2f} MB/s")
80+ print(" Decoding...")
81+
82+ decode_time, decode_mb_per_sec, decoded = run_timed_trials(
83+ data, decode_fn, encoded, dtype, data.shape
84+ )
85+
86+ if verbose:
87+ print(f" Decode time: {decode_time*1000:.2f}ms")
88+ print(f" Decode throughput: {decode_mb_per_sec:.2f} MB/s")
89+
90+ # Verify correctness
91+ if len(data) != len(decoded):
92+ raise ValueError(
93+ f"Decompression failed: decoded length {len(decoded)} != original length {len(data)}"
94+ )
95+
96+ if not np.array_equal(data, decoded):
97+ print(data[:100])
98+ print(decoded[:100])
99+ for j in range(len(data)):
100+ if data[j] != decoded[j]:
101+ print(f"Error at index {j}: {data[j]} != {decoded[j]}")
102+ break
103+ raise ValueError(f"Decompression verification failed for {algorithm_name}")
104+
105+ if verbose:
106+ print(" Verification successful!")
107+
108+ result = {
109+ "compression_ratio": compression_ratio,
110+ "encode_time": encode_time,
111+ "decode_time": decode_time,
112+ "encode_mb_per_sec": encode_mb_per_sec,
113+ "decode_mb_per_sec": decode_mb_per_sec,
114+ "original_size": original_size,
115+ "compressed_size": compressed_size,
116+ "array_shape": data.shape,
117+ "array_dtype": dtype,
118+ "timestamp": time.time(),
119+ "cache_status": "new",
120+ }
121+
122+ return result, encoded
benchcompress/src/benchcompress/run_benchmarks/cache_management.pyadded+118−0View file
@@ -0,0 +1,118 @@
1+import os
2+import json
3+from typing import Optional, Dict, Any
4+from ._memobin import (
5+ construct_memobin_url,
6+ download_from_memobin,
7+)
8+
9+
10+def check_cached_result(
11+ cache_dir: str,
12+ dataset_name: str,
13+ algorithm_name: str,
14+ algorithm_version: str,
15+ dataset_version: str,
16+ system_version: str,
17+ force: bool = False,
18+ verbose: bool = True,
19+) -> Optional[Dict[str, Any]]:
20+ """Check for cached benchmark results locally and in memobin.
21+
22+ Args:
23+ cache_dir: Directory containing cached results
24+ dataset_name: Name of the dataset
25+ algorithm_name: Name of the algorithm
26+ algorithm_version: Version of the algorithm
27+ dataset_version: Version of the dataset
28+ system_version: Version of the system
29+ force: If True, ignore cached results
30+ verbose: Whether to print progress messages
31+
32+ Returns:
33+ Cached result dictionary if found and valid, None otherwise
34+ """
35+ test_dir = os.path.join(cache_dir, dataset_name, algorithm_name)
36+ metadata_file = os.path.join(test_dir, "metadata.json")
37+
38+ # First try local cache (unless force flag is set)
39+ cached_data = None
40+ if not force and os.path.exists(metadata_file):
41+ with open(metadata_file, "r") as f:
42+ cached_data = json.load(f)
43+ # if versions do not match, then set to None
44+ if isinstance(cached_data, dict) and "result" in cached_data:
45+ result = cached_data["result"]
46+ if (
47+ result["algorithm_version"] != algorithm_version
48+ or result["dataset_version"] != dataset_version
49+ or result.get("system_version", "") != system_version
50+ ):
51+ cached_data = None
52+
53+ # If not in local cache, try memobin (unless force flag is set)
54+ if cached_data is None and not force:
55+ memobin_url = construct_memobin_url(
56+ algorithm_name,
57+ dataset_name,
58+ algorithm_version,
59+ dataset_version,
60+ system_version,
61+ "metadata.json",
62+ )
63+ if verbose:
64+ print(" Looking for cached result in memobin...")
65+ cached_data = download_from_memobin(memobin_url)
66+ if cached_data is not None:
67+ if verbose:
68+ print(" Found result in memobin, saving locally...")
69+ # Save to local cache
70+ os.makedirs(test_dir, exist_ok=True)
71+ with open(metadata_file, "w") as f:
72+ json.dump(cached_data, f, indent=2)
73+
74+ if (
75+ cached_data is not None
76+ and isinstance(cached_data, dict)
77+ and "result" in cached_data
78+ ):
79+ result = cached_data["result"]
80+ if (
81+ isinstance(result, dict)
82+ and result.get("algorithm_version") == algorithm_version
83+ and result.get("dataset_version") == dataset_version
84+ and result.get("system_version", "") == system_version
85+ ):
86+ result["cache_status"] = "cached"
87+ return result
88+
89+ return None
90+
91+
92+def save_result_to_cache(
93+ result: Dict[str, Any],
94+ encoded_data: bytes,
95+ cache_dir: str,
96+ dataset_name: str,
97+ algorithm_name: str,
98+) -> None:
99+ """Save benchmark result and compressed data to cache.
100+
101+ Args:
102+ result: Benchmark result dictionary
103+ encoded_data: Compressed data bytes
104+ cache_dir: Directory to store cached results
105+ dataset_name: Name of the dataset
106+ algorithm_name: Name of the algorithm
107+ """
108+ test_dir = os.path.join(cache_dir, dataset_name, algorithm_name)
109+ metadata_file = os.path.join(test_dir, "metadata.json")
110+ compressed_file = os.path.join(test_dir, "compressed.dat")
111+
112+ os.makedirs(test_dir, exist_ok=True)
113+ cache_data = {"result": result}
114+
115+ with open(metadata_file, "w") as f:
116+ json.dump(cache_data, f, indent=2)
117+ with open(compressed_file, "wb") as f:
118+ f.write(encoded_data)
benchcompress/src/benchcompress/run_benchmarks/collect_info.pyadded+62−0View file
@@ -0,0 +1,62 @@
1+from typing import List, Dict, Any
2+from ._memobin import construct_dataset_url
3+
4+GITHUB_ALGORITHMS_PREFIX = "https://github.com/magland/benchcompress/blob/main/benchcompress/src/benchcompress/algorithms/"
5+GITHUB_DATASETS_PREFIX = "https://github.com/magland/benchcompress/blob/main/benchcompress/src/benchcompress/datasets/"
6+
7+
8+def collect_algorithm_info(algorithms: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
9+ """Collect information about compression algorithms.
10+
11+ Args:
12+ algorithms: List of algorithm dictionaries
13+
14+ Returns:
15+ List of algorithm information dictionaries
16+ """
17+ algorithm_info = []
18+ for algorithm in algorithms:
19+ info = {
20+ "name": algorithm["name"],
21+ "description": algorithm.get("description", ""),
22+ "long_description": algorithm.get("long_description", ""),
23+ "version": algorithm["version"],
24+ "tags": algorithm.get("tags", []),
25+ }
26+ if "source_file" in algorithm:
27+ info["source_file"] = GITHUB_ALGORITHMS_PREFIX + algorithm["source_file"]
28+ algorithm_info.append(info)
29+ return algorithm_info
30+
31+
32+def collect_dataset_info(datasets: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
33+ """Collect information about benchmark datasets.
34+
35+ Args:
36+ datasets: List of dataset dictionaries
37+
38+ Returns:
39+ List of dataset information dictionaries
40+ """
41+ dataset_info = []
42+ for dataset in datasets:
43+ info = {
44+ "name": dataset["name"],
45+ "description": dataset.get("description", ""),
46+ "long_description": dataset.get("long_description", ""),
47+ "version": dataset["version"],
48+ "tags": dataset.get("tags", []),
49+ "data_url_raw": construct_dataset_url(
50+ dataset["name"], dataset["version"], "dat"
51+ ),
52+ "data_url_npy": construct_dataset_url(
53+ dataset["name"], dataset["version"], "npy"
54+ ),
55+ "data_url_json": construct_dataset_url(
56+ dataset["name"], dataset["version"], "json"
57+ ),
58+ }
59+ if "source_file" in dataset:
60+ info["source_file"] = GITHUB_DATASETS_PREFIX + dataset["source_file"]
61+ dataset_info.append(info)
62+ return dataset_info
benchcompress/src/benchcompress/run_benchmarks/is_compatible.pyadded+43−0View file
@@ -0,0 +1,43 @@
1+from typing import List
2+
3+
4+def is_compatible(algorithm_tags: List[str], dataset_tags: List[str]) -> bool:
5+ """Check if an algorithm is compatible with a dataset based on their tags.
6+
7+ Args:
8+ algorithm_tags: List of tags for the algorithm
9+ dataset_tags: List of tags for the dataset
10+
11+ Returns:
12+ True if the algorithm should be applied to the dataset
13+ """
14+ # If algorithm has delta_encoding or lpc_prediction, dataset must have continuous, timeseries, 1d, integer
15+ if "delta_encoding" in algorithm_tags or "lpc_prediction" in algorithm_tags:
16+ if (
17+ "correlated" not in dataset_tags
18+ or "timeseries" not in dataset_tags
19+ or "1d" not in dataset_tags
20+ or "integer" not in dataset_tags
21+ ):
22+ return False
23+
24+ # If algorithm has zero_rle, dataset must have sparse, timeseries, 1d
25+ if "zero_rle" in algorithm_tags:
26+ if (
27+ "sparse" not in dataset_tags
28+ or "timeseries" not in dataset_tags
29+ or "1d" not in dataset_tags
30+ ):
31+ return False
32+
33+ # If algorithm has integer, dataset must have integer
34+ if "integer" in algorithm_tags:
35+ if "integer" not in dataset_tags:
36+ return False
37+
38+ # If algorithm has "no_bernoulli", dataset must not have "bernoulli"
39+ if "no_bernoulli" in algorithm_tags:
40+ if "bernoulli" in dataset_tags:
41+ return False
42+
43+ return True
benchcompress/src/benchcompress/run_benchmarks/run_benchmarks.pyadded+225−0View file
@@ -0,0 +1,225 @@
1+import os
2+import time
3+from typing import Dict, Any, List, Optional
4+import numpy as np
5+
6+from ..algorithms import algorithms
7+from ..datasets import datasets
8+from ._memobin import construct_memobin_url, upload_to_memobin
9+from .upload_dataset import upload_dataset_to_memobin
10+from .cache_management import check_cached_result, save_result_to_cache
11+from .benchmark_timing import run_compression_benchmark
12+from .collect_info import collect_algorithm_info, collect_dataset_info
13+from .is_compatible import is_compatible
14+from .upload_benchmark_status import upload_benchmark_status
15+
16+system_version = "v6"
17+
18+
19+def run_benchmarks(
20+ cache_dir: str = ".benchmark_cache",
21+ verbose: bool = True,
22+ selected_algorithms: Optional[List[dict]] = None,
23+ selected_datasets: Optional[List[dict]] = None,
24+ force: bool = False,
25+) -> Dict[str, Any]:
26+ """Run all benchmarks, with caching based on algorithm and dataset versions.
27+
28+ Results are stored in separate directories for each dataset/algorithm combination:
29+ cache_dir/
30+ dataset_name/
31+ algorithm_name/
32+ metadata.json # Contains algorithm version, dataset version, and results
33+ compressed.dat # The actual compressed data
34+
35+ Args:
36+ cache_dir: Directory to store cached results
37+ verbose: Whether to print progress messages
38+ selected_algorithms: Optional list of specific algorithms to run
39+ selected_datasets: Optional list of specific datasets to run
40+ force: If True, ignore cached results
41+
42+ Returns:
43+ Dictionary containing benchmark results and metadata
44+ """
45+ print("\n=== Starting Benchmark Run ===")
46+ print(f"Cache directory: {cache_dir}")
47+
48+ os.makedirs(cache_dir, exist_ok=True)
49+
50+ start_time = time.time()
51+ last_status_upload = 0 # Track last status upload time
52+ results = []
53+ print("\nRunning benchmarks for all dataset-algorithm combinations...")
54+
55+ # Use selected datasets/algorithms or fall back to all
56+ datasets_to_run = selected_datasets if selected_datasets is not None else datasets
57+ algorithms_to_run = (
58+ selected_algorithms if selected_algorithms is not None else algorithms
59+ )
60+
61+ # Calculate total number of benchmarks
62+ total_benchmarks = sum(
63+ 1
64+ for dataset in datasets_to_run
65+ for algorithm in algorithms_to_run
66+ if is_compatible(algorithm.get("tags", []), dataset.get("tags", []))
67+ )
68+
69+ # Run benchmarks for each dataset and algorithm combination
70+ memobin_api_key = os.environ.get("MEMOBIN_API_KEY")
71+ upload_enabled = os.environ.get("UPLOAD_TO_MEMOBIN") == "1"
72+
73+ for dataset in datasets_to_run:
74+ dataset_tags = dataset.get("tags", [])
75+ print(f"\n*** Dataset: {dataset['name']} (tags: {dataset_tags}) ***")
76+
77+ # only create the dataset if it is needed
78+ data = None
79+
80+ for algorithm in algorithms_to_run:
81+ alg_name = algorithm["name"]
82+ alg_tags = algorithm.get("tags", [])
83+
84+ # Skip if algorithm and dataset are not compatible based on tags
85+ if not is_compatible(alg_tags, dataset_tags):
86+ if verbose:
87+ print(
88+ f"\nSkipping algorithm {alg_name} (tags: {alg_tags}) - incompatible with dataset tags"
89+ )
90+ continue
91+
92+ print(f"\nTesting algorithm: {alg_name} on dataset: {dataset['name']}")
93+
94+ # Upload current status to memobin if enabled (once per minute)
95+ current_time = time.time()
96+ if (
97+ memobin_api_key
98+ and upload_enabled
99+ and (current_time - last_status_upload >= 60)
100+ ): # Check if 60 seconds have passed
101+ try:
102+ upload_benchmark_status(
103+ memobin_api_key,
104+ dataset["name"],
105+ alg_name,
106+ results,
107+ total_benchmarks,
108+ start_time,
109+ )
110+ last_status_upload = current_time # Update last upload time
111+ except Exception as e:
112+ print(f" Warning: Failed to upload status to memobin: {str(e)}")
113+
114+ # Check if we can use cached result
115+ cached_result = check_cached_result(
116+ cache_dir,
117+ dataset["name"],
118+ alg_name,
119+ algorithm["version"],
120+ dataset["version"],
121+ system_version,
122+ force,
123+ verbose,
124+ )
125+
126+ if cached_result is not None:
127+ print(" Using cached result")
128+ results.append(cached_result)
129+ continue
130+
131+ print(f" Running benchmark for {alg_name} on {dataset['name']}...")
132+ if data is None:
133+ data = dataset["create"]()
134+ print(f"Created dataset: shape={data.shape}, dtype={data.dtype}")
135+ else:
136+ print("Dataset already created")
137+
138+ # Upload dataset to memobin if enabled
139+ if memobin_api_key and upload_enabled:
140+ try:
141+ upload_dataset_to_memobin(
142+ data,
143+ dataset["name"],
144+ dataset["version"],
145+ memobin_api_key,
146+ cache_dir,
147+ verbose,
148+ )
149+ except Exception as e:
150+ print(f" Warning: Failed to upload dataset to memobin: {str(e)}")
151+
152+ # Run the benchmark
153+ result, encoded = run_compression_benchmark(
154+ data,
155+ alg_name,
156+ algorithm["encode"],
157+ algorithm["decode"],
158+ verbose,
159+ )
160+
161+ # Add metadata to result
162+ result.update(
163+ {
164+ "dataset": dataset["name"],
165+ "algorithm": alg_name,
166+ "algorithm_version": algorithm["version"],
167+ "dataset_version": dataset["version"],
168+ "system_version": system_version,
169+ }
170+ )
171+ results.append(result)
172+
173+ # Save result and compressed data
174+ save_result_to_cache(
175+ result,
176+ encoded,
177+ cache_dir,
178+ dataset["name"],
179+ alg_name,
180+ )
181+ print(
182+ f" Results saved to: {os.path.join(cache_dir, dataset['name'], alg_name)}"
183+ )
184+
185+ # Upload to memobin if enabled
186+ if memobin_api_key and upload_enabled:
187+ try:
188+ memobin_url = construct_memobin_url(
189+ alg_name,
190+ dataset["name"],
191+ algorithm["version"],
192+ dataset["version"],
193+ system_version,
194+ )
195+ upload_to_memobin(
196+ {"result": result},
197+ memobin_url,
198+ memobin_api_key,
199+ )
200+ if verbose:
201+ print(" Successfully uploaded to memobin")
202+ except Exception as e:
203+ print(f" Warning: Failed to upload to memobin: {str(e)}")
204+
205+ print("\n=== Benchmark Run Complete ===\n")
206+
207+ # Collect algorithm and dataset information
208+ algorithm_info = collect_algorithm_info(algorithms)
209+ dataset_info = collect_dataset_info(datasets)
210+
211+ # Upload final benchmark status
212+ if memobin_api_key and upload_enabled:
213+ try:
214+ upload_benchmark_status(
215+ memobin_api_key,
216+ "All datasets",
217+ "All algorithms",
218+ results,
219+ total_benchmarks,
220+ start_time,
221+ )
222+ except Exception as e:
223+ print(f" Warning: Failed to upload final status to memobin: {str(e)}")
224+
225+ return {"results": results, "algorithms": algorithm_info, "datasets": dataset_info}
benchcompress/src/benchcompress/run_benchmarks/upload_benchmark_status.pyadded+39−0View file
@@ -0,0 +1,39 @@
1+from typing import Any, Dict, List
2+import time
3+from datetime import datetime
4+from ._memobin import (
5+ upload_to_memobin,
6+)
7+
8+
9+def upload_benchmark_status(
10+ memobin_api_key: str,
11+ current_dataset: str,
12+ current_algorithm: str,
13+ completed_benchmarks: List[Dict[str, Any]],
14+ total_benchmarks: int,
15+ start_time: float,
16+) -> None:
17+ """Upload current benchmark status to memobin.
18+
19+ Args:
20+ memobin_api_key: API key for memobin authentication
21+ current_dataset: Name of the current dataset being processed
22+ current_algorithm: Name of the current algorithm being tested
23+ completed_benchmarks: List of completed benchmark results
24+ total_benchmarks: Total number of benchmarks to run
25+ start_time: Timestamp when the benchmark run started
26+ """
27+ status = {
28+ "current_dataset": current_dataset,
29+ "current_algorithm": current_algorithm,
30+ "completed_count": len(completed_benchmarks),
31+ "total_count": total_benchmarks,
32+ "progress_percentage": (len(completed_benchmarks) / total_benchmarks) * 100,
33+ "elapsed_time": time.time() - start_time,
34+ "last_update": datetime.now().isoformat(),
35+ "completed_benchmarks": completed_benchmarks,
36+ }
37+
38+ status_url = "https://tempory.net/f/memobin/benchmark_status/current.json"
39+ upload_to_memobin(status, status_url, memobin_api_key)
benchcompress/src/benchcompress/run_benchmarks/upload_dataset.pyadded+79−0View file
@@ -0,0 +1,79 @@
1+import os
2+import numpy as np
3+from ._memobin import (
4+ construct_dataset_url,
5+ exists_in_memobin,
6+ upload_to_memobin,
7+)
8+
9+
10+def upload_dataset_to_memobin(
11+ data: np.ndarray,
12+ dataset_name: str,
13+ dataset_version: str,
14+ memobin_api_key: str,
15+ cache_dir: str,
16+ verbose: bool = True,
17+) -> None:
18+ """Upload dataset to memobin in multiple formats.
19+
20+ Args:
21+ data: The numpy array dataset to upload
22+ dataset_name: Name of the dataset
23+ dataset_version: Version of the dataset
24+ memobin_api_key: API key for memobin
25+ cache_dir: Directory for temporary files
26+ verbose: Whether to print progress messages
27+ """
28+ try:
29+ # Upload array metadata as JSON
30+ dataset_url_json = construct_dataset_url(dataset_name, dataset_version, "json")
31+ if not exists_in_memobin(dataset_url_json):
32+ if verbose:
33+ print(" Uploading dataset metadata to memobin...")
34+ metadata = {"dtype": str(data.dtype), "shape": data.shape}
35+ upload_to_memobin(
36+ metadata,
37+ dataset_url_json,
38+ memobin_api_key,
39+ content_type="application/json",
40+ )
41+ if verbose:
42+ print(" Successfully uploaded metadata")
43+
44+ # Upload raw .dat format
45+ dataset_url_raw = construct_dataset_url(dataset_name, dataset_version, "dat")
46+ if not exists_in_memobin(dataset_url_raw):
47+ if verbose:
48+ print(" Uploading dataset (raw) to memobin...")
49+ upload_to_memobin(
50+ data.tobytes(),
51+ dataset_url_raw,
52+ memobin_api_key,
53+ content_type="application/octet-stream",
54+ )
55+ if verbose:
56+ print(" Successfully uploaded raw dataset")
57+
58+ # Upload .npy format
59+ dataset_url_npy = construct_dataset_url(dataset_name, dataset_version, "npy")
60+ if not exists_in_memobin(dataset_url_npy):
61+ if verbose:
62+ print(" Uploading dataset (npy) to memobin...")
63+ # Save array to a temporary .npy file
64+ temp_npy = os.path.join(cache_dir, "temp.npy")
65+ np.save(temp_npy, data)
66+ with open(temp_npy, "rb") as f:
67+ npy_bytes = f.read()
68+ os.remove(temp_npy) # Clean up temp file
69+
70+ upload_to_memobin(
71+ npy_bytes,
72+ dataset_url_npy,
73+ memobin_api_key,
74+ content_type="application/octet-stream",
75+ )
76+ if verbose:
77+ print(" Successfully uploaded npy dataset")
78+ except Exception as e:
79+ print(f" Warning: Failed to upload dataset to memobin: {str(e)}")
benchmark_results/results.jsonadded+26648−0View file
This diff is 26,654 lines long and is not shown.
devel/check_format.shadded+36−0View file
@@ -0,0 +1,36 @@
1+#!/bin/bash
2+
3+# Get the directory where the script is located
4+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
5+PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )"
6+
7+# Check Python code formatting
8+echo "Checking Python code formatting..."
9+if ! black --check "$PROJECT_ROOT/benchcompress/src/benchcompress"; then
10+ echo "Python code is not properly formatted!"
11+ echo "Please run './devel/format_code.sh' to format the code"
12+ exit 1
13+fi
14+
15+# Check TypeScript/JavaScript code formatting
16+echo "Checking TypeScript/JavaScript code formatting..."
17+cd "$PROJECT_ROOT/web-ui"
18+if ! npm run format:check; then
19+ echo "TypeScript/JavaScript code is not properly formatted!"
20+ echo "Please run './devel/format_code.sh' to format the code"
21+ exit 1
22+fi
23+
24+# Check C++ code formatting
25+echo "Checking C++ code formatting..."
26+cpp_files=$(find "$PROJECT_ROOT" -type f \( -name "*.cpp" -o -name "*.h" -o -name "*.hpp" \))
27+for file in $cpp_files; do
28+ if ! clang-format --dry-run -Werror "$file" > /dev/null 2>&1; then
29+ echo "C++ code is not properly formatted!"
30+ echo "Please run './devel/format_code.sh' to format the code"
31+ exit 1
32+ fi
33+done
34+
35+echo "All code is properly formatted!"
36+exit 0
devel/compile_markov_bench_cpp.shadded+3−0View file
@@ -0,0 +1,3 @@
1+# sudo apt-get install libeigen3-dev
2+
3+g++ -std=c++17 -O2 -I /usr/include/eigen3 lpc_bench.cpp -o lpc_bench.out
devel/compile_paper.shadded+10−0View file
@@ -0,0 +1,10 @@
1+#!/bin/bash
2+
3+cd "$(dirname "$0")/.."
4+cd paper
5+
6+pandoc paper.md \
7+ --bibliography=references.bib \
8+ -o paper.pdf
9+
10+echo "Paper compiled successfully to paper.pdf"
devel/copy_paper_to_web.shadded+15−0View file
@@ -0,0 +1,15 @@
1+#!/bin/bash
2+
3+# Exit on error
4+set -e
5+
6+# Get the directory containing this script
7+cd "$(dirname "$0")/.."
8+
9+# First compile the paper
10+./devel/compile_paper.sh
11+
12+# Then copy it to web-ui/public/
13+cp paper/paper.pdf web-ui/public/
14+
15+echo "Paper copied to web-ui/public/paper.pdf"
devel/format_code.shadded+19−0View file
@@ -0,0 +1,19 @@
1+#!/bin/bash
2+
3+# Get the directory where the script is located
4+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
5+PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )"
6+
7+# Format Python code
8+echo "Formatting Python code..."
9+black "$PROJECT_ROOT/benchcompress/src/benchcompress"
10+
11+# Format TypeScript/JavaScript code
12+echo "Formatting TypeScript/JavaScript code..."
13+cd "$PROJECT_ROOT/web-ui" && npm run format
14+
15+# Format C++ code
16+echo "Formatting C++ code..."
17+find "$PROJECT_ROOT" -type f \( -name "*.cpp" -o -name "*.h" -o -name "*.hpp" \) -exec clang-format -i {} \;
18+
19+echo "Code formatting complete!"
devel/generate_posts_index.shadded+16−0View file
@@ -0,0 +1,16 @@
1+#!/bin/bash
2+
3+# Directory containing this script
4+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
5+
6+# Path to the posts directory relative to the script location
7+POSTS_DIR="$SCRIPT_DIR/../web-ui/public/posts"
8+INDEX_FILE="$POSTS_DIR/index.txt"
9+
10+# Create posts directory if it doesn't exist
11+mkdir -p "$POSTS_DIR"
12+
13+# Find all .md files in the posts directory and write to index.txt
14+find "$POSTS_DIR" -maxdepth 1 -name "*.md" -type f -printf "%f\n" | sort > "$INDEX_FILE"
15+
16+echo "Generated posts index at $INDEX_FILE"
devel/markov_bench.cppadded+224−0View file
@@ -0,0 +1,224 @@
1+/*************************************************
2+ * lpc_bench.cpp
3+ *
4+ * A C++ program demonstrating:
5+ * 1. Generating int16_t data (size N).
6+ * 2. Building a design matrix in float32.
7+ * 3. Solving for coefficients in float32 with Eigen.
8+ * 4. Predicting a new sequence (float32 -> round to int16_t).
9+ * 5. Measuring times & throughput (MB/s).
10+ *
11+ * Requires Eigen for linear algebra:
12+ * https://eigen.tuxfamily.org/
13+ *************************************************/
14+
15+#include <Eigen/Dense>
16+#include <chrono>
17+#include <cmath>
18+#include <cstdint>
19+#include <iostream>
20+#include <random>
21+#include <vector>
22+
23+//---------------------------------------------------------------------------
24+// 1) Helper for timing
25+//---------------------------------------------------------------------------
26+inline double
27+secondsBetween(const std::chrono::high_resolution_clock::time_point &start,
28+ const std::chrono::high_resolution_clock::time_point &end) {
29+ return std::chrono::duration<double>(end - start).count();
30+}
31+
32+//---------------------------------------------------------------------------
33+// 2) Generate random int16_t data
34+// We create N random integers in [-30000, 30000].
35+//---------------------------------------------------------------------------
36+std::vector<int16_t> generateData(std::size_t N, unsigned seed = 0) {
37+ std::mt19937_64 rng(seed);
38+ std::uniform_int_distribution<int> dist(-30000, 30000);
39+
40+ std::vector<int16_t> x(N);
41+ for (std::size_t i = 0; i < N; ++i) {
42+ x[i] = static_cast<int16_t>(dist(rng));
43+ }
44+ return x;
45+}
46+
47+//---------------------------------------------------------------------------
48+// 3) Build the design matrix A (float32) in a naive manner.
49+//
50+// For a Markov model of order M, we want rows for j in [M..N-1]:
51+// [1, x_{j-1}, x_{j-2}, ..., x_{j-M} ]
52+// We'll store these in A, shape: (N - M) x (M + 1).
53+//---------------------------------------------------------------------------
54+Eigen::MatrixXf buildDesignMatrixNaive(const std::vector<int16_t> &x,
55+ std::size_t M) {
56+ const std::size_t N = x.size();
57+ const std::size_t rows = N - M;
58+ const std::size_t cols = M + 1;
59+
60+ Eigen::MatrixXf A(rows, cols); // float32
61+
62+ // Fill A
63+ for (std::size_t row = 0; row < rows; ++row) {
64+ // First column = 1.0
65+ A(row, 0) = 1.0f;
66+ // Next columns: x_{(row + M) - k}, for k=1..M
67+ for (std::size_t k = 1; k <= M; ++k) {
68+ int16_t val = x[(row + M) - k];
69+ A(row, k) = static_cast<float>(val); // int16 -> float32
70+ }
71+ }
72+
73+ return A;
74+}
75+
76+//---------------------------------------------------------------------------
77+// 4) Solve for coefficients c in float32 using Eigen's SVD.
78+//
79+// c = argmin_c ||A c - y||^2
80+// We store and return c as Eigen::VectorXf (float32).
81+//---------------------------------------------------------------------------
82+Eigen::VectorXf solveCoeffsEigen(const Eigen::MatrixXf &A,
83+ const Eigen::VectorXf &y) {
84+ // SVD-based solve (ComputeThinU|V for full solution in least-squares sense)
85+ // Alternatively: A.colPivHouseholderQr().solve(y), etc.
86+ Eigen::VectorXf c =
87+ A.bdcSvd(Eigen::ComputeThinU | Eigen::ComputeThinV).solve(y);
88+ return c;
89+}
90+
91+//---------------------------------------------------------------------------
92+// 5) Predict a new sequence using naive float32 arithmetic and round to
93+// int16_t.
94+//
95+// - coeffs: c0..cM (length M+1)
96+// - seed: last M real data points (int16_t)
97+// - Output: N predicted points, as int16_t.
98+//
99+// predicted[j] = c0 + c1*predicted[j-1] + ... + cM*predicted[j-M]
100+//---------------------------------------------------------------------------
101+std::vector<int16_t> predictNaive(const Eigen::VectorXf &coeffs,
102+ const std::vector<int16_t> &seed,
103+ std::size_t N) {
104+ std::size_t M = coeffs.size() - 1; // since c has M+1 entries
105+ std::vector<float> predFloat(N, 0.0f); // store intermediate in float32
106+ std::vector<int16_t> predInt(N, 0); // final integer output
107+
108+ // Initialize first M from seed (cast to float)
109+ for (std::size_t i = 0; i < M; ++i) {
110+ predFloat[i] = static_cast<float>(seed[i]);
111+ }
112+
113+ // Predict forward
114+ for (std::size_t j = M; j < N; ++j) {
115+ float val = coeffs(0); // c0
116+ for (std::size_t k = 1; k <= M; ++k) {
117+ val += coeffs(k) * predFloat[j - k];
118+ }
119+ predFloat[j] = val;
120+ }
121+
122+ // Round to int16_t
123+ for (std::size_t i = 0; i < N; ++i) {
124+ float r = std::round(predFloat[i]);
125+ // clamp into int16 range if you want to be safe, but ignoring extremes
126+ // here:
127+ if (r > 32767.f)
128+ r = 32767.f;
129+ if (r < -32768.f)
130+ r = -32768.f;
131+ predInt[i] = static_cast<int16_t>(r);
132+ }
133+
134+ return predInt;
135+}
136+
137+//---------------------------------------------------------------------------
138+// 6) Main benchmark function
139+//---------------------------------------------------------------------------
140+int main() {
141+ // Parameters
142+ std::size_t N = 5 * 1000 * 1000;
143+ std::size_t M = 5;
144+ unsigned seed = 42;
145+
146+ // Generate data
147+ auto startAll = std::chrono::high_resolution_clock::now();
148+ auto x = generateData(N, seed);
149+ auto endAll = std::chrono::high_resolution_clock::now();
150+ double dtGen = secondsBetween(startAll, endAll);
151+
152+ // We'll define "bytes processed" as 2*N for throughput (since int16_t=2
153+ // bytes).
154+ double bytesProcessed = double(2 * N);
155+
156+ std::cout << "\n=== C++ Markov Model Benchmark ===\n"
157+ << "N=" << N << ", M=" << M << ", data=int16_t, coeffs=float32\n\n";
158+
159+ std::cout << "Data generation: " << dtGen << " s\n";
160+
161+ //----------------------------------------------------------------------
162+ // Build design matrix (naive)
163+ //----------------------------------------------------------------------
164+ auto t0 = std::chrono::high_resolution_clock::now();
165+ Eigen::MatrixXf A = buildDesignMatrixNaive(x, M);
166+ auto t1 = std::chrono::high_resolution_clock::now();
167+ double dtBuild = secondsBetween(t0, t1);
168+
169+ // Prepare y (float32) = x[M..N-1]
170+ // We'll store it in an Eigen vector
171+ std::size_t rows = N - M;
172+ Eigen::VectorXf y(rows);
173+ for (std::size_t i = 0; i < rows; ++i) {
174+ y(i) = static_cast<float>(x[i + M]);
175+ }
176+
177+ //----------------------------------------------------------------------
178+ // Solve for coefficients
179+ //----------------------------------------------------------------------
180+ auto t2 = std::chrono::high_resolution_clock::now();
181+ Eigen::VectorXf coeffs = solveCoeffsEigen(A, y);
182+ auto t3 = std::chrono::high_resolution_clock::now();
183+ double dtSolve = secondsBetween(t2, t3);
184+
185+ double dtFittingTotal = dtBuild + dtSolve;
186+
187+ double buildThroughput = bytesProcessed / (dtBuild * 1.0e6);
188+ double solveThroughput = bytesProcessed / (dtSolve * 1.0e6);
189+ double totalThroughput = bytesProcessed / (dtFittingTotal * 1.0e6);
190+
191+ std::cout << "--- Fitting (Naive build + Eigen solve) ---\n";
192+ std::cout << " Build matrix: " << dtBuild << " s, ~" << buildThroughput
193+ << " MB/s\n";
194+ std::cout << " Solve: " << dtSolve << " s, ~" << solveThroughput
195+ << " MB/s\n";
196+ std::cout << " TOTAL: " << dtFittingTotal << " s, ~"
197+ << totalThroughput << " MB/s\n";
198+
199+ //----------------------------------------------------------------------
200+ // Prediction
201+ //----------------------------------------------------------------------
202+ // We'll seed from the last M points of x
203+ std::vector<int16_t> seedVals(M);
204+ for (std::size_t i = 0; i < M; ++i) {
205+ seedVals[i] = x[N - M + i];
206+ }
207+
208+ auto t4 = std::chrono::high_resolution_clock::now();
209+ auto predictions = predictNaive(coeffs, seedVals, N);
210+ auto t5 = std::chrono::high_resolution_clock::now();
211+ double dtPredict = secondsBetween(t4, t5);
212+ double predThroughput = bytesProcessed / (dtPredict * 1.0e6);
213+
214+ std::cout << "\n--- Prediction (Naive float32 -> round int16) ---\n";
215+ std::cout << " Predict: " << dtPredict << " s, ~" << predThroughput
216+ << " MB/s\n";
217+
218+ //----------------------------------------------------------------------
219+ // Done
220+ //----------------------------------------------------------------------
221+ std::cout << "\n=== Done ===\n\n";
222+
223+ return 0;
224+}
devel/markov_bench.pyadded+265−0View file
@@ -0,0 +1,265 @@
1+import numpy as np
2+import time
3+from sklearn.linear_model import LinearRegression
4+
5+########################################################################
6+# 1) Functions to build the design matrix in float32
7+########################################################################
8+
9+def build_design_matrix_naive(x: np.ndarray, M: int):
10+ """
11+ Naive Python approach to build design matrix A for Markov model.
12+ x is int16, but we'll cast to float32 in A.
13+ A has shape (N - M, M + 1).
14+ Row j = [1, x_{j+M-1}, x_{j+M-2}, ..., x_{j+M-M}],
15+ for j in 0..(N-M-1).
16+ """
17+ N = len(x)
18+ A = np.ones((N - M, M + 1), dtype=np.float32) # store as float32
19+ for row_idx in range(N - M):
20+ for k in range(1, M + 1):
21+ # cast x (int16) to float32 on assignment
22+ A[row_idx, k] = x[M + row_idx - k]
23+ return A
24+
25+def build_design_matrix_vectorized(x: np.ndarray, M: int):
26+ """
27+ Vectorized approach (using slicing) to build the same design matrix in float32.
28+ x is int16; cast slices to float32.
29+ """
30+ N = len(x)
31+ rows = N - M
32+ # First column of 1's in float32
33+ A_cols = [np.ones(rows, dtype=np.float32)]
34+ # Next columns are shifted versions of x, cast to float32
35+ for shift in range(1, M + 1):
36+ A_cols.append(x[M - shift : N - shift].astype(np.float32))
37+ A = np.column_stack(A_cols)
38+ return A
39+
40+########################################################################
41+# 2) Functions to solve for coefficients (stored as float32)
42+########################################################################
43+
44+def solve_lstsq_numpy(A: np.ndarray, y: np.ndarray):
45+ """
46+ Solve for coefficients using NumPy's lstsq solver in float32.
47+ Returns a 1D array of shape (M+1,) in float32.
48+ """
49+ # lstsq may do computations in float64 internally, but we can cast back
50+ c, _, _, _ = np.linalg.lstsq(A, y, rcond=None)
51+ return c.astype(np.float32)
52+
53+def solve_lstsq_sklearn(A: np.ndarray, y: np.ndarray):
54+ """
55+ Solve for coefficients using scikit-learn's LinearRegression.
56+ (fit_intercept=False because we already have the 1-column in A).
57+ We store final coefficients as float32.
58+ """
59+ reg = LinearRegression(fit_intercept=False)
60+ reg.fit(A, y)
61+ # sklearn might do double precision internally; cast to float32
62+ return reg.coef_.astype(np.float32)
63+
64+########################################################################
65+# 3) Functions to predict a new sequence
66+########################################################################
67+
68+def predict_naive(coeffs: np.ndarray, seed: np.ndarray, N: int):
69+ """
70+ Naive Python prediction loop with float32 arithmetic.
71+ - coeffs: float32 array of shape (M+1,) => [c0, c1, ..., cM].
72+ - seed: last M known values (int16), but we'll cast to float32 internally.
73+ - N: number of output points to produce.
74+ The final predictions are rounded to int16.
75+ """
76+ M = len(coeffs) - 1
77+ # We'll store predictions in float32, then round to int16 at the end
78+ pred = np.zeros(N, dtype=np.float32)
79+ # Copy seed (cast to float32)
80+ pred[:M] = seed.astype(np.float32)
81+
82+ # Predict forward
83+ for j in range(M, N):
84+ val = coeffs[0]
85+ for k in range(1, M + 1):
86+ val += coeffs[k] * pred[j - k]
87+ pred[j] = val
88+
89+ # Now round to int16
90+ return np.round(pred).astype(np.int16)
91+
92+# Optional: Numba-accelerated version
93+try:
94+ from numba import njit
95+
96+ @njit
97+ def _predict_lpc_numba(pred, coeffs, M):
98+ for j in range(M, len(pred)):
99+ val = coeffs[0]
100+ for k in range(1, M + 1):
101+ val += coeffs[k] * pred[j - k]
102+ pred[j] = val
103+
104+ def predict_numba(coeffs: np.ndarray, seed: np.ndarray, N: int):
105+ """
106+ Numba-accelerated version of predict_naive (float32 internal arithmetic).
107+ Rounds final results to int16.
108+ """
109+ M = len(coeffs) - 1
110+ pred = np.zeros(N, dtype=np.float32)
111+ pred[:M] = seed.astype(np.float32)
112+ _predict_lpc_numba(pred, coeffs, M)
113+ return np.round(pred).astype(np.int16)
114+
115+except ImportError:
116+ def predict_numba(coeffs: np.ndarray, seed: np.ndarray, N: int):
117+ """
118+ Fallback if Numba is not installed: use naive approach.
119+ """
120+ print("Numba not installed. Falling back to naive prediction.")
121+ return predict_naive(coeffs, seed, N)
122+
123+########################################################################
124+# 4) Main benchmarking function
125+########################################################################
126+
127+def benchmark_lpc_model(N=1_000_000, M=5, seed=0):
128+ """
129+ 1) Generate random data x of length N as int16.
130+ 2) Fit the Markov model in multiple ways:
131+ - Naive (build + solve) in float32
132+ - Vectorized+NumPy (float32)
133+ - Vectorized+sklearn (float32)
134+ Print total fitting time for each method, plus throughput in MB/s
135+ (using 2*N bytes as the data size).
136+ 3) Predict a new sequence in multiple ways (naive vs numba-accelerated),
137+ using float32 arithmetic internally, then rounding to int16.
138+ Print total prediction time + throughput in MB/s.
139+ """
140+ np.random.seed(seed)
141+ # Generate random integers in range [-32768, 32767], but let's just do typical range
142+ x = np.random.randint(-30000, 30000, size=N, dtype=np.int16)
143+
144+ # We define "bytes processed" as 2*N because each x_i is an int16 (2 bytes).
145+ bytes_processed = 2 * N
146+ # Build target array (still in float32 for the solver)
147+ # y corresponds to x[M:], but we cast to float32
148+ y = x[M:].astype(np.float32)
149+
150+ print(f"\n=== Benchmarking Markov Model (N={N}, M={M}, data=int16, coeffs=float32) ===")
151+
152+ ########################################################################
153+ # Fitting: Naive approach
154+ ########################################################################
155+ fit_start = time.perf_counter()
156+
157+ # a) build design matrix (naive)
158+ t0 = time.perf_counter()
159+ A_naive = build_design_matrix_naive(x, M)
160+ dt_build_naive = time.perf_counter() - t0
161+
162+ # b) solve
163+ t0 = time.perf_counter()
164+ c_naive = solve_lstsq_numpy(A_naive, y)
165+ dt_solve_naive = time.perf_counter() - t0
166+
167+ total_naive = time.perf_counter() - fit_start
168+
169+ throughput_build_naive = bytes_processed / (dt_build_naive * 1e6)
170+ throughput_solve_naive = bytes_processed / (dt_solve_naive * 1e6)
171+ throughput_total_naive = bytes_processed / (total_naive * 1e6)
172+
173+ print("\n--- Fitting (Naive) ---")
174+ print(f" Build matrix: {dt_build_naive:.3f}s, ~{throughput_build_naive:.2f} MB/s")
175+ print(f" Solve: {dt_solve_naive:.3f}s, ~{throughput_solve_naive:.2f} MB/s")
176+ print(f" TOTAL: {total_naive:.3f}s, ~{throughput_total_naive:.2f} MB/s")
177+
178+ ########################################################################
179+ # Fitting: Vectorized + NumPy
180+ ########################################################################
181+ fit_start = time.perf_counter()
182+
183+ # a) build design matrix (vectorized)
184+ t0 = time.perf_counter()
185+ A_vec = build_design_matrix_vectorized(x, M)
186+ dt_build_vec = time.perf_counter() - t0
187+
188+ # b) solve
189+ t0 = time.perf_counter()
190+ c_vec = solve_lstsq_numpy(A_vec, y)
191+ dt_solve_vec = time.perf_counter() - t0
192+
193+ total_vec = time.perf_counter() - fit_start
194+
195+ throughput_build_vec = bytes_processed / (dt_build_vec * 1e6)
196+ throughput_solve_vec = bytes_processed / (dt_solve_vec * 1e6)
197+ throughput_total_vec = bytes_processed / (total_vec * 1e6)
198+
199+ print("\n--- Fitting (Vectorized + NumPy) ---")
200+ print(f" Build matrix: {dt_build_vec:.3f}s, ~{throughput_build_vec:.2f} MB/s")
201+ print(f" Solve: {dt_solve_vec:.3f}s, ~{throughput_solve_vec:.2f} MB/s")
202+ print(f" TOTAL: {total_vec:.3f}s, ~{throughput_total_vec:.2f} MB/s")
203+
204+ ########################################################################
205+ # Fitting: Vectorized + scikit-learn
206+ ########################################################################
207+ fit_start = time.perf_counter()
208+
209+ # We can re-use A_vec from above, but to measure a true "total time"
210+ # for the vectorized+sklearn path, let's rebuild it.
211+ t0 = time.perf_counter()
212+ A_vec_sklearn = build_design_matrix_vectorized(x, M)
213+ dt_build_vec_sklearn = time.perf_counter() - t0
214+
215+ t0 = time.perf_counter()
216+ c_sklearn = solve_lstsq_sklearn(A_vec_sklearn, y)
217+ dt_solve_sklearn = time.perf_counter() - t0
218+
219+ total_sklearn = time.perf_counter() - fit_start
220+
221+ throughput_build_sklearn = bytes_processed / (dt_build_vec_sklearn * 1e6)
222+ throughput_solve_sklearn = bytes_processed / (dt_solve_sklearn * 1e6)
223+ throughput_total_sklearn = bytes_processed / (total_sklearn * 1e6)
224+
225+ print("\n--- Fitting (Vectorized + Sklearn) ---")
226+ print(f" Build matrix: {dt_build_vec_sklearn:.3f}s, ~{throughput_build_sklearn:.2f} MB/s")
227+ print(f" Solve: {dt_solve_sklearn:.3f}s, ~{throughput_solve_sklearn:.2f} MB/s")
228+ print(f" TOTAL: {total_sklearn:.3f}s, ~{throughput_total_sklearn:.2f} MB/s")
229+
230+ ########################################################################
231+ # Prediction
232+ ########################################################################
233+ # We can pick c_vec from above as "our" coefficients for prediction.
234+ # We'll seed from the last M points of x (int16).
235+ seed_vals = x[-M:]
236+
237+ # 1) Naive Prediction
238+ t0 = time.perf_counter()
239+ pred_naive = predict_naive(c_vec, seed_vals, N)
240+ dt_pred_naive = time.perf_counter() - t0
241+ throughput_pred_naive = bytes_processed / (dt_pred_naive * 1e6)
242+
243+ print("\n--- Prediction (Naive) ---")
244+ print(f" Predict: {dt_pred_naive:.3f}s, ~{throughput_pred_naive:.2f} MB/s")
245+
246+ # Warm up Numba
247+ _ = predict_numba(c_vec, seed_vals, len(seed_vals) + 1)
248+
249+ t0 = time.perf_counter()
250+ pred_numba = predict_numba(c_vec, seed_vals, N)
251+ dt_pred_numba = time.perf_counter() - t0
252+ throughput_pred_numba = bytes_processed / (dt_pred_numba * 1e6)
253+
254+ print("\n--- Prediction (Numba) ---")
255+ print(f" Predict: {dt_pred_numba:.3f}s, ~{throughput_pred_numba:.2f} MB/s")
256+
257+ print("\n=== Done ===\n")
258+
259+########################################################################
260+# Script entry point
261+########################################################################
262+if __name__ == "__main__":
263+ # Example usage
264+ benchmark_lpc_model(N=5_000_000, M=5, seed=42)
265+ # Adjust N, M, and seed as needed.
devel/prepare_lim_et_al_2024.pyadded+120−0View file
@@ -0,0 +1,120 @@
1+#!/usr/bin/env python
2+
3+"""
4+This script prepares a seismic dataset from Lim et al. 2024 for use in the benchcompress project.
5+It downloads seismic waveform data from Zenodo, extracts .sac files from a specific folder,
6+and concatenates them into a single numpy array. The resulting dataset will be used to evaluate
7+various compression algorithms in benchcompress to determine their effectiveness on real-world
8+seismic waveform data.
9+
10+The original data is from:
11+Lim, H., & Zhang, M. (2024). Machine Learning Phase Picker Training Dataset in Shanghai.
12+Zenodo. https://doi.org/10.5281/zenodo.10457508
13+"""
14+
15+import os
16+import sys
17+from urllib.request import urlretrieve
18+import tarfile
19+import glob
20+from obspy import read
21+import numpy as np
22+
23+def download_file(url, local_filename):
24+ if os.path.exists(local_filename):
25+ print(f"File {local_filename} already exists locally")
26+ return True
27+
28+ print(f"Downloading {url} to {local_filename}...")
29+ try:
30+ urlretrieve(url, local_filename)
31+ print("Download completed successfully")
32+ return True
33+ except Exception as e:
34+ print(f"Error downloading file: {e}")
35+ return False
36+
37+def extract_data_folder(archive_path):
38+ if not os.path.exists(archive_path):
39+ print(f"Archive file {archive_path} not found")
40+ return False
41+
42+ print(f"Extracting /data/01 folder from {archive_path}...")
43+ try:
44+ with tarfile.open(archive_path, 'r:gz') as tar:
45+ # Extract only files in the /data/01 directory
46+ members = [m for m in tar.getmembers() if m.name.startswith('data/01/')]
47+ for member in members:
48+ tar.extract(member)
49+ print("Extraction completed successfully")
50+ return True
51+ except Exception as e:
52+ print(f"Error extracting archive: {e}")
53+ return False
54+
55+def load_and_concatenate_sac_files(directory):
56+ # Find all .sac files in the directory
57+ sac_files = glob.glob(f"{directory}/*.sac")
58+ if not sac_files:
59+ print(f"No .sac files found in {directory}")
60+ return None
61+
62+ print(f"Found {len(sac_files)} .sac files")
63+
64+ all_data = []
65+ for i, filename in enumerate(sac_files, 1):
66+ try:
67+ print(f"Loading file {i}/{len(sac_files)}: {filename}")
68+ st = read(filename)
69+ tr = st[0] # Get the first trace
70+ all_data.append(tr.data)
71+ print(f" Shape: {tr.data.shape}, dtype: {tr.data.dtype}")
72+ except Exception as e:
73+ print(f"Error loading {filename}: {e}")
74+ continue
75+
76+ if not all_data:
77+ print("No data was successfully loaded")
78+ return None
79+
80+ # Concatenate all data arrays
81+ concatenated_data = np.concatenate(all_data)
82+ print(f"\nConcatenation complete!")
83+ print(f"Final array shape: {concatenated_data.shape}")
84+ print(f"Final array dtype: {concatenated_data.dtype}")
85+
86+ return concatenated_data
87+
88+def main():
89+ url = "https://zenodo.org/records/10457508/files/data.tar.gz?download=1"
90+ local_file = "lim_et_al_2024.zip"
91+ sac_directory = "data/01"
92+ output_file = "lim_et_al_2024.01.concat.npy"
93+
94+ # Download if needed
95+ if not download_file(url, local_file):
96+ print("Failed to download file")
97+ sys.exit(1)
98+
99+ # Extract data folder
100+ if not extract_data_folder(local_file):
101+ print("Failed to extract data folder")
102+ sys.exit(1)
103+
104+ # Load and concatenate SAC files
105+ concatenated_data = load_and_concatenate_sac_files(sac_directory)
106+ if concatenated_data is None:
107+ print("Failed to process SAC files")
108+ sys.exit(1)
109+
110+ # Save concatenated data
111+ print(f"\nSaving concatenated data to {output_file}...")
112+ try:
113+ np.save(output_file, concatenated_data)
114+ print("Data saved successfully")
115+ except Exception as e:
116+ print(f"Error saving data: {e}")
117+ sys.exit(1)
118+
119+if __name__ == "__main__":
120+ main()
figures/load_results.pyadded+42−0View file
@@ -0,0 +1,42 @@
1+import requests
2+import json
3+from typing import List
4+import numpy as np
5+import matplotlib.pyplot as plt
6+from matplotlib.axes import Axes
7+from pydantic import BaseModel
8+from adjustText import adjust_text
9+
10+results_url = 'https://tempory.net/f/memobin/benchcompress/global/results.json'
11+
12+class BenchmarkResult(BaseModel):
13+ compression_ratio: float
14+ encode_time: float
15+ decode_time: float
16+ encode_mb_per_sec: float
17+ decode_mb_per_sec: float
18+ original_size: int
19+ compressed_size: int
20+ array_shape: List[int]
21+ array_dtype: str
22+ timestamp: float
23+ cache_status: str
24+ dataset: str
25+ algorithm: str
26+ algorithm_version: str
27+ dataset_version: str
28+ system_version: str
29+
30+class BenchmarkResults(BaseModel):
31+ results: List[BenchmarkResult]
32+
33+def download_results() -> dict:
34+ url = results_url
35+ print(f"Downloading results from {url}...")
36+ response = requests.get(url)
37+ response.raise_for_status()
38+ return response.json()
39+
40+def load_results_into_models(data: dict) -> BenchmarkResults:
41+ """Load the downloaded data into pydantic models."""
42+ return BenchmarkResults(**data)
figures/make_figures_iid.pyadded+282−0View file
@@ -0,0 +1,282 @@
1+import requests
2+import json
3+import os
4+import numpy as np
5+import matplotlib.pyplot as plt
6+from matplotlib.axes import Axes
7+from adjustText import adjust_text
8+from load_results import download_results, load_results_into_models, BenchmarkResults
9+
10+# Font size configurations
11+FONT_SIZES = {
12+ 'base': 14,
13+ 'title': 18,
14+ 'axis_label': 16,
15+ 'tick_label': 14,
16+ 'legend': 14,
17+ 'annotation': 14
18+}
19+
20+figheight = 9
21+figheight2 = 9
22+
23+# Set base font size for all plots
24+plt.rcParams.update({'font.size': FONT_SIZES['base']})
25+
26+def ensure_image_dir():
27+ """Ensure the images directory exists relative to this script."""
28+ script_dir = os.path.dirname(os.path.abspath(__file__))
29+ image_dir = os.path.join(script_dir, 'images')
30+ os.makedirs(image_dir, exist_ok=True)
31+ return image_dir
32+
33+def get_theoretical_ratio(dataset_name: str) -> float | None:
34+ """Get theoretical compression ratio for a dataset if applicable."""
35+ if dataset_name.startswith('bernoulli-'):
36+ try:
37+ p = float(dataset_name.split('-')[1])
38+ probs = np.array([p, 1-p])
39+ entropy = -np.sum(probs * np.log2(probs)) # Shannon entropy
40+ return 8 / entropy # 8 bits per byte divided by entropy
41+ except (IndexError, ValueError):
42+ return None
43+ elif dataset_name.startswith('gaussian-q'):
44+ try:
45+ sigma = float(dataset_name.split('-q')[1])
46+ # Calculate probabilities for integers in a reasonable range around mean (0)
47+ # For 99.7% coverage, use ±4*sigma
48+ range_max = int(4 * sigma)
49+ x = np.arange(-range_max, range_max + 1)
50+ # Calculate probabilities using normal distribution PDF
51+ probs = (1/(sigma * np.sqrt(2*np.pi))) * np.exp(-(x**2)/(2*sigma**2))
52+ # Normalize probabilities to sum to 1
53+ probs = probs / np.sum(probs)
54+ # Calculate entropy
55+ nonzero_probs = probs[probs > 0] # Avoid log(0)
56+ entropy = -np.sum(nonzero_probs * np.log2(nonzero_probs))
57+ return 16 / entropy # 8 bits per byte divided by entropy
58+ except (IndexError, ValueError):
59+ return None
60+ return None
61+
62+def plot_single_compression_ratio_comparison(results: BenchmarkResults, dataset_name: str, ax: Axes):
63+ """Create a horizontal bar plot for a specific dataset."""
64+ # Filter for the specified dataset
65+ dataset_results = [r for r in results.results if r.dataset == dataset_name]
66+
67+ # Get unique algorithms and their best compression ratios
68+ algorithm_ratios = {}
69+ for r in dataset_results:
70+ current_ratio = r.compression_ratio
71+ if r.algorithm not in algorithm_ratios or current_ratio < algorithm_ratios[r.algorithm]:
72+ algorithm_ratios[r.algorithm] = current_ratio
73+
74+ # Sort by compression ratio
75+ sorted_items = sorted(algorithm_ratios.items(), key=lambda x: x[1])
76+ algorithms = [item[0] for item in sorted_items]
77+ ratios = [item[1] for item in sorted_items]
78+
79+ # Create horizontal bar plot
80+ ax.barh(algorithms, ratios)
81+ ax.set_xlabel('Compression Ratio', fontsize=FONT_SIZES['axis_label'])
82+ ax.set_ylabel('Algorithm', fontsize=FONT_SIZES['axis_label'])
83+ ax.set_title(f'Compression Ratios for {dataset_name}', fontsize=FONT_SIZES['title'])
84+ ax.tick_params(axis='both', which='major', labelsize=FONT_SIZES['tick_label'])
85+
86+ # Add theoretical limit line if applicable
87+ theoretical_ratio = get_theoretical_ratio(dataset_name)
88+ if theoretical_ratio is not None:
89+ ax.axvline(x=theoretical_ratio, color='red', linestyle='--')
90+ ax.legend()
91+
92+def figure_tradeoff_comparison(results: BenchmarkResults, datasets: list[str]):
93+ """Create scatter plots showing compression ratio vs speed tradeoffs."""
94+ # Create two figures - one for encode, one for decode
95+ for speed_type in ['encode', 'decode']:
96+ n_plots = len(datasets)
97+ fig, axes = plt.subplots(1, n_plots, figsize=(10*n_plots, figheight2))
98+ fig.suptitle(f'Compression Ratio vs {speed_type.capitalize()} Speed Tradeoff')
99+
100+ # Handle single dataset case
101+ if n_plots == 1:
102+ axes = [axes]
103+
104+ for dataset_name, ax in zip(datasets, axes):
105+ # Filter for specified dataset
106+ dataset_results = [r for r in results.results if r.dataset == dataset_name]
107+
108+ # Get best results for each algorithm
109+ algorithm_metrics = {}
110+ for r in dataset_results:
111+ speed = r.encode_mb_per_sec if speed_type == 'encode' else r.decode_mb_per_sec
112+ if r.algorithm not in algorithm_metrics:
113+ algorithm_metrics[r.algorithm] = {
114+ 'ratio': r.compression_ratio,
115+ 'speed': speed
116+ }
117+ else:
118+ # Update if this instance has better trade-off score
119+ current_score = speed / r.compression_ratio
120+ existing_score = (algorithm_metrics[r.algorithm]['speed'] /
121+ algorithm_metrics[r.algorithm]['ratio'])
122+ if current_score > existing_score:
123+ algorithm_metrics[r.algorithm] = {
124+ 'ratio': r.compression_ratio,
125+ 'speed': speed
126+ }
127+
128+ # Extract data for plotting
129+ ratios = [metrics['ratio'] for metrics in algorithm_metrics.values()]
130+ speeds = [metrics['speed'] for metrics in algorithm_metrics.values()]
131+ algorithms = list(algorithm_metrics.keys())
132+
133+ # Create scatter plot
134+ ax.scatter(ratios, speeds, alpha=0.6)
135+
136+ texts = [ax.text(ratios[i], speeds[i], alg, fontsize=FONT_SIZES['annotation']) for i, alg in enumerate(algorithms)]
137+ adjust_text(texts,
138+ arrowprops=dict(arrowstyle='->', color='gray', linewidth=0.5),
139+ ensure_inside_axes=False,
140+ ax=ax)
141+
142+ # Customize plot
143+ ax.set_xlabel('Compression Ratio (higher is better)', fontsize=FONT_SIZES['axis_label'])
144+ ax.set_ylabel(f'{speed_type.capitalize()} Speed MB/sec (higher is better)', fontsize=FONT_SIZES['axis_label'])
145+ ax.set_title(f'{dataset_name}', fontsize=FONT_SIZES['title'])
146+ ax.tick_params(axis='both', which='major', labelsize=FONT_SIZES['tick_label'])
147+ ax.grid(True, alpha=0.3)
148+
149+ # Add theoretical limit line if applicable
150+ theoretical_ratio = get_theoretical_ratio(dataset_name)
151+ if theoretical_ratio is not None:
152+ ax.axvline(x=theoretical_ratio, color='red', linestyle='--')
153+ ax.legend()
154+
155+ plt.tight_layout()
156+ # Save figure
157+ plt.savefig(os.path.join(ensure_image_dir(), f'iid_tradeoff_{speed_type}_{"-".join(datasets)}.png'))
158+ plt.close()
159+
160+def figure_speeds_comparison(results: BenchmarkResults, datasets: list[str]):
161+ """Create horizontal bar plots comparing encode/decode speeds for the given datasets."""
162+ # Create figure with enough subplots for all datasets
163+ n_plots = len(datasets)
164+ fig, axes = plt.subplots(1, n_plots, figsize=(10*n_plots, figheight))
165+
166+ # Handle single dataset case
167+ if n_plots == 1:
168+ axes = [axes]
169+
170+ for dataset_name, ax in zip(datasets, axes):
171+ # Filter for specified dataset
172+ dataset_results = [r for r in results.results if r.dataset == dataset_name]
173+
174+ # Get unique algorithms and their speeds
175+ algorithm_speeds = {}
176+ for r in dataset_results:
177+ if r.algorithm not in algorithm_speeds:
178+ algorithm_speeds[r.algorithm] = {
179+ 'encode': r.encode_mb_per_sec,
180+ 'decode': r.decode_mb_per_sec
181+ }
182+ else:
183+ # Update if this instance has better average speed
184+ current_avg = (r.encode_mb_per_sec + r.decode_mb_per_sec) / 2
185+ existing_avg = (algorithm_speeds[r.algorithm]['encode'] +
186+ algorithm_speeds[r.algorithm]['decode']) / 2
187+ if current_avg > existing_avg:
188+ algorithm_speeds[r.algorithm] = {
189+ 'encode': r.encode_mb_per_sec,
190+ 'decode': r.decode_mb_per_sec
191+ }
192+
193+ # Sort alphabetically by algorithm name
194+ sorted_items = sorted(algorithm_speeds.items(), key=lambda x: x[0])
195+ algorithms = [item[0] for item in sorted_items]
196+ encode_speeds = [item[1]['encode'] for item in sorted_items]
197+ decode_speeds = [item[1]['decode'] for item in sorted_items]
198+
199+ # Set up positions for the bars
200+ y_pos = np.arange(len(algorithms))
201+ bar_height = 0.35
202+
203+ # Create bars
204+ ax.barh(y_pos - bar_height/2, encode_speeds, bar_height,
205+ label='Encode Speed', color='royalblue', alpha=0.8)
206+ ax.barh(y_pos + bar_height/2, decode_speeds, bar_height,
207+ label='Decode Speed', color='seagreen', alpha=0.8)
208+
209+ # Customize plot
210+ ax.set_yticks(y_pos)
211+ ax.set_yticklabels(algorithms)
212+ ax.set_xlabel('Speed (MB/sec)', fontsize=FONT_SIZES['axis_label'])
213+ ax.set_title(f'Encode/Decode Speeds for {dataset_name}', fontsize=FONT_SIZES['title'])
214+ ax.tick_params(axis='both', which='major', labelsize=FONT_SIZES['tick_label'])
215+ ax.legend(fontsize=FONT_SIZES['legend'])
216+
217+ # Add grid for readability
218+ ax.grid(True, axis='x', alpha=0.3)
219+
220+ plt.tight_layout()
221+ # Save figure
222+ plt.savefig(os.path.join(ensure_image_dir(), f'iid_speeds_comparison_{"-".join(datasets)}.png'))
223+ plt.close()
224+
225+def figure_compression_ratios_comparison(results: BenchmarkResults, datasets: list[str]):
226+ """Create horizontal bar plots comparing compression ratios for given datasets."""
227+ # Create figure with enough subplots for all datasets
228+ n_plots = len(datasets)
229+ fig, axes = plt.subplots(1, n_plots, figsize=(10*n_plots, figheight))
230+
231+ # Handle single dataset case
232+ if n_plots == 1:
233+ axes = [axes]
234+
235+ # Plot comparisons for each dataset
236+ for dataset, ax in zip(datasets, axes):
237+ plot_single_compression_ratio_comparison(results, dataset, ax)
238+
239+ plt.tight_layout()
240+ # Save figure
241+ plt.savefig(os.path.join(ensure_image_dir(), f'iid_compression_ratios_{"-".join(datasets)}.png'))
242+ plt.close()
243+
244+def main():
245+ try:
246+ # Download the results
247+ raw_data = download_results()
248+
249+ # Load into pydantic models
250+ results = load_results_into_models(raw_data)
251+
252+ print(f"Successfully loaded {len(results.results)} benchmark results")
253+
254+ datasets_list = [
255+ ['bernoulli-0.1', 'bernoulli-0.5'],
256+ ['gaussian-q2', 'gaussian-q5']
257+ ]
258+
259+ for datasets in datasets_list:
260+ print("\nGenerating compression ratios plot...")
261+ figure_compression_ratios_comparison(results, datasets)
262+
263+ print("\nGenerating speed comparison plot...")
264+ figure_speeds_comparison(results, datasets)
265+
266+ print("\nGenerating tradeoff plots...")
267+ figure_tradeoff_comparison(results, datasets)
268+
269+ return results
270+
271+ except requests.RequestException as e:
272+ print(f"Error downloading results: {e}")
273+ return None
274+ except json.JSONDecodeError as e:
275+ print(f"Error parsing JSON: {e}")
276+ return None
277+ except Exception as e:
278+ print(f"Error loading data into models: {e}")
279+ return None
280+
281+if __name__ == '__main__':
282+ main()
paper/.gitignoreadded+1−0View file
@@ -0,0 +1 @@
1+*.pdf
paper/README.mdadded+5−0View file
@@ -0,0 +1,5 @@
1+# Paper Directory
2+
3+Contains the main paper content (paper.md) for the Benchcompress project.
4+
5+For compiling and copying to the public directory, see the scripts in devel/
paper/paper.mdadded+136−0View file
@@ -0,0 +1,136 @@
1+---
2+title: "Benchmarking Compression Algorithms for Scientific Data Arrays"
3+author: Jeremy Magland, Center for Computational Mathematics, Flatiron Institute
4+date: "January 2025"
5+bibliography: references.bib
6+geometry:
7+ - margin=1in
8+---
9+
10+**Early draft, work in progress.**
11+
12+## Abstract
13+
14+:::abstract
15+*Benchcompress* is a benchmarking framework designed to evaluate the performance of various compression algorithms on scientific data arrays, with a special focus on electrophysiology traces. The framework automates the benchmarking process, measuring compression ratio, encoding throughput, and decoding throughput for each algorithm-dataset pair. Results are verified through decompression and comparison with the original data, ensuring accuracy. The benchmark results are stored and visualized through an interactive web interface, allowing users to filter, sort, and explore the data. This paper presents the design, implementation, and preliminary results of Benchcompress, highlighting its utility in identifying optimal compression techniques for scientific datasets.
16+:::
17+
18+## Introduction
19+
20+Scientific research often generates large volumes of data that must be stored, shared, and analyzed efficiently. In fields ranging from electrophysiology to climate modeling, these datasets can be enormous, making data compression essential for reducing file sizes, accelerating data transfer, and facilitating both short- and long-term storage. While the performance of standard compression strategies for electrophysiology data has been studied [@buccino2023compression], the specialized nature of numeric data arrays poses unique challenges that many general-purpose compression algorithms, often optimized for text, images, or video, do not address effectively.
21+
22+Researchers are typically wary of introducing lossy compression to raw data collected from laboratory devices. Rather than experimenting with a wide range of lossy methods and analyzing their impact on scientific results, we adopt a more transparent strategy for lossy compression: apply well-defined, carefully documented transformations, such as quantization, filtering, or normalization, and then apply strictly lossless compression. For instance, floating-point data often contains bits beyond the precision actually needed, so it frequently compresses poorly unless a preliminary quantization step is used. By estimating the measurement resolution of the acquisition process, we can convert floating-point values to a suitably scaled integer representation, effectively removing these superfluous bits. This approach typically imposes negligible loss in subsequent analyses while significantly improving compressibility. Similarly, filtering (to focus on a specific band of frequencies) and normalization may further enhance compression while remaining straightforward to analyze.
23+
24+Unlike these preprocessing transformations, some techniques are strictly reversible and should therefore be considered part of the lossless method, rather than part of the preprocessing. For example, delta encoding significantly reduces sample magnitudes in data drawn from continuous signals, improving compression ratios in most lossless compressors. Delta encoding plus a lossless method should then be viewed as a compound lossless technique. For datasets exhibiting even smoother behavior, a more powerful variant, which we will call linear Markov prediction, extends delta encoding to higher-order autoregressive models, often resulting in superior compression.
25+
26+When assessing compression performance, it is important to consider not only final compression ratios but also how quickly data can be encoded and decoded. Unlike typical web delivery scenarios where data are compressed once and then decompressed repeatedly, scientific workflows may require efficient encoding because data are often compressed at the point of acquisition and then decompressed only once prior to archival. Moreover, scientific requirements vary widely, from long-term preservation to on-demand, cloud-based visualization. In some cases, significant data loss may be acceptable for a high-level overview, whereas in others, minimal alteration of raw measurements is paramount.
27+
28+To help researchers navigate these questions, we introduce Benchcompress, a benchmarking framework designed to systematically evaluate and compare compression algorithms for numeric data arrays. By automating the testing and providing tools for analysis, Benchcompress simplifies the process of identifying which methods work best for particular datasets. In the sections that follow, we describe the system in detail, outline both standard and specialized compression strategies, and present preliminary results from our benchmarks.
29+
30+## Traditional Compression Methods and Scientific Data
31+
32+Traditional compression algorithms like LZW (Lempel-Ziv-Welch) [@welch1984technique], LZ77 [@ziv1977universal], BWT (Burrows-Wheeler Transform) [@burrows1994block], and Huffman coding [@huffman1952method] were originally designed for text data where patterns manifest as repeated sequences of characters or words. While these methods can compress numeric data from scientific instruments, they may not be optimal for this use case. Text compression excels at identifying exact matches of recurring patterns, whereas scientific measurements often exhibit more complex relationships between values.
33+
34+For example, in text compression, finding repeated instances of common words or phrases leads to efficient encoding. In contrast, scientific data arrays often contain continuous variations where measurement noise and uncertainties mean exact repetition is rare. Even when numeric values are similar, their byte-level representations may share some common bits while differing in others. This characteristic of scientific data suggests that methods explicitly accounting for numerical relationships between values may achieve better compression ratios than general-purpose algorithms.
35+
36+## Independent identically distributed samples
37+
38+Moving from text to numeric data, we begin with the fundamental case of independently and identically distributed (i.i.d.) discrete data. For such data, where each sample is drawn from a discrete probability distribution (e.g., Bernoulli sampling or quantized Gaussian noise), the theoretical compressed size in bits per sample is determined by Shannon's entropy formula [@shannon1948mathematical]:
39+
40+$$
41+H(X) = -\sum_{i} p(x_i) \log_2 p(x_i).
42+$$
43+
44+Here, $p(x_i)$ represents the probability of occurrence of the $i$-th symbol $x_i$ in the discrete distribution. So for example, if we have a Bernoulli distribution with $p=0.5$, the entropy is
45+
46+$$
47+H_{\text{Bernoulli, p=0.5}} = -0.5 \log_2 0.5 - 0.5 \log_2 0.5 = 1
48+$$
49+
50+bits per sample. This means that the optimal compression ratio for such a dataset is 8, assuming the samples are stored as 8-bit integers. On the other hand, if $p\neq 0.5$, the entropy becomes lower and we can achieve compression at a rate of less than 1 bit per sample (e.g., for $p=0.1$, the entropy is around 0.47 bits per sample, so the compression ratio would be around 17).
51+
52+## Asymmetric numeral systems
53+
54+In practice, achieving this theoretical compression ratio requires sophisticated encoding techniques. Arithmetic coding [@witten1987arithmetic] is one such method, but it is challenging to implement and can be computationally inefficient. A more modern and efficient alternative is asymmetric numeral systems (ANS) [@duda2015use], which closely approaches the theoretical limit and is incorporated into state-of-the-art compressors such as ZStandard [@collet2018zstandard]. However, these large, general-use packages are primarily optimized for structured data types, such as text, rather than for numeric scientific data. In our benchmarks, we evaluate ANS using a simple, no-frills, implementation using a Python package we developed for this purpose called `simple_ans`. As anticipated, we show that ANS demonstrates superior performance when compressing i.i.d. samples from a discrete distribution.
55+
56+## Delta encoding
57+
58+The efficiency of pure entropy coders (such as ANS) diminishes when handling more structured data, such as continuous signals (e.g., voltage traces in electrophysiology). Applying delta encoding partially mitigates this limitation by leveraging the continuity properties of the data through differencing. This reversible preprocessing step enhances ANS performance because the deltas are typically smaller than the original samples, leading to lower entropy when assuming independence between samples.
59+
60+## Linear Predictive Coding
61+
62+Even with delta encoding for real or realistic datasets, ANS still can fall short of the compression achieved by methods like ZStandard. To improve its performance by further exploiting temporal correlations in the data, we consider a generalization of delta encoding which we call linear Markov predictive modeling. The Markov prediction scheme employs a linear autoregressive model where each sample is predicted as a linear combination of $M$ previous samples. For a given integer time series $x[t]$, the prediction $\hat{x}[t]$ is computed as:
63+
64+$$
65+\hat{x}[t] = \text{round}\left(\sum_{i=1}^M c_i x[t-i] + b\right)
66+$$
67+
68+where the coefficients $c_i$ and bias term $b$ are determined through least squares regression on a subset of the data. The rounding operation ensures integer predictions. Rather than compressing the original signal directly, the algorithm compresses the integer residual error sequence $r[t] = x[t] - \hat{x}[t]$. This approach proves effective because the residuals typically have smaller magnitude compared with the original signal or with the deltas. The compression process stores three components: the floating-point model coefficients, a small set of initial integer values required to begin prediction, and the compressed integer residual sequence. During reconstruction, the original signal is recovered by computing predictions and adding back the residuals:
69+
70+$$
71+x[t] = \text{round}\left(\sum_{i=1}^M c_i x[t-i] + b\right) + r[t]
72+$$
73+
74+This predictive preprocessing step improves compression performance compared to applying ANS (or other algorithms) directly to either the raw signal or delta-encoded data.
75+
76+## Compression Algorithms
77+
78+The benchmark suite evaluates a diverse set of compression algorithms, ranging from established general-purpose methods to specialized techniques.
79+
80+Zstandard (zstd) is a fast real-time compression algorithm developed by Facebook that provides a broad spectrum of compression levels, from fast compression (level 4) to maximum compression (level 22).
81+
82+LZMA (Lempel-Ziv-Markov chain Algorithm) employs a dictionary compression scheme similar to LZ77 but with sophisticated modeling of repeating patterns using Markov chains. It focuses on achieving high compression ratios through larger dictionary sizes and complex modeling, generally resulting in slower processing but smaller file sizes.
83+
84+LZ4, designed for speed, is particularly well-suited for real-time compression scenarios. It belongs to the LZ77 family of byte-oriented compression schemes and offers various compression levels, from its fastest mode (level 0) to maximum compression (level 16).
85+
86+Zlib, implementing the DEFLATE algorithm, combines LZ77 and Huffman coding to provide a balance between compression ratio and speed across its different compression levels (1-9).
87+
88+Bzip2 employs the Burrows-Wheeler transform along with Huffman coding, typically achieving better compression than traditional LZ77-based algorithms but at reduced speed.
89+
90+Brotli, developed by Google, combines LZ77, Huffman coding, and context modeling to provide effective general-purpose compression across multiple compression levels.
91+
92+We also include a custom implementation of ANS, as described above. The implementation uses the simple_ans Python package developed specifically for this project.
93+
94+To enhance the performance of these basic compression algorithms, we also test reversible preprocessing steps before compression. These include, where appropriate, delta encoding, linear Markov prediction, and zero run length encoding for sparse data.
95+
96+## Datasets
97+
98+Our benchmark suite combines synthetic test cases and real scientific measurements to evaluate compression performance across diverse scenarios. The synthetic datasets include Bernoulli sequences with varying probabilities (p=0.1 to 0.5) providing well-defined theoretical entropy bounds, and Gaussian-distributed data in both quantized integer and floating-point formats with different standard deviations ($\sigma=1$ to $8$).
99+
100+For real-world data, we draw from multiple sources in neuroscience and geophysics. Extracellular electrophysiology samples from the DANDI Archive [@dandi_archive] are provided in three forms: raw 30 kHz recordings, bandpass filtered (300-6000 Hz) and normalized traces, and sparse versions using activity-based suppression. We also include intracranial EEG data from OpenNeuro dataset ds005592 [@markiewicz2021openneuro], consisting of concatenated recordings from ten channels as 32-bit floating-point voltage measurements.
101+
102+The marine seismic dataset, collected during the Roger Revelle voyage RR1508 [@gorman_2023_8152964], provides both original floating-point measurements and quantized integer versions from a gas hydrate system survey.
103+
104+Finally, we include a sample of functional MRI data [@ds005880] consisting of 16-bit integer BOLD signal time series, representing neural activity patterns as 3D brain volumes over time.
105+
106+## System Architecture
107+
108+The framework is implemented primarily in Python for the core benchmarking functionality and TypeScript/React for the web interface. Performance-critical components, such as the Markov prediction algorithm, are implemented in C++ with Python bindings.
109+
110+The system is organized around two primary registries in an open-source repository: compression algorithms and test datasets. Both are defined through Python modules, enabling the scientific community to contribute through GitHub pull requests. Each algorithm specifies encoding and decoding functions along with tags indicating its capabilities, while datasets specify generation parameters and tags describing their characteristics like data type, dimensionality, and properties (continuous, sparse, etc.).
111+
112+The tag system manages algorithm-dataset compatibility. For example, algorithms with delta encoding or Markov prediction tags require datasets marked as continuous integer time series, while those using zero run-length encoding are matched with sparse datasets. This ensures algorithms are only tested on appropriate dataset types.
113+
114+The benchmarking process runs automatically through GitHub Actions when changes are pushed to the main branch. For compatible algorithm-dataset pairs, the system measures compression ratio and processing speeds, verifying results through decompression. A cloud-based caching system prevents redundant benchmark runs, only re-evaluating when components change.
115+
116+Results are presented through an interactive web interface where users can explore dynamically generated visualizations comparing compression ratios and processing speeds across different algorithms and datasets. The interface supports filtering by dataset characteristics and algorithm properties, enabling focused analysis of specific use cases.
117+
118+## Results
119+
120+*NOTE: There are no figures in this section because the reader is expected to browse the results interactively on the Benchcompress website. But eventually we can include some curated figures here.*
121+
122+We first report on compression ratio performance, and then we will move on to encoding and decoding speeds.
123+
124+For the synthetic datasets that involve i.i.d. samples, ANS demonstrates superior compression performance compared to other algorithms, achieving compression ratios close to the theoretical entropy bounds. For example, in the Bernoulli sequence with $p=0.1$ ([https://magland.github.io/benchcompress/dataset/bernoulli-0.1](https://magland.github.io/benchcompress/dataset/bernoulli-0.1)), ANS achieves a compression ratio of ~16.9 whereas the next best algorithms (bzip2, lzma, btrotli-11, zstd-19, and zstd-22) achieve ratios around 13.9. Zlib struggles for this dataset, with a compression ratio of only ~12.2 at the highest compression level (9), and much lower values for lower levels. Brotli and Zstandard at the lower compression levels also perform poorly (for example ~9.6 for zstd-7). The results show a similar trend for the other Bernoulli datasets with different probabilities, but the spread is less pronounced as the entropy increases. The story is similar for the quantized Gaussian datasets, with ANS outperforming the other algorithms, especially for lower standard deviations (e.g., $\sigma=3$, [https://magland.github.io/benchcompress/dataset/gaussian-q3](https://magland.github.io/benchcompress/dataset/gaussian-q3)).
125+
126+
127+
128+## Discussion
129+
130+[Discussion to be added]
131+
132+## Conclusion
133+
134+[Conclusion to be added]
135+
136+## References
paper/references.bibadded+128−0View file
@@ -0,0 +1,128 @@
1+@article{witten1987arithmetic,
2+ title={Arithmetic coding for data compression},
3+ author={Witten, Ian H and Neal, Radford M and Cleary, John G},
4+ journal={Communications of the ACM},
5+ volume={30},
6+ number={6},
7+ pages={520--540},
8+ year={1987},
9+ publisher={ACM New York, NY, USA}
10+}
11+
12+@inproceedings{duda2015use,
13+ title={The use of asymmetric numeral systems as an accurate replacement for Huffman coding},
14+ author={Duda, Jarek and Tahboub, Khalid and Gadgil, Neeraj J and Delp, Edward J},
15+ booktitle={2015 Picture Coding Symposium (PCS)},
16+ pages={65--69},
17+ year={2015},
18+ organization={IEEE}
19+}
20+
21+@article{buccino2023compression,
22+ title={Compression strategies for large-scale electrophysiology data},
23+ author={Buccino, Alessio P and Winter, Olivier and Bryant, David and Feng, David and Svoboda, Karel and Siegle, Joshua H},
24+ journal={Journal of Neural Engineering},
25+ volume={20},
26+ number={5},
27+ pages={056009},
28+ year={2023},
29+ publisher={IOP Publishing}
30+}
31+
32+@techreport{collet2018zstandard,
33+ title={Zstandard Compression and the application/zstd Media Type},
34+ author={Collet, Yann and Kucherawy, Murray},
35+ year={2018}
36+}
37+
38+@article{welch1984technique,
39+ title={A technique for high-performance data compression},
40+ author={Welch, Terry A.},
41+ journal={Computer},
42+ volume={17},
43+ number={06},
44+ pages={8--19},
45+ year={1984},
46+ publisher={IEEE Computer Society}
47+}
48+
49+@article{ziv1977universal,
50+ title={A universal algorithm for sequential data compression},
51+ author={Ziv, Jacob and Lempel, Abraham},
52+ journal={IEEE Transactions on information theory},
53+ volume={23},
54+ number={3},
55+ pages={337--343},
56+ year={1977},
57+ publisher={IEEE}
58+}
59+
60+@article{burrows1994block,
61+ title={A block-sorting lossless data compression algorithm},
62+ author={Burrows, Michael},
63+ journal={SRS Research Report},
64+ volume={124},
65+ year={1994}
66+}
67+
68+@article{shannon1948mathematical,
69+ title={A mathematical theory of communication},
70+ author={Shannon, Claude Elwood},
71+ journal={The Bell system technical journal},
72+ volume={27},
73+ number={3},
74+ pages={379--423},
75+ year={1948},
76+ publisher={Nokia Bell Labs}
77+}
78+
79+@article{huffman1952method,
80+ title={A method for the construction of minimum-redundancy codes},
81+ author={Huffman, David A},
82+ journal={Proceedings of the IRE},
83+ volume={40},
84+ number={9},
85+ pages={1098--1101},
86+ year={1952},
87+ publisher={IEEE}
88+}
89+
90+@misc{dandi_archive,
91+ author = {DANDI Archive},
92+ title = {Distributed Archives for Neurophysiology Data Integration (DANDI)},
93+ howpublished = {\url{https://dandiarchive.org/}},
94+ note = {Accessed: 2025-01-31}
95+}
96+
97+@article{markiewicz2021openneuro,
98+ title={The OpenNeuro resource for sharing of neuroscience data},
99+ author={Markiewicz, Christopher J and Gorgolewski, Krzysztof J and Feingold, Franklin and Blair, Ross and Halchenko, Yaroslav O and Miller, Eric and Hardcastle, Nell and Wexler, Joe and Esteban, Oscar and Goncavles, Mathias and others},
100+ journal={Elife},
101+ volume={10},
102+ pages={e71774},
103+ year={2021},
104+ publisher={eLife Sciences Publications Limited}
105+}
106+
107+@dataset{ds005880,
108+ title = {BIDS Dataset for the Diminished Seventh Chord},
109+ author = {Wu, Chia-Ying and Chang, Yi-Pei and Lee, Yu-Ying and Tsai, Chen-Gia},
110+ year = 2024,
111+ version = {1.0.1},
112+ publisher = {OpenNeuro},
113+ doi = {10.18112/openneuro.ds005880.v1.0.1},
114+ url = {https://openneuro.org/datasets/ds005880/versions/1.0.1}
115+}
116+
117+@dataset{gorman_2023_8152964,
118+ author = {Gorman, Andrew R.},
119+ title = {Datasets associated with Uruti Basin gas hydrate
120+ heat flow feature investigated as part of Roger
121+ Revelle voyage RR1508
122+ },
123+ month = jul,
124+ year = 2023,
125+ publisher = {Zenodo},
126+ doi = {10.5281/zenodo.8152964},
127+ url = {https://doi.org/10.5281/zenodo.8152964},
128+}
scripts/run_benchmarks.pyadded+38−0View file
@@ -0,0 +1,38 @@
1+#!/usr/bin/env python3
2+
3+import json
4+import os
5+from pathlib import Path
6+from benchcompress import run_benchmarks
7+from benchcompress.run_benchmarks._memobin import upload_to_memobin, construct_memobin_url
8+
9+def main():
10+ # Run benchmarks
11+ print("Running benchmarks...")
12+ results = run_benchmarks()
13+
14+ # Save detailed results to JSON
15+ output_dir = Path("benchmark_results")
16+ output_dir.mkdir(exist_ok=True)
17+ output_file = output_dir / "results.json"
18+
19+ with open(output_file, "w") as f:
20+ json.dump(results, f, indent=2)
21+
22+ print(f"\nDetailed results saved to {output_file}")
23+
24+ # Upload results to memobin if enabled
25+ memobin_api_key = os.environ.get("MEMOBIN_API_KEY")
26+ upload_enabled = os.environ.get("UPLOAD_TO_MEMOBIN") == "1"
27+
28+ if memobin_api_key and upload_enabled:
29+ try:
30+ # Construct URL for the global results file
31+ url = "https://tempory.net/f/memobin/benchcompress/global/results.json"
32+ upload_to_memobin(results, url, memobin_api_key)
33+ print("Successfully uploaded results to memobin")
34+ except Exception as e:
35+ print(f"Warning: Failed to upload results to memobin: {str(e)}")
36+
37+if __name__ == "__main__":
38+ main()
test1.pyadded+180−0View file
@@ -0,0 +1,180 @@
1+# %%
2+import numpy as np
3+from benchcompress._filters import bandpass_filter, highpass_filter
4+from benchcompress._compress_ints_lossless import compress_ints_lossless
5+from benchcompress.datasets import datasets
6+# %%
7+import numpy as np
8+
9+# %%
10+import numpy as np
11+from benchcompress._filters import bandpass_filter, highpass_filter
12+from benchcompress._data_loaders import load_real_000876, load_real_000409, load_real_001290
13+from benchcompress._compress_ints_lossless import compress_ints_lossless
14+from benchcompress._analysis import linear_fit, compute_entropy_per_sample, estimate_noise_level
15+import matplotlib.pyplot as plt
16+
17+# %%
18+# Find the real-000409-ch101 dataset
19+real_dataset = next(d for d in datasets if d['name'] == 'real-000409-ch101')
20+X = real_dataset['create']().flatten()
21+
22+X = X.astype(np.int16)
23+
24+# %%
25+plt.figure(figsize=(12, 4))
26+plt.plot(X[:2400])
27+# %%
28+def print_ideal_compression_ratio(X):
29+ ee = compute_entropy_per_sample(X)
30+ print(f'Ideal compression ratio: {X.itemsize * 8 / ee:.2f} ({ee:.2f} bits per sample)')
31+
32+def print_actual_compression_ratios(X):
33+ buf_zstd = compress_ints_lossless(X, method='zstd')
34+ buf_zlib = compress_ints_lossless(X, method='zlib')
35+ buf_lzma = compress_ints_lossless(X, method='lzma')
36+ buf_ans = compress_ints_lossless(X, method='ans')
37+ print(f'Zstd compression ratio: {len(X) * X.itemsize / len(buf_zstd):.2f}')
38+ print(f'Zlib compression ratio: {len(X) * X.itemsize / len(buf_zlib):.2f}')
39+ print(f'Lzma compression ratio: {len(X) * X.itemsize / len(buf_lzma):.2f}')
40+ print(f'ANS compression ratio: {len(X) * X.itemsize / len(buf_ans):.2f}')
41+
42+def get_marcovian_prediction_residual(X, M):
43+ sequences = np.array([X[i:i+M] for i in range(len(X) - 2 * M + 1)])
44+ predictors = sequences[:, :M - 1]
45+ target = sequences[:, M - 1]
46+
47+ coeffs, predict = linear_fit(predictors, target)
48+ predictions = predict(predictors)
49+ predictions = np.round(predictions)
50+ residuals = target - predictions
51+ residuals = residuals.astype(np.int16)
52+ return residuals
53+
54+# %%
55+print('RAW')
56+print_ideal_compression_ratio(X)
57+
58+# %%
59+print('RAW DELTA ENCODING')
60+print_ideal_compression_ratio(np.diff(X))
61+
62+# %%
63+print('RAW DELTA ENCODING - actual compression ratios')
64+print_actual_compression_ratios(np.diff(X))
65+print_ideal_compression_ratio(np.diff(X))
66+
67+# %%
68+X_mr = get_marcovian_prediction_residual(X, 20)
69+print('RAW MARCOVIAN')
70+print_ideal_compression_ratio(X_mr)
71+
72+# %%
73+v = 0.25 # step size for quantization
74+lowcut = 300
75+highcut = 6000
76+X_filt = bandpass_filter(X - np.median(X), sampling_frequency=30000, lowcut=lowcut, highcut=highcut)
77+noise_level = estimate_noise_level(X_filt, sampling_frequency=30000)
78+X_filt_normalized = X_filt / noise_level
79+X2b = X_filt_normalized / v
80+X2 = np.round(X2b).astype(np.int16)
81+
82+# %%
83+plt.figure(figsize=(12, 4))
84+plt.plot(X[:2400])
85+
86+# %%
87+print('FILTERED (and quantized)')
88+print_ideal_compression_ratio(X2)
89+
90+# %%
91+print('FILTERED DELTA ENCODING')
92+print_ideal_compression_ratio(np.diff(X2))
93+
94+# %%
95+residuals2 = get_marcovian_prediction_residual(X2, 20)
96+print('FILTERED MARCOVIAN')
97+print_ideal_compression_ratio(residuals2)
98+
99+# %%
100+import matplotlib.pyplot as plt
101+plt.figure(figsize=(12, 4))
102+plt.plot(X[:2400])
103+plt.title('RAW')
104+
105+plt.figure(figsize=(12, 4))
106+plt.plot(X2[:2400])
107+plt.title('FILTERED')
108+
109+plt.figure(figsize=(12, 4))
110+plt.plot(residuals2[:2400])
111+plt.title('FILTERED MARCOVIAN')
112+
113+# %%
114+def sliding_max(x, delta):
115+ y = np.zeros_like(x)
116+ for i in range(len(x)):
117+ y[i] = np.max(x[max(0, i - delta):min(len(x), i + delta + 1)])
118+ return y
119+
120+def smoothed(x, delta):
121+ y = np.zeros_like(x)
122+ for i in range(len(x)):
123+ y[i] = np.mean(x[max(0, i - delta):min(len(x), i + delta + 1)])
124+ return y
125+
126+cc = [3, 6]
127+Y = sliding_max(np.abs(X_filt_normalized), 50)
128+Y = smoothed(Y, 20)
129+Y = np.minimum(1, np.maximum(0, (Y - cc[0]) / (cc[1] - cc[0])))
130+# Y = highpass_filter(Y, sampling_frequency=30000, lowcut=3)
131+Y_scaled = X_filt_normalized * Y
132+X3b = Y_scaled / v
133+X3 = np.round(X3b).astype(np.int16)
134+
135+plt.figure(figsize=(12, 4))
136+plt.plot(X2[:2400], color='lightgray')
137+# plt.plot(Y[4000:5000])
138+plt.plot(X3b[:2400])
139+# %%
140+print('FILTERED (and quantized) with suppression')
141+print_ideal_compression_ratio(X3)
142+print('')
143+print('FILTERED DELTA ENCODING with suppression')
144+print_ideal_compression_ratio(np.diff(X3))
145+print('')
146+print('FILTERED MARCOVIAN with suppression')
147+residuals3 = get_marcovian_prediction_residual(X3, 20)
148+print_ideal_compression_ratio(residuals3)
149+print('ACTUAL FILTERED MARCOVIAN with suppression')
150+print_actual_compression_ratios(residuals3)
151+# %%
152+def get_run_lengths(x):
153+ runs = []
154+ i = 0
155+ current_nonzero_run_length = 0
156+ while i < len(x):
157+ if np.all(x[i:i+10] == 0):
158+ runs.append(current_nonzero_run_length)
159+ current_nonzero_run_length = 0
160+ j = i
161+ while j < len(x) and x[j] == 0:
162+ j += 1
163+ runs.append(j - i)
164+ i = j
165+ else:
166+ current_nonzero_run_length += 1
167+ i += 1
168+ if np.max(runs) < 256:
169+ return np.array(runs, dtype=np.uint8)
170+ if np.max(runs) < 2 ** 16:
171+ return np.array(runs, dtype=np.uint16)
172+ return np.array(runs, dtype=np.uint32)
173+
174+AA = residuals3[residuals3 != 0]
175+run_lengths = get_run_lengths(residuals3)
176+print(run_lengths, run_lengths.itemsize)
177+ee = compute_entropy_per_sample(AA)
178+theoretical_size = (len(AA) * ee / 8) + run_lengths.nbytes
179+theoretical_compression_ratio = len(residuals3) * X.itemsize / theoretical_size
180+print(f'Theoretical compression ratio: {theoretical_compression_ratio:.2f}')
web-ui/.gitignoreadded+24−0View file
@@ -0,0 +1,24 @@
1+# Logs
2+logs
3+*.log
4+npm-debug.log*
5+yarn-debug.log*
6+yarn-error.log*
7+pnpm-debug.log*
8+lerna-debug.log*
9+
10+node_modules
11+dist
12+dist-ssr
13+*.local
14+
15+# Editor directories and files
16+.vscode/*
17+!.vscode/extensions.json
18+.idea
19+.DS_Store
20+*.suo
21+*.ntvs*
22+*.njsproj
23+*.sln
24+*.sw?
web-ui/README.mdadded+50−0View file
@@ -0,0 +1,50 @@
1+# React + TypeScript + Vite
2+
3+This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
4+
5+Currently, two official plugins are available:
6+
7+- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
8+- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
9+
10+## Expanding the ESLint configuration
11+
12+If you are developing a production application, we recommend updating the configuration to enable type aware lint rules:
13+
14+- Configure the top-level `parserOptions` property like this:
15+
16+```js
17+export default tseslint.config({
18+ languageOptions: {
19+ // other options...
20+ parserOptions: {
21+ project: ['./tsconfig.node.json', './tsconfig.app.json'],
22+ tsconfigRootDir: import.meta.dirname,
23+ },
24+ },
25+})
26+```
27+
28+- Replace `tseslint.configs.recommended` to `tseslint.configs.recommendedTypeChecked` or `tseslint.configs.strictTypeChecked`
29+- Optionally add `...tseslint.configs.stylisticTypeChecked`
30+- Install [eslint-plugin-react](https://github.com/jsx-eslint/eslint-plugin-react) and update the config:
31+
32+```js
33+// eslint.config.js
34+import react from 'eslint-plugin-react'
35+
36+export default tseslint.config({
37+ // Set the react version
38+ settings: { react: { version: '18.3' } },
39+ plugins: {
40+ // Add the react plugin
41+ react,
42+ },
43+ rules: {
44+ // other rules...
45+ // Enable its recommended rules
46+ ...react.configs.recommended.rules,
47+ ...react.configs['jsx-runtime'].rules,
48+ },
49+})
50+```
web-ui/eslint.config.jsadded+28−0View file
@@ -0,0 +1,28 @@
1+import js from '@eslint/js'
2+import globals from 'globals'
3+import reactHooks from 'eslint-plugin-react-hooks'
4+import reactRefresh from 'eslint-plugin-react-refresh'
5+import tseslint from 'typescript-eslint'
6+
7+export default tseslint.config(
8+ { ignores: ['dist'] },
9+ {
10+ extends: [js.configs.recommended, ...tseslint.configs.recommended],
11+ files: ['**/*.{ts,tsx}'],
12+ languageOptions: {
13+ ecmaVersion: 2020,
14+ globals: globals.browser,
15+ },
16+ plugins: {
17+ 'react-hooks': reactHooks,
18+ 'react-refresh': reactRefresh,
19+ },
20+ rules: {
21+ ...reactHooks.configs.recommended.rules,
22+ 'react-refresh/only-export-components': [
23+ 'warn',
24+ { allowConstantExport: true },
25+ ],
26+ },
27+ },
28+)
web-ui/index.htmladded+26−0View file
@@ -0,0 +1,26 @@
1+<!doctype html>
2+<html lang="en">
3+ <head>
4+ <meta charset="UTF-8" />
5+ <script>
6+ // Single Page Apps for GitHub Pages
7+ // MIT License
8+ // https://github.com/rafgraph/spa-github-pages
9+ (function(l) {
10+ if (l.search[1] === '/' ) {
11+ var decoded = l.search.slice(1).split('&')[0].split('=')[0].replace(/~and~/g, '&');
12+ window.history.replaceState(null, null,
13+ l.pathname.slice(0, -1) + decoded + l.hash
14+ );
15+ }
16+ }(window.location))
17+ </script>
18+ <link rel="icon" type="image/svg+xml" href="./favicon.svg" />
19+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
20+ <title>Benchcompress</title>
21+ </head>
22+ <body>
23+ <div id="root"></div>
24+ <script type="module" src="./src/main.tsx"></script>
25+ </body>
26+</html>
web-ui/package-lock.jsonadded+7968−0View file
This diff is 7,973 lines long and is not shown.
web-ui/package.jsonadded+47−0View file
@@ -0,0 +1,47 @@
1+{
2+ "name": "web-ui",
3+ "private": true,
4+ "version": "0.0.0",
5+ "type": "module",
6+ "scripts": {
7+ "dev": "../devel/generate_posts_index.sh && vite",
8+ "build": "tsc -b && vite build",
9+ "lint": "eslint .",
10+ "preview": "vite preview",
11+ "format": "prettier --write \"src/**/*.{ts,tsx,js,jsx}\"",
12+ "format:check": "prettier --check \"src/**/*.{ts,tsx,js,jsx}\""
13+ },
14+ "dependencies": {
15+ "@tanstack/react-table": "^8.20.6",
16+ "@types/plotly.js": "^2.35.2",
17+ "axios": "^1.7.9",
18+ "katex": "^0.16.21",
19+ "plotly.js-dist-min": "^2.35.3",
20+ "react": "^18.3.1",
21+ "react-dom": "^18.3.1",
22+ "react-markdown": "^9.0.3",
23+ "react-plotly.js": "^2.6.0",
24+ "react-router-dom": "^7.1.3",
25+ "rehype-katex": "^7.0.1",
26+ "remark-math": "^6.0.0",
27+ "yaml": "^2.7.0"
28+ },
29+ "devDependencies": {
30+ "@eslint/js": "^9.17.0",
31+ "@types/react": "^18.3.18",
32+ "@types/react-dom": "^18.3.5",
33+ "@types/react-plotly.js": "^2.6.3",
34+ "@vitejs/plugin-react": "^4.3.4",
35+ "autoprefixer": "^10.4.20",
36+ "eslint": "^9.17.0",
37+ "eslint-plugin-react-hooks": "^5.0.0",
38+ "eslint-plugin-react-refresh": "^0.4.16",
39+ "globals": "^15.14.0",
40+ "postcss": "^8.5.1",
41+ "prettier": "^3.4.2",
42+ "tailwindcss": "^4.0.0",
43+ "typescript": "~5.6.2",
44+ "typescript-eslint": "^8.18.2",
45+ "vite": "^6.0.5"
46+ }
47+}
web-ui/public/.gitignoreadded+1−0View file
@@ -0,0 +1 @@
1+*.pdf
web-ui/public/404.htmladded+26−0View file
@@ -0,0 +1,26 @@
1+<!DOCTYPE html>
2+<html>
3+ <head>
4+ <meta charset="utf-8">
5+ <title>Benchcompress</title>
6+ <script>
7+ // Single Page Apps for GitHub Pages
8+ // MIT License
9+ // https://github.com/rafgraph/spa-github-pages
10+ (function(){
11+ var pathSegmentsToKeep = 1;
12+
13+ var l = window.location;
14+ l.replace(
15+ l.protocol + '//' + l.hostname + (l.port ? ':' + l.port : '') +
16+ l.pathname.split('/').slice(0, 1 + pathSegmentsToKeep).join('/') + '/?/' +
17+ l.pathname.slice(1).split('/').slice(pathSegmentsToKeep).join('/').replace(/&/g, '~and~') +
18+ (l.search ? '&' + l.search.slice(1).replace(/&/g, '~and~') : '') +
19+ l.hash
20+ );
21+ }());
22+ </script>
23+ </head>
24+ <body>
25+ </body>
26+</html>
web-ui/public/favicon.svgadded+31−0View file
@@ -0,0 +1,31 @@
1+<?xml version="1.0" encoding="UTF-8"?>
2+<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
3+ <!-- Background shape -->
4+ <rect width="32" height="32" rx="6" fill="#f8f9fa"/>
5+
6+ <!-- Scientific waveform representing data -->
7+ <path d="M6 16 L10 16 L12 12 L14 20 L16 12 L18 20 L20 16 L24 16"
8+ stroke="#2563eb"
9+ stroke-width="2"
10+ fill="none"
11+ stroke-linecap="round"
12+ stroke-linejoin="round"/>
13+
14+ <!-- Compression brackets -->
15+ <path d="M4 8 L4 24 L8 20 M8 12 L4 8"
16+ stroke="#1e40af"
17+ stroke-width="2"
18+ fill="none"
19+ stroke-linecap="round"
20+ stroke-linejoin="round"/>
21+
22+ <path d="M28 8 L28 24 L24 20 M24 12 L28 8"
23+ stroke="#1e40af"
24+ stroke-width="2"
25+ fill="none"
26+ stroke-linecap="round"
27+ stroke-linejoin="round"/>
28+
29+ <!-- Central dot -->
30+ <circle cx="16" cy="16" r="1.5" fill="#1e40af"/>
31+</svg>
web-ui/public/logo.svgadded+31−0View file
@@ -0,0 +1,31 @@
1+<?xml version="1.0" encoding="UTF-8"?>
2+<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
3+ <!-- Background shape -->
4+ <rect width="32" height="32" rx="6" fill="#f8f9fa"/>
5+
6+ <!-- Scientific waveform representing data -->
7+ <path d="M6 16 L10 16 L12 12 L14 20 L16 12 L18 20 L20 16 L24 16"
8+ stroke="#2563eb"
9+ stroke-width="2"
10+ fill="none"
11+ stroke-linecap="round"
12+ stroke-linejoin="round"/>
13+
14+ <!-- Compression brackets -->
15+ <path d="M4 8 L4 24 L8 20 M8 12 L4 8"
16+ stroke="#1e40af"
17+ stroke-width="2"
18+ fill="none"
19+ stroke-linecap="round"
20+ stroke-linejoin="round"/>
21+
22+ <path d="M28 8 L28 24 L24 20 M24 12 L28 8"
23+ stroke="#1e40af"
24+ stroke-width="2"
25+ fill="none"
26+ stroke-linecap="round"
27+ stroke-linejoin="round"/>
28+
29+ <!-- Central dot -->
30+ <circle cx="16" cy="16" r="1.5" fill="#1e40af"/>
31+</svg>
web-ui/public/posts/2025-01-27-dataset-timeseries-view.mdadded+5−0View file
@@ -0,0 +1,5 @@
1+# Dataset timeseries view
2+
3+For each dataset, you can see an interactive view of the timeseries data on the Dataset page. Here's an example:
4+
5+[/benchcompress/dataset/gaussian-8](/benchcompress/dataset/gaussian-8)
web-ui/public/posts/index.txtadded+1−0View file
@@ -0,0 +1 @@
1+2025-01-27-dataset-timeseries-view.md
web-ui/public/vite.svgadded+1−0View file
@@ -0,0 +1 @@
1+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>
web-ui/src/App.cssadded+35−0View file
@@ -0,0 +1,35 @@
1+/* Existing styles */
2+
3+/* Markdown content styles */
4+.markdown-content {
5+ font-family: system-ui, -apple-system, sans-serif;
6+ line-height: 1.6;
7+ color: #333;
8+}
9+
10+.markdown-content h1 {
11+ font-size: 2.2rem;
12+ margin-bottom: 1.5rem;
13+ color: #1a1a1a;
14+}
15+
16+.markdown-content h2 {
17+ font-size: 1.8rem;
18+ margin: 2rem 0 1rem;
19+ color: #1a1a1a;
20+}
21+
22+.markdown-content p {
23+ margin-bottom: 1.2rem;
24+ font-size: 1.1rem;
25+}
26+
27+.markdown-content ul {
28+ margin: 1rem 0;
29+ padding-left: 2rem;
30+}
31+
32+.markdown-content li {
33+ margin: 0.5rem 0;
34+ font-size: 1.1rem;
35+}
web-ui/src/App.tsxadded+185−0View file
@@ -0,0 +1,185 @@
1+import { BrowserRouter, Routes, Route, Link } from "react-router-dom";
2+import { useEffect, useState } from "react";
3+import axios from "axios";
4+import { ScrollToTop } from "./components/ScrollToTop";
5+import "./components/AppHeader.css";
6+import Home from "./pages/Home";
7+import BenchmarkView from "./pages/BenchmarkView";
8+import Paper from "./pages/Paper";
9+import Monitor from "./pages/Monitor";
10+import Submit from "./pages/Submit";
11+import { BenchmarkData } from "./types";
12+
13+function App() {
14+ const [benchmarkData, setBenchmarkData] = useState<BenchmarkData | null>(
15+ null,
16+ );
17+ const [isLoading, setIsLoading] = useState(true);
18+ const [error, setError] = useState<string | null>(null);
19+
20+ useEffect(() => {
21+ const fetchData = async () => {
22+ try {
23+ setIsLoading(true);
24+ setError(null);
25+ const cacheBust = Math.random().toString(36).substring(2, 15);
26+ const response = await axios.get(
27+ `https://tempory.net/f/memobin/benchcompress/global/results.json?cachebust=${cacheBust}`,
28+ );
29+ setBenchmarkData(response.data);
30+ } catch (error) {
31+ const message =
32+ error instanceof Error ? error.message : "Failed to fetch data";
33+ setError(message);
34+ console.error("Error fetching benchmark data:", error);
35+ } finally {
36+ setIsLoading(false);
37+ }
38+ };
39+
40+ fetchData();
41+ }, []);
42+
43+ return (
44+ <BrowserRouter basename="/benchcompress/">
45+ <ScrollToTop />
46+ <div
47+ style={{
48+ paddingTop: "3rem",
49+ padding: "3rem 2rem 2rem 2rem",
50+ }}
51+ >
52+ <nav
53+ style={{
54+ position: "fixed",
55+ top: 0,
56+ left: 0,
57+ right: 0,
58+ padding: "0.35rem min(2rem, 4%)",
59+ backgroundColor: "white",
60+ zIndex: 1000,
61+ }}
62+ >
63+ <div
64+ style={{
65+ display: "flex",
66+ justifyContent: "space-between",
67+ alignItems: "center",
68+ minHeight: "32px",
69+ }}
70+ >
71+ <Link
72+ to="/"
73+ style={{
74+ display: "flex",
75+ alignItems: "center",
76+ textDecoration: "none",
77+ minWidth: 0,
78+ maxWidth: "calc(100% - 80px)",
79+ }}
80+ >
81+ <img
82+ src="/benchcompress/logo.svg"
83+ alt="Benchcompress Logo"
84+ style={{
85+ width: "28px",
86+ height: "28px",
87+ marginRight: "10px",
88+ flexShrink: 0,
89+ }}
90+ />
91+ <span
92+ style={{
93+ minWidth: 0,
94+ whiteSpace: "nowrap",
95+ overflow: "hidden",
96+ textOverflow: "ellipsis",
97+ }}
98+ >
99+ <span
100+ style={{
101+ fontSize: "1rem",
102+ fontWeight: "500",
103+ color: "#2c2c2c",
104+ }}
105+ >
106+ Benchcompress
107+ </span>
108+ <span className="app-header-subtitle">
109+ {" · "}
110+ <span style={{ fontSize: "1rem", color: "#777" }}>
111+ Comparing compression algorithms for scientific data arrays
112+ </span>
113+ </span>
114+ </span>
115+ </Link>
116+ <div style={{ display: "flex", gap: "1.5rem" }}>
117+ <Link
118+ to="/datasets"
119+ style={{
120+ color: "#0066cc",
121+ textDecoration: "none",
122+ fontWeight: "500",
123+ }}
124+ >
125+ Datasets
126+ </Link>
127+ <Link
128+ to="/algorithms"
129+ style={{
130+ color: "#0066cc",
131+ textDecoration: "none",
132+ fontWeight: "500",
133+ }}
134+ >
135+ Algorithms
136+ </Link>
137+ <Link
138+ to="/paper"
139+ style={{
140+ color: "#0066cc",
141+ textDecoration: "none",
142+ fontWeight: "500",
143+ }}
144+ >
145+ Paper
146+ </Link>
147+ </div>
148+ </div>
149+ </nav>
150+ <main>
151+ {isLoading ? (
152+ <div>Loading benchmark data...</div>
153+ ) : error ? (
154+ <div>Error: {error}</div>
155+ ) : (
156+ <Routes>
157+ <Route path="/" element={<Home />} />
158+ <Route
159+ path="/datasets"
160+ element={<BenchmarkView benchmarkData={benchmarkData} />}
161+ />
162+ <Route
163+ path="/algorithms"
164+ element={<BenchmarkView benchmarkData={benchmarkData} />}
165+ />
166+ <Route
167+ path="/dataset/:datasetName"
168+ element={<BenchmarkView benchmarkData={benchmarkData} />}
169+ />
170+ <Route
171+ path="/algorithm/:algorithmName"
172+ element={<BenchmarkView benchmarkData={benchmarkData} />}
173+ />
174+ <Route path="/paper" element={<Paper />} />
175+ <Route path="/monitor" element={<Monitor />} />
176+ <Route path="/submit" element={<Submit />} />
177+ </Routes>
178+ )}
179+ </main>
180+ </div>
181+ </BrowserRouter>
182+ );
183+}
184+
185+export default App;
web-ui/src/assets/react.svgadded+1−0View file
@@ -0,0 +1 @@
1+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>
web-ui/src/components/AppHeader.cssadded+9−0View file
@@ -0,0 +1,9 @@
1+.app-header-subtitle {
2+ display: none;
3+}
4+
5+@media (min-width: 640px) {
6+ .app-header-subtitle {
7+ display: inline;
8+ }
9+}
web-ui/src/components/BenchmarkTable.tsxadded+1−0View file
@@ -0,0 +1 @@
1+export { BenchmarkTable } from "./benchmark/table/BenchmarkTable";
web-ui/src/components/Button.cssadded+18−0View file
@@ -0,0 +1,18 @@
1+.soft-button {
2+ display: inline-block;
3+ padding: 0.6rem 1.2rem;
4+ background-color: #2b7de9;
5+ color: white;
6+ text-decoration: none;
7+ border-radius: 20px;
8+ font-weight: 500;
9+ box-shadow: 0 2px 4px rgba(0,0,0,0.1);
10+ transition: all 0.2s ease;
11+ cursor: pointer;
12+}
13+
14+.soft-button:hover {
15+ background-color: #1a68d4;
16+ transform: translateY(-1px);
17+ box-shadow: 0 4px 8px rgba(0,0,0,0.15);
18+}
web-ui/src/components/ScrollToTop.tsxadded+12−0View file
@@ -0,0 +1,12 @@
1+import { useEffect } from "react";
2+import { useLocation } from "react-router-dom";
3+
4+export function ScrollToTop() {
5+ const { pathname } = useLocation();
6+
7+ useEffect(() => {
8+ window.scrollTo(0, 0);
9+ }, [pathname]);
10+
11+ return null;
12+}
web-ui/src/components/TagFilter.tsxadded+45−0View file
@@ -0,0 +1,45 @@
1+interface TagFilterProps {
2+ availableTags: string[];
3+ selectedTags: string[];
4+ onTagToggle: (tag: string) => void;
5+ label: string;
6+}
7+
8+export function TagFilter({
9+ availableTags,
10+ selectedTags,
11+ onTagToggle,
12+ label,
13+}: TagFilterProps) {
14+ return (
15+ <div style={{ marginTop: "1rem" }}>
16+ <div style={{ display: "flex", alignItems: "center", gap: "1rem" }}>
17+ <div
18+ style={{ fontSize: "0.9rem", color: "#666", whiteSpace: "nowrap" }}
19+ >
20+ {label}:
21+ </div>
22+ <div style={{ display: "flex", flexWrap: "wrap", gap: "0.5rem" }}>
23+ {availableTags.map((tag) => (
24+ <button
25+ key={tag}
26+ onClick={() => onTagToggle(tag)}
27+ style={{
28+ padding: "4px 8px",
29+ border: "1px solid #ddd",
30+ borderRadius: "4px",
31+ background: selectedTags.includes(tag) ? "#0066cc" : "white",
32+ color: selectedTags.includes(tag) ? "white" : "#333",
33+ cursor: "pointer",
34+ fontSize: "0.8rem",
35+ transition: "all 0.2s ease",
36+ }}
37+ >
38+ {tag}
39+ </button>
40+ ))}
41+ </div>
42+ </div>
43+ </div>
44+ );
45+}
web-ui/src/components/algorithm/AlgorithmContent.tsxadded+33−0View file
@@ -0,0 +1,33 @@
1+import { Algorithm, BenchmarkData } from "../../types";
2+import { BaseContent } from "../shared/BaseContent";
3+import "../shared/ContentStyles.css";
4+
5+interface AlgorithmContentProps {
6+ algorithm: Algorithm;
7+ benchmarkData: BenchmarkData | null;
8+ chartData: Array<{
9+ algorithmOrDataset: string;
10+ compression_ratio: number;
11+ reference_compression_ratio: number | null;
12+ encode_speed: number;
13+ decode_speed: number;
14+ }>;
15+}
16+
17+export const AlgorithmContent = ({
18+ algorithm,
19+ benchmarkData,
20+ chartData,
21+}: AlgorithmContentProps) => {
22+ return (
23+ <BaseContent
24+ item={algorithm}
25+ benchmarkData={benchmarkData}
26+ chartData={chartData}
27+ tagNavigationPrefix="/algorithms"
28+ filterKey="algorithm"
29+ showSortByCompressionRatio={false}
30+ showNormalizeByReference={true}
31+ />
32+ );
33+};
web-ui/src/components/benchmark/charts/BenchmarkCharts.tsxadded+187−0View file
@@ -0,0 +1,187 @@
1+import { useState } from "react";
2+import Plot from "react-plotly.js";
3+
4+interface BenchmarkBarChartProps {
5+ title: string;
6+ data: ChartData[];
7+ dataKey: keyof Pick<
8+ ChartData,
9+ "compression_ratio" | "encode_speed" | "decode_speed"
10+ >;
11+ color: string;
12+ xAxisTitle: string;
13+ normalize?: boolean;
14+}
15+
16+function BenchmarkBarChart({
17+ title,
18+ data,
19+ dataKey,
20+ color,
21+ xAxisTitle,
22+ normalize,
23+}: BenchmarkBarChartProps) {
24+ const normalizedData =
25+ normalize && dataKey === "compression_ratio"
26+ ? data.map((d) => ({
27+ ...d,
28+ compression_ratio: d.reference_compression_ratio
29+ ? d.compression_ratio / d.reference_compression_ratio
30+ : d.compression_ratio,
31+ reference_compression_ratio: d.reference_compression_ratio ? 1 : null,
32+ }))
33+ : data;
34+
35+ return (
36+ <div style={{ margin: "0 20px 20px 0" }}>
37+ <h3 style={{ marginBottom: "10px" }}>{title}</h3>
38+ <Plot
39+ data={[
40+ {
41+ type: "bar",
42+ orientation: "h",
43+ y: normalizedData.map((d) => d.algorithmOrDataset),
44+ x: normalizedData.map((d) => d[dataKey]),
45+ marker: { color },
46+ name: title,
47+ hovertemplate:
48+ normalize && dataKey === "compression_ratio"
49+ ? "%{x:.3f}×<extra></extra>"
50+ : "%{x:.2f}<extra></extra>",
51+ },
52+ ...(dataKey === "compression_ratio" &&
53+ normalizedData.some((d) => d.reference_compression_ratio !== null)
54+ ? [
55+ ...normalizedData
56+ .filter((d) => d.reference_compression_ratio !== null)
57+ .flatMap((d) => [
58+ {
59+ type: "scatter" as const,
60+ mode: "lines" as const,
61+ y: [d.algorithmOrDataset, d.algorithmOrDataset],
62+ x: [0, d.reference_compression_ratio],
63+ line: { color, width: 1 },
64+ showlegend: false,
65+ hoverinfo: "skip" as const,
66+ },
67+ {
68+ type: "scatter" as const,
69+ mode: "markers" as const,
70+ y: [d.algorithmOrDataset],
71+ x: [d.reference_compression_ratio],
72+ marker: { color: "#aaaaaa", size: 8 },
73+ name: "Best Compression",
74+ hovertemplate: normalize
75+ ? "Best: 1.000×<extra></extra>"
76+ : "Best: %{x:.2f}<extra></extra>",
77+ showlegend:
78+ d.algorithmOrDataset ===
79+ normalizedData[0].algorithmOrDataset,
80+ },
81+ ]),
82+ ]
83+ : []),
84+ ]}
85+ layout={{
86+ width: 700,
87+ height: Math.max(300, data.length * 23 + 40),
88+ margin: { t: 5, r: 30, l: 200, b: 30 },
89+ xaxis: { title: xAxisTitle },
90+ yaxis: { automargin: true, ticksuffix: " " },
91+ dragmode: false,
92+ }}
93+ config={{ displayModeBar: false }}
94+ />
95+ </div>
96+ );
97+}
98+
99+interface ChartData {
100+ algorithmOrDataset: string;
101+ compression_ratio: number;
102+ reference_compression_ratio: number | null; // the highest compression ratio for the dataset (if algorithmOrDataset is a dataset)
103+ encode_speed: number;
104+ decode_speed: number;
105+}
106+
107+interface BenchmarkChartsProps {
108+ chartData: ChartData[];
109+ showSortByCompressionRatio?: boolean;
110+ showNormalizeByReference?: boolean;
111+}
112+
113+export function BenchmarkCharts({
114+ chartData,
115+ showSortByCompressionRatio,
116+ showNormalizeByReference,
117+}: BenchmarkChartsProps) {
118+ const [sortByRatio, setSortByRatio] = useState(
119+ showSortByCompressionRatio ? true : false,
120+ );
121+ const [normalize, setNormalize] = useState(false);
122+
123+ if (!chartData.length) return null;
124+
125+ const sortedData = sortByRatio
126+ ? [...chartData].sort((a, b) => a.compression_ratio - b.compression_ratio)
127+ : chartData;
128+
129+ return (
130+ <div>
131+ {showSortByCompressionRatio && (
132+ <div style={{ marginBottom: "10px" }}>
133+ <label style={{ display: "flex", alignItems: "center", gap: "8px" }}>
134+ <input
135+ type="checkbox"
136+ checked={sortByRatio}
137+ onChange={(e) => setSortByRatio(e.target.checked)}
138+ />
139+ Sort by compression ratio
140+ </label>
141+ </div>
142+ )}
143+ {showNormalizeByReference && (
144+ <div style={{ marginBottom: "10px" }}>
145+ <label style={{ display: "flex", alignItems: "center", gap: "8px" }}>
146+ <input
147+ type="checkbox"
148+ checked={normalize}
149+ onChange={(e) => setNormalize(e.target.checked)}
150+ />
151+ Normalize to best compression
152+ </label>
153+ </div>
154+ )}
155+ <div
156+ style={{
157+ display: "flex",
158+ flexWrap: "wrap",
159+ gap: "20px",
160+ }}
161+ >
162+ <BenchmarkBarChart
163+ title="Compression Ratio"
164+ data={sortedData}
165+ dataKey="compression_ratio"
166+ color="#8884d8"
167+ xAxisTitle={normalize ? "Fraction of Best Compression" : "Ratio"}
168+ normalize={normalize}
169+ />
170+ <BenchmarkBarChart
171+ title="Encode Speed (MB/s)"
172+ data={sortedData}
173+ dataKey="encode_speed"
174+ color="#82ca9d"
175+ xAxisTitle="MB/s"
176+ />
177+ <BenchmarkBarChart
178+ title="Decode Speed (MB/s)"
179+ data={sortedData}
180+ dataKey="decode_speed"
181+ color="#ff7300"
182+ xAxisTitle="MB/s"
183+ />
184+ </div>
185+ </div>
186+ );
187+}
web-ui/src/components/benchmark/charts/BenchmarkScatterPlots.tsxadded+161−0View file
@@ -0,0 +1,161 @@
1+import Plot from "react-plotly.js";
2+import { useState } from "react";
3+
4+interface ChartData {
5+ algorithmOrDataset: string;
6+ compression_ratio: number;
7+ encode_speed: number;
8+ decode_speed: number;
9+}
10+
11+interface BenchmarkScatterPlotsProps {
12+ chartData: ChartData[];
13+}
14+
15+export function BenchmarkScatterPlots({
16+ chartData,
17+}: BenchmarkScatterPlotsProps) {
18+ const [showLabels, setShowLabels] = useState(false);
19+
20+ if (!chartData.length) return null;
21+
22+ const uniqueAlgorithms = Array.from(
23+ new Set(chartData.map((d) => d.algorithmOrDataset)),
24+ );
25+
26+ const colors = [
27+ "#1f77b4", // blue
28+ "#ff7f0e", // orange
29+ "#2ca02c", // green
30+ "#d62728", // red
31+ "#9467bd", // purple
32+ "#8c564b", // brown
33+ "#e377c2", // pink
34+ "#7f7f7f", // gray
35+ ];
36+
37+ const markers = ["circle", "square", "diamond", "triangle-up", "star"];
38+
39+ // Create traces for each algorithm
40+ const traces = uniqueAlgorithms.flatMap((algo, i) => {
41+ const algoData = chartData.filter((d) => d.algorithmOrDataset === algo);
42+ const baseTrace = {
43+ name: algo,
44+ mode: showLabels ? ("markers+text" as const) : ("markers" as const),
45+ marker: {
46+ color: colors[i % colors.length],
47+ symbol: markers[Math.floor(i / colors.length) % markers.length],
48+ size: 10,
49+ },
50+ text: showLabels ? algoData.map(() => algo) : [],
51+ textposition: "top center" as const,
52+ showlegend: true,
53+ legendgroup: algo,
54+ };
55+
56+ return [
57+ // Compression Ratio vs Decode Speed (upper left)
58+ {
59+ ...baseTrace,
60+ x: algoData.map((d) => d.compression_ratio),
61+ y: algoData.map((d) => d.decode_speed),
62+ xaxis: "x" as const,
63+ yaxis: "y" as const,
64+ showlegend: true,
65+ },
66+ // Compression Ratio vs Encode Speed (lower left)
67+ {
68+ ...baseTrace,
69+ x: algoData.map((d) => d.compression_ratio),
70+ y: algoData.map((d) => d.encode_speed),
71+ xaxis: "x2" as const,
72+ yaxis: "y2" as const,
73+ showlegend: false,
74+ },
75+ // Decode Speed vs Encode Speed (lower right)
76+ {
77+ ...baseTrace,
78+ x: algoData.map((d) => d.decode_speed),
79+ y: algoData.map((d) => d.encode_speed),
80+ xaxis: "x3" as const,
81+ yaxis: "y3" as const,
82+ showlegend: false,
83+ },
84+ ];
85+ });
86+
87+ return (
88+ <div style={{ margin: "20px 0" }}>
89+ <div style={{ marginBottom: "10px" }}>
90+ <h2 style={{ marginBottom: "10px" }}>Performance Relationships</h2>
91+ <label style={{ display: "flex", alignItems: "center", gap: "8px" }}>
92+ <input
93+ type="checkbox"
94+ checked={showLabels}
95+ onChange={(e) => setShowLabels(e.target.checked)}
96+ />
97+ Show point labels
98+ </label>
99+ </div>
100+ <Plot
101+ data={traces}
102+ layout={{
103+ width: 1000,
104+ height: 670,
105+ grid: {
106+ rows: 2,
107+ columns: 2,
108+ pattern: "independent",
109+ },
110+ xaxis: {
111+ title: "Compression Ratio",
112+ domain: [0, 0.45],
113+ },
114+ yaxis: {
115+ title: "Decode Speed (MB/s)",
116+ domain: [0.55, 1],
117+ },
118+ xaxis2: {
119+ title: "Compression Ratio",
120+ domain: [0, 0.45],
121+ },
122+ yaxis2: {
123+ title: "Encode Speed (MB/s)",
124+ domain: [0, 0.45],
125+ },
126+ xaxis3: {
127+ title: "Decode Speed (MB/s)",
128+ domain: [0.55, 1],
129+ },
130+ yaxis3: {
131+ title: "Encode Speed (MB/s)",
132+ domain: [0, 0.45],
133+ },
134+ showlegend: true,
135+ legend: {
136+ x: 1.08,
137+ y: 1,
138+ xanchor: "left" as const,
139+ yanchor: "top" as const,
140+ },
141+ margin: {
142+ l: 60,
143+ r: 40,
144+ t: 20,
145+ b: 60,
146+ },
147+ }}
148+ config={{
149+ displayModeBar: true,
150+ displaylogo: false,
151+ modeBarButtonsToRemove: [
152+ "lasso2d",
153+ "select2d",
154+ "hoverClosestCartesian",
155+ "hoverCompareCartesian",
156+ ],
157+ }}
158+ />
159+ </div>
160+ );
161+}
web-ui/src/components/benchmark/export/csvExport.tsadded+49−0View file
@@ -0,0 +1,49 @@
1+import { BenchmarkResult } from "../../../types";
2+import { formatNumber, formatSize } from "../utils/formatters";
3+import { columns } from "../table/columns";
4+
5+export const exportToCsv = (
6+ data: BenchmarkResult[],
7+ selectedDataset: string,
8+) => {
9+ // Convert data to CSV
10+ const headers = columns.map((col) => col.header).join(",");
11+ const rows = data
12+ .map((row) =>
13+ columns
14+ .map((col) => {
15+ const value = row[col.accessorKey as keyof BenchmarkResult];
16+ // Format numbers according to their display format
17+ if (col.accessorKey === "compression_ratio") {
18+ return `${formatNumber(value as number)}x`;
19+ } else if (
20+ col.accessorKey === "encode_time" ||
21+ col.accessorKey === "decode_time"
22+ ) {
23+ return formatNumber(value as number, 4);
24+ } else if (
25+ col.accessorKey === "original_size" ||
26+ col.accessorKey === "compressed_size"
27+ ) {
28+ return formatSize(value as number);
29+ } else if (typeof value === "number") {
30+ return formatNumber(value);
31+ }
32+ return value;
33+ })
34+ .join(","),
35+ )
36+ .join("\n");
37+ const csv = `${headers}\n${rows}`;
38+
39+ // Create and trigger download
40+ const blob = new Blob([csv], { type: "text/csv" });
41+ const url = window.URL.createObjectURL(blob);
42+ const a = document.createElement("a");
43+ a.href = url;
44+ a.download = `benchmark-results${selectedDataset ? `-${selectedDataset}` : ""}.csv`;
45+ document.body.appendChild(a);
46+ a.click();
47+ document.body.removeChild(a);
48+ window.URL.revokeObjectURL(url);
49+};
web-ui/src/components/benchmark/table/BenchmarkTable.tsxadded+98−0View file
@@ -0,0 +1,98 @@
1+import {
2+ flexRender,
3+ getCoreRowModel,
4+ getSortedRowModel,
5+ useReactTable,
6+} from "@tanstack/react-table";
7+import { BenchmarkResult } from "../../../types";
8+import { exportToCsv } from "../export/csvExport";
9+import { columns } from "./columns";
10+
11+interface BenchmarkTableProps {
12+ results: BenchmarkResult[];
13+}
14+
15+export function BenchmarkTable({ results }: BenchmarkTableProps) {
16+ const table = useReactTable({
17+ data: results,
18+ columns,
19+ getCoreRowModel: getCoreRowModel(),
20+ getSortedRowModel: getSortedRowModel(),
21+ });
22+
23+ return (
24+ <div className="table-container">
25+ <table>
26+ <thead>
27+ {table.getHeaderGroups().map((headerGroup) => (
28+ <tr key={headerGroup.id}>
29+ {headerGroup.headers.map((header) => (
30+ <th
31+ key={header.id}
32+ onClick={header.column.getToggleSortingHandler()}
33+ style={{ cursor: "pointer" }}
34+ >
35+ {flexRender(
36+ header.column.columnDef.header,
37+ header.getContext(),
38+ )}
39+ {header.column.getIsSorted() && (
40+ <span style={{ marginLeft: "4px" }}>
41+ {header.column.getIsSorted() === "asc" ? "↑" : "↓"}
42+ </span>
43+ )}
44+ </th>
45+ ))}
46+ </tr>
47+ ))}
48+ </thead>
49+ <tbody>
50+ {table.getRowModel().rows.map((row) => (
51+ <tr key={row.id}>
52+ {row.getVisibleCells().map((cell) => (
53+ <td key={cell.id}>
54+ {flexRender(cell.column.columnDef.cell, cell.getContext())}
55+ </td>
56+ ))}
57+ </tr>
58+ ))}
59+ </tbody>
60+ </table>
61+
62+ <div
63+ style={{
64+ marginTop: "12px",
65+ display: "flex",
66+ justifyContent: "flex-end",
67+ }}
68+ >
69+ <button
70+ onClick={() => exportToCsv(results, "benchmark_results")}
71+ style={{
72+ padding: "8px 16px",
73+ backgroundColor: "#4CAF50",
74+ color: "white",
75+ border: "none",
76+ borderRadius: "4px",
77+ cursor: "pointer",
78+ display: "flex",
79+ alignItems: "center",
80+ gap: "8px",
81+ }}
82+ >
83+ <svg
84+ width="16"
85+ height="16"
86+ viewBox="0 0 16 16"
87+ fill="none"
88+ xmlns="http://www.w3.org/2000/svg"
89+ >
90+ <path d="M8 12L3 7H6V1H10V7H13L8 12Z" fill="currentColor" />
91+ <path d="M2 14V15H14V14H2Z" fill="currentColor" />
92+ </svg>
93+ Download CSV
94+ </button>
95+ </div>
96+ </div>
97+ );
98+}
web-ui/src/components/benchmark/table/columns.tsxadded+102−0View file
@@ -0,0 +1,102 @@
1+import { createColumnHelper } from "@tanstack/react-table";
2+import { BenchmarkResult } from "../../../types";
3+import { formatNumber, formatSize } from "../utils/formatters";
4+import { Link } from "react-router-dom";
5+
6+const columnHelper = createColumnHelper<BenchmarkResult>();
7+
8+export const columns = [
9+ columnHelper.accessor("dataset", {
10+ header: "Dataset",
11+ cell: (info) => (
12+ <Link
13+ to={`/dataset/${info.getValue()}`}
14+ style={{ color: "#2563eb", textDecoration: "none" }}
15+ onMouseEnter={(e) =>
16+ (e.currentTarget.style.textDecoration = "underline")
17+ }
18+ onMouseLeave={(e) => (e.currentTarget.style.textDecoration = "none")}
19+ >
20+ {info.getValue()}
21+ </Link>
22+ ),
23+ }),
24+ columnHelper.accessor("algorithm", {
25+ header: "Algorithm",
26+ cell: (info) => (
27+ <Link
28+ to={`/algorithm/${info.getValue()}`}
29+ style={{ color: "#2563eb", textDecoration: "none" }}
30+ onMouseEnter={(e) =>
31+ (e.currentTarget.style.textDecoration = "underline")
32+ }
33+ onMouseLeave={(e) => (e.currentTarget.style.textDecoration = "none")}
34+ >
35+ {info.getValue()}
36+ </Link>
37+ ),
38+ }),
39+ columnHelper.accessor("compression_ratio", {
40+ header: "Compression Ratio",
41+ cell: (info) => `${formatNumber(info.getValue())}`,
42+ sortingFn: (rowA, rowB) => {
43+ const a = rowA.original.compression_ratio;
44+ const b = rowB.original.compression_ratio;
45+ return a - b;
46+ },
47+ }),
48+ columnHelper.accessor("encode_time", {
49+ header: "Encode Time (s)",
50+ cell: (info) => formatNumber(info.getValue(), 4),
51+ sortingFn: (rowA, rowB) => {
52+ const a = rowA.original.encode_time;
53+ const b = rowB.original.encode_time;
54+ return a - b;
55+ },
56+ }),
57+ columnHelper.accessor("decode_time", {
58+ header: "Decode Time (s)",
59+ cell: (info) => formatNumber(info.getValue(), 4),
60+ sortingFn: (rowA, rowB) => {
61+ const a = rowA.original.decode_time;
62+ const b = rowB.original.decode_time;
63+ return a - b;
64+ },
65+ }),
66+ columnHelper.accessor("encode_mb_per_sec", {
67+ header: "Encode Speed (MB/s)",
68+ cell: (info) => formatNumber(info.getValue()),
69+ sortingFn: (rowA, rowB) => {
70+ const a = rowA.original.encode_mb_per_sec;
71+ const b = rowB.original.encode_mb_per_sec;
72+ return a - b;
73+ },
74+ }),
75+ columnHelper.accessor("decode_mb_per_sec", {
76+ header: "Decode Speed (MB/s)",
77+ cell: (info) => formatNumber(info.getValue()),
78+ sortingFn: (rowA, rowB) => {
79+ const a = rowA.original.decode_mb_per_sec;
80+ const b = rowB.original.decode_mb_per_sec;
81+ return a - b;
82+ },
83+ }),
84+ columnHelper.accessor("original_size", {
85+ header: "Original Size",
86+ cell: (info) => formatSize(info.getValue()),
87+ sortingFn: (rowA, rowB) => {
88+ const a = rowA.original.original_size;
89+ const b = rowB.original.original_size;
90+ return a - b;
91+ },
92+ }),
93+ columnHelper.accessor("compressed_size", {
94+ header: "Compressed Size",
95+ cell: (info) => formatSize(info.getValue()),
96+ sortingFn: (rowA, rowB) => {
97+ const a = rowA.original.compressed_size;
98+ const b = rowB.original.compressed_size;
99+ return a - b;
100+ },
101+ }),
102+];
web-ui/src/components/benchmark/utils/formatters.tsadded+11−0View file
@@ -0,0 +1,11 @@
1+export const formatNumber = (num: number, decimals = 2) => {
2+ return new Intl.NumberFormat("en-US", {
3+ minimumFractionDigits: decimals,
4+ maximumFractionDigits: decimals,
5+ }).format(num);
6+};
7+
8+export const formatSize = (bytes: number) => {
9+ const mb = bytes / (1024 * 1024);
10+ return `${formatNumber(mb)} MB`;
11+};
web-ui/src/components/dataset/DatasetContent.tsxadded+99−0View file
@@ -0,0 +1,99 @@
1+import { Dataset, BenchmarkData } from "../../types";
2+import { useEffect, useRef, useState } from "react";
3+import TimeseriesView from "./TimeseriesView";
4+import { BaseContent } from "../shared/BaseContent";
5+import "../shared/ContentStyles.css";
6+
7+interface DatasetContentProps {
8+ dataset: Dataset;
9+ benchmarkData: BenchmarkData | null;
10+ chartData: Array<{
11+ algorithmOrDataset: string;
12+ compression_ratio: number;
13+ reference_compression_ratio: number | null;
14+ encode_speed: number;
15+ decode_speed: number;
16+ }>;
17+}
18+
19+export const DatasetContent = ({
20+ dataset,
21+ benchmarkData,
22+ chartData,
23+}: DatasetContentProps) => {
24+ const containerRef = useRef<HTMLDivElement>(null);
25+ const [containerWidth, setContainerWidth] = useState(1200);
26+
27+ useEffect(() => {
28+ if (!containerRef.current) return;
29+
30+ const resizeObserver = new ResizeObserver((entries) => {
31+ for (const entry of entries) {
32+ setContainerWidth(entry.contentRect.width - 32);
33+ }
34+ });
35+
36+ resizeObserver.observe(containerRef.current);
37+
38+ return () => {
39+ resizeObserver.disconnect();
40+ };
41+ }, []);
42+
43+ const downloadSection =
44+ dataset.data_url_npy || dataset.data_url_raw ? (
45+ <div>
46+ <span className="metadata-label">Download: </span>
47+ <span style={{ display: "inline-flex", gap: "0.5rem" }}>
48+ {dataset.data_url_npy && (
49+ <a
50+ href={dataset.data_url_npy}
51+ download={`${dataset.name}-${dataset.version}.npy`}
52+ className="download-link"
53+ >
54+ NPY
55+ </a>
56+ )}
57+ {dataset.data_url_raw && (
58+ <a
59+ href={dataset.data_url_raw}
60+ download={`${dataset.name}-${dataset.version}.dat`}
61+ className="download-link"
62+ >
63+ RAW
64+ </a>
65+ )}
66+ </span>
67+ </div>
68+ ) : null;
69+
70+ const timeseriesSection = (
71+ <div className="content-container">
72+ <div
73+ ref={containerRef}
74+ style={{
75+ width: "100%",
76+ height: "300px",
77+ backgroundColor: "#f5f5f5",
78+ borderRadius: "4px",
79+ padding: "1rem",
80+ }}
81+ >
82+ <TimeseriesView width={containerWidth} height={250} dataset={dataset} />
83+ </div>
84+ </div>
85+ );
86+
87+ return (
88+ <BaseContent
89+ item={dataset}
90+ benchmarkData={benchmarkData}
91+ chartData={chartData}
92+ tagNavigationPrefix="/datasets"
93+ filterKey="dataset"
94+ downloadSection={downloadSection}
95+ additionalContent={timeseriesSection}
96+ showSortByCompressionRatio={true}
97+ />
98+ );
99+};
web-ui/src/components/dataset/TimeseriesNavigationBar.tsxadded+98−0View file
@@ -0,0 +1,98 @@
1+import React, { useRef } from "react";
2+import { Range } from "./WorkerTypes";
3+
4+interface TimeseriesNavigationBarProps {
5+ width: number;
6+ height: number;
7+ totalRange: Range;
8+ viewRange: Range;
9+ onViewRangeChange: (range: Range) => void;
10+}
11+
12+const TimeseriesNavigationBar: React.FC<TimeseriesNavigationBarProps> = ({
13+ width,
14+ height,
15+ totalRange,
16+ viewRange,
17+ onViewRangeChange,
18+}) => {
19+ const containerRef = useRef<HTMLDivElement>(null);
20+
21+ // Constants
22+ const minMarkerWidth = 25; // Minimum width of the marker in pixels
23+ const padding = 10; // Padding on left and right
24+ const barWidth = width - 2 * padding;
25+
26+ // Convert data range to pixel coordinates
27+ const rangeToPixel = (value: number): number => {
28+ const ratio = (value - totalRange.min) / (totalRange.max - totalRange.min);
29+ return padding + ratio * barWidth;
30+ };
31+
32+ // Convert pixel coordinates to data range
33+ const pixelToRange = (pixel: number): number => {
34+ const ratio = (pixel - padding) / barWidth;
35+ return totalRange.min + ratio * (totalRange.max - totalRange.min);
36+ };
37+
38+ // Calculate marker position and width
39+ const markerLeft = rangeToPixel(viewRange.min);
40+ const rawMarkerWidth = rangeToPixel(viewRange.max) - markerLeft;
41+ const markerWidth = Math.max(rawMarkerWidth, minMarkerWidth);
42+
43+ const handleClick = (e: React.MouseEvent) => {
44+ if (!containerRef.current) return;
45+
46+ const rect = containerRef.current.getBoundingClientRect();
47+ const clickX = e.clientX - rect.left;
48+
49+ // Click on the bar - center the view on click position
50+ const clickedValue = pixelToRange(clickX);
51+ const currentSize = viewRange.max - viewRange.min;
52+ const halfSize = currentSize / 2;
53+
54+ let newMin = clickedValue - halfSize;
55+ let newMax = clickedValue + halfSize;
56+
57+ // Clamp to total range bounds
58+ if (newMin < totalRange.min) {
59+ newMin = totalRange.min;
60+ newMax = newMin + currentSize;
61+ }
62+ if (newMax > totalRange.max) {
63+ newMax = totalRange.max;
64+ newMin = newMax - currentSize;
65+ }
66+
67+ onViewRangeChange({ min: newMin, max: newMax });
68+ };
69+
70+ return (
71+ <div
72+ ref={containerRef}
73+ style={{
74+ width,
75+ height,
76+ position: "relative",
77+ backgroundColor: "#f0f0f0",
78+ borderRadius: 4,
79+ cursor: "pointer",
80+ }}
81+ onClick={handleClick}
82+ >
83+ <div
84+ style={{
85+ position: "absolute",
86+ left: markerLeft,
87+ width: markerWidth,
88+ height: "100%",
89+ backgroundColor: "#007bff",
90+ borderRadius: 4,
91+ pointerEvents: "none",
92+ }}
93+ />
94+ </div>
95+ );
96+};
97+
98+export default TimeseriesNavigationBar;
web-ui/src/components/dataset/TimeseriesView.tsxadded+437−0View file
@@ -0,0 +1,437 @@
1+import { useEffect, useMemo, useReducer, useState, useCallback } from "react";
2+import TimeseriesNavigationBar from "./TimeseriesNavigationBar";
3+import { SupportedTypedArray } from "../../hooks/TimeseriesDataClient";
4+import { useTimeseriesDataClient } from "../../hooks/useTimeseriesDataClient";
5+import { Dataset } from "../../types";
6+import { Margins, Range, WorkerMessage } from "./WorkerTypes";
7+import { initialState, timeseriesViewReducer } from "./timeseriesViewReducer";
8+
9+interface TimeseriesViewProps {
10+ width: number;
11+ height: number;
12+ dataset: Dataset;
13+}
14+
15+const TimeseriesView: React.FC<TimeseriesViewProps> = ({
16+ width,
17+ height,
18+ dataset,
19+}) => {
20+ const { client, error: clientError } = useTimeseriesDataClient(dataset);
21+ const [dataT, setDataT] = useState<number[] | null>(null);
22+ const [dataY, setDataY] = useState<SupportedTypedArray | null>(null);
23+ const [error, setError] = useState<string | null>(clientError);
24+ const [isLoading, setIsLoading] = useState(false);
25+
26+ const [canvasElement, setCanvasElement] = useState<HTMLCanvasElement | null>(
27+ null,
28+ );
29+ const [overlayCanvasElement, setOverlayCanvasElement] =
30+ useState<HTMLCanvasElement | null>(null);
31+ const [state, dispatch] = useReducer(timeseriesViewReducer, initialState);
32+ const { selectedIndex, isDragging, lastDragX, xRange } = state;
33+ const [isWheelEnabled, setIsWheelEnabled] = useState(false);
34+ const [showHint, setShowHint] = useState(true);
35+
36+ // Hide hint when user interacts with the graph
37+ const hideHint = useCallback(() => {
38+ setShowHint(false);
39+ }, []);
40+
41+ // Auto-hide hint after 4 seconds
42+ useEffect(() => {
43+ if (showHint) {
44+ const timer = setTimeout(() => {
45+ setShowHint(false);
46+ }, 5000);
47+ return () => clearTimeout(timer);
48+ }
49+ }, [showHint]);
50+
51+ const [container, setContainer] = useState<HTMLDivElement | null>(null);
52+ const [worker, setWorker] = useState<Worker | null>(null);
53+ const [margins] = useState<Margins>({
54+ left: 50,
55+ right: 20,
56+ top: 20,
57+ bottom: 50,
58+ });
59+
60+ // Load data for current range
61+ useEffect(() => {
62+ if (!client || !xRange) return;
63+
64+ const loadRangeData = async () => {
65+ try {
66+ setIsLoading(true);
67+ const start = Math.floor(xRange.min);
68+ const end = Math.ceil(xRange.max) + 1;
69+ const rangeData = await client.fetchRange(start, end);
70+ setDataY(rangeData);
71+ const dT = Array.from(
72+ { length: rangeData.length },
73+ (_, i) => i + start,
74+ );
75+ setDataT(dT);
76+ setError(null);
77+ } catch (err) {
78+ setError(
79+ err instanceof Error ? err.message : "Failed to load data range",
80+ );
81+ } finally {
82+ setIsLoading(false);
83+ }
84+ };
85+
86+ loadRangeData();
87+ }, [client, xRange]);
88+
89+ // Update xRange when client is initialized
90+ useEffect(() => {
91+ if (client) {
92+ const shape = client.getShape();
93+ dispatch({
94+ type: "SET_X_RANGE",
95+ range: { min: 0, max: Math.min(999, shape - 1) },
96+ });
97+ }
98+ }, [client]);
99+
100+ // Set up wheel event listener
101+ useEffect(() => {
102+ if (!container || !client) return;
103+
104+ const handleWheel = (e: WheelEvent) => {
105+ if (!isWheelEnabled) {
106+ return; // Allow page scrolling if wheel zoom not enabled
107+ }
108+ e.preventDefault();
109+
110+ const rect = container.getBoundingClientRect();
111+ const x = e.clientX - rect.left;
112+ const xRatio =
113+ (x - margins.left) / (width - margins.left - margins.right);
114+
115+ // Calculate zoom center in data coordinates
116+ const zoomCenter = xRange.min + (xRange.max - xRange.min) * xRatio;
117+
118+ // Calculate new range
119+ const zoomFactor = e.deltaY > 0 ? 1.1 : 1 / 1.1;
120+ const shape = client.getShape();
121+
122+ // Ensure we don't zoom out beyond data bounds
123+ const newMin = Math.max(
124+ 0,
125+ zoomCenter - (zoomCenter - xRange.min) * zoomFactor,
126+ );
127+ const newMax = Math.min(
128+ shape - 1,
129+ zoomCenter + (xRange.max - zoomCenter) * zoomFactor,
130+ );
131+
132+ dispatch({ type: "SET_X_RANGE", range: { min: newMin, max: newMax } });
133+ };
134+
135+ container.addEventListener("wheel", handleWheel, { passive: false });
136+ return () => {
137+ container.removeEventListener("wheel", handleWheel);
138+ };
139+ }, [container, client, width, margins, xRange, isWheelEnabled]);
140+
141+ // Set up mouse event listeners for panning
142+ useEffect(() => {
143+ if (!container || !client) return;
144+
145+ const handleMouseDown = (e: MouseEvent) => {
146+ dispatch({ type: "SET_IS_DRAGGING", isDragging: true });
147+ dispatch({ type: "SET_LAST_DRAG_X", x: e.clientX });
148+ };
149+
150+ const handleMouseMove = (e: MouseEvent) => {
151+ if (!isDragging || lastDragX === 0) return;
152+
153+ const deltaX = e.clientX - lastDragX;
154+ const xRatio = deltaX / (width - margins.left - margins.right);
155+ const dataDelta = (xRange.max - xRange.min) * xRatio;
156+ const shape = client.getShape();
157+
158+ if (xRange.min - dataDelta < 0) return;
159+ if (xRange.max - dataDelta > shape - 1) return;
160+
161+ const newMin = xRange.min - dataDelta;
162+ const newMax = xRange.max - dataDelta;
163+
164+ // Only update if we're still within bounds
165+ if (newMin >= 0 && newMax <= shape - 1) {
166+ dispatch({ type: "SET_X_RANGE", range: { min: newMin, max: newMax } });
167+ }
168+
169+ dispatch({ type: "SET_LAST_DRAG_X", x: e.clientX });
170+ };
171+
172+ const handleMouseUp = () => {
173+ dispatch({ type: "SET_IS_DRAGGING", isDragging: false });
174+ dispatch({ type: "SET_LAST_DRAG_X", x: 0 });
175+ };
176+
177+ container.addEventListener("mousedown", handleMouseDown);
178+ window.addEventListener("mousemove", handleMouseMove);
179+ window.addEventListener("mouseup", handleMouseUp);
180+
181+ return () => {
182+ container.removeEventListener("mousedown", handleMouseDown);
183+ window.removeEventListener("mousemove", handleMouseMove);
184+ window.removeEventListener("mouseup", handleMouseUp);
185+ };
186+ }, [container, client, width, margins, xRange, isDragging, lastDragX]);
187+
188+ // Set worker
189+ useEffect(() => {
190+ if (!canvasElement) return;
191+ const worker = new Worker(
192+ new URL("./TimeseriesViewWorker", import.meta.url),
193+ {
194+ type: "module",
195+ },
196+ );
197+ let offscreenCanvas: OffscreenCanvas;
198+ try {
199+ offscreenCanvas = canvasElement.transferControlToOffscreen();
200+ } catch (err) {
201+ console.warn(err);
202+ console.warn(
203+ "Unable to transfer control to offscreen canvas (expected during dev)",
204+ );
205+ return;
206+ }
207+ const msg: WorkerMessage = {
208+ type: "initialize",
209+ canvas: offscreenCanvas,
210+ };
211+ worker.postMessage(msg, [offscreenCanvas]);
212+
213+ setWorker(worker);
214+
215+ return () => {
216+ worker.terminate();
217+ };
218+ }, [canvasElement]);
219+
220+ // Calculate yRange from data
221+ const yRange = useMemo<Range>(() => {
222+ if (!dataY) return { min: 0, max: 1 };
223+ return {
224+ min: computeMin(dataY),
225+ max: computeMax(dataY),
226+ };
227+ }, [dataY]);
228+
229+ // Handle dimension changes
230+ useEffect(() => {
231+ if (!worker) return;
232+ if (!dataY) return;
233+ if (!dataT) return;
234+
235+ const msg: WorkerMessage = {
236+ type: "render",
237+ timeseriesT: dataT,
238+ timeseriesY: Array.from(dataY),
239+ width,
240+ height,
241+ margins,
242+ xRange,
243+ yRange,
244+ };
245+ worker.postMessage(msg);
246+ }, [width, height, dataT, dataY, worker, margins, xRange, yRange]);
247+
248+ // Render cursor on overlay canvas
249+ useEffect(() => {
250+ if (!overlayCanvasElement || selectedIndex === null || !dataY) return;
251+ const ctx = overlayCanvasElement.getContext("2d");
252+ if (!ctx) return;
253+
254+ // Clear overlay canvas
255+ ctx.clearRect(0, 0, width, height);
256+
257+ // Draw cursor line
258+ const xRatio = (selectedIndex - xRange.min) / (xRange.max - xRange.min);
259+ const x = margins.left + xRatio * (width - margins.left - margins.right);
260+ ctx.beginPath();
261+ ctx.strokeStyle = "#ff0000";
262+ ctx.lineWidth = 1;
263+ ctx.setLineDash([4, 4]);
264+ ctx.moveTo(x, margins.top);
265+ ctx.lineTo(x, height - margins.bottom);
266+ ctx.stroke();
267+ }, [
268+ selectedIndex,
269+ overlayCanvasElement,
270+ width,
271+ height,
272+ margins,
273+ dataT,
274+ dataY,
275+ xRange,
276+ ]);
277+
278+ const selectedValue = useMemo(() => {
279+ if (selectedIndex === -1 || !dataT || !dataY) return null;
280+ for (let i = 0; i < dataT.length; i++) {
281+ if (dataT[i] === selectedIndex) {
282+ return dataY[i];
283+ }
284+ }
285+ return null;
286+ }, [selectedIndex, dataT, dataY]);
287+
288+ if (error || clientError) {
289+ return <div>Error loading data: {error || clientError}</div>;
290+ }
291+
292+ if (isLoading && !dataY) {
293+ return <div>Loading...</div>;
294+ }
295+
296+ const handleCanvasClick = (e: React.MouseEvent<HTMLDivElement>) => {
297+ if (!overlayCanvasElement || !dataY || isDragging) return;
298+
299+ // Enable wheel zooming on first click
300+ if (!isWheelEnabled) {
301+ setIsWheelEnabled(true);
302+ }
303+
304+ const rect = overlayCanvasElement.getBoundingClientRect();
305+ const x = e.clientX - rect.left;
306+ const xRatio = (x - margins.left) / (width - margins.left - margins.right);
307+ const index = Math.round(xRange.min + xRatio * (xRange.max - xRange.min));
308+ if (index >= 0) {
309+ dispatch({ type: "SET_SELECTED_INDEX", index });
310+ }
311+ };
312+
313+ return (
314+ <div style={{ position: "relative", width, height: height + 50 }}>
315+ <div style={{ marginBottom: 10, height: 20 }}>
316+ <TimeseriesNavigationBar
317+ width={width}
318+ height={20}
319+ totalRange={{ min: 0, max: client ? client.getShape() - 1 : 999 }}
320+ viewRange={xRange}
321+ onViewRangeChange={(range) =>
322+ dispatch({ type: "SET_X_RANGE", range })
323+ }
324+ />
325+ </div>
326+ {showHint && (
327+ <div
328+ style={{
329+ position: "absolute",
330+ top: margins.top + 10,
331+ right: margins.right + 10,
332+ display: "flex",
333+ flexDirection: "column",
334+ alignItems: "flex-end",
335+ gap: "8px",
336+ zIndex: 10,
337+ opacity: showHint ? 0.8 : 0,
338+ transition: "opacity 0.5s ease-out",
339+ pointerEvents: "none",
340+ fontSize: "12px",
341+ color: "#666",
342+ }}
343+ >
344+ <div
345+ style={{
346+ display: "flex",
347+ alignItems: "center",
348+ gap: "4px",
349+ backgroundColor: "rgba(255, 255, 255, 0.9)",
350+ padding: "2px 6px",
351+ borderRadius: "4px",
352+ }}
353+ >
354+ <span>Drag to pan</span>
355+ <svg width="14" height="14" viewBox="0 0 24 24" fill="#666">
356+ <path d="M15 3h2v5h-2V3zm4 0h2v5h-2V3zm-4 7h2v5h-2v-5zm4 0h2v5h-2v-5zm-4 7h2v5h-2v-5zm4 0h2v5h-2v-5z" />
357+ </svg>
358+ </div>
359+ <div
360+ style={{
361+ display: "flex",
362+ alignItems: "center",
363+ gap: "4px",
364+ backgroundColor: "rgba(255, 255, 255, 0.9)",
365+ padding: "2px 6px",
366+ borderRadius: "4px",
367+ }}
368+ >
369+ <span>Scroll to zoom</span>
370+ <svg width="14" height="14" viewBox="0 0 24 24" fill="#666">
371+ <path d="M12 3c-4.97 0-9 4.03-9 9s4.03 9 9 9 9-4.03 9-9-4.03-9-9-9zm0 16c-3.86 0-7-3.14-7-7s3.14-7 7-7 7 3.14 7 7-3.14 7-7 7zm1-11h-2v3H8v2h3v3h2v-3h3v-2h-3V8z" />
372+ </svg>
373+ </div>
374+ </div>
375+ )}
376+ <div
377+ ref={setContainer}
378+ style={{ position: "relative", width, height }}
379+ onClick={(e) => {
380+ handleCanvasClick(e);
381+ hideHint();
382+ }}
383+ onMouseDown={hideHint}
384+ >
385+ <canvas
386+ ref={setCanvasElement}
387+ key={`canvas-${width}-${height}`}
388+ width={width}
389+ height={height}
390+ style={{
391+ position: "absolute",
392+ width: "100%",
393+ height: "100%",
394+ }}
395+ />
396+ <canvas
397+ ref={setOverlayCanvasElement}
398+ width={width}
399+ height={height}
400+ style={{
401+ position: "absolute",
402+ width: "100%",
403+ height: "100%",
404+ pointerEvents: "none",
405+ }}
406+ />
407+ </div>
408+ {selectedIndex !== -1 && dataY && (
409+ <div style={{ height: 30, padding: "5px 0", color: "#666" }}>
410+ Index: {selectedIndex}, Value: {selectedValue?.toFixed(3)}
411+ </div>
412+ )}
413+ </div>
414+ );
415+};
416+
417+const computeMin = (data: SupportedTypedArray) => {
418+ let min = Infinity;
419+ for (let i = 0; i < data.length; i++) {
420+ if (data[i] < min) {
421+ min = data[i];
422+ }
423+ }
424+ return min;
425+};
426+
427+const computeMax = (data: SupportedTypedArray) => {
428+ let max = -Infinity;
429+ for (let i = 0; i < data.length; i++) {
430+ if (data[i] > max) {
431+ max = data[i];
432+ }
433+ }
434+ return max;
435+};
436+
437+export default TimeseriesView;
web-ui/src/components/dataset/TimeseriesViewWorker.tsadded+239−0View file
@@ -0,0 +1,239 @@
1+// Web worker for rendering timeseries data to canvas
2+
3+import { Margins, Range, WorkerMessage } from "./WorkerTypes";
4+
5+// Helper function to find a nice integer tick interval
6+function getNiceTickInterval(range: number, maxTicks: number): number {
7+ const minInterval = Math.ceil(range / maxTicks);
8+ if (minInterval <= 1) return 1;
9+
10+ const magnitude = Math.pow(10, Math.floor(Math.log10(minInterval)));
11+ const niceIntervals = [1, 2, 5, 10];
12+
13+ for (const interval of niceIntervals) {
14+ const tickInterval = interval * magnitude;
15+ if (tickInterval >= minInterval) {
16+ return Math.ceil(tickInterval);
17+ }
18+ }
19+ return Math.ceil(niceIntervals[niceIntervals.length - 1] * magnitude * 10);
20+}
21+
22+// Helper function to estimate the width of a number in pixels
23+// This is an approximation since we can't measure text width directly in a worker
24+function estimateNumberWidth(num: number): number {
25+ const numStr = Math.abs(num).toString();
26+ const digitWidth = 8; // Approximate width of a digit in pixels
27+ const padding = 4; // Padding between numbers
28+ return (numStr.length + (num < 0 ? 1 : 0)) * digitWidth + padding;
29+}
30+
31+// Helper function to get tick positions
32+function getTickPositions(
33+ range: Range,
34+ width: number,
35+ considerNumberWidth = false, // Only true for x-axis where we need to handle large integers
36+): { value: number; x: number }[] {
37+ let pixelsPerTick = 20; // Default minimum pixels between ticks
38+
39+ if (considerNumberWidth) {
40+ // For x-axis, calculate spacing based on largest number width
41+ const maxAbsValue = Math.max(Math.abs(range.min), Math.abs(range.max));
42+ const maxNumberWidth = estimateNumberWidth(maxAbsValue);
43+ pixelsPerTick = Math.max(maxNumberWidth, 20); // Use the larger of estimated width or minimum spacing
44+ }
45+ const maxTicks = Math.floor(width / pixelsPerTick);
46+ const tickInterval = getNiceTickInterval(range.max - range.min, maxTicks);
47+
48+ const firstTick = Math.ceil(range.min / tickInterval) * tickInterval;
49+ const lastTick = Math.floor(range.max);
50+
51+ const ticks: { value: number; x: number }[] = [];
52+ for (let value = firstTick; value <= lastTick; value += tickInterval) {
53+ const x = (value - range.min) / (range.max - range.min);
54+ if (Number.isInteger(value)) {
55+ ticks.push({ value, x });
56+ }
57+ }
58+
59+ return ticks;
60+}
61+
62+let canvas: OffscreenCanvas | null = null;
63+let ctx: OffscreenCanvasRenderingContext2D | null = null;
64+
65+function renderTimeseries(
66+ timeseriesT: number[],
67+ timeseriesY: number[],
68+ width: number,
69+ height: number,
70+ margins: Margins,
71+ xRange: Range,
72+ yRange: Range,
73+) {
74+ if (!ctx || !canvas) return;
75+
76+ const context = ctx; // Create a stable reference to satisfy TypeScript
77+
78+ // Clear canvas
79+ context.clearRect(0, 0, width, height);
80+
81+ // Draw axes
82+ context.strokeStyle = "#666666";
83+ context.lineWidth = 1;
84+ context.beginPath();
85+
86+ // Y axis
87+ context.moveTo(margins.left, margins.top);
88+ context.lineTo(margins.left, height - margins.bottom);
89+
90+ // X axis
91+ context.moveTo(margins.left, height - margins.bottom);
92+ context.lineTo(width - margins.right, height - margins.bottom);
93+
94+ context.stroke();
95+
96+ // Calculate the drawing area dimensions
97+ const drawingWidth = width - margins.left - margins.right;
98+ const drawingHeight = height - margins.top - margins.bottom;
99+
100+ // Set up clipping region for timeseries
101+ context.save();
102+ context.beginPath();
103+ context.rect(margins.left, margins.top, drawingWidth, drawingHeight);
104+ context.clip();
105+
106+ // Set up drawing style for timeseries
107+ context.strokeStyle = "#2196f3";
108+ context.lineWidth = 2;
109+ context.beginPath();
110+
111+ // Calculate scaling factors
112+ const xScale = drawingWidth / (xRange.max - xRange.min);
113+ const yScale = drawingHeight / (yRange.max - yRange.min);
114+
115+ // Draw the path
116+ let isFirst = true;
117+ for (let i = 0; i < timeseriesT.length; i++) {
118+ const x = margins.left + (timeseriesT[i] - xRange.min) * xScale;
119+ const y =
120+ margins.top + drawingHeight - (timeseriesY[i] - yRange.min) * yScale;
121+ if (isFirst) {
122+ context.moveTo(x, y);
123+ isFirst = false;
124+ } else {
125+ context.lineTo(x, y);
126+ }
127+ }
128+
129+ context.stroke();
130+
131+ // Remove clipping before drawing ticks
132+ context.restore();
133+
134+ // Draw Y-axis ticks and labels
135+ const yTicks = getTickPositions(yRange, drawingHeight);
136+
137+ context.textAlign = "right";
138+ context.textBaseline = "middle";
139+ context.fillStyle = "#666666";
140+ context.font = "12px Arial";
141+
142+ yTicks.forEach((tick) => {
143+ const y = margins.top + drawingHeight - tick.x * drawingHeight;
144+
145+ // Draw tick mark
146+ context.beginPath();
147+ context.moveTo(margins.left - 6, y);
148+ context.lineTo(margins.left, y);
149+ context.stroke();
150+
151+ // Draw label
152+ context.fillText(tick.value.toString(), margins.left - 8, y);
153+ });
154+
155+ // Draw X-axis ticks and labels
156+ const ticks = getTickPositions(xRange, drawingWidth, true); // Consider number width for x-axis
157+
158+ context.textAlign = "center";
159+ context.textBaseline = "top";
160+ context.fillStyle = "#666666";
161+ context.font = "12px Arial";
162+
163+ ticks.forEach((tick) => {
164+ const x = margins.left + tick.x * drawingWidth;
165+
166+ // Draw tick mark
167+ context.beginPath();
168+ context.moveTo(x, height - margins.bottom);
169+ context.lineTo(x, height - margins.bottom + 6);
170+ context.stroke();
171+
172+ // Draw label
173+ context.fillText(tick.value.toString(), x, height - margins.bottom + 8);
174+ });
175+}
176+
177+self.onmessage = (evt: MessageEvent) => {
178+ const message = evt.data as WorkerMessage;
179+
180+ if (message.type === "initialize") {
181+ canvas = message.canvas;
182+ ctx = canvas.getContext("2d");
183+ if (!ctx) {
184+ self.postMessage({
185+ type: "error",
186+ error: "Failed to get canvas context",
187+ });
188+ return;
189+ }
190+ self.postMessage({ type: "initialized" });
191+ return;
192+ }
193+
194+ if (message.type === "render") {
195+ throttleRender(() => {
196+ const {
197+ timeseriesT,
198+ timeseriesY,
199+ width,
200+ height,
201+ margins,
202+ xRange,
203+ yRange,
204+ } = message;
205+ renderTimeseries(
206+ timeseriesT,
207+ timeseriesY,
208+ width,
209+ height,
210+ margins,
211+ xRange,
212+ yRange,
213+ );
214+ self.postMessage({ type: "render_complete" });
215+ });
216+ return;
217+ }
218+};
219+
220+let renderStack: (() => void)[] = [];
221+let lastRenderTime = 0;
222+
223+const throttleRender = (callback: () => void) => {
224+ renderStack.push(callback);
225+ const checkRender = () => {
226+ if (renderStack.length === 0) return;
227+ const elapsed = Date.now() - lastRenderTime;
228+ if (elapsed > 100) {
229+ lastRenderTime = Date.now();
230+ renderStack[renderStack.length - 1]();
231+ renderStack = [];
232+ } else {
233+ setTimeout(checkRender, 150);
234+ }
235+ };
236+ checkRender();
237+};
238+
239+export {}; // Needed for TypeScript modules
web-ui/src/components/dataset/WorkerTypes.tsadded+24−0View file
@@ -0,0 +1,24 @@
1+export interface Range {
2+ min: number;
3+ max: number;
4+}
5+
6+export interface Margins {
7+ left: number;
8+ right: number;
9+ top: number;
10+ bottom: number;
11+}
12+
13+export type WorkerMessage =
14+ | { type: "initialize"; canvas: OffscreenCanvas }
15+ | {
16+ type: "render";
17+ timeseriesT: number[];
18+ timeseriesY: number[];
19+ width: number;
20+ height: number;
21+ margins: Margins;
22+ xRange: Range;
23+ yRange: Range;
24+ };
web-ui/src/components/dataset/timeseriesViewReducer.tsadded+55−0View file
@@ -0,0 +1,55 @@
1+import { Range } from "./WorkerTypes";
2+
3+// State Type
4+export interface TimeseriesViewState {
5+ selectedIndex: number;
6+ isDragging: boolean;
7+ lastDragX: number;
8+ xRange: Range;
9+}
10+
11+// Initial State
12+export const initialState: TimeseriesViewState = {
13+ selectedIndex: -1,
14+ isDragging: false,
15+ lastDragX: 0,
16+ xRange: { min: 0, max: 999 },
17+};
18+
19+// Action Types Union
20+type TimeseriesViewAction =
21+ | { type: "SET_SELECTED_INDEX"; index: number }
22+ | { type: "SET_IS_DRAGGING"; isDragging: boolean }
23+ | { type: "SET_LAST_DRAG_X"; x: number }
24+ | { type: "SET_X_RANGE"; range: Range };
25+
26+// Reducer
27+export const timeseriesViewReducer = (
28+ state: TimeseriesViewState = initialState,
29+ action: TimeseriesViewAction,
30+): TimeseriesViewState => {
31+ switch (action.type) {
32+ case "SET_SELECTED_INDEX":
33+ return {
34+ ...state,
35+ selectedIndex: action.index,
36+ };
37+ case "SET_IS_DRAGGING":
38+ return {
39+ ...state,
40+ isDragging: action.isDragging,
41+ };
42+ case "SET_LAST_DRAG_X":
43+ return {
44+ ...state,
45+ lastDragX: action.x,
46+ };
47+ case "SET_X_RANGE":
48+ return {
49+ ...state,
50+ xRange: action.range,
51+ };
52+ default:
53+ return state;
54+ }
55+};
web-ui/src/components/shared/BaseContent.tsxadded+137−0View file
@@ -0,0 +1,137 @@
1+import { useNavigate } from "react-router-dom";
2+import { useState } from "react";
3+import ReactMarkdown from "react-markdown";
4+import remarkMath from "remark-math";
5+import rehypeKatex from "rehype-katex";
6+import { BenchmarkData } from "../../types";
7+import { BenchmarkCharts } from "../benchmark/charts/BenchmarkCharts";
8+import { BenchmarkScatterPlots } from "../benchmark/charts/BenchmarkScatterPlots";
9+import { BenchmarkTable } from "../benchmark/table/BenchmarkTable";
10+import "./ContentStyles.css";
11+
12+export interface BaseItem {
13+ name: string;
14+ description: string;
15+ long_description?: string;
16+ version: string;
17+ tags: string[];
18+ source_file?: string;
19+}
20+
21+interface BaseContentProps {
22+ item: BaseItem;
23+ benchmarkData: BenchmarkData | null;
24+ chartData: Array<{
25+ algorithmOrDataset: string;
26+ compression_ratio: number;
27+ reference_compression_ratio: number | null;
28+ encode_speed: number;
29+ decode_speed: number;
30+ }>;
31+ tagNavigationPrefix: string;
32+ filterKey: "dataset" | "algorithm";
33+ downloadSection?: React.ReactNode;
34+ additionalContent?: React.ReactNode;
35+ showSortByCompressionRatio?: boolean;
36+ showNormalizeByReference?: boolean;
37+}
38+
39+export const BaseContent = ({
40+ item,
41+ benchmarkData,
42+ chartData,
43+ tagNavigationPrefix,
44+ filterKey,
45+ downloadSection,
46+ additionalContent,
47+ showSortByCompressionRatio,
48+ showNormalizeByReference,
49+}: BaseContentProps) => {
50+ const navigate = useNavigate();
51+ const [isExpanded, setIsExpanded] = useState(false);
52+
53+ return (
54+ <div>
55+ <div className="content-container">
56+ <p className="content-header">
57+ <strong>{item.name}</strong> | {item.description}
58+ </p>
59+ {item.long_description && (
60+ <>
61+ <button
62+ className="description-toggle"
63+ onClick={() => setIsExpanded(!isExpanded)}
64+ >
65+ {isExpanded ? "View less" : "Read more"}
66+ </button>
67+ {isExpanded && (
68+ <div className="long-description">
69+ <ReactMarkdown
70+ remarkPlugins={[remarkMath]}
71+ rehypePlugins={[rehypeKatex]}
72+ >
73+ {item.long_description}
74+ </ReactMarkdown>
75+ </div>
76+ )}
77+ </>
78+ )}
79+ </div>
80+ <div className="metadata-section">
81+ <div>
82+ <span className="metadata-label">Version: </span>
83+ <span className="metadata-value">{item.version}</span>
84+ </div>
85+ <div>
86+ <span className="metadata-label">Tags: </span>
87+ {item.tags.map((tag) => (
88+ <span
89+ key={tag}
90+ className="tag"
91+ onClick={() => navigate(`${tagNavigationPrefix}?tag=${tag}`)}
92+ >
93+ {tag}
94+ </span>
95+ ))}
96+ </div>
97+ {downloadSection}
98+ {item.source_file && (
99+ <div>
100+ <span className="metadata-label">Source: </span>
101+ <a
102+ href={item.source_file}
103+ target="_blank"
104+ rel="noopener noreferrer"
105+ className="source-link"
106+ >
107+ View
108+ </a>
109+ </div>
110+ )}
111+ </div>
112+ {additionalContent}
113+ {benchmarkData && (
114+ <>
115+ <div className="benchmark-section">
116+ <h2 className="benchmark-title">Benchmark Results</h2>
117+ <BenchmarkCharts
118+ chartData={chartData}
119+ showSortByCompressionRatio={showSortByCompressionRatio}
120+ showNormalizeByReference={showNormalizeByReference}
121+ />
122+ {filterKey === "dataset" && (
123+ <BenchmarkScatterPlots chartData={chartData} />
124+ )}
125+ </div>
126+ <div className="benchmark-section">
127+ <BenchmarkTable
128+ results={benchmarkData.results.filter(
129+ (result) => result[filterKey] === item.name,
130+ )}
131+ />
132+ </div>
133+ </>
134+ )}
135+ </div>
136+ );
137+};
web-ui/src/components/shared/ContentStyles.cssadded+178−0View file
@@ -0,0 +1,178 @@
1+.content-container {
2+ margin-bottom: 1.5rem;
3+}
4+
5+.content-header {
6+ font-size: 0.9rem;
7+ line-height: 1.5;
8+ margin-bottom: 0.5rem;
9+}
10+
11+.description-toggle {
12+ color: #0066cc;
13+ text-decoration: none;
14+ font-size: 0.8rem;
15+ cursor: pointer;
16+ background: none;
17+ border: none;
18+ padding: 0;
19+ margin-top: 0.25rem;
20+ display: block;
21+}
22+
23+.description-toggle:hover {
24+ text-decoration: underline;
25+}
26+
27+.long-description {
28+ font-size: 0.9rem;
29+ line-height: 1.5;
30+ margin-top: 0.5rem;
31+ padding: 0.5rem;
32+ background-color: #f8f8f8;
33+ border-radius: 4px;
34+}
35+
36+/* Markdown styles */
37+.long-description h1,
38+.long-description h2,
39+.long-description h3,
40+.long-description h4,
41+.long-description h5,
42+.long-description h6 {
43+ margin-top: 1.5em;
44+ margin-bottom: 0.5em;
45+ font-weight: 600;
46+}
47+
48+.long-description h1 { font-size: 1.5em; }
49+.long-description h2 { font-size: 1.3em; }
50+.long-description h3 { font-size: 1.1em; }
51+
52+.long-description p {
53+ margin-bottom: 1em;
54+}
55+
56+.long-description ul,
57+.long-description ol {
58+ margin: 1em 0;
59+ padding-left: 2em;
60+}
61+
62+.long-description li {
63+ margin: 0.5em 0;
64+}
65+
66+.long-description code {
67+ background-color: #eee;
68+ padding: 0.2em 0.4em;
69+ border-radius: 3px;
70+ font-family: monospace;
71+ font-size: 0.9em;
72+}
73+
74+.long-description pre {
75+ background-color: #eee;
76+ padding: 1em;
77+ border-radius: 4px;
78+ overflow-x: auto;
79+ margin: 1em 0;
80+}
81+
82+.long-description pre code {
83+ background-color: transparent;
84+ padding: 0;
85+}
86+
87+.long-description blockquote {
88+ border-left: 4px solid #ddd;
89+ margin: 1em 0;
90+ padding-left: 1em;
91+ color: #666;
92+}
93+
94+.long-description a {
95+ color: #0066cc;
96+ text-decoration: none;
97+}
98+
99+.long-description a:hover {
100+ text-decoration: underline;
101+}
102+
103+.long-description img {
104+ max-width: 100%;
105+ height: auto;
106+ margin: 1em 0;
107+}
108+
109+.long-description table {
110+ border-collapse: collapse;
111+ width: 100%;
112+ margin: 1em 0;
113+}
114+
115+.long-description th,
116+.long-description td {
117+ border: 1px solid #ddd;
118+ padding: 0.5em;
119+ text-align: left;
120+}
121+
122+.long-description th {
123+ background-color: #f0f0f0;
124+}
125+
126+.metadata-section {
127+ margin-bottom: 1.5rem;
128+ display: flex;
129+ gap: 2rem;
130+ flex-wrap: wrap;
131+}
132+
133+.metadata-label {
134+ font-weight: bold;
135+ font-size: 0.9rem;
136+}
137+
138+.metadata-value {
139+ font-size: 0.9rem;
140+}
141+
142+.tag {
143+ color: #0066cc;
144+ text-decoration: none;
145+ padding: 2px 6px;
146+ background-color: #f0f0f0;
147+ border-radius: 4px;
148+ font-size: 0.9rem;
149+ cursor: pointer;
150+}
151+
152+.source-link {
153+ color: #0066cc;
154+ text-decoration: none;
155+ padding: 2px 6px;
156+ background-color: #f0f0f0;
157+ border-radius: 4px;
158+ font-size: 0.9rem;
159+}
160+
161+.benchmark-section {
162+ margin-bottom: 1.5rem;
163+}
164+
165+.benchmark-title {
166+ font-size: 1.2rem;
167+ font-weight: bold;
168+ margin-bottom: 0.5rem;
169+}
170+
171+.download-link {
172+ color: #0066cc;
173+ text-decoration: none;
174+ padding: 2px 6px;
175+ background-color: #f0f0f0;
176+ border-radius: 4px;
177+ font-size: 0.9rem;
178+}
web-ui/src/components/tables/DatasetAlgorithmTables.tsxadded+432−0View file
@@ -0,0 +1,432 @@
1+import { Link } from "react-router-dom";
2+import { TagFilter } from "../TagFilter";
3+import { Dataset, Algorithm, BenchmarkResult } from "../../types";
4+
5+interface DatasetTableProps {
6+ filteredDatasets: Dataset[];
7+ availableDatasetTags: string[];
8+ selectedTags: string[];
9+ toggleTag: (tag: string) => void;
10+ benchmarkResults: BenchmarkResult[];
11+}
12+
13+interface AlgorithmTableProps {
14+ filteredAlgorithms: Algorithm[];
15+ availableAlgorithmTags: string[];
16+ selectedTags: string[];
17+ toggleTag: (tag: string) => void;
18+}
19+
20+export const DatasetTable = ({
21+ filteredDatasets,
22+ availableDatasetTags,
23+ selectedTags,
24+ toggleTag,
25+ benchmarkResults,
26+}: DatasetTableProps) => {
27+ const getBestCompressionResult = (datasetName: string) => {
28+ const datasetResults = benchmarkResults.filter(
29+ (result) => result.dataset === datasetName,
30+ );
31+ if (datasetResults.length === 0) return { algorithm: "N/A", ratio: 0 };
32+
33+ const bestResult = datasetResults.reduce((best, current) =>
34+ current.compression_ratio > best.compression_ratio ? current : best,
35+ );
36+ return {
37+ algorithm: bestResult.algorithm,
38+ ratio: bestResult.compression_ratio,
39+ };
40+ };
41+
42+ return (
43+ <div style={{ overflowX: "auto" }}>
44+ <div style={{ marginBottom: "1rem" }}>
45+ <TagFilter
46+ availableTags={availableDatasetTags}
47+ selectedTags={selectedTags}
48+ onTagToggle={toggleTag}
49+ label="Filter datasets"
50+ />
51+ </div>
52+ <table style={{ width: "100%", borderCollapse: "collapse" }}>
53+ <thead>
54+ <tr style={{ backgroundColor: "#f5f5f5" }}>
55+ <th
56+ style={{
57+ padding: "8px 12px",
58+ textAlign: "left",
59+ borderBottom: "1px solid #ddd",
60+ fontSize: "0.9rem",
61+ whiteSpace: "nowrap",
62+ }}
63+ >
64+ Name
65+ </th>
66+ <th
67+ style={{
68+ padding: "8px 12px",
69+ textAlign: "left",
70+ borderBottom: "1px solid #ddd",
71+ fontSize: "0.9rem",
72+ whiteSpace: "nowrap",
73+ }}
74+ >
75+ Version
76+ </th>
77+ <th
78+ style={{
79+ padding: "8px 12px",
80+ textAlign: "left",
81+ borderBottom: "1px solid #ddd",
82+ fontSize: "0.9rem",
83+ whiteSpace: "nowrap",
84+ }}
85+ >
86+ Description
87+ </th>
88+ <th
89+ style={{
90+ padding: "8px 12px",
91+ textAlign: "left",
92+ borderBottom: "1px solid #ddd",
93+ fontSize: "0.9rem",
94+ whiteSpace: "nowrap",
95+ }}
96+ >
97+ Tags
98+ </th>
99+ <th
100+ style={{
101+ padding: "8px 12px",
102+ textAlign: "left",
103+ borderBottom: "1px solid #ddd",
104+ fontSize: "0.9rem",
105+ whiteSpace: "nowrap",
106+ }}
107+ >
108+ Source
109+ </th>
110+ <th
111+ style={{
112+ padding: "8px 12px",
113+ textAlign: "left",
114+ borderBottom: "1px solid #ddd",
115+ fontSize: "0.9rem",
116+ whiteSpace: "nowrap",
117+ }}
118+ >
119+ Data
120+ </th>
121+ <th
122+ style={{
123+ padding: "8px 12px",
124+ textAlign: "left",
125+ borderBottom: "1px solid #ddd",
126+ fontSize: "0.9rem",
127+ whiteSpace: "nowrap",
128+ }}
129+ >
130+ Best Compression
131+ </th>
132+ </tr>
133+ </thead>
134+ <tbody>
135+ {filteredDatasets.map((dataset, index) => (
136+ <tr
137+ key={`${dataset.name}-${dataset.version}`}
138+ style={{ backgroundColor: index % 2 === 0 ? "white" : "#fafafa" }}
139+ >
140+ <td
141+ style={{
142+ padding: "6px 12px",
143+ borderBottom: "1px solid #ddd",
144+ fontSize: "0.9rem",
145+ }}
146+ >
147+ <Link
148+ to={`/dataset/${encodeURIComponent(dataset.name)}`}
149+ style={{
150+ color: "#0066cc",
151+ textDecoration: "none",
152+ fontWeight: "500",
153+ }}
154+ >
155+ {dataset.name}
156+ </Link>
157+ </td>
158+ <td
159+ style={{
160+ padding: "6px 12px",
161+ borderBottom: "1px solid #ddd",
162+ fontSize: "0.9rem",
163+ }}
164+ >
165+ {dataset.version}
166+ </td>
167+ <td
168+ style={{
169+ padding: "6px 12px",
170+ borderBottom: "1px solid #ddd",
171+ fontSize: "0.9rem",
172+ }}
173+ >
174+ {dataset.description}
175+ </td>
176+ <td
177+ style={{
178+ padding: "6px 12px",
179+ borderBottom: "1px solid #ddd",
180+ fontSize: "0.9rem",
181+ }}
182+ >
183+ {dataset.tags.map((tag) => (
184+ <span
185+ key={tag}
186+ style={{
187+ display: "inline-block",
188+ backgroundColor: "#e1e1e1",
189+ padding: "2px 6px",
190+ borderRadius: "3px",
191+ margin: "1px",
192+ fontSize: "0.8rem",
193+ }}
194+ >
195+ {tag}
196+ </span>
197+ ))}
198+ </td>
199+ <td
200+ style={{
201+ padding: "6px 12px",
202+ borderBottom: "1px solid #ddd",
203+ fontSize: "0.9rem",
204+ }}
205+ >
206+ {dataset.source_file && (
207+ <a
208+ href={dataset.source_file}
209+ target="_blank"
210+ rel="noopener noreferrer"
211+ style={{ color: "#0066cc", textDecoration: "none" }}
212+ >
213+ View Source
214+ </a>
215+ )}
216+ </td>
217+ <td
218+ style={{
219+ padding: "6px 12px",
220+ borderBottom: "1px solid #ddd",
221+ fontSize: "0.9rem",
222+ }}
223+ >
224+ {dataset.data_url_npy && (
225+ <a
226+ href={dataset.data_url_npy}
227+ download={`${dataset.name}-${dataset.version}.npy`}
228+ style={{ color: "#0066cc", textDecoration: "none" }}
229+ >
230+ Download
231+ </a>
232+ )}
233+ </td>
234+ <td
235+ style={{
236+ padding: "6px 12px",
237+ borderBottom: "1px solid #ddd",
238+ fontSize: "0.9rem",
239+ }}
240+ >
241+ {(() => {
242+ const result = getBestCompressionResult(dataset.name);
243+ return (
244+ <>
245+ <Link
246+ to={`/algorithm/${encodeURIComponent(result.algorithm)}`}
247+ style={{
248+ color: "#0066cc",
249+ textDecoration: "none",
250+ fontWeight: "500",
251+ }}
252+ >
253+ {result.algorithm}
254+ </Link>
255+ {result.algorithm !== "N/A" &&
256+ ` (${result.ratio.toFixed(1)})`}
257+ </>
258+ );
259+ })()}
260+ </td>
261+ </tr>
262+ ))}
263+ </tbody>
264+ </table>
265+ </div>
266+ );
267+};
268+
269+export const AlgorithmTable = ({
270+ filteredAlgorithms,
271+ availableAlgorithmTags,
272+ selectedTags,
273+ toggleTag,
274+}: AlgorithmTableProps) => (
275+ <div style={{ overflowX: "auto" }}>
276+ <div style={{ marginBottom: "1rem" }}>
277+ <TagFilter
278+ availableTags={availableAlgorithmTags}
279+ selectedTags={selectedTags}
280+ onTagToggle={toggleTag}
281+ label="Filter algorithms"
282+ />
283+ </div>
284+ <table style={{ width: "100%", borderCollapse: "collapse" }}>
285+ <thead>
286+ <tr style={{ backgroundColor: "#f5f5f5" }}>
287+ <th
288+ style={{
289+ padding: "8px 12px",
290+ textAlign: "left",
291+ borderBottom: "1px solid #ddd",
292+ fontSize: "0.9rem",
293+ whiteSpace: "nowrap",
294+ }}
295+ >
296+ Name
297+ </th>
298+ <th
299+ style={{
300+ padding: "8px 12px",
301+ textAlign: "left",
302+ borderBottom: "1px solid #ddd",
303+ fontSize: "0.9rem",
304+ whiteSpace: "nowrap",
305+ }}
306+ >
307+ Version
308+ </th>
309+ <th
310+ style={{
311+ padding: "8px 12px",
312+ textAlign: "left",
313+ borderBottom: "1px solid #ddd",
314+ fontSize: "0.9rem",
315+ whiteSpace: "nowrap",
316+ }}
317+ >
318+ Description
319+ </th>
320+ <th
321+ style={{
322+ padding: "8px 12px",
323+ textAlign: "left",
324+ borderBottom: "1px solid #ddd",
325+ fontSize: "0.9rem",
326+ whiteSpace: "nowrap",
327+ }}
328+ >
329+ Tags
330+ </th>
331+ <th
332+ style={{
333+ padding: "8px 12px",
334+ textAlign: "left",
335+ borderBottom: "1px solid #ddd",
336+ fontSize: "0.9rem",
337+ whiteSpace: "nowrap",
338+ }}
339+ >
340+ Source
341+ </th>
342+ </tr>
343+ </thead>
344+ <tbody>
345+ {filteredAlgorithms.map((algorithm, index) => (
346+ <tr
347+ key={`${algorithm.name}-${algorithm.version}`}
348+ style={{ backgroundColor: index % 2 === 0 ? "white" : "#fafafa" }}
349+ >
350+ <td
351+ style={{
352+ padding: "6px 12px",
353+ borderBottom: "1px solid #ddd",
354+ fontSize: "0.9rem",
355+ }}
356+ >
357+ <Link
358+ to={`/algorithm/${encodeURIComponent(algorithm.name)}`}
359+ style={{
360+ color: "#0066cc",
361+ textDecoration: "none",
362+ fontWeight: "500",
363+ }}
364+ >
365+ {algorithm.name}
366+ </Link>
367+ </td>
368+ <td
369+ style={{
370+ padding: "6px 12px",
371+ borderBottom: "1px solid #ddd",
372+ fontSize: "0.9rem",
373+ }}
374+ >
375+ {algorithm.version}
376+ </td>
377+ <td
378+ style={{
379+ padding: "6px 12px",
380+ borderBottom: "1px solid #ddd",
381+ fontSize: "0.9rem",
382+ }}
383+ >
384+ {algorithm.description}
385+ </td>
386+ <td
387+ style={{
388+ padding: "6px 12px",
389+ borderBottom: "1px solid #ddd",
390+ fontSize: "0.9rem",
391+ }}
392+ >
393+ {algorithm.tags.map((tag) => (
394+ <span
395+ key={tag}
396+ style={{
397+ display: "inline-block",
398+ backgroundColor: "#e1e1e1",
399+ padding: "2px 6px",
400+ borderRadius: "3px",
401+ margin: "1px",
402+ fontSize: "0.8rem",
403+ }}
404+ >
405+ {tag}
406+ </span>
407+ ))}
408+ </td>
409+ <td
410+ style={{
411+ padding: "6px 12px",
412+ borderBottom: "1px solid #ddd",
413+ fontSize: "0.9rem",
414+ }}
415+ >
416+ {algorithm.source_file && (
417+ <a
418+ href={algorithm.source_file}
419+ target="_blank"
420+ rel="noopener noreferrer"
421+ style={{ color: "#0066cc", textDecoration: "none" }}
422+ >
423+ View Source
424+ </a>
425+ )}
426+ </td>
427+ </tr>
428+ ))}
429+ </tbody>
430+ </table>
431+ </div>
432+);
web-ui/src/content/home-content.ymladded+58−0View file
@@ -0,0 +1,58 @@
1+title: Welcome to Benchcompress
2+description:
3+ Benchcompress is an open-source project for comparing compression algorithms
4+ on scientific data arrays, with a special focus on electrophysiology
5+ recordings.
6+
7+sections:
8+ algorithms:
9+ title: Algorithms
10+ description:
11+ Explore the supported compression algorithms, including traditional methods
12+ like zlib and modern approaches like ANS.
13+ link: /algorithms
14+ linkText: View Algorithms
15+
16+ datasets:
17+ title: Datasets
18+ description:
19+ Explore the curated collection of scientific datasets, including
20+ electrophysiology recordings, seismic data, and more. See the performance
21+ of the various compression algorithms on each dataset.
22+ link: /datasets
23+ linkText: View Datasets
24+
25+ paper:
26+ title: Research Paper
27+ description:
28+ Read the work-in-progress paper describing the methodology,
29+ implementation, and findings of this research.
30+ link: /paper
31+ linkText: Read Paper
32+
33+ monitor:
34+ title: Live Monitor
35+ description:
36+ The benchmarking runs automatically in a GitHub Actions workflow. Track
37+ current progress, view completed benchmarks, and monitor performance
38+ metrics as they are generated.
39+ link: /monitor
40+ linkText: View Monitor
41+
42+ source:
43+ title: Source Code
44+ description:
45+ Explore the GitHub repository to view the source code, contribute to the
46+ project, or run your own benchmarks locally.
47+ link: https://github.com/magland/benchcompress
48+ linkText: View on GitHub
49+ external: true
50+
51+ submit:
52+ title: Submit an Algorithm or Dataset
53+ description:
54+ Want to contribute? Learn how to add your own compression algorithm
55+ or scientific dataset to the benchmark suite. Follow the step-by-step
56+ guide for preparing and submitting your contribution.
57+ link: /submit
58+ linkText: Submission Guide
web-ui/src/env.d.tsadded+1−0View file
@@ -0,0 +1 @@
1+declare const __BUILD_DATE__: string;
web-ui/src/hooks/TimeseriesDataClient.tsadded+175−0View file
@@ -0,0 +1,175 @@
1+export type SupportedTypedArray =
2+ | Uint8Array
3+ | Uint16Array
4+ | Uint32Array
5+ | Int16Array
6+ | Int32Array
7+ | Float32Array;
8+
9+interface ChunkCache {
10+ [key: number]: SupportedTypedArray;
11+}
12+
13+type DType = "uint8" | "uint16" | "uint32" | "int16" | "int32" | "float32";
14+
15+const TypedArrayConstructors = {
16+ uint8: Uint8Array,
17+ uint16: Uint16Array,
18+ uint32: Uint32Array,
19+ int16: Int16Array,
20+ int32: Int32Array,
21+ float32: Float32Array,
22+} as const;
23+
24+export class TimeseriesDataClient {
25+ private shape: number = 0;
26+ private dtype: DType | null = null;
27+ private chunkSize: number;
28+ private cache: ChunkCache = {};
29+ private inProgressFetches: { [key: number]: Promise<SupportedTypedArray> } =
30+ {};
31+ private datasetJsonUrl: string;
32+ private datasetDataUrl: string;
33+
34+ constructor(
35+ datasetJsonUrl: string,
36+ datasetDataUrl: string,
37+ chunkSize: number = 100000,
38+ ) {
39+ this.datasetJsonUrl = datasetJsonUrl;
40+ this.datasetDataUrl = datasetDataUrl;
41+ this.chunkSize = chunkSize;
42+ }
43+
44+ static async create(
45+ datasetJsonUrl: string,
46+ datasetDataUrl: string,
47+ chunkSize: number = 1000,
48+ ): Promise<TimeseriesDataClient> {
49+ const client = new TimeseriesDataClient(
50+ datasetJsonUrl,
51+ datasetDataUrl,
52+ chunkSize,
53+ );
54+ await client.initialize();
55+ return client;
56+ }
57+
58+ private async initialize() {
59+ const infoUrl = this.datasetJsonUrl;
60+ const response = await fetch(infoUrl);
61+ if (!response.ok) {
62+ throw new Error(`Failed to fetch dataset info: ${response.statusText}`);
63+ }
64+ const info = await response.json();
65+ this.shape = info.shape[0];
66+
67+ if (!this.isValidDType(info.dtype)) {
68+ throw new Error(`Unsupported data type: ${info.dtype}`);
69+ }
70+ this.dtype = info.dtype;
71+ }
72+
73+ private isValidDType(dtype: string): dtype is DType {
74+ return dtype in TypedArrayConstructors;
75+ }
76+
77+ private getChunkIndices(start: number, end: number): number[] {
78+ const startChunk = Math.floor(start / this.chunkSize);
79+ const endChunk = Math.floor(end / this.chunkSize);
80+ const chunks: number[] = [];
81+ for (let i = startChunk; i <= endChunk; i++) {
82+ chunks.push(i);
83+ }
84+ return chunks;
85+ }
86+
87+ private async fetchChunk(chunkIndex: number): Promise<SupportedTypedArray> {
88+ // Return cached chunk if available
89+ if (this.cache[chunkIndex]) {
90+ return this.cache[chunkIndex];
91+ }
92+
93+ // If this chunk is already being fetched, wait for it to complete
94+ const inProgressFetch = this.inProgressFetches[chunkIndex];
95+ if (inProgressFetch !== undefined) {
96+ return inProgressFetch;
97+ }
98+
99+ // Start new fetch and track it
100+ const fetchPromise = (async () => {
101+ if (!this.dtype) {
102+ throw new Error("Data type not initialized");
103+ }
104+
105+ const start = chunkIndex * this.chunkSize;
106+ const end = Math.min(start + this.chunkSize, this.shape);
107+ const url = this.datasetDataUrl;
108+ const itemSize = TypedArrayConstructors[this.dtype].BYTES_PER_ELEMENT;
109+ const byteStart = start * itemSize;
110+ const byteEnd = end * itemSize;
111+
112+ try {
113+ const response = await fetch(url, {
114+ headers: {
115+ Range: `bytes=${byteStart}-${byteEnd - 1}`,
116+ },
117+ });
118+ if (!response.ok) {
119+ throw new Error(`Failed to fetch chunk: ${response.statusText}`);
120+ }
121+
122+ const buffer = await response.arrayBuffer();
123+ const ArrayConstructor = TypedArrayConstructors[this.dtype];
124+ const data = new ArrayConstructor(buffer);
125+ this.cache[chunkIndex] = data;
126+ return data;
127+ } finally {
128+ // Clean up the in-progress fetch regardless of success/failure
129+ delete this.inProgressFetches[chunkIndex];
130+ }
131+ })();
132+
133+ // Store the promise for other requests to wait on
134+ this.inProgressFetches[chunkIndex] = fetchPromise;
135+ return fetchPromise;
136+ }
137+
138+ async fetchRange(start: number, end: number): Promise<SupportedTypedArray> {
139+ if (!this.dtype) {
140+ throw new Error("Data type not initialized");
141+ }
142+
143+ const chunkIndices = this.getChunkIndices(start, end);
144+ const chunks = await Promise.all(
145+ chunkIndices.map((idx) => this.fetchChunk(idx)),
146+ );
147+
148+ // Calculate total length needed
149+ const length = end - start;
150+ const ArrayConstructor = TypedArrayConstructors[this.dtype];
151+ const result = new ArrayConstructor(length);
152+
153+ // Copy data from chunks into result array
154+ let resultOffset = 0;
155+ for (let i = 0; i < chunks.length; i++) {
156+ const chunk = chunks[i];
157+ const chunkStart = chunkIndices[i] * this.chunkSize;
158+ const copyStart = Math.max(0, start - chunkStart);
159+ const copyEnd = Math.min(chunk.length, end - chunkStart);
160+ const copyLength = copyEnd - copyStart;
161+ result.set(chunk.subarray(copyStart, copyEnd), resultOffset);
162+ resultOffset += copyLength;
163+ }
164+
165+ return result;
166+ }
167+
168+ getShape(): number {
169+ return this.shape;
170+ }
171+
172+ getDType(): DType | null {
173+ return this.dtype;
174+ }
175+}
web-ui/src/hooks/useBenchmarkChartData.tsadded+37−0View file
@@ -0,0 +1,37 @@
1+import { useMemo } from "react";
2+import { BenchmarkResult } from "../types";
3+
4+export function useBenchmarkChartData(
5+ results: BenchmarkResult[],
6+ selectedDataset?: string | null,
7+ selectedAlgorithm?: string | null,
8+) {
9+ return useMemo(() => {
10+ if (selectedDataset) {
11+ return results
12+ .filter((row) => row.dataset === selectedDataset)
13+ .map((row) => ({
14+ algorithmOrDataset: row.algorithm,
15+ compression_ratio: row.compression_ratio,
16+ reference_compression_ratio: null,
17+ encode_speed: row.encode_mb_per_sec,
18+ decode_speed: row.decode_mb_per_sec,
19+ }));
20+ } else if (selectedAlgorithm) {
21+ return results
22+ .filter((row) => row.algorithm === selectedAlgorithm)
23+ .map((row) => ({
24+ algorithmOrDataset: row.dataset,
25+ compression_ratio: row.compression_ratio,
26+ reference_compression_ratio: Math.max(
27+ ...results
28+ .filter((r) => r.dataset === row.dataset)
29+ .map((r) => r.compression_ratio),
30+ ),
31+ encode_speed: row.encode_mb_per_sec,
32+ decode_speed: row.decode_mb_per_sec,
33+ }));
34+ }
35+ return [];
36+ }, [results, selectedDataset, selectedAlgorithm]);
37+}
web-ui/src/hooks/useMarkdownContent.tsadded+31−0View file
@@ -0,0 +1,31 @@
1+import { useState, useEffect } from "react";
2+
3+export const useMarkdownContent = (path: string) => {
4+ const [content, setContent] = useState<string>("");
5+ const [error, setError] = useState<string | null>(null);
6+
7+ useEffect(() => {
8+ const fetchContent = async () => {
9+ try {
10+ const response = await fetch(path);
11+ if (!response.ok) {
12+ throw new Error(
13+ `Failed to load markdown content: ${response.statusText}`,
14+ );
15+ }
16+ const text = await response.text();
17+ setContent(text);
18+ } catch (err) {
19+ setError(
20+ err instanceof Error
21+ ? err.message
22+ : "Failed to load markdown content",
23+ );
24+ }
25+ };
26+
27+ fetchContent();
28+ }, [path]);
29+
30+ return { content, error };
31+};
web-ui/src/hooks/useMarkdownPosts.tsadded+62−0View file
@@ -0,0 +1,62 @@
1+import { useState, useEffect } from "react";
2+
3+interface Post {
4+ path: string;
5+ content: string;
6+ date: Date;
7+}
8+
9+export const useMarkdownPosts = (directory: string) => {
10+ const [posts, setPosts] = useState<Post[]>([]);
11+ const [error, setError] = useState<string | null>(null);
12+ const [loading, setLoading] = useState(true);
13+
14+ useEffect(() => {
15+ const fetchPosts = async () => {
16+ try {
17+ // First fetch the index
18+ const indexResponse = await fetch(`${directory}/index.txt`);
19+ if (!indexResponse.ok) {
20+ throw new Error(
21+ `Failed to load post index: ${indexResponse.statusText}`,
22+ );
23+ }
24+ const indexContent = await indexResponse.text();
25+ const paths = indexContent.trim().split("\n");
26+
27+ // Then fetch all posts in parallel
28+ const postPromises = paths.map(async (path) => {
29+ const response = await fetch(`${directory}/${path}`);
30+ if (!response.ok) {
31+ throw new Error(
32+ `Failed to load post ${path}: ${response.statusText}`,
33+ );
34+ }
35+ const content = await response.text();
36+
37+ // Parse date from filename (format: YYYY-MM-DD-title.md)
38+ const dateMatch = path.match(/^(\d{4}-\d{2}-\d{2})/);
39+ if (!dateMatch) {
40+ throw new Error(`Invalid post filename format: ${path}`);
41+ }
42+ const date = new Date(dateMatch[1]);
43+
44+ return { path, content, date };
45+ });
46+
47+ const loadedPosts = await Promise.all(postPromises);
48+ // Sort posts by date, newest first
49+ loadedPosts.sort((a, b) => b.date.getTime() - a.date.getTime());
50+ setPosts(loadedPosts);
51+ setLoading(false);
52+ } catch (err) {
53+ setError(err instanceof Error ? err.message : "Failed to load posts");
54+ setLoading(false);
55+ }
56+ };
57+
58+ fetchPosts();
59+ }, [directory]);
60+
61+ return { posts, error, loading };
62+};
web-ui/src/hooks/useTagFilter.tsadded+30−0View file
@@ -0,0 +1,30 @@
1+import { useMemo } from "react";
2+
3+interface TaggableItem {
4+ tags: string[];
5+}
6+
7+export function useTagFilter<T extends TaggableItem>(
8+ items: T[],
9+ selectedTags: string[],
10+) {
11+ const availableTags = useMemo(() => {
12+ const tagSet = new Set<string>();
13+ items.forEach((item) => {
14+ item.tags.forEach((tag) => tagSet.add(tag));
15+ });
16+ return Array.from(tagSet).sort();
17+ }, [items]);
18+
19+ const filteredItems = useMemo(() => {
20+ if (selectedTags.length === 0) return items;
21+ return items.filter((item) =>
22+ selectedTags.every((tag) => item.tags.includes(tag)),
23+ );
24+ }, [items, selectedTags]);
25+
26+ return {
27+ availableTags,
28+ filteredItems,
29+ };
30+}
web-ui/src/hooks/useTimeseriesData.tsadded+89−0View file
@@ -0,0 +1,89 @@
1+import { useEffect, useState } from "react";
2+import { Dataset } from "../types";
3+
4+const getDtypeSize = (dtype: string): number => {
5+ switch (dtype) {
6+ case "uint8":
7+ return 1;
8+ case "uint16":
9+ return 2;
10+ case "uint32":
11+ return 4;
12+ case "int16":
13+ return 2;
14+ case "int32":
15+ return 4;
16+ default:
17+ throw new Error(`Unsupported dtype: ${dtype}`);
18+ }
19+};
20+
21+const createTypedArray = (buffer: ArrayBuffer, dtype: string): number[] => {
22+ switch (dtype) {
23+ case "uint8":
24+ return Array.from(new Uint8Array(buffer));
25+ case "uint16":
26+ return Array.from(new Uint16Array(buffer));
27+ case "uint32":
28+ return Array.from(new Uint32Array(buffer));
29+ case "int16":
30+ return Array.from(new Int16Array(buffer));
31+ case "int32":
32+ return Array.from(new Int32Array(buffer));
33+ default:
34+ throw new Error(`Unsupported dtype: ${dtype}`);
35+ }
36+};
37+
38+export const useTimeseriesData = (dataset: Dataset) => {
39+ const [data, setData] = useState<number[] | null>(null);
40+ const [error, setError] = useState<string | null>(null);
41+
42+ useEffect(() => {
43+ const fetchData = async () => {
44+ if (!dataset.data_url_raw) {
45+ setError("No raw data URL available");
46+ return;
47+ }
48+
49+ const metaJsonUrl = dataset.data_url_json;
50+ if (!metaJsonUrl) {
51+ setError("No JSON metadata URL available");
52+ return;
53+ }
54+
55+ try {
56+ const metaResponse = await fetch(metaJsonUrl);
57+ if (!metaResponse.ok) {
58+ throw new Error(`HTTP error! status: ${metaResponse.status}`);
59+ }
60+
61+ const metaJson = await metaResponse.json();
62+
63+ const dtype = metaJson.dtype;
64+ const bytesPerElement = getDtypeSize(dtype);
65+
66+ const numBytes = bytesPerElement * 1000;
67+
68+ const response = await fetch(dataset.data_url_raw, {
69+ headers: {
70+ Range: `bytes=0-${numBytes - 1}`, // First 1000 elements
71+ },
72+ });
73+
74+ if (!response.ok) {
75+ throw new Error(`HTTP error! status: ${response.status}`);
76+ }
77+
78+ const buffer = await response.arrayBuffer();
79+ const data = createTypedArray(buffer, dtype);
80+ setData(data);
81+ } catch (err) {
82+ setError(err instanceof Error ? err.message : "Failed to fetch data");
83+ }
84+ };
85+ fetchData();
86+ }, [dataset]);
87+
88+ return { data, error };
89+};
web-ui/src/hooks/useTimeseriesDataClient.tsadded+39−0View file
@@ -0,0 +1,39 @@
1+import { useEffect, useState } from "react";
2+import { Dataset } from "../types";
3+import { TimeseriesDataClient } from "./TimeseriesDataClient";
4+
5+interface UseTimeseriesDataClientResult {
6+ client: TimeseriesDataClient | null;
7+ error: string | null;
8+}
9+
10+export const useTimeseriesDataClient = (
11+ dataset: Dataset,
12+ chunkSize: number = 1000,
13+): UseTimeseriesDataClientResult => {
14+ const [client, setClient] = useState<TimeseriesDataClient | null>(null);
15+ const [error, setError] = useState<string | null>(null);
16+
17+ useEffect(() => {
18+ const initClient = async () => {
19+ try {
20+ const newClient = await TimeseriesDataClient.create(
21+ dataset.data_url_json || "",
22+ dataset.data_url_raw || "",
23+ chunkSize,
24+ );
25+ setClient(newClient);
26+ setError(null);
27+ } catch (err) {
28+ setError(
29+ err instanceof Error ? err.message : "Failed to initialize client",
30+ );
31+ setClient(null);
32+ }
33+ };
34+
35+ initClient();
36+ }, [dataset.data_url_json, dataset.data_url_raw, chunkSize]);
37+
38+ return { client, error };
39+};
web-ui/src/index.cssadded+68−0View file
@@ -0,0 +1,68 @@
1+:root {
2+ font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
3+ line-height: 1.5;
4+ font-weight: 400;
5+}
6+
7+body {
8+ margin: 0;
9+ min-width: 320px;
10+ min-height: 100vh;
11+ background-color: #f5f5f5;
12+}
13+
14+table {
15+ width: 100%;
16+ border-collapse: collapse;
17+ background: white;
18+ box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
19+ border-radius: 4px;
20+ overflow: hidden;
21+ font-size: 0.9rem;
22+ line-height: 1.3;
23+}
24+
25+th {
26+ background-color: #f8f9fa;
27+ padding: 8px 12px;
28+ text-align: left;
29+ font-weight: 600;
30+ color: #333;
31+ border-bottom: 1px solid #dee2e6;
32+ cursor: pointer;
33+ white-space: nowrap;
34+}
35+
36+th:hover {
37+ background-color: #e9ecef;
38+}
39+
40+td {
41+ padding: 6px 12px;
42+ border-bottom: 1px solid #dee2e6;
43+ color: #444;
44+}
45+
46+tr:hover {
47+ background-color: #f8f9fa;
48+}
49+
50+tr:last-child td {
51+ border-bottom: none;
52+}
53+
54+.table-container {
55+ margin: 12px 0;
56+ overflow-x: auto;
57+}
58+
59+/* Make filter controls more compact */
60+select {
61+ padding: 4px 8px !important;
62+ font-size: 0.9rem !important;
63+ min-width: 150px !important;
64+}
65+
66+.table-container label {
67+ font-size: 0.9rem;
68+}
web-ui/src/main.tsxadded+10−0View file
@@ -0,0 +1,10 @@
1+import { StrictMode } from "react";
2+import { createRoot } from "react-dom/client";
3+import "./index.css";
4+import App from "./App.tsx";
5+
6+createRoot(document.getElementById("root")!).render(
7+ <StrictMode>
8+ <App />
9+ </StrictMode>,
10+);
web-ui/src/pages/BenchmarkView.tsxadded+251−0View file
@@ -0,0 +1,251 @@
1+import { useLocation, useNavigate, useParams } from "react-router-dom";
2+import { useReducer, useEffect } from "react";
3+import { tabsReducer } from "../reducers/tabsReducer";
4+import { AlgorithmContent } from "../components/algorithm/AlgorithmContent";
5+import { DatasetContent } from "../components/dataset/DatasetContent";
6+import {
7+ AlgorithmTable,
8+ DatasetTable,
9+} from "../components/tables/DatasetAlgorithmTables";
10+import { useBenchmarkChartData } from "../hooks/useBenchmarkChartData";
11+import { useTagFilter } from "../hooks/useTagFilter";
12+import { BenchmarkData } from "../types";
13+
14+interface BenchmarkViewProps {
15+ benchmarkData: BenchmarkData | null;
16+}
17+
18+export default function BenchmarkView({ benchmarkData }: BenchmarkViewProps) {
19+ const location = useLocation();
20+ const navigate = useNavigate();
21+ const { datasetName, algorithmName } = useParams<{
22+ datasetName?: string;
23+ algorithmName?: string;
24+ }>();
25+
26+ const [tabsState, dispatch] = useReducer(tabsReducer, {
27+ tabs: [
28+ { id: "datasets", label: "Datasets", route: "/datasets" },
29+ { id: "algorithms", label: "Algorithms", route: "/algorithms" },
30+ ],
31+ activeTabId: "datasets",
32+ });
33+
34+ // Effect to handle URL changes and update tabs
35+ useEffect(() => {
36+ if (datasetName) {
37+ dispatch({
38+ type: "ADD_TAB",
39+ payload: {
40+ id: `dataset-${datasetName}`,
41+ label: datasetName,
42+ route: `/dataset/${datasetName}`,
43+ },
44+ });
45+ } else if (algorithmName) {
46+ dispatch({
47+ type: "ADD_TAB",
48+ payload: {
49+ id: `algorithm-${algorithmName}`,
50+ label: algorithmName,
51+ route: `/algorithm/${algorithmName}`,
52+ },
53+ });
54+ } else if (location.pathname.includes("/algorithms")) {
55+ dispatch({ type: "SET_ACTIVE_TAB", payload: "algorithms" });
56+ } else if (location.pathname.includes("/datasets")) {
57+ dispatch({ type: "SET_ACTIVE_TAB", payload: "datasets" });
58+ }
59+ }, [datasetName, algorithmName, location.pathname]);
60+
61+ // Handle tab click
62+ const handleTabClick = (tabId: string, route: string) => {
63+ dispatch({ type: "SET_ACTIVE_TAB", payload: tabId });
64+ navigate(route);
65+ };
66+
67+ // Get specific dataset or algorithm if viewing one
68+ const dataset = datasetName
69+ ? benchmarkData?.datasets.find((d) => d.name === datasetName)
70+ : undefined;
71+ const algorithm = algorithmName
72+ ? benchmarkData?.algorithms.find((a) => a.name === algorithmName)
73+ : undefined;
74+
75+ // Get chart data for specific dataset or algorithm view
76+ const chartData = useBenchmarkChartData(
77+ benchmarkData?.results || [],
78+ dataset?.name || null,
79+ algorithm?.name || null,
80+ );
81+
82+ // Get selected tags from URL
83+ const searchParams = new URLSearchParams(location.search);
84+ const selectedTags = searchParams.get("tag")?.split(",") || [];
85+
86+ // Set up tag filtering for datasets and algorithms
87+ const {
88+ availableTags: availableDatasetTags,
89+ filteredItems: filteredDatasets,
90+ } = useTagFilter(
91+ benchmarkData?.datasets || [],
92+ location.pathname.includes("/datasets") ? selectedTags : [],
93+ );
94+
95+ const {
96+ availableTags: availableAlgorithmTags,
97+ filteredItems: filteredAlgorithms,
98+ } = useTagFilter(
99+ benchmarkData?.algorithms || [],
100+ location.pathname.includes("/algorithms") ? selectedTags : [],
101+ );
102+
103+ // Handle tag toggling by updating URL
104+ const handleTagToggle = (tag: string) => {
105+ const newTags = selectedTags.includes(tag)
106+ ? selectedTags.filter((t) => t !== tag)
107+ : [...selectedTags, tag];
108+
109+ const params = new URLSearchParams();
110+ if (newTags.length > 0) {
111+ params.set("tag", newTags.join(","));
112+ }
113+ navigate({ search: params.toString() });
114+ };
115+
116+ return (
117+ <div>
118+ <main>
119+ <div
120+ style={{
121+ position: "fixed",
122+ top: "3rem",
123+ left: 0,
124+ right: 0,
125+ backgroundColor: "white",
126+ zIndex: 999,
127+ padding: "0 2rem 0 2rem",
128+ marginTop: "-4px",
129+ boxShadow: "0 2px 4px rgba(0,0,0,0.1)",
130+ borderBottom: "1px solid #eaeaea",
131+ }}
132+ >
133+ <div
134+ style={{
135+ paddingBottom: "2px",
136+ display: "flex",
137+ gap: "4px",
138+ overflowX: "auto",
139+ width: "100%",
140+ backgroundColor: "white",
141+ }}
142+ >
143+ {tabsState.tabs.map((tab) => (
144+ <div
145+ key={tab.id}
146+ style={{
147+ display: "flex",
148+ alignItems: "center",
149+ gap: "4px",
150+ }}
151+ >
152+ <button
153+ onClick={() => handleTabClick(tab.id, tab.route)}
154+ style={{
155+ padding: "8px 16px",
156+ border: "none",
157+ background: "none",
158+ borderBottom:
159+ tabsState.activeTabId === tab.id
160+ ? "2px solid #0066cc"
161+ : "none",
162+ color:
163+ tabsState.activeTabId === tab.id ? "#0066cc" : "#666",
164+ fontWeight:
165+ tabsState.activeTabId === tab.id ? "600" : "normal",
166+ cursor: "pointer",
167+ textDecoration: "none",
168+ whiteSpace: "nowrap",
169+ }}
170+ >
171+ {tab.label}
172+ </button>
173+ {tab.id !== "datasets" && tab.id !== "algorithms" && (
174+ <button
175+ onClick={(e) => {
176+ e.stopPropagation();
177+ const newActiveTab =
178+ tab.id === tabsState.activeTabId
179+ ? tabsState.tabs[0].id // Default to first tab if closing active
180+ : tabsState.activeTabId;
181+ dispatch({ type: "CLOSE_TAB", payload: tab.id });
182+ // Navigate if closing active tab
183+ if (tab.id === tabsState.activeTabId) {
184+ const defaultTab = tabsState.tabs.find(
185+ (t) => t.id === newActiveTab,
186+ );
187+ if (defaultTab) {
188+ navigate(defaultTab.route);
189+ }
190+ }
191+ }}
192+ style={{
193+ padding: "4px",
194+ border: "none",
195+ background: "none",
196+ color: "#666",
197+ cursor: "pointer",
198+ fontSize: "12px",
199+ display: "flex",
200+ alignItems: "center",
201+ justifyContent: "center",
202+ width: "20px",
203+ height: "20px",
204+ borderRadius: "50%",
205+ marginRight: "4px",
206+ marginLeft: "-4px",
207+ }}
208+ aria-label="Close tab"
209+ >
210+ ×
211+ </button>
212+ )}
213+ </div>
214+ ))}
215+ </div>
216+ </div>
217+
218+ <div style={{ padding: "3rem 0 1rem 0" }}>
219+ {dataset ? (
220+ <DatasetContent
221+ dataset={dataset}
222+ benchmarkData={benchmarkData}
223+ chartData={chartData}
224+ />
225+ ) : algorithm ? (
226+ <AlgorithmContent
227+ algorithm={algorithm}
228+ benchmarkData={benchmarkData}
229+ chartData={chartData}
230+ />
231+ ) : tabsState.activeTabId === "datasets" ? (
232+ <DatasetTable
233+ filteredDatasets={filteredDatasets}
234+ availableDatasetTags={availableDatasetTags}
235+ selectedTags={selectedTags}
236+ toggleTag={handleTagToggle}
237+ benchmarkResults={benchmarkData?.results || []}
238+ />
239+ ) : (
240+ <AlgorithmTable
241+ filteredAlgorithms={filteredAlgorithms}
242+ availableAlgorithmTags={availableAlgorithmTags}
243+ selectedTags={selectedTags}
244+ toggleTag={handleTagToggle}
245+ />
246+ )}
247+ </div>
248+ </main>
249+ </div>
250+ );
251+}
web-ui/src/pages/Home.tsxadded+192−0View file
@@ -0,0 +1,192 @@
1+import React, { useEffect, useState } from "react";
2+import axios from "axios";
3+import { Link } from "react-router-dom";
4+import yaml from "yaml";
5+import contentYaml from "../content/home-content.yml?raw";
6+import { HomeContent, HomeSection } from "../types/home-content";
7+import "../components/Button.css";
8+
9+const content = yaml.parse(contentYaml) as HomeContent;
10+
11+const SectionCard: React.FC<{ section: HomeSection }> = ({ section }) => {
12+ if (section.external) {
13+ return (
14+ <div
15+ style={{
16+ padding: "0.75rem",
17+ border: "1px solid #eaeaea",
18+ borderRadius: "8px",
19+ backgroundColor: "#f9f9f9",
20+ display: "flex",
21+ flexDirection: "column",
22+ height: "100%",
23+ }}
24+ >
25+ <h2 style={{ fontSize: "1.25rem", marginBottom: "0.5rem" }}>
26+ {section.title}
27+ </h2>
28+ <p style={{ marginBottom: "0.5rem" }}>{section.description}</p>
29+ <a
30+ href={section.link}
31+ target="_blank"
32+ rel="noopener noreferrer"
33+ className="soft-button"
34+ style={{ marginTop: "auto", alignSelf: "flex-start" }}
35+ >
36+ {section.linkText}
37+ </a>
38+ </div>
39+ );
40+ }
41+
42+ return (
43+ <div
44+ style={{
45+ padding: "0.75rem",
46+ border: "1px solid #eaeaea",
47+ borderRadius: "8px",
48+ backgroundColor: "#f9f9f9",
49+ display: "flex",
50+ flexDirection: "column",
51+ height: "100%",
52+ }}
53+ >
54+ <h2 style={{ fontSize: "1.25rem", marginBottom: "0.5rem" }}>
55+ {section.title}
56+ </h2>
57+ <p style={{ marginBottom: "0.5rem" }}>{section.description}</p>
58+ <Link
59+ to={section.link}
60+ className="soft-button"
61+ style={{ marginTop: "auto", alignSelf: "flex-start" }}
62+ >
63+ {section.linkText}
64+ </Link>
65+ </div>
66+ );
67+};
68+
69+interface BenchmarkStatus {
70+ current_dataset: string;
71+ current_algorithm: string;
72+ completed_count: number;
73+ total_count: number;
74+ progress_percentage: number;
75+ elapsed_time: number;
76+ last_update: string;
77+ completed_benchmarks: Array<{
78+ dataset: string;
79+ algorithm: string;
80+ compression_ratio: number;
81+ encode_time: number;
82+ decode_time: number;
83+ cache_status: string;
84+ }>;
85+}
86+
87+export default function Home() {
88+ const [status, setStatus] = useState<BenchmarkStatus | null>(null);
89+
90+ useEffect(() => {
91+ const fetchStatus = async () => {
92+ try {
93+ const cacheBust = Math.random().toString(36).substring(2, 15);
94+ const response = await axios.get(
95+ `https://tempory.net/f/memobin/benchmark_status/current.json?cachebust=${cacheBust}`,
96+ );
97+ setStatus(response.data);
98+ } catch (error) {
99+ console.error("Error fetching benchmark status:", error);
100+ }
101+ };
102+
103+ fetchStatus();
104+ }, []);
105+
106+ return (
107+ <div style={{ maxWidth: "1200px", margin: "0 auto", padding: "2rem" }}>
108+ <h1
109+ style={{
110+ marginBottom: "2rem",
111+ display: "flex",
112+ alignItems: "center",
113+ gap: "1rem",
114+ }}
115+ >
116+ <img
117+ src={`${import.meta.env.BASE_URL}logo.svg`}
118+ alt="BenchCompress Logo"
119+ style={{ width: "40px", height: "auto" }}
120+ />
121+ {content.title}
122+ </h1>
123+
124+ <p
125+ style={{ fontSize: "1.1rem", lineHeight: "1.6", marginBottom: "2rem" }}
126+ >
127+ {content.description}
128+ </p>
129+
130+ <div
131+ style={{
132+ display: "grid",
133+ gap: "2rem",
134+ gridTemplateColumns: "repeat(auto-fit, minmax(300px, 1fr))",
135+ }}
136+ >
137+ {Object.entries(content.sections).map(([key, section]) => (
138+ <SectionCard key={key} section={section} />
139+ ))}
140+ </div>
141+
142+ <hr
143+ style={{
144+ margin: "3rem 0",
145+ border: "none",
146+ borderTop: "1px solid #eaeaea",
147+ }}
148+ />
149+
150+ <footer
151+ style={{ textAlign: "center", color: "#666", fontSize: "0.9rem" }}
152+ >
153+ <p>
154+ Last UI update: {__BUILD_DATE__}
155+ <br />
156+ {status && (
157+ <>
158+ <div
159+ style={{
160+ margin: "1rem 0",
161+ padding: "0.5rem",
162+ border: "1px solid #eaeaea",
163+ borderRadius: "4px",
164+ display: "inline-block",
165+ }}
166+ >
167+ Last benchmark run:{" "}
168+ {new Date(status.last_update).toLocaleString()}
169+ <br />
170+ Status:{" "}
171+ {status.progress_percentage === 100
172+ ? "Completed"
173+ : "In Progress"}{" "}
174+ ({status.completed_count}/{status.total_count} benchmarks)
175+ </div>
176+ <br />
177+ </>
178+ )}
179+ Released under{" "}
180+ <a
181+ href="https://github.com/magland/benchcompress/blob/main/LICENSE"
182+ target="_blank"
183+ rel="noopener noreferrer"
184+ style={{ color: "#666", textDecoration: "underline" }}
185+ >
186+ Apache License 2.0
187+ </a>
188+ </p>
189+ </footer>
190+ </div>
191+ );
192+}
web-ui/src/pages/Monitor.tsxadded+287−0View file
@@ -0,0 +1,287 @@
1+import { useEffect, useState } from "react";
2+import axios from "axios";
3+
4+interface BenchmarkStatus {
5+ current_dataset: string;
6+ current_algorithm: string;
7+ completed_count: number;
8+ total_count: number;
9+ progress_percentage: number;
10+ elapsed_time: number;
11+ last_update: string;
12+ completed_benchmarks: Array<{
13+ dataset: string;
14+ algorithm: string;
15+ compression_ratio: number;
16+ encode_time: number;
17+ decode_time: number;
18+ cache_status: string;
19+ }>;
20+}
21+
22+export default function Monitor() {
23+ const [status, setStatus] = useState<BenchmarkStatus | null>(null);
24+ const [error, setError] = useState<string | null>(null);
25+ const [loading, setLoading] = useState(true);
26+
27+ const fetchStatus = async () => {
28+ setLoading(true);
29+ try {
30+ const cacheBust = Math.random().toString(36).substring(2, 15);
31+ const response = await axios.get(
32+ `https://tempory.net/f/memobin/benchmark_status/current.json?cachebust=${cacheBust}`,
33+ );
34+ setStatus(response.data);
35+ setError(null);
36+ } catch (error) {
37+ const message =
38+ error instanceof Error ? error.message : "Failed to fetch status";
39+ setError(message);
40+ console.error("Error fetching benchmark status:", error);
41+ } finally {
42+ setLoading(false);
43+ }
44+ };
45+
46+ useEffect(() => {
47+ fetchStatus();
48+ }, []);
49+
50+ if (loading) {
51+ return <div>Loading benchmark status...</div>;
52+ }
53+
54+ if (error) {
55+ return <div>Error: {error}</div>;
56+ }
57+
58+ if (!status) {
59+ return <div>No active benchmark run found.</div>;
60+ }
61+
62+ const formatTime = (seconds: number) => {
63+ const hours = Math.floor(seconds / 3600);
64+ const minutes = Math.floor((seconds % 3600) / 60);
65+ const remainingSeconds = Math.floor(seconds % 60);
66+ return `${hours}h ${minutes}m ${remainingSeconds}s`;
67+ };
68+
69+ return (
70+ <div style={{ padding: "20px" }}>
71+ <div
72+ style={{
73+ display: "flex",
74+ alignItems: "center",
75+ gap: "20px",
76+ marginBottom: "20px",
77+ }}
78+ >
79+ <h1 style={{ margin: 0 }}>Benchmark Progress</h1>
80+ <button
81+ onClick={fetchStatus}
82+ style={{
83+ padding: "8px 16px",
84+ backgroundColor: "#4CAF50",
85+ color: "white",
86+ border: "none",
87+ borderRadius: "4px",
88+ cursor: "pointer",
89+ display: "flex",
90+ alignItems: "center",
91+ gap: "8px",
92+ }}
93+ disabled={loading}
94+ >
95+ {loading ? "Refreshing..." : "Refresh"}
96+ </button>
97+ </div>
98+
99+ <div style={{ marginBottom: "20px" }}>
100+ <h2>Current Status</h2>
101+ <div
102+ style={{
103+ border: "1px solid #eee",
104+ padding: "20px",
105+ borderRadius: "8px",
106+ backgroundColor: "#f9f9f9",
107+ }}
108+ >
109+ <p>
110+ <strong>Current Dataset:</strong> {status.current_dataset}
111+ </p>
112+ <p>
113+ <strong>Current Algorithm:</strong> {status.current_algorithm}
114+ </p>
115+ <p>
116+ <strong>Progress:</strong> {status.completed_count} /{" "}
117+ {status.total_count} ({status.progress_percentage.toFixed(1)}%)
118+ </p>
119+ <p>
120+ <strong>Elapsed Time:</strong> {formatTime(status.elapsed_time)}
121+ </p>
122+ <p>
123+ <strong>Last Update:</strong>{" "}
124+ {new Date(status.last_update).toLocaleString()}
125+ </p>
126+
127+ <div style={{ marginTop: "10px" }}>
128+ <div
129+ style={{
130+ width: "100%",
131+ height: "20px",
132+ backgroundColor: "#eee",
133+ borderRadius: "10px",
134+ overflow: "hidden",
135+ }}
136+ >
137+ <div
138+ style={{
139+ width: `${status.progress_percentage}%`,
140+ height: "100%",
141+ backgroundColor: "#4CAF50",
142+ transition: "width 0.5s ease-in-out",
143+ }}
144+ />
145+ </div>
146+ </div>
147+ </div>
148+ </div>
149+
150+ <div>
151+ <h2>Completed Benchmarks</h2>
152+ <div style={{ overflowX: "auto" }}>
153+ <table
154+ style={{
155+ width: "100%",
156+ borderCollapse: "collapse",
157+ marginTop: "10px",
158+ }}
159+ >
160+ <thead>
161+ <tr style={{ backgroundColor: "#f5f5f5" }}>
162+ <th
163+ style={{
164+ padding: "12px",
165+ textAlign: "left",
166+ borderBottom: "2px solid #ddd",
167+ }}
168+ >
169+ Dataset
170+ </th>
171+ <th
172+ style={{
173+ padding: "12px",
174+ textAlign: "left",
175+ borderBottom: "2px solid #ddd",
176+ }}
177+ >
178+ Algorithm
179+ </th>
180+ <th
181+ style={{
182+ padding: "12px",
183+ textAlign: "right",
184+ borderBottom: "2px solid #ddd",
185+ }}
186+ >
187+ Compression Ratio
188+ </th>
189+ <th
190+ style={{
191+ padding: "12px",
192+ textAlign: "right",
193+ borderBottom: "2px solid #ddd",
194+ }}
195+ >
196+ Encode Time (ms)
197+ </th>
198+ <th
199+ style={{
200+ padding: "12px",
201+ textAlign: "right",
202+ borderBottom: "2px solid #ddd",
203+ }}
204+ >
205+ Decode Time (ms)
206+ </th>
207+ <th
208+ style={{
209+ padding: "12px",
210+ textAlign: "center",
211+ borderBottom: "2px solid #ddd",
212+ }}
213+ >
214+ Cache Status
215+ </th>
216+ </tr>
217+ </thead>
218+ <tbody>
219+ {status.completed_benchmarks.map((benchmark, index) => (
220+ <tr
221+ key={index}
222+ style={{
223+ backgroundColor:
224+ benchmark.cache_status === "cached"
225+ ? "#f5f5f5"
226+ : index % 2 === 0
227+ ? "white"
228+ : "#fafafa",
229+ }}
230+ >
231+ <td
232+ style={{ padding: "12px", borderBottom: "1px solid #ddd" }}
233+ >
234+ {benchmark.dataset}
235+ </td>
236+ <td
237+ style={{ padding: "12px", borderBottom: "1px solid #ddd" }}
238+ >
239+ {benchmark.algorithm}
240+ </td>
241+ <td
242+ style={{
243+ padding: "12px",
244+ textAlign: "right",
245+ borderBottom: "1px solid #ddd",
246+ }}
247+ >
248+ {benchmark.compression_ratio.toFixed(2)}x
249+ </td>
250+ <td
251+ style={{
252+ padding: "12px",
253+ textAlign: "right",
254+ borderBottom: "1px solid #ddd",
255+ }}
256+ >
257+ {(benchmark.encode_time * 1000).toFixed(2)}
258+ </td>
259+ <td
260+ style={{
261+ padding: "12px",
262+ textAlign: "right",
263+ borderBottom: "1px solid #ddd",
264+ }}
265+ >
266+ {(benchmark.decode_time * 1000).toFixed(2)}
267+ </td>
268+ <td
269+ style={{
270+ padding: "12px",
271+ textAlign: "center",
272+ borderBottom: "1px solid #ddd",
273+ color:
274+ benchmark.cache_status === "cached" ? "#888" : "#000",
275+ }}
276+ >
277+ {benchmark.cache_status}
278+ </td>
279+ </tr>
280+ ))}
281+ </tbody>
282+ </table>
283+ </div>
284+ </div>
285+ </div>
286+ );
287+}
web-ui/src/pages/Paper.cssadded+53−0View file
@@ -0,0 +1,53 @@
1+.paper-container {
2+ max-width: 800px;
3+ margin: 0 auto;
4+ padding: 1.5rem;
5+ background-color: white;
6+ box-shadow: 0 0 10px rgba(0,0,0,0.1);
7+ font-family: 'Roboto', sans-serif;
8+}
9+
10+.paper-content h1 {
11+ font-size: 22px;
12+ font-weight: bold;
13+ margin-bottom: 0.5rem;
14+ text-align: center;
15+}
16+
17+.paper-content h2 {
18+ font-size: 16px;
19+ font-weight: bold;
20+ margin-top: 1rem;
21+ margin-bottom: 0.5rem;
22+}
23+
24+.paper-content h3 {
25+ font-size: 14px;
26+ font-weight: bold;
27+ margin-top: 1rem;
28+ margin-bottom: 0.25rem;
29+}
30+
31+.paper-content p {
32+ font-size: 15px;
33+ line-height: 1.3;
34+ margin-bottom: 0.5rem;
35+ text-align: justify;
36+}
37+
38+.paper-content em {
39+ font-style: italic;
40+}
41+
42+.paper-content .math-display {
43+ margin: 0.5rem 0;
44+ overflow-x: auto;
45+ text-align: center;
46+}
47+
48+.paper-content code {
49+ font-size: 12px;
50+ background-color: #f5f5f5;
51+ padding: 1px 3px;
52+ border-radius: 2px;
53+}
web-ui/src/pages/Paper.tsxadded+19−0View file
@@ -0,0 +1,19 @@
1+import React from "react";
2+import "./Paper.css";
3+
4+const Paper: React.FC = () => {
5+ return (
6+ <div className="paper-container">
7+ <div className="paper-content">
8+ <iframe
9+ src="/benchcompress/paper.pdf#toolbar=0&navpanes=0"
10+ width="100%"
11+ height="100%"
12+ style={{ minHeight: "100vh", border: "none" }}
13+ />
14+ </div>
15+ </div>
16+ );
17+};
18+
19+export default Paper;
web-ui/src/pages/Submit.tsxadded+14−0View file
@@ -0,0 +1,14 @@
1+import submitContent from "./submit.md?raw";
2+import ReactMarkdown from "react-markdown";
3+import remarkMath from "remark-math";
4+import rehypeKatex from "rehype-katex";
5+
6+export default function Submit() {
7+ return (
8+ <div className="content-container">
9+ <ReactMarkdown remarkPlugins={[remarkMath]} rehypePlugins={[rehypeKatex]}>
10+ {submitContent}
11+ </ReactMarkdown>
12+ </div>
13+ );
14+}
web-ui/src/pages/submit.mdadded+143−0View file
@@ -0,0 +1,143 @@
1+# Contributing to Benchcompress
2+
3+This guide explains how to contribute new algorithms or datasets to Benchcompress.
4+
5+## Overview
6+
7+Benchcompress welcomes contributions of new compression algorithms and scientific datasets. The framework is designed to make it easy to add new components while ensuring consistent benchmarking and evaluation.
8+
9+## Getting Started
10+
11+1. Fork and clone the repository:
12+```bash
13+git clone https://github.com/[your-username]/benchcompress.git
14+cd benchcompress
15+```
16+
17+2. Install dependencies:
18+```bash
19+# Install Python package
20+cd benchcompress
21+pip install -e .
22+
23+# Install pre-commit hooks for code compliance checks
24+pip install pre-commit
25+pre-commit install
26+```
27+
28+## Adding a New Algorithm
29+
30+New algorithms are added in `benchcompress/src/benchcompress/algorithms/`. Each algorithm should:
31+
32+1. Create a new directory with:
33+ - `__init__.py`: Algorithm implementation
34+ - `algorithm-name.md`: Documentation and description
35+
36+2. In `__init__.py`:
37+ - Implement compression/decompression functions
38+ - Define metadata (version, tags, compatibility)
39+ - Follow existing algorithms as examples
40+
41+Example structure:
42+```
43+algorithms/
44+└── my_algorithm/
45+ ├── __init__.py
46+ └── my_algorithm.md
47+```
48+
49+## Adding a New Dataset
50+
51+New datasets are added in `benchcompress/src/benchcompress/datasets/`. Each dataset should:
52+
53+1. Create a new directory with:
54+ - `__init__.py`: Dataset generation/loading code
55+ - `dataset-name.md`: Documentation and description
56+
57+2. In `__init__.py`:
58+ - Implement data generation/loading
59+ - Define metadata (version, tags)
60+ - Follow existing datasets as examples
61+
62+Example structure:
63+```
64+datasets/
65+└── my_dataset/
66+ ├── __init__.py
67+ └── my_dataset.md
68+```
69+
70+## Testing Locally
71+
72+Run benchmarks for your new component:
73+```bash
74+benchcompress run --algorithm my_algorithm --dataset my_dataset
75+```
76+
77+The framework will automatically:
78+- Run the benchmarks
79+- Verify results by decompressing and comparing with original data
80+- Measure compression ratios and throughput
81+
82+## Code Formatting
83+
84+The project uses specific formatters for each language:
85+- Python: black formatter
86+- TypeScript/JavaScript: ESLint + Prettier
87+- C++: clang-format
88+
89+To format your code before committing:
90+```bash
91+# From project root
92+./devel/format_code.sh
93+```
94+
95+This will format all code according to project standards.
96+
97+## Code Compliance
98+
99+Pre-commit hooks will check code compliance when you commit changes. They verify:
100+- Code formatting
101+- Import ordering
102+- Type checking
103+- Other project-specific rules
104+
105+If checks fail, format your code using the format script and try again.
106+
107+## Creating a Pull Request
108+
109+1. Create a new branch:
110+```bash
111+git checkout -b add-my-component
112+```
113+
114+2. Format code and ensure it passes compliance checks:
115+```bash
116+./devel/format_code.sh
117+```
118+
119+3. Commit your changes:
120+```bash
121+git add .
122+git commit -m "Add new algorithm/dataset: [name]"
123+```
124+
125+4. Push to your fork:
126+```bash
127+git push origin add-my-component
128+```
129+
130+5. Open a pull request on GitHub with:
131+ - Clear description of the new component
132+ - Any relevant background or references
133+ - Local benchmark results
134+ - Confirmation that code is formatted and passes checks
135+
136+## Guidelines
137+
138+- Follow existing code structure and patterns
139+- Include thorough documentation
140+- Add appropriate tags for filtering
141+- Test compatibility with existing components
142+- Format code using provided script
143+- Ensure all pre-commit checks pass
web-ui/src/reducers/tabsReducer.tsadded+85−0View file
@@ -0,0 +1,85 @@
1+export type TabAction =
2+ | { type: "ADD_TAB"; payload: { id: string; label: string; route: string } }
3+ | { type: "SET_ACTIVE_TAB"; payload: string }
4+ | { type: "CLOSE_TAB"; payload: string }
5+ | { type: "REORDER_TABS"; payload: { fromIndex: number; toIndex: number } };
6+
7+export interface TabItem {
8+ id: string;
9+ label: string;
10+ route: string;
11+}
12+
13+export interface TabsState {
14+ tabs: TabItem[];
15+ activeTabId: string;
16+}
17+
18+const initialState: TabsState = {
19+ tabs: [
20+ { id: "datasets", label: "Datasets", route: "/datasets" },
21+ { id: "algorithms", label: "Algorithms", route: "/algorithms" },
22+ ],
23+ activeTabId: "datasets",
24+};
25+
26+export function tabsReducer(
27+ state: TabsState = initialState,
28+ action: TabAction,
29+): TabsState {
30+ switch (action.type) {
31+ case "ADD_TAB": {
32+ // If tab already exists, just set it as active
33+ if (state.tabs.some((tab) => tab.id === action.payload.id)) {
34+ return {
35+ ...state,
36+ activeTabId: action.payload.id,
37+ };
38+ }
39+
40+ // Otherwise add new tab
41+ return {
42+ ...state,
43+ tabs: [...state.tabs, action.payload],
44+ activeTabId: action.payload.id,
45+ };
46+ }
47+ case "SET_ACTIVE_TAB":
48+ return {
49+ ...state,
50+ activeTabId: action.payload,
51+ };
52+ case "REORDER_TABS": {
53+ const newTabs = [...state.tabs];
54+ const [movedTab] = newTabs.splice(action.payload.fromIndex, 1);
55+ newTabs.splice(action.payload.toIndex, 0, movedTab);
56+ return {
57+ ...state,
58+ tabs: newTabs,
59+ };
60+ }
61+ case "CLOSE_TAB": {
62+ // Don't allow closing of datasets or algorithms tabs
63+ if (action.payload === "datasets" || action.payload === "algorithms") {
64+ return state;
65+ }
66+
67+ const newTabs = state.tabs.filter((tab) => tab.id !== action.payload);
68+
69+ // If we're closing the active tab, activate the last tab in the list
70+ if (state.activeTabId === action.payload) {
71+ return {
72+ tabs: newTabs,
73+ activeTabId: newTabs[newTabs.length - 1].id,
74+ };
75+ }
76+
77+ return {
78+ ...state,
79+ tabs: newTabs,
80+ };
81+ }
82+ default:
83+ return state;
84+ }
85+}
web-ui/src/types.tsadded+60−0View file
@@ -0,0 +1,60 @@
1+export interface BenchmarkResult {
2+ dataset: string;
3+ algorithm: string;
4+ algorithm_version: string;
5+ dataset_version: string;
6+ system_version: string;
7+ compression_ratio: number;
8+ encode_time: number;
9+ decode_time: number;
10+ encode_mb_per_sec: number;
11+ decode_mb_per_sec: number;
12+ original_size: number;
13+ compressed_size: number;
14+ array_shape: number[];
15+ array_dtype: string;
16+ timestamp: number;
17+}
18+
19+export interface Algorithm {
20+ name: string;
21+ description: string;
22+ long_description?: string;
23+ version: string;
24+ tags: string[];
25+ source_file?: string;
26+}
27+
28+export interface Dataset {
29+ name: string;
30+ description: string;
31+ long_description?: string;
32+ version: string;
33+ tags: string[];
34+ source_file?: string;
35+ data_url_npy?: string; // URL to download the dataset as .npy
36+ data_url_raw?: string; // URL to download the raw dataset as .dat
37+ data_url_json?: string; // URL to download the dataset info as .json (dtype and shape)
38+}
39+
40+export interface BenchmarkData {
41+ results: BenchmarkResult[];
42+ algorithms: Algorithm[];
43+ datasets: Dataset[];
44+}
45+
46+export interface TabItem {
47+ id: string;
48+ label: string;
49+ route: string;
50+}
51+
52+export interface TabsState {
53+ tabs: TabItem[];
54+ activeTabId: string;
55+}
56+
57+export type TabAction =
58+ | { type: "ADD_TAB"; payload: { id: string; label: string; route: string } }
59+ | { type: "SET_ACTIVE_TAB"; payload: string }
60+ | { type: "REORDER_TABS"; payload: { fromIndex: number; toIndex: number } };
web-ui/src/types/home-content.tsadded+15−0View file
@@ -0,0 +1,15 @@
1+export interface HomeSection {
2+ title: string;
3+ description: string;
4+ link: string;
5+ linkText: string;
6+ external?: boolean;
7+}
8+
9+export interface HomeContent {
10+ title: string;
11+ description: string;
12+ sections: {
13+ [key: string]: HomeSection;
14+ };
15+}
web-ui/src/vite-env.d.tsadded+1−0View file
@@ -0,0 +1 @@
1+/// <reference types="vite/client" />
web-ui/tsconfig.app.jsonadded+24−0View file
@@ -0,0 +1,24 @@
1+{
2+ "compilerOptions": {
3+ "target": "ES2020",
4+ "useDefineForClassFields": true,
5+ "lib": ["ES2020", "DOM", "DOM.Iterable"],
6+ "module": "ESNext",
7+ "skipLibCheck": true,
8+
9+ /* Bundler mode */
10+ "moduleResolution": "bundler",
11+ "allowImportingTsExtensions": true,
12+ "isolatedModules": true,
13+ "moduleDetection": "force",
14+ "noEmit": true,
15+ "jsx": "react-jsx",
16+
17+ /* Linting */
18+ "strict": true,
19+ "noUnusedLocals": true,
20+ "noUnusedParameters": true,
21+ "noFallthroughCasesInSwitch": true
22+ },
23+ "include": ["src"]
24+}
web-ui/tsconfig.app.tsbuildinfoadded+1−0View file
@@ -0,0 +1 @@
1+{"root":["./src/App.tsx","./src/env.d.ts","./src/main.tsx","./src/types.ts","./src/vite-env.d.ts","./src/components/BenchmarkTable.tsx","./src/components/ScrollToTop.tsx","./src/components/TagFilter.tsx","./src/components/algorithm/AlgorithmContent.tsx","./src/components/benchmark/charts/BenchmarkCharts.tsx","./src/components/benchmark/export/csvExport.ts","./src/components/benchmark/table/BenchmarkTable.tsx","./src/components/benchmark/table/columns.tsx","./src/components/benchmark/utils/formatters.ts","./src/components/dataset/DatasetContent.tsx","./src/components/dataset/TimeseriesNavigationBar.tsx","./src/components/dataset/TimeseriesView.tsx","./src/components/dataset/TimeseriesViewWorker.ts","./src/components/dataset/WorkerTypes.ts","./src/components/dataset/timeseriesViewReducer.ts","./src/components/shared/BaseContent.tsx","./src/components/tables/DatasetAlgorithmTables.tsx","./src/hooks/TimeseriesDataClient.ts","./src/hooks/useBenchmarkChartData.ts","./src/hooks/useMarkdownContent.ts","./src/hooks/useMarkdownPosts.ts","./src/hooks/useTagFilter.ts","./src/hooks/useTimeseriesData.ts","./src/hooks/useTimeseriesDataClient.ts","./src/pages/Algorithm.tsx","./src/pages/BenchmarkView.tsx","./src/pages/Dataset.tsx","./src/pages/Home.tsx","./src/pages/Monitor.tsx","./src/pages/Paper.tsx","./src/pages/Submit.tsx","./src/reducers/tabsReducer.ts","./src/types/home-content.ts"],"version":"5.6.3"}
web-ui/tsconfig.jsonadded+7−0View file
@@ -0,0 +1,7 @@
1+{
2+ "files": [],
3+ "references": [
4+ { "path": "./tsconfig.app.json" },
5+ { "path": "./tsconfig.node.json" }
6+ ]
7+}
web-ui/tsconfig.node.jsonadded+24−0View file
@@ -0,0 +1,24 @@
1+{
2+ "compilerOptions": {
3+ "incremental": true,
4+ "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.node.tsbuildinfo",
5+ "target": "ES2022",
6+ "lib": ["ES2023"],
7+ "module": "ESNext",
8+ "skipLibCheck": true,
9+
10+ /* Bundler mode */
11+ "moduleResolution": "bundler",
12+ "allowImportingTsExtensions": true,
13+ "isolatedModules": true,
14+ "moduleDetection": "force",
15+ "noEmit": true,
16+
17+ /* Linting */
18+ "strict": true,
19+ "noUnusedLocals": true,
20+ "noUnusedParameters": true,
21+ "noFallthroughCasesInSwitch": true
22+ },
23+ "include": ["vite.config.ts"]
24+}
web-ui/vite.config.tsadded+11−0View file
@@ -0,0 +1,11 @@
1+import { defineConfig } from 'vite'
2+import react from '@vitejs/plugin-react'
3+
4+// https://vite.dev/config/
5+export default defineConfig({
6+ plugins: [react()],
7+ base: '/benchcompress/', // Base URL for GitHub Pages deployment
8+ define: {
9+ __BUILD_DATE__: JSON.stringify(new Date().toLocaleDateString())
10+ }
11+})