Update benchmark results from 2026-06-16 14:21:30 [skip ci]
108 changed files+25497−0
.github/workflows/benchmark.ymladded+69−0View file
@@ -0,0 +1,69 @@
1+name: Run Benchmarks
2+
3+# Add concurrency to cancel in-progress jobs
4+concurrency:
5+ group: ${{ github.workflow }}-${{ github.ref }}
6+ cancel-in-progress: true
7+
8+on:
9+ workflow_dispatch: # Manual trigger
10+ push:
11+ branches: [ main ] # Run on main branch pushes
12+ paths:
13+ - 'python/**' # Run only if benchmarks are updated
14+
15+permissions:
16+ contents: write
17+
18+jobs:
19+ benchmark:
20+ name: Run Benchmarks
21+ runs-on: ubuntu-latest
22+
23+ steps:
24+ - uses: actions/checkout@v4
25+
26+ - name: Set up Python
27+ uses: actions/setup-python@v4
28+ with:
29+ python-version: '3.12'
30+
31+ - name: Install package and dependencies
32+ run: |
33+ cd python
34+ pip install -e .
35+
36+ - name: Run benchmarks
37+ env:
38+ MEMOBIN_API_KEY: ${{ secrets.MEMOBIN_API_KEY }}
39+ UPLOAD_TO_MEMOBIN: '1'
40+ run: |
41+ python scripts/run_benchmarks.py
42+
43+ - name: Upload benchmark results as artifacts
44+ uses: actions/upload-artifact@v4
45+ with:
46+ name: benchmark-results
47+ path: |
48+ python/benchmark_results/results.json
49+
50+ - name: Configure Git
51+ if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
52+ run: |
53+ git config user.name "GitHub Actions Bot"
54+ git config user.email "actions@github.com"
55+
56+ - name: Create fresh results branch
57+ if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
58+ run: |
59+ git checkout --orphan benchmark-results
60+
61+ - name: Commit benchmark results
62+ if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
63+ env:
64+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
65+ run: |
66+ # need to force add because it's in the .gitignore
67+ git add -f benchmark_results/
68+ git commit -m "Update benchmark results from $(date +'%Y-%m-%d %H:%M:%S') [skip ci]"
69+ git push -f https://${GITHUB_TOKEN}@github.com/${{ github.repository }} benchmark-results
.github/workflows/deploy-gh-pages.ymladded+61−0View file
@@ -0,0 +1,61 @@
1+name: Deploy to GitHub Pages
2+
3+on:
4+ push:
5+ branches: ['main']
6+ paths:
7+ - 'web-ui/**'
8+ workflow_dispatch:
9+
10+# Sets the GITHUB_TOKEN permissions to allow deployment to GitHub Pages
11+permissions:
12+ contents: read
13+ pages: write
14+ id-token: write
15+
16+# Allow one concurrent deployment
17+concurrency:
18+ group: 'pages'
19+ cancel-in-progress: true
20+
21+jobs:
22+ deploy:
23+ environment:
24+ name: github-pages
25+ url: ${{ steps.deployment.outputs.page_url }}
26+ runs-on: ubuntu-latest
27+ steps:
28+ - name: Checkout
29+ uses: actions/checkout@v4
30+
31+ - name: Install pandoc and texlive
32+ run: |
33+ sudo apt-get update
34+ sudo apt-get install -y pandoc texlive-latex-recommended
35+
36+ - name: Setup Node.js
37+ uses: actions/setup-node@v3
38+ with:
39+ node-version: '20'
40+ cache: 'npm'
41+ cache-dependency-path: web-ui/package-lock.json
42+
43+ - name: Install dependencies
44+ working-directory: web-ui
45+ run: npm ci
46+
47+ - name: Build website
48+ working-directory: web-ui
49+ run: npm run build
50+
51+ - name: Setup Pages
52+ uses: actions/configure-pages@v4
53+
54+ - name: Upload artifact
55+ uses: actions/upload-pages-artifact@v3
56+ with:
57+ path: ./web-ui/dist
58+
59+ - name: Deploy to GitHub Pages
60+ id: deployment
61+ uses: actions/deploy-pages@v4
.gitignoreadded+2−0View file
@@ -0,0 +1,2 @@
1+.benchmark_cache
2+benchmark_results
benchmark_results/results.jsonadded+8780−0View file
This diff is 8,786 lines long and is not shown.
prepare_datasets/.gitignoreadded+7−0View file
@@ -0,0 +1,7 @@
1+*.si
2+*.npy
3+__pycache__
4+*.pyc
5+*.pyo
6+*.pyd
7+
prepare_datasets/aind/list_s3_directories.pyadded+77−0View file
@@ -0,0 +1,77 @@
1+#!/usr/bin/env python3
2+"""
3+Script to list directories in s3://aind-benchmark-data/ephys-compression
4+Lists directories up to depth 2
5+"""
6+import boto3
7+from botocore import UNSIGNED
8+from botocore.config import Config
9+
10+
11+def list_s3_directories(bucket_name: str, prefix: str = '', depth: int = 1):
12+ """
13+ List directories (common prefixes) in an S3 bucket recursively up to a certain depth
14+
15+ Args:
16+ bucket_name: Name of the S3 bucket
17+ prefix: Prefix path to list directories under (default: root)
18+ depth: How deep to recurse (1 = only immediate subdirs)
19+ """
20+ # S3 configuration for public bucket (no credentials needed)
21+ s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED))
22+
23+ # Ensure prefix ends with / if not empty
24+ if prefix and not prefix.endswith('/'):
25+ prefix += '/'
26+
27+ # List objects with delimiter to get "directories"
28+ paginator = s3.get_paginator('list_objects_v2')
29+ pages = paginator.paginate(Bucket=bucket_name, Prefix=prefix, Delimiter='/')
30+
31+ directories = []
32+
33+ for page in pages:
34+ # CommonPrefixes contains the "directories"
35+ if 'CommonPrefixes' in page:
36+ for prefix_obj in page['CommonPrefixes']:
37+ dir_path = prefix_obj['Prefix']
38+ # Remove the base prefix to get relative path
39+ relative_dir = dir_path[len(prefix):] if prefix else dir_path
40+ directories.append(relative_dir)
41+
42+ # Sort directories
43+ directories = sorted(directories)
44+
45+ # Print current level directories and their subdirectories
46+ for directory in directories:
47+ print(f" {directory}")
48+
49+ # Recurse if depth > 1
50+ if depth > 1:
51+ # List subdirectories
52+ full_prefix = prefix + directory
53+ subpages = paginator.paginate(Bucket=bucket_name, Prefix=full_prefix, Delimiter='/')
54+
55+ subdirs = []
56+ for page in subpages:
57+ if 'CommonPrefixes' in page:
58+ for prefix_obj in page['CommonPrefixes']:
59+ dir_path = prefix_obj['Prefix']
60+ relative_dir = dir_path[len(full_prefix):] if full_prefix else dir_path
61+ subdirs.append(relative_dir)
62+
63+ # Print subdirectories with indentation
64+ for subdir in sorted(subdirs):
65+ print(f" {subdir}")
66+
67+ return directories
68+
69+
70+if __name__ == '__main__':
71+ bucket_name = 'aind-benchmark-data'
72+ prefix = 'ephys-compression'
73+
74+ print(f"Listing directories in s3://{bucket_name}/{prefix} (depth 2)")
75+ print()
76+
77+ list_s3_directories(bucket_name, prefix, depth=2)
prepare_datasets/aind/prepare_aind_compression_datasets.pyadded+48−0View file
@@ -0,0 +1,48 @@
1+import os
2+import spikeinterface as si
3+import numpy as np
4+from s3_utils import download_s3_folder
5+
6+s3_base_url = "s3://aind-benchmark-data/ephys-compression"
7+
8+folder_names = [
9+ # ("aind-np2/612962_2022-04-13_19-18-04_ProbeB", "aind-np2-probeB", "CH"),
10+ # ("aind-np1/625749_2022-08-03_15-15-06_ProbeA", "aind-np1-probeA", "AP")
11+ ("ibl-np1/CSHZAD026_2020-09-04_probe00", "ibl-np1-probe00", "AP"),
12+]
13+
14+for folder_name, name0, channel_prefix in folder_names:
15+ s3_folder_name = f"{s3_base_url}/{folder_name}"
16+ local_folder_name = f"{folder_name}.si"
17+
18+ if not os.path.exists(local_folder_name):
19+ # make parent directories if needed
20+ os.makedirs(os.path.dirname(local_folder_name), exist_ok=True)
21+ print(f'Downloading {s3_folder_name} to {local_folder_name}...')
22+ # IMPORTANT NOTE: we may interrupt this download early because we really only need the first part.
23+ download_s3_folder(s3_folder_name, local_folder_name)
24+
25+ # For now this only works with spikeinterface 0.102
26+ recording = si.load(
27+ local_folder_name
28+ )
29+
30+ channel_ids = [
31+ f'{channel_prefix}101',
32+ f'{channel_prefix}102',
33+ f'{channel_prefix}103',
34+ f'{channel_prefix}104',
35+ f'{channel_prefix}105',
36+ f'{channel_prefix}106',
37+ f'{channel_prefix}107',
38+ f'{channel_prefix}108',
39+ f'{channel_prefix}109',
40+ f'{channel_prefix}110'
41+ ]
42+
43+ fname = f'{name0}-ch101-110.raw.npy'
44+ if not os.path.exists(fname):
45+ print(f'Writing {fname}...')
46+ X = recording.get_traces(channel_ids=channel_ids, start_frame=30000, end_frame=30000 + 30000 * 10)
47+ print(f'X.shape = {X.shape}')
48+ np.save(fname, X)
prepare_datasets/aind/s3_utils.pyadded+132−0View file
@@ -0,0 +1,132 @@
1+"""
2+Utility functions for downloading from S3 public buckets
3+"""
4+import boto3
5+from botocore import UNSIGNED
6+from botocore.config import Config
7+import os
8+from pathlib import Path
9+import sys
10+
11+
12+class ProgressCallback:
13+ """Callback to show download progress"""
14+ def __init__(self, filename, filesize):
15+ self._filename = filename
16+ self._size = filesize
17+ self._seen_so_far = 0
18+
19+ def __call__(self, bytes_amount):
20+ self._seen_so_far += bytes_amount
21+ percentage = (self._seen_so_far / self._size) * 100 if self._size > 0 else 0
22+ sys.stdout.write(
23+ f"\r Progress: {self._seen_so_far:,} / {self._size:,} bytes ({percentage:.1f}%)"
24+ )
25+ sys.stdout.flush()
26+
27+
28+def download_s3_folder(s3_url: str, local_dir: str, skip_confirmation: bool = False):
29+ """
30+ Download entire folder from S3 public bucket
31+
32+ Args:
33+ s3_url: S3 URL in format s3://bucket-name/path/to/folder/
34+ local_dir: Local directory path to download files to
35+ skip_confirmation: If True, skip user confirmation prompt
36+ """
37+ # Parse S3 URL
38+ if not s3_url.startswith('s3://'):
39+ raise ValueError(f"S3 URL must start with 's3://': {s3_url}")
40+
41+ s3_url = s3_url.rstrip('/')
42+ s3_parts = s3_url[5:].split('/', 1)
43+ bucket_name = s3_parts[0]
44+ prefix = s3_parts[1] + '/' if len(s3_parts) > 1 else ''
45+
46+ # S3 configuration for public bucket (no credentials needed)
47+ s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED))
48+
49+ # Create local directory
50+ Path(local_dir).mkdir(parents=True, exist_ok=True)
51+
52+ print(f"Scanning s3://{bucket_name}/{prefix}")
53+ print(f"Will download to local directory: {local_dir}/")
54+ print()
55+
56+ # List all objects in the folder to calculate total size
57+ paginator = s3.get_paginator('list_objects_v2')
58+ pages = paginator.paginate(Bucket=bucket_name, Prefix=prefix)
59+
60+ files_to_download = []
61+ total_size = 0
62+
63+ for page in pages:
64+ if 'Contents' not in page:
65+ print("No files found in the specified path")
66+ return
67+
68+ for obj in page['Contents']:
69+ s3_key = obj['Key']
70+ file_size = obj['Size']
71+
72+ # Skip if it's just the directory itself
73+ if s3_key == prefix or s3_key == prefix.rstrip('/'):
74+ continue
75+
76+ # Get the relative path (remove the prefix)
77+ relative_path = s3_key[len(prefix):]
78+ if not relative_path:
79+ continue
80+
81+ files_to_download.append({
82+ 's3_key': s3_key,
83+ 'relative_path': relative_path,
84+ 'size': file_size
85+ })
86+ total_size += file_size
87+
88+ print(f"Found {len(files_to_download)} files")
89+ print(f"Total size: {total_size:,} bytes ({total_size / (1024**2):.2f} MB, {total_size / (1024**3):.2f} GB)")
90+ print()
91+
92+ if not skip_confirmation:
93+ response = input("Continue with download? (y/n): ")
94+ if response.lower() != 'y':
95+ print("Download cancelled")
96+ return
97+
98+ print()
99+ print("Starting download...")
100+ print()
101+
102+ # Download all files
103+ downloaded_bytes = 0
104+ for idx, file_info in enumerate(files_to_download, 1):
105+ s3_key = file_info['s3_key']
106+ relative_path = file_info['relative_path']
107+ file_size = file_info['size']
108+
109+ # Local file path
110+ local_file = os.path.join(local_dir, relative_path)
111+
112+ # Create subdirectories if needed
113+ local_file_dir = os.path.dirname(local_file)
114+ if local_file_dir:
115+ Path(local_file_dir).mkdir(parents=True, exist_ok=True)
116+
117+ # Download the file with progress callback
118+ # Using download_fileobj to write directly to file for partial download support
119+ print(f"[{idx}/{len(files_to_download)}] {relative_path} ({file_size:,} bytes)")
120+ progress = ProgressCallback(relative_path, file_size)
121+ with open(local_file, 'wb') as f:
122+ s3.download_fileobj(bucket_name, s3_key, f, Callback=progress)
123+ print() # New line after progress
124+
125+ downloaded_bytes += file_size
126+ overall_progress = (downloaded_bytes / total_size) * 100
127+ print(f" Overall progress: {downloaded_bytes:,} / {total_size:,} bytes ({overall_progress:.1f}%)")
128+ print()
129+
130+ print(f"Download complete!")
131+ print(f"Total files: {len(files_to_download)}")
132+ print(f"Total size: {total_size:,} bytes ({total_size / (1024**3):.2f} GB)")
python/.gitignoreadded+41−0View file
@@ -0,0 +1,41 @@
1+# Python
2+__pycache__/
3+*.py[cod]
4+*$py.class
5+*.so
6+.Python
7+build/
8+develop-eggs/
9+dist/
10+downloads/
11+eggs/
12+.eggs/
13+lib/
14+lib64/
15+parts/
16+sdist/
17+var/
18+wheels/
19+*.egg-info/
20+.installed.cfg
21+*.egg
22+
23+# Virtual Environment
24+venv/
25+env/
26+ENV/
27+
28+# IDE
29+.idea/
30+.vscode/
31+*.swp
32+*.swo
33+
34+# Testing
35+.coverage
36+htmlcov/
37+.pytest_cache/
38+.mypy_cache/
39+
40+# Misc
41+.DS_Store
python/LICENSEadded+201−0View file
@@ -0,0 +1,201 @@
1+ Apache License
2+ Version 2.0, January 2004
3+ http://www.apache.org/licenses/
4+
5+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6+
7+ 1. Definitions.
8+
9+ "License" shall mean the terms and conditions for use, reproduction,
10+ and distribution as defined by Sections 1 through 9 of this document.
11+
12+ "Licensor" shall mean the copyright owner or entity authorized by
13+ the copyright owner that is granting the License.
14+
15+ "Legal Entity" shall mean the union of the acting entity and all
16+ other entities that control, are controlled by, or are under common
17+ control with that entity. For the purposes of this definition,
18+ "control" means (i) the power, direct or indirect, to cause the
19+ direction or management of such entity, whether by contract or
20+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21+ outstanding shares, or (iii) beneficial ownership of such entity.
22+
23+ "You" (or "Your") shall mean an individual or Legal Entity
24+ exercising permissions granted by this License.
25+
26+ "Source" form shall mean the preferred form for making modifications,
27+ including but not limited to software source code, documentation
28+ source, and configuration files.
29+
30+ "Object" form shall mean any form resulting from mechanical
31+ transformation or translation of a Source form, including but
32+ not limited to compiled object code, generated documentation,
33+ and conversions to other media types.
34+
35+ "Work" shall mean the work of authorship, whether in Source or
36+ Object form, made available under the License, as indicated by a
37+ copyright notice that is included in or attached to the work
38+ (an example is provided in the Appendix below).
39+
40+ "Derivative Works" shall mean any work, whether in Source or Object
41+ form, that is based on (or derived from) the Work and for which the
42+ editorial revisions, annotations, elaborations, or other modifications
43+ represent, as a whole, an original work of authorship. For the purposes
44+ of this License, Derivative Works shall not include works that remain
45+ separable from, or merely link (or bind by name) to the interfaces of,
46+ the Work and Derivative Works thereof.
47+
48+ "Contribution" shall mean any work of authorship, including
49+ the original version of the Work and any modifications or additions
50+ to that Work or Derivative Works thereof, that is intentionally
51+ submitted to Licensor for inclusion in the Work by the copyright owner
52+ or by an individual or Legal Entity authorized to submit on behalf of
53+ the copyright owner. For the purposes of this definition, "submitted"
54+ means any form of electronic, verbal, or written communication sent
55+ to the Licensor or its representatives, including but not limited to
56+ communication on electronic mailing lists, source code control systems,
57+ and issue tracking systems that are managed by, or on behalf of, the
58+ Licensor for the purpose of discussing and improving the Work, but
59+ excluding communication that is conspicuously marked or otherwise
60+ designated in writing by the copyright owner as "Not a Contribution."
61+
62+ "Contributor" shall mean Licensor and any individual or Legal Entity
63+ on behalf of whom a Contribution has been received by Licensor and
64+ subsequently incorporated within the Work.
65+
66+ 2. Grant of Copyright License. Subject to the terms and conditions of
67+ this License, each Contributor hereby grants to You a perpetual,
68+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69+ copyright license to reproduce, prepare Derivative Works of,
70+ publicly display, publicly perform, sublicense, and distribute the
71+ Work and such Derivative Works in Source or Object form.
72+
73+ 3. Grant of Patent License. Subject to the terms and conditions of
74+ this License, each Contributor hereby grants to You a perpetual,
75+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76+ (except as stated in this section) patent license to make, have made,
77+ use, offer to sell, sell, import, and otherwise transfer the Work,
78+ where such license applies only to those patent claims licensable
79+ by such Contributor that are necessarily infringed by their
80+ Contribution(s) alone or by combination of their Contribution(s)
81+ with the Work to which such Contribution(s) was submitted. If You
82+ institute patent litigation against any entity (including a
83+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84+ or a Contribution incorporated within the Work constitutes direct
85+ or contributory patent infringement, then any patent licenses
86+ granted to You under this License for that Work shall terminate
87+ as of the date such litigation is filed.
88+
89+ 4. Redistribution. You may reproduce and distribute copies of the
90+ Work or Derivative Works thereof in any medium, with or without
91+ modifications, and in Source or Object form, provided that You
92+ meet the following conditions:
93+
94+ (a) You must give any other recipients of the Work or
95+ Derivative Works a copy of this License; and
96+
97+ (b) You must cause any modified files to carry prominent notices
98+ stating that You changed the files; and
99+
100+ (c) You must retain, in the Source form of any Derivative Works
101+ that You distribute, all copyright, patent, trademark, and
102+ attribution notices from the Source form of the Work,
103+ excluding those notices that do not pertain to any part of
104+ the Derivative Works; and
105+
106+ (d) If the Work includes a "NOTICE" text file as part of its
107+ distribution, then any Derivative Works that You distribute must
108+ include a readable copy of the attribution notices contained
109+ within such NOTICE file, excluding those notices that do not
110+ pertain to any part of the Derivative Works, in at least one
111+ of the following places: within a NOTICE text file distributed
112+ as part of the Derivative Works; within the Source form or
113+ documentation, if provided along with the Derivative Works; or,
114+ within a display generated by the Derivative Works, if and
115+ wherever such third-party notices normally appear. The contents
116+ of the NOTICE file are for informational purposes only and
117+ do not modify the License. You may add Your own attribution
118+ notices within Derivative Works that You distribute, alongside
119+ or as an addendum to the NOTICE text from the Work, provided
120+ that such additional attribution notices cannot be construed
121+ as modifying the License.
122+
123+ You may add Your own copyright statement to Your modifications and
124+ may provide additional or different license terms and conditions
125+ for use, reproduction, or distribution of Your modifications, or
126+ for any such Derivative Works as a whole, provided Your use,
127+ reproduction, and distribution of the Work otherwise complies with
128+ the conditions stated in this License.
129+
130+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131+ any Contribution intentionally submitted for inclusion in the Work
132+ by You to the Licensor shall be under the terms and conditions of
133+ this License, without any additional terms or conditions.
134+ Notwithstanding the above, nothing herein shall supersede or modify
135+ the terms of any separate license agreement you may have executed
136+ with Licensor regarding such Contributions.
137+
138+ 6. Trademarks. This License does not grant permission to use the trade
139+ names, trademarks, service marks, or product names of the Licensor,
140+ except as required for reasonable and customary use in describing the
141+ origin of the Work and reproducing the content of the NOTICE file.
142+
143+ 7. Disclaimer of Warranty. Unless required by applicable law or
144+ agreed to in writing, Licensor provides the Work (and each
145+ Contributor provides its Contributions) on an "AS IS" BASIS,
146+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147+ implied, including, without limitation, any warranties or conditions
148+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149+ PARTICULAR PURPOSE. You are solely responsible for determining the
150+ appropriateness of using or redistributing the Work and assume any
151+ risks associated with Your exercise of permissions under this License.
152+
153+ 8. Limitation of Liability. In no event and under no legal theory,
154+ whether in tort (including negligence), contract, or otherwise,
155+ unless required by applicable law (such as deliberate and grossly
156+ negligent acts) or agreed to in writing, shall any Contributor be
157+ liable to You for damages, including any direct, indirect, special,
158+ incidental, or consequential damages of any character arising as a
159+ result of this License or out of the use or inability to use the
160+ Work (including but not limited to damages for loss of goodwill,
161+ work stoppage, computer failure or malfunction, or any and all
162+ other commercial damages or losses), even if such Contributor
163+ has been advised of the possibility of such damages.
164+
165+ 9. Accepting Warranty or Additional Liability. While redistributing
166+ the Work or Derivative Works thereof, You may choose to offer,
167+ and charge a fee for, acceptance of support, warranty, indemnity,
168+ or other liability obligations and/or rights consistent with this
169+ License. However, in accepting such obligations, You may act only
170+ on Your own behalf and on Your sole responsibility, not on behalf
171+ of any other Contributor, and only if You agree to indemnify,
172+ defend, and hold each Contributor harmless for any liability
173+ incurred by, or claims asserted against, such Contributor by reason
174+ of your accepting any such warranty or additional liability.
175+
176+ END OF TERMS AND CONDITIONS
177+
178+ APPENDIX: How to apply the Apache License to your work.
179+
180+ To apply the Apache License to your work, attach the following
181+ boilerplate notice, with the fields enclosed by brackets "[]"
182+ replaced with your own identifying information. (Don't include
183+ the brackets!) The text should be enclosed in the appropriate
184+ comment syntax for the file format. We also recommend that a
185+ file or class name and description of purpose be included on the
186+ same "printed page" as the copyright notice for easier
187+ identification within third-party archives.
188+
189+ Copyright [yyyy] [name of copyright owner]
190+
191+ Licensed under the Apache License, Version 2.0 (the "License");
192+ you may not use this file except in compliance with the License.
193+ You may obtain a copy of the License at
194+
195+ http://www.apache.org/licenses/LICENSE-2.0
196+
197+ Unless required by applicable law or agreed to in writing, software
198+ distributed under the License is distributed on an "AS IS" BASIS,
199+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200+ See the License for the specific language governing permissions and
201+ limitations under the License.
python/MANIFEST.inadded+1−0View file
@@ -0,0 +1 @@
1+recursive-include ephys_compression_tests *.md
python/README.mdadded+1−0View file
@@ -0,0 +1 @@
1+# ephys_compression_tests
python/ephys_compression_tests/__init__.pyadded+1−0View file
@@ -0,0 +1 @@
1+from .run_benchmarks import run_benchmarks
python/ephys_compression_tests/_filters.pyadded+62−0View file
@@ -0,0 +1,62 @@
1+from typing import cast
2+import numpy as np
3+from scipy.signal import butter, lfilter
4+
5+
6+def bandpass_filter(
7+ array: np.ndarray, *, sampling_frequency: float, lowcut: float, highcut: float
8+) -> np.ndarray:
9+ """Apply a bandpass filter to the input array.
10+
11+ Args:
12+ array: Input signal array
13+ sampling_frequency: Sampling frequency in Hz
14+ lowcut: Lower cutoff frequency in Hz
15+ highcut: Higher cutoff frequency in Hz
16+
17+ Returns:
18+ Filtered signal array
19+ """
20+ nyquist = 0.5 * sampling_frequency
21+ low = lowcut / nyquist
22+ high = highcut / nyquist
23+ b, a = butter(5, [low, high], btype="band")
24+ return cast(np.ndarray, lfilter(b, a, array, axis=0))
25+
26+
27+def lowpass_filter(
28+ array: np.ndarray, *, sampling_frequency: float, highcut: float
29+) -> np.ndarray:
30+ """Apply a lowpass filter to the input array.
31+
32+ Args:
33+ array: Input signal array
34+ sampling_frequency: Sampling frequency in Hz
35+ highcut: Cutoff frequency in Hz
36+
37+ Returns:
38+ Filtered signal array
39+ """
40+ nyquist = 0.5 * sampling_frequency
41+ high = highcut / nyquist
42+ b, a = butter(5, high, btype="low")
43+ return cast(np.ndarray, lfilter(b, a, array, axis=0))
44+
45+
46+def highpass_filter(
47+ array: np.ndarray, *, sampling_frequency: float, lowcut: float
48+) -> np.ndarray:
49+ """Apply a highpass filter to the input array.
50+
51+ Args:
52+ array: Input signal array
53+ sampling_frequency: Sampling frequency in Hz
54+ lowcut: Cutoff frequency in Hz
55+
56+ Returns:
57+ Filtered signal array
58+ """
59+ nyquist = 0.5 * sampling_frequency
60+ low = lowcut / nyquist
61+ b, a = butter(5, low, btype="high")
62+ return cast(np.ndarray, lfilter(b, a, array, axis=0))
python/ephys_compression_tests/algorithms/__init__.pyadded+14−0View file
@@ -0,0 +1,14 @@
1+from .blosc2 import algorithms as blosc2_algorithms
2+from .ans import algorithms as ans_algorithms
3+from .wavpack import algorithms as wavpack_algorithms
4+from .lzma import algorithms as lzma_algorithms
5+from .zlib import algorithms as zlib_algorithms
6+from ..types import Algorithm
7+
8+algorithms: list[Algorithm] = (
9+ blosc2_algorithms
10+ + ans_algorithms
11+ + wavpack_algorithms
12+ + lzma_algorithms
13+ + zlib_algorithms
14+)
python/ephys_compression_tests/algorithms/ans/__init__.pyadded+376−0View file
@@ -0,0 +1,376 @@
1+import numpy as np
2+import os
3+from . import lpc_numba
4+from ...types import Algorithm
5+
6+
7+# Adapter functions
8+def encode_lpc(data: np.ndarray, order: int):
9+ """Encode using LPC model - adapter for lpc_numba."""
10+ coeffs, initial_points = lpc_numba.fit_lpc_model(data, k=order, subsample_factor=100, min_samples=1000)
11+ residuals_full = lpc_numba.compute_residuals(data, coeffs, initial_points)
12+ # Extract residuals excluding the initial points (first 'order' rows)
13+ residuals = residuals_full[order:, :]
14+ # Transpose initial_points to match old API: (order, channels)
15+ initial_values = initial_points.T
16+ return coeffs, residuals, initial_values
17+
18+
19+def encode_lpc_lossy(data: np.ndarray, order: int, step: int):
20+ """Encode using LPC model with lossy quantization - adapter for lpc_numba."""
21+ # Fit the LPC model
22+ coeffs, initial_points = lpc_numba.fit_lpc_model(data, k=order, subsample_factor=100, min_samples=1000)
23+
24+ # Compute residuals with quantization
25+ residuals_full = lpc_numba.compute_residuals_lossy(data, coeffs, initial_points, step=step)
26+
27+ # Extract residuals excluding the initial points (first 'order' rows)
28+ residuals = residuals_full[order:, :]
29+
30+ # Transpose initial_points to match old API: (order, channels)
31+ initial_values = initial_points.T
32+ return coeffs, residuals, initial_values
33+
34+
35+def decode_lpc(coeffs: np.ndarray, residuals: np.ndarray, initial_values: np.ndarray):
36+ """Decode LPC encoded data - adapter for lpc_numba."""
37+ # Transpose initial_values from (order, channels) to (channels, order)
38+ initial_points = initial_values.T
39+
40+ # Create full residuals array including initial points
41+ order = coeffs.shape[1]
42+ n_residuals, n_channels = residuals.shape
43+ n_timepoints = n_residuals + order
44+
45+ residuals_full = np.zeros((n_timepoints, n_channels), dtype=np.int16)
46+ residuals_full[:order, :] = initial_points.T
47+ residuals_full[order:, :] = residuals
48+
49+ return lpc_numba.reconstruct_from_residuals(residuals_full, coeffs, initial_points)
50+
51+SOURCE_FILE = "ans/__init__.py"
52+
53+
54+def _load_long_description():
55+ current_dir = os.path.dirname(os.path.abspath(__file__))
56+ md_path = os.path.join(current_dir, "ans.md")
57+ with open(md_path, "r", encoding="utf-8") as f:
58+ return f.read()
59+
60+
61+LONG_DESCRIPTION = _load_long_description()
62+
63+def create_ans_header(
64+ dtype_code: int,
65+ num_words: int,
66+ signal_length: int,
67+ state: np.uint64,
68+ symbol_counts: np.ndarray,
69+ symbol_values: np.ndarray,
70+ shape: tuple
71+) -> bytes:
72+ ndim = len(shape)
73+ section0 = np.array([ndim] + list(shape), dtype=np.uint32)
74+ section1 = np.array([dtype_code, num_words, signal_length, len(symbol_counts)], dtype=np.uint32)
75+ section2 = np.array([state], dtype=np.uint64)
76+ symbol_counts_bytes = symbol_counts.astype(np.uint32).tobytes()
77+ symbol_values_bytes = symbol_values.tobytes()
78+
79+ return section0.tobytes() + section1.tobytes() + section2.tobytes() + symbol_counts_bytes + symbol_values_bytes
80+
81+def unpack_ans_header(header_bytes: bytes) -> dict:
82+ # read section 0
83+ ndim = np.frombuffer(header_bytes[:4], dtype=np.uint32)[0]
84+ shape = tuple(np.frombuffer(header_bytes[4 : 4 + ndim * 4], dtype=np.uint32))
85+ offset = 4 + ndim * 4
86+ header_bytes = header_bytes[offset:]
87+ # read section 1
88+ section1_size = 4 * 4 # 4 uint32
89+ section1 = np.frombuffer(header_bytes[:section1_size], dtype=np.uint32)
90+ dtype_code = int(section1[0])
91+ num_words = int(section1[1])
92+ signal_length = int(section1[2])
93+ num_symbols = int(section1[3])
94+ # read section 2
95+ section2_size = 8 # 1 uint64
96+ section2 = np.frombuffer(header_bytes[section1_size : section1_size + section2_size], dtype=np.uint64)
97+ state = np.uint64(section2[0])
98+ # read symbol counts and values
99+ remaining_bytes = header_bytes[section1_size + section2_size :]
100+ symbol_counts = np.frombuffer(remaining_bytes[: num_symbols * 4], dtype=np.uint32)
101+
102+ symbol_values_dtype = {0: np.uint8, 1: np.uint16, 2: np.uint32, 3: np.int16, 4: np.int32}.get(dtype_code)
103+ num_bytes_per_value = np.dtype(symbol_values_dtype).itemsize
104+ if symbol_values_dtype is None:
105+ raise ValueError(f"Unsupported dtype code: {dtype_code}")
106+ symbol_values = np.frombuffer(remaining_bytes[num_symbols * 4 : num_symbols * 4 + num_symbols * num_bytes_per_value], dtype=symbol_values_dtype)
107+
108+ if len(symbol_counts) != len(symbol_values):
109+ raise ValueError("Mismatch between number of symbol counts and symbol values")
110+
111+ return {
112+ "dtype_code": dtype_code,
113+ "num_words": num_words,
114+ "signal_length": signal_length,
115+ "state": state,
116+ "symbol_counts": symbol_counts,
117+ "symbol_values": symbol_values,
118+ "shape": shape
119+ }
120+
121+
122+def ans_encode_0(x: np.ndarray) -> bytes:
123+ from simple_ans import ans_encode
124+
125+ shape0 = x.shape
126+ if x.ndim == 2:
127+ # flatten
128+ x = x.reshape(-1)
129+
130+ encoded = ans_encode(x)
131+ if x.dtype == np.uint8:
132+ dtype_code = 0
133+ elif x.dtype == np.uint16:
134+ dtype_code = 1
135+ elif x.dtype == np.uint32:
136+ dtype_code = 2
137+ elif x.dtype == np.int16:
138+ dtype_code = 3
139+ elif x.dtype == np.int32:
140+ dtype_code = 4
141+ else:
142+ raise ValueError(f"Unsupported dtype: {x.dtype}")
143+
144+ # Use the new header utilities
145+ header_bytes = create_ans_header(
146+ dtype_code=dtype_code,
147+ num_words=len(encoded.words),
148+ signal_length=encoded.signal_length,
149+ state=encoded.state,
150+ symbol_counts=encoded.symbol_counts,
151+ symbol_values=encoded.symbol_values,
152+ shape=shape0
153+ )
154+
155+ header_size = np.array([len(header_bytes)], dtype="uint32")
156+
157+ return header_size.tobytes() + header_bytes + encoded.words.tobytes()
158+
159+
160+
161+def ans_decode_0(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
162+ from simple_ans import ans_decode, EncodedSignal
163+
164+ header_size = np.frombuffer(x[:4], dtype=np.uint32)[0]
165+
166+ # Use the new header utilities
167+ header_dict = unpack_ans_header(x[4 : 4 + header_size])
168+
169+ dtype_code = header_dict["dtype_code"]
170+ num_words = header_dict["num_words"]
171+ signal_length = header_dict["signal_length"]
172+ state = header_dict["state"]
173+ symbol_counts = header_dict["symbol_counts"]
174+ symbol_values = header_dict["symbol_values"]
175+ shape_from_header = header_dict["shape"]
176+ if shape != shape_from_header:
177+ raise ValueError("Shape mismatch between provided shape and shape in header")
178+
179+ words_bytes = x[4 + header_size :]
180+
181+ if dtype_code == 0:
182+ assert dtype == "uint8"
183+ elif dtype_code == 1:
184+ assert dtype == "uint16"
185+ elif dtype_code == 2:
186+ assert dtype == "uint32"
187+ elif dtype_code == 3:
188+ assert dtype == "int16"
189+ elif dtype_code == 4:
190+ assert dtype == "int32"
191+ else:
192+ raise ValueError(f"Unsupported dtype code: {dtype_code}")
193+
194+ encoded = EncodedSignal(
195+ signal_length=int(signal_length),
196+ state=np.uint64(state),
197+ symbol_counts=symbol_counts.astype(np.uint32),
198+ symbol_values=symbol_values.astype(dtype),
199+ words=np.frombuffer(words_bytes, dtype=np.uint32, count=num_words),
200+ )
201+ return ans_decode(encoded).reshape(shape)
202+
203+algorithm_dicts_base = [
204+ {
205+ "name": "ans",
206+ "version": "1",
207+ "encode": lambda x: ans_encode_0(x),
208+ "decode": lambda x, dtype, shape: ans_decode_0(x, dtype, shape),
209+ "description": "ANS",
210+ "tags": ["ans"],
211+ "source_file": SOURCE_FILE,
212+ "long_description": LONG_DESCRIPTION,
213+ }
214+]
215+
216+algorithm_dicts = []
217+for a in algorithm_dicts_base:
218+ algorithm_dicts.append(a)
219+
220+# add delta encoding
221+for a in algorithm_dicts_base:
222+ def encode0(x: np.ndarray, a=a) -> bytes:
223+ assert x.ndim == 2 and x.shape[0] > 1, "Input array must be 2D with more than one timepoint"
224+ x_diff = np.diff(x, axis=0)
225+ first_timepoint = x[0:1, :].flatten()
226+ encoded_diff = a["encode"](x_diff)
227+ # Store the first value at the start
228+ first_timepoint_bytes = first_timepoint.tobytes()
229+ return first_timepoint_bytes + encoded_diff
230+ def decode0(x: bytes, dtype: str, shape: tuple, a=a) -> np.ndarray:
231+ dtype_np = np.dtype(dtype)
232+ num_bytes_first_timepoint = dtype_np.itemsize * shape[1]
233+ first_timepoint_bytes = x[:num_bytes_first_timepoint]
234+ first_timepoint = np.frombuffer(first_timepoint_bytes, dtype=dtype_np)
235+ encoded_diff = x[num_bytes_first_timepoint:]
236+ x_diff = a["decode"](encoded_diff, dtype, (shape[0]-1, shape[1]))
237+ x_reconstructed = np.empty(shape, dtype=dtype_np)
238+ x_reconstructed[0] = first_timepoint
239+ x_reconstructed[1:] = first_timepoint + np.cumsum(x_diff, axis=0)
240+ return x_reconstructed
241+ algorithm_dicts.append({
242+ "name": a["name"] + "-delta",
243+ "version": a["version"],
244+ "encode": encode0,
245+ "decode": decode0,
246+ "description": a["description"] + " with delta encoding",
247+ "tags": a["tags"] + ["delta"],
248+ "source_file": a["source_file"],
249+ "long_description": a["long_description"]
250+ })
251+
252+# add delta2 encoding
253+for a in algorithm_dicts_base:
254+ def encode0_lpc_lossy(x: np.ndarray, a=a) -> bytes:
255+ assert x.ndim == 2 and x.shape[0] > 2, "Input array must be 2D with more than two timepoints"
256+ x_diff = np.diff(np.diff(x, axis=0), axis=0)
257+ first_timepoint = x[0:1, :].flatten()
258+ second_timepoint = x[1:2, :].flatten()
259+ encoded_diff = a["encode"](x_diff)
260+ # Store the first value at the start
261+ first_timepoint_bytes = first_timepoint.tobytes()
262+ second_timepoint_bytes = second_timepoint.tobytes()
263+ return first_timepoint_bytes + second_timepoint_bytes + encoded_diff
264+ def decode0_lpc_lossy(x: bytes, dtype: str, shape: tuple, a=a) -> np.ndarray:
265+ dtype_np = np.dtype(dtype)
266+ num_bytes_first_timepoint = dtype_np.itemsize * shape[1]
267+ first_timepoint_bytes = x[:num_bytes_first_timepoint]
268+ second_timepoint_bytes = x[num_bytes_first_timepoint:2*num_bytes_first_timepoint]
269+ x0 = np.frombuffer(first_timepoint_bytes, dtype=dtype_np)
270+ x1 = np.frombuffer(second_timepoint_bytes, dtype=dtype_np)
271+ encoded_diff2 = x[2*num_bytes_first_timepoint:]
272+ x_diff2 = a["decode"](encoded_diff2, dtype, (shape[0]-2, shape[1]))
273+ x_recon1 = np.empty((shape[0]-1,shape[1]), dtype=dtype_np)
274+ x_recon1[0] = x1 - x0
275+ x_recon1[1:] = x_recon1[0] + np.cumsum(x_diff2, axis=0)
276+ x_reconstructed = np.empty(shape, dtype=dtype_np)
277+ x_reconstructed[0] = x0
278+ x_reconstructed[1:] = x0 + np.cumsum(x_recon1, axis=0)
279+ return x_reconstructed
280+ algorithm_dicts.append({
281+ "name": a["name"] + "-delta2",
282+ "version": a["version"],
283+ "encode": encode0_lpc_lossy,
284+ "decode": decode0_lpc_lossy,
285+ "description": a["description"] + " with delta2 encoding",
286+ "tags": a["tags"] + ["delta2"],
287+ "source_file": a["source_file"],
288+ "long_description": a["long_description"]
289+ })
290+
291+# Add auto-regressive prediction encoding
292+for a in algorithm_dicts_base:
293+ for order in [2, 8]:
294+ def encode0_lpc(x: np.ndarray, a=a, order=order) -> bytes:
295+ assert x.ndim == 2 and x.shape[0] > order, f"Input array must be 2D (timepoints x channels) with more than {order} timepoints"
296+ coeffs, residuals, initial_values = encode_lpc(x, order=order)
297+ # coeffs: (n_channels x order), residuals: (n_timepoints-order x n_channels), initial_values: (order x n_channels)
298+ encoded_residuals = a["encode"](residuals)
299+ coeffs_bytes = coeffs.astype(np.float32).tobytes()
300+ initial_values_bytes = initial_values.astype(np.int16).tobytes()
301+ return coeffs_bytes + initial_values_bytes + encoded_residuals
302+ def decode0_lpc(x: bytes, dtype: str, shape: tuple, a=a, order=order) -> np.ndarray:
303+ assert len(shape) == 2, f"Shape must be 2D (timepoints x channels)"
304+ dtype_np = np.dtype(dtype)
305+ n_channels = shape[1]
306+ # coeffs is (n_channels x order)
307+ num_bytes_coeffs = n_channels * order * np.dtype(np.float32).itemsize
308+ coeffs_bytes = x[:num_bytes_coeffs]
309+ coeffs = np.frombuffer(coeffs_bytes, dtype=np.float32).reshape((n_channels, order))
310+ # initial_values is (order x n_channels)
311+ num_bytes_initial_values = order * n_channels * dtype_np.itemsize
312+ initial_values_bytes = x[num_bytes_coeffs : num_bytes_coeffs + num_bytes_initial_values]
313+ initial_values = np.frombuffer(initial_values_bytes, dtype=dtype_np).reshape((order, n_channels))
314+ encoded_residuals = x[num_bytes_coeffs + num_bytes_initial_values :]
315+ # residuals is ((shape[0]-order) x n_channels)
316+ residuals = a["decode"](encoded_residuals, dtype, (shape[0]-order, n_channels))
317+ reconstructed = decode_lpc(coeffs, residuals, initial_values)
318+ return reconstructed
319+ algorithm_dicts.append({
320+ "name": a["name"] + f"-lpc{order}",
321+ "version": a["version"] + f".5",
322+ "encode": encode0_lpc,
323+ "decode": decode0_lpc,
324+ "description": a["description"] + f" with auto-regressive prediction encoding of order {order}",
325+ "tags": a["tags"] + [f"lpc{order}"],
326+ "source_file": a["source_file"],
327+ "long_description": a["long_description"]
328+ })
329+
330+# Add lossy lpc
331+for lpc_order in [2, 8]:
332+ for tolerance in [1, 2, 3, 4, 6, 8, 12, 16]:
333+ def make_encode_lpc_lossy(tolerance=tolerance, order=lpc_order):
334+ def encode0_lpc_lossy(x: np.ndarray) -> bytes:
335+ assert x.ndim == 2, f"Input array must be 2D (timepoints x channels)"
336+ coeffs, residuals, initial_values = encode_lpc_lossy(x, order=order, step=tolerance * 2 + 1)
337+ # coeffs: (n_channels x order), residuals: (n_timepoints-order x n_channels), initial_values: (order x n_channels)
338+ encoded_residuals = ans_encode_0(residuals)
339+ coeffs_bytes = coeffs.astype(np.float32).tobytes()
340+ initial_values_bytes = initial_values.astype(np.int16).tobytes()
341+ return coeffs_bytes + initial_values_bytes + encoded_residuals
342+ return encode0_lpc_lossy
343+ def make_decode_lpc_lossy(order=lpc_order):
344+ def decode0_lpc_lossy(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
345+ assert len(shape) == 2, f"Shape must be 2D (timepoints x channels)"
346+ dtype_np = np.dtype(dtype)
347+ n_channels = shape[1]
348+ # coeffs is (n_channels x order)
349+ num_bytes_coeffs = n_channels * order * np.dtype(np.float32).itemsize
350+ coeffs_bytes = x[:num_bytes_coeffs]
351+ coeffs = np.frombuffer(coeffs_bytes, dtype=np.float32).reshape((n_channels, order))
352+ # initial_values is (order x n_channels)
353+ num_bytes_initial_values = order * n_channels * dtype_np.itemsize
354+ initial_values_bytes = x[num_bytes_coeffs : num_bytes_coeffs + num_bytes_initial_values]
355+ initial_values = np.frombuffer(initial_values_bytes, dtype=dtype_np).reshape((order, n_channels))
356+ encoded_residuals = x[num_bytes_coeffs + num_bytes_initial_values :]
357+ # residuals is ((shape[0]-order) x n_channels)
358+ residuals = ans_decode_0(encoded_residuals, dtype, (shape[0]-order, n_channels))
359+ reconstructed = decode_lpc(coeffs, residuals, initial_values)
360+ return reconstructed
361+ return decode0_lpc_lossy
362+ algorithm_dicts.append({
363+ "name": f"ans-lpc{lpc_order}-lossy-tol{tolerance}",
364+ "version": "14",
365+ "encode": make_encode_lpc_lossy(),
366+ "decode": make_decode_lpc_lossy(),
367+ "description": f"ANS with lossy linear predictive coding of order {lpc_order} and tolerance {tolerance}",
368+ "tags": ["ans", "lossy", f"lpc{lpc_order}"],
369+ "source_file": SOURCE_FILE,
370+ "long_description": LONG_DESCRIPTION
371+ })
372+
373+algorithms = [
374+ Algorithm(**a)
375+ for a in algorithm_dicts
376+]
python/ephys_compression_tests/algorithms/ans/ans.mdadded+1−0View file
@@ -0,0 +1 @@
1+# ANS
python/ephys_compression_tests/algorithms/ans/lpc_numba.pyadded+247−0View file
@@ -0,0 +1,247 @@
1+"""
2+Numba-accelerated implementation of linear predictive coding (LPC) model operations.
3+All operations work with int16 data.
4+"""
5+
6+import numpy as np
7+from numba import jit, prange, njit
8+
9+
10+@njit
11+def _create_design_matrix_channel(data: np.ndarray, order: int, subsample_factor: int = 1) -> tuple[np.ndarray, np.ndarray]:
12+ """Numba-optimized design matrix creation for LPC model (single channel) with optional subsampling."""
13+ n = len(data)
14+ n_samples = (n - order + subsample_factor - 1) // subsample_factor
15+ X_design = np.zeros((n_samples, order))
16+ y_target = np.zeros(n_samples)
17+
18+ sample_idx = 0
19+ for i in range(0, n - order, subsample_factor):
20+ for j in range(order):
21+ X_design[sample_idx, j] = data[i + order - j - 1]
22+ y_target[sample_idx] = data[i + order]
23+ sample_idx += 1
24+
25+ return X_design[:sample_idx], y_target[:sample_idx]
26+
27+
28+def fit_lpc_model(data: np.ndarray, k: int, subsample_factor: int = 1,
29+ min_samples: int = 1000) -> tuple[np.ndarray, np.ndarray]:
30+ """
31+ Fit a linear predictive coding (LPC) model of order k to multi-channel time series data.
32+
33+ Args:
34+ data: 2D array of shape (timepoints, channels) with dtype int16
35+ k: Order of the LPC model
36+ subsample_factor: Use every Nth sample for fitting (default: 1)
37+ min_samples: Minimum number of samples to use for fitting (default: 1000)
38+
39+ Returns:
40+ coefficients: Array of shape (channels, k) with dtype float32
41+ initial_points: Array of shape (channels, k) with dtype int16 (first k samples per channel)
42+ """
43+ n_timepoints, n_channels = data.shape
44+ if k >= n_timepoints:
45+ raise ValueError(f"LPC order {k} must be less than data length {n_timepoints}")
46+
47+ # Store initial k points for each channel
48+ initial_points = data[:k, :].T.copy().astype(np.int16) # Shape: (n_channels, k)
49+
50+ # Adjust subsample_factor if needed to ensure we have at least min_samples
51+ effective_subsample_factor = subsample_factor
52+ n_subsampled = (n_timepoints - k) // effective_subsample_factor
53+ if n_subsampled < min_samples:
54+ # Adjust subsample_factor to meet min_samples requirement
55+ effective_subsample_factor = max(1, (n_timepoints - k) // min_samples)
56+
57+ # Fit LPC model for each channel separately
58+ coefficients = np.zeros((n_channels, k), dtype=np.float32)
59+
60+ for ch in range(n_channels):
61+ # Create design matrix using numba-optimized function
62+ # This subsamples the target points but uses full history for predictors
63+ X_design, y_target = _create_design_matrix_channel(data[:, ch], k, effective_subsample_factor)
64+
65+ # Use faster solve via normal equations: (X^T X) coeffs = X^T y
66+ # This is faster than lstsq for overdetermined systems
67+ XtX = X_design.T @ X_design
68+ Xty = X_design.T @ y_target
69+ coefficients[ch] = np.linalg.solve(XtX, Xty).astype(np.float32)
70+
71+ return coefficients, initial_points
72+
73+
74+@jit(nopython=True, parallel=True, fastmath=True)
75+def _compute_residuals_jit(data: np.ndarray, coefficients: np.ndarray,
76+ initial_points: np.ndarray, k: int) -> np.ndarray:
77+ """
78+ JIT-compiled residuals computation.
79+ """
80+ n_timepoints, n_channels = data.shape
81+ residuals = np.zeros((n_timepoints, n_channels), dtype=np.int16)
82+
83+ # First k points are copied as-is
84+ residuals[:k, :] = initial_points.T
85+
86+ # Compute residuals for each channel in parallel
87+ for ch in prange(n_channels):
88+ coef = coefficients[ch, :]
89+
90+ for t in range(k, n_timepoints):
91+ # Predict from previous k samples
92+ predicted = np.float32(0.0)
93+ for i in range(k):
94+ predicted += coef[i] * np.float32(data[t - 1 - i, ch])
95+
96+ # Residual = actual - predicted (rounded)
97+ residuals[t, ch] = data[t, ch] - np.int16(np.round(predicted))
98+
99+ return residuals
100+
101+
102+@jit(nopython=True, parallel=True, fastmath=True)
103+def _compute_residuals_lossy_jit(data: np.ndarray, coefficients: np.ndarray,
104+ initial_points: np.ndarray, k: int, step: int) -> np.ndarray:
105+ """
106+ JIT-compiled lossy residuals computation with quantization feedback.
107+ """
108+ n_timepoints, n_channels = data.shape
109+ residuals = np.zeros((n_timepoints, n_channels), dtype=np.int16)
110+ reconstructed = np.zeros((n_timepoints, n_channels), dtype=np.int16)
111+
112+ # First k points are copied as-is
113+ residuals[:k, :] = initial_points.T
114+ reconstructed[:k, :] = initial_points.T
115+
116+ step_f32 = np.float32(step)
117+
118+ # Compute residuals for each channel in parallel
119+ for ch in prange(n_channels):
120+ coef = coefficients[ch, :]
121+
122+ for t in range(k, n_timepoints):
123+ # Predict from previous k reconstructed samples
124+ predicted = np.float32(0.0)
125+ for i in range(k):
126+ predicted += coef[i] * np.float32(reconstructed[t - 1 - i, ch])
127+
128+ prediction_int = np.int16(np.round(predicted))
129+
130+ # Compute residual from original data
131+ residual = data[t, ch] - prediction_int
132+
133+ # Quantize residual to nearest multiple of step
134+ quantized_residual = np.int16(np.round(np.float32(residual) / step_f32) * step_f32)
135+ residuals[t, ch] = quantized_residual
136+
137+ # Reconstruct sample using quantized residual for future predictions
138+ reconstructed[t, ch] = prediction_int + quantized_residual
139+
140+ return residuals
141+
142+
143+def compute_residuals(data: np.ndarray, coefficients: np.ndarray,
144+ initial_points: np.ndarray) -> np.ndarray:
145+ """
146+ Compute residuals given data and LPC model coefficients.
147+
148+ Args:
149+ data: 2D array of shape (timepoints, channels) with dtype int16
150+ coefficients: Array of shape (channels, k) with dtype float32
151+ initial_points: Array of shape (channels, k) with dtype int16
152+
153+ Returns:
154+ residuals: Array of shape (timepoints, channels) with dtype int16
155+ """
156+ k = coefficients.shape[1]
157+ return _compute_residuals_jit(data, coefficients, initial_points, k)
158+
159+
160+def compute_residuals_lossy(data: np.ndarray, coefficients: np.ndarray,
161+ initial_points: np.ndarray, step: int) -> np.ndarray:
162+ """
163+ Compute lossy residuals with quantization given data and AR model coefficients.
164+
165+ Args:
166+ data: 2D array of shape (timepoints, channels) with dtype int16
167+ coefficients: Array of shape (channels, k) with dtype float32
168+ initial_points: Array of shape (channels, k) with dtype int16
169+ step: Quantization step size
170+
171+ Returns:
172+ residuals: Array of shape (timepoints, channels) with dtype int16
173+ """
174+ k = coefficients.shape[1]
175+ return _compute_residuals_lossy_jit(data, coefficients, initial_points, k, step)
176+
177+
178+@jit(nopython=True, parallel=True, fastmath=True)
179+def _reconstruct_from_residuals_jit(residuals: np.ndarray, coefficients: np.ndarray,
180+ initial_points: np.ndarray, k: int) -> np.ndarray:
181+ """
182+ JIT-compiled reconstruction.
183+ """
184+ n_timepoints, n_channels = residuals.shape
185+ reconstructed = np.zeros((n_timepoints, n_channels), dtype=np.int16)
186+
187+ # First k points are copied from initial_points
188+ reconstructed[:k, :] = initial_points.T
189+
190+ # Reconstruct each channel in parallel
191+ for ch in prange(n_channels):
192+ coef = coefficients[ch, :]
193+
194+ for t in range(k, n_timepoints):
195+ # Predict from previous k reconstructed samples
196+ predicted = np.float32(0.0)
197+ for i in range(k):
198+ predicted += coef[i] * np.float32(reconstructed[t - 1 - i, ch])
199+
200+ # Reconstruct: actual = predicted (rounded) + residual
201+ reconstructed[t, ch] = np.int16(np.round(predicted)) + residuals[t, ch]
202+
203+ return reconstructed
204+
205+
206+def reconstruct_from_residuals(residuals: np.ndarray, coefficients: np.ndarray,
207+ initial_points: np.ndarray) -> np.ndarray:
208+ """
209+ Reconstruct original data from residuals and LPC model coefficients.
210+
211+ Args:
212+ residuals: 2D array of shape (timepoints, channels) with dtype int16
213+ coefficients: Array of shape (channels, k) with dtype float32
214+ initial_points: Array of shape (channels, k) with dtype int16
215+
216+ Returns:
217+ reconstructed: Array of shape (timepoints, channels) with dtype int16
218+ """
219+ k = coefficients.shape[1]
220+ return _reconstruct_from_residuals_jit(residuals, coefficients, initial_points, k)
221+
222+
223+def warmup(n_channels: int = 10, k: int = 10):
224+ """
225+ Warm up Numba JIT compilation for all functions.
226+
227+ Args:
228+ n_channels: Number of channels for warmup data
229+ k: LPC model order for warmup
230+ """
231+ print("Warming up JIT...", end="", flush=True)
232+ # Create small warmup data
233+ warmup_data = np.random.randint(-1000, 1000, size=(1000, n_channels), dtype=np.int16)
234+
235+ # Warm up fit_lpc_model
236+ coefficients, initial_points = fit_lpc_model(warmup_data, k)
237+
238+ # Warm up compute_residuals
239+ residuals = compute_residuals(warmup_data, coefficients, initial_points)
240+
241+ # Warm up compute_residuals_lossy
242+ _ = compute_residuals_lossy(warmup_data, coefficients, initial_points, step=2)
243+
244+ # Warm up reconstruct_from_residuals
245+ _ = reconstruct_from_residuals(residuals, coefficients, initial_points)
246+
247+ print(" done")
python/ephys_compression_tests/algorithms/blosc2/__init__.pyadded+91−0View file
@@ -0,0 +1,91 @@
1+import numpy as np
2+import os
3+import blosc2
4+from ...types import Algorithm
5+
6+SOURCE_FILE = "blosc2/__init__.py"
7+
8+
9+def _load_long_description():
10+ current_dir = os.path.dirname(os.path.abspath(__file__))
11+ md_path = os.path.join(current_dir, "blosc2.md")
12+ with open(md_path, "r", encoding="utf-8") as f:
13+ return f.read()
14+
15+
16+LONG_DESCRIPTION = _load_long_description()
17+
18+
19+def blosc2_encode(x: np.ndarray, clevel: int, codec, filter: int = 2) -> bytes:
20+ import blosc2
21+
22+ # Convert filter int to proper enum
23+ if filter == 2:
24+ blosc_filter = blosc2.Filter.BITSHUFFLE
25+ elif filter == 1:
26+ blosc_filter = blosc2.Filter.SHUFFLE
27+ else:
28+ blosc_filter = blosc2.Filter.NOFILTER
29+
30+ # Get typesize from numpy array
31+ typesize = x.dtype.itemsize
32+
33+ # Compress data
34+ compressed = blosc2.compress(
35+ x, # numpy arrays support buffer interface
36+ typesize=typesize,
37+ clevel=clevel,
38+ filter=blosc_filter,
39+ codec=codec,
40+ )
41+ assert isinstance(compressed, bytes) # Type assertion
42+ return compressed
43+
44+
45+def blosc2_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
46+ import blosc2
47+
48+ decompressed = blosc2.decompress(x)
49+ assert isinstance(decompressed, (bytes, bytearray)) # Type assertion
50+ arr = np.frombuffer(decompressed, dtype=np.dtype(dtype))
51+ return arr.reshape(shape)
52+
53+zstd_codec = blosc2.Codec.ZSTD
54+
55+algorithms_dicts = [
56+ {
57+ "name": "blosc2-zstd-1",
58+ "version": "1b",
59+ "encode": lambda x: blosc2_encode(x, clevel=1, codec=zstd_codec),
60+ "decode": lambda x, dtype, shape: blosc2_decode(x, dtype, shape),
61+ "description": "Blosc2 compression at level 1 (fastest compression).",
62+ "tags": ["blosc2"],
63+ "source_file": SOURCE_FILE,
64+ "long_description": LONG_DESCRIPTION,
65+ },
66+ {
67+ "name": "blosc2-zstd-5",
68+ "version": "1",
69+ "encode": lambda x: blosc2_encode(x, clevel=5, codec=zstd_codec),
70+ "decode": lambda x, dtype, shape: blosc2_decode(x, dtype, shape),
71+ "description": "Blosc2 compression at level 5 (balanced speed/compression).",
72+ "tags": ["blosc2"],
73+ "source_file": SOURCE_FILE,
74+ "long_description": LONG_DESCRIPTION,
75+ },
76+ {
77+ "name": "blosc2-zstd-9",
78+ "version": "1",
79+ "encode": lambda x: blosc2_encode(x, clevel=9, codec=zstd_codec),
80+ "decode": lambda x, dtype, shape: blosc2_decode(x, dtype, shape),
81+ "description": "Blosc2 compression at level 9 (maximum compression).",
82+ "tags": ["blosc2"],
83+ "source_file": SOURCE_FILE,
84+ "long_description": LONG_DESCRIPTION,
85+ }
86+]
87+
88+algorithms = [
89+ Algorithm(**a)
90+ for a in algorithms_dicts
91+]
python/ephys_compression_tests/algorithms/blosc2/blosc2.mdadded+3−0View file
@@ -0,0 +1,3 @@
1+# Blosc2 Algorithm
2+
3+Blosc2 is a modern, fast data compression library that builds upon the original Blosc library. It is designed for efficient compression of binary data, particularly optimized for in-memory compression of numerical arrays. Blosc2 uses block-oriented compression with support for multithreading and SIMD instructions.
python/ephys_compression_tests/algorithms/lzma/__init__.pyadded+130−0View file
@@ -0,0 +1,130 @@
1+import numpy as np
2+import os
3+import lzma
4+from ...types import Algorithm
5+
6+SOURCE_FILE = "lzma/__init__.py"
7+
8+
9+def _load_long_description():
10+ current_dir = os.path.dirname(os.path.abspath(__file__))
11+ md_path = os.path.join(current_dir, "lzma.md")
12+ with open(md_path, "r", encoding="utf-8") as f:
13+ return f.read()
14+
15+
16+LONG_DESCRIPTION = _load_long_description()
17+
18+
19+def lzma_encode(x: np.ndarray, preset: int = 9) -> bytes:
20+ """Encode numpy array using LZMA compression.
21+
22+ Args:
23+ x: Input numpy array
24+ preset: Compression level (0-9, default 9 for maximum compression)
25+
26+ Returns:
27+ Compressed bytes
28+ """
29+ # Store dtype and shape information
30+ dtype_str = str(x.dtype)
31+ shape_bytes = np.array(x.shape, dtype=np.int64).tobytes()
32+ dtype_bytes = dtype_str.encode('utf-8')
33+ dtype_len = np.array([len(dtype_bytes)], dtype=np.uint32).tobytes()
34+
35+ # Compress the array data
36+ data_bytes = x.tobytes()
37+ compressed_data = lzma.compress(data_bytes, preset=preset)
38+
39+ # Combine metadata and compressed data
40+ return dtype_len + dtype_bytes + shape_bytes + compressed_data
41+
42+
43+def lzma_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
44+ """Decode LZMA compressed bytes back to numpy array.
45+
46+ Args:
47+ x: Compressed bytes
48+ dtype: Expected numpy dtype
49+ shape: Expected array shape
50+
51+ Returns:
52+ Decompressed numpy array
53+ """
54+ # Read dtype length
55+ dtype_len = np.frombuffer(x[:4], dtype=np.uint32)[0]
56+ offset = 4
57+
58+ # Read dtype string (not used but stored for completeness)
59+ # dtype_str = x[offset:offset + dtype_len].decode('utf-8')
60+ offset += dtype_len
61+
62+ # Read shape (not used but stored for completeness)
63+ # Determine number of dimensions from shape parameter
64+ num_dims = len(shape)
65+ shape_size = num_dims * 8 # int64
66+ # stored_shape = np.frombuffer(x[offset:offset + shape_size], dtype=np.int64)
67+ offset += shape_size
68+
69+ # Decompress the data
70+ compressed_data = x[offset:]
71+ decompressed_data = lzma.decompress(compressed_data)
72+
73+ # Reconstruct array
74+ arr = np.frombuffer(decompressed_data, dtype=np.dtype(dtype))
75+ return arr.reshape(shape)
76+
77+
78+algorithm_dicts_base = [
79+ {
80+ "name": "lzma",
81+ "version": "1",
82+ "encode": lambda x: lzma_encode(x, preset=9),
83+ "decode": lambda x, dtype, shape: lzma_decode(x, dtype, shape),
84+ "description": "LZMA compression at level 9 (maximum compression)",
85+ "tags": ["lzma"],
86+ "source_file": SOURCE_FILE,
87+ "long_description": LONG_DESCRIPTION,
88+ }
89+]
90+
91+algorithm_dicts = []
92+for a in algorithm_dicts_base:
93+ algorithm_dicts.append(a)
94+
95+# add delta encoding
96+for a in algorithm_dicts_base:
97+ def encode0(x: np.ndarray, a=a) -> bytes:
98+ assert x.ndim == 2 and x.shape[0] > 1, "Input array must be 2D with more than one timepoint"
99+ x_diff = np.diff(x, axis=0)
100+ first_timepoint = x[0:1, :].flatten()
101+ encoded_diff = a["encode"](x_diff)
102+ # Store the first value at the start
103+ first_timepoint_bytes = first_timepoint.tobytes()
104+ return first_timepoint_bytes + encoded_diff
105+ def decode0(x: bytes, dtype: str, shape: tuple, a=a) -> np.ndarray:
106+ dtype_np = np.dtype(dtype)
107+ num_bytes_first_timepoint = dtype_np.itemsize * shape[1]
108+ first_timepoint_bytes = x[:num_bytes_first_timepoint]
109+ first_timepoint = np.frombuffer(first_timepoint_bytes, dtype=dtype_np)
110+ encoded_diff = x[num_bytes_first_timepoint:]
111+ x_diff = a["decode"](encoded_diff, dtype, (shape[0]-1, shape[1]))
112+ x_reconstructed = np.empty(shape, dtype=dtype_np)
113+ x_reconstructed[0] = first_timepoint
114+ x_reconstructed[1:] = first_timepoint + np.cumsum(x_diff, axis=0)
115+ return x_reconstructed
116+ algorithm_dicts.append({
117+ "name": a["name"] + "-delta",
118+ "version": a["version"],
119+ "encode": encode0,
120+ "decode": decode0,
121+ "description": a["description"] + " with delta encoding",
122+ "tags": a["tags"] + ["delta"],
123+ "source_file": a["source_file"],
124+ "long_description": a["long_description"]
125+ })
126+
127+algorithms = [
128+ Algorithm(**a)
129+ for a in algorithm_dicts
130+]
python/ephys_compression_tests/algorithms/lzma/lzma.mdadded+5−0View file
@@ -0,0 +1,5 @@
1+# LZMA Algorithm
2+
3+LZMA (Lempel-Ziv-Markov chain Algorithm) is a lossless data compression algorithm that provides a high compression ratio. It is the default and general compression method of 7z format in the 7-Zip program. LZMA uses a dictionary compression scheme and features a high compression ratio with variable dictionary size, while still maintaining fast decompression speed.
4+
5+The LZMA algorithm is particularly effective for compressing large files and is widely used in various applications including file archivers, software distribution, and embedded systems. Python's built-in `lzma` module provides access to this compression algorithm.
python/ephys_compression_tests/algorithms/wavpack/__init__.pyadded+104−0View file
@@ -0,0 +1,104 @@
1+import numpy as np
2+import os
3+from ...types import Algorithm
4+
5+SOURCE_FILE = "wavpack/__init__.py"
6+
7+
8+def _load_long_description():
9+ current_dir = os.path.dirname(os.path.abspath(__file__))
10+ md_path = os.path.join(current_dir, "wavpack.md")
11+ with open(md_path, "r", encoding="utf-8") as f:
12+ return f.read()
13+
14+
15+LONG_DESCRIPTION = _load_long_description()
16+
17+
18+def wavpack_encode(x: np.ndarray, bps: float=None) -> bytes:
19+ from wavpack_numcodecs import WavPack
20+ if bps is not None:
21+ codec = WavPack(bps=bps)
22+ else:
23+ codec = WavPack()
24+ encoded = codec.encode(x)
25+ assert isinstance(encoded, bytes)
26+ return encoded
27+
28+def wavpack_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
29+ from wavpack_numcodecs import WavPack
30+ codec = WavPack()
31+ decoded = codec.decode(x)
32+ arr = np.frombuffer(decoded, dtype=np.dtype(dtype))
33+ return arr.reshape(shape)
34+
35+algorithm_dicts_base = [
36+ {
37+ "name": "wavpack",
38+ "version": "1",
39+ "encode": lambda x: wavpack_encode(x),
40+ "decode": lambda x, dtype, shape: wavpack_decode(x, dtype, shape),
41+ "description": "WavPack",
42+ "tags": ["wavpack"],
43+ "source_file": SOURCE_FILE,
44+ "long_description": LONG_DESCRIPTION,
45+ }
46+]
47+
48+algorithm_dicts = []
49+for a in algorithm_dicts_base:
50+ algorithm_dicts.append(a)
51+
52+# add delta encoding
53+for a in algorithm_dicts_base:
54+ def encode0(x: np.ndarray, a=a) -> bytes:
55+ assert x.ndim == 2 and x.shape[0] > 1, "Input array must be 2D with more than one timepoint"
56+ x_diff = np.diff(x, axis=0)
57+ first_timepoint = x[0:1, :].flatten()
58+ encoded_diff = a["encode"](x_diff)
59+ # Store the first value at the start
60+ first_timepoint_bytes = first_timepoint.tobytes()
61+ return first_timepoint_bytes + encoded_diff
62+ def decode0(x: bytes, dtype: str, shape: tuple, a=a) -> np.ndarray:
63+ dtype_np = np.dtype(dtype)
64+ num_bytes_first_timepoint = dtype_np.itemsize * shape[1]
65+ first_timepoint_bytes = x[:num_bytes_first_timepoint]
66+ first_timepoint = np.frombuffer(first_timepoint_bytes, dtype=dtype_np)
67+ encoded_diff = x[num_bytes_first_timepoint:]
68+ x_diff = a["decode"](encoded_diff, dtype, (shape[0]-1, shape[1]))
69+ x_reconstructed = np.empty(shape, dtype=dtype_np)
70+ x_reconstructed[0] = first_timepoint
71+ x_reconstructed[1:] = first_timepoint + np.cumsum(x_diff, axis=0)
72+ return x_reconstructed
73+ algorithm_dicts.append({
74+ "name": a["name"] + "-delta",
75+ "version": a["version"],
76+ "encode": encode0,
77+ "decode": decode0,
78+ "description": a["description"] + " with delta encoding",
79+ "tags": a["tags"] + ["delta"],
80+ "source_file": a["source_file"],
81+ "long_description": a["long_description"]
82+ })
83+
84+# Add lossy versions
85+for bps in [2.25, 3, 4, 5, 6]:
86+ def encode_lossy(x: np.ndarray, bps=bps) -> bytes:
87+ return wavpack_encode(x, bps=bps)
88+ def decode_lossy(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
89+ return wavpack_decode(x, dtype, shape)
90+ algorithm_dicts.append({
91+ "name": f"wavpack-lossy-bps{bps}",
92+ "version": "3",
93+ "encode": encode_lossy,
94+ "decode": decode_lossy,
95+ "description": f"WavPack lossy with {bps} bits per sample",
96+ "tags": ["wavpack", "lossy"],
97+ "source_file": SOURCE_FILE,
98+ "long_description": LONG_DESCRIPTION,
99+ })
100+
101+algorithms = [
102+ Algorithm(**a)
103+ for a in algorithm_dicts
104+]
python/ephys_compression_tests/algorithms/wavpack/wavpack.mdadded+1−0View file
@@ -0,0 +1 @@
1+# wavpack
python/ephys_compression_tests/algorithms/zlib/__init__.pyadded+130−0View file
@@ -0,0 +1,130 @@
1+import numpy as np
2+import os
3+import zlib
4+from ...types import Algorithm
5+
6+SOURCE_FILE = "zlib/__init__.py"
7+
8+
9+def _load_long_description():
10+ current_dir = os.path.dirname(os.path.abspath(__file__))
11+ md_path = os.path.join(current_dir, "zlib.md")
12+ with open(md_path, "r", encoding="utf-8") as f:
13+ return f.read()
14+
15+
16+LONG_DESCRIPTION = _load_long_description()
17+
18+
19+def zlib_encode(x: np.ndarray, level: int = 9) -> bytes:
20+ """Encode numpy array using zlib compression.
21+
22+ Args:
23+ x: Input numpy array
24+ level: Compression level (0-9, default 9 for maximum compression)
25+
26+ Returns:
27+ Compressed bytes
28+ """
29+ # Store dtype and shape information
30+ dtype_str = str(x.dtype)
31+ shape_bytes = np.array(x.shape, dtype=np.int64).tobytes()
32+ dtype_bytes = dtype_str.encode('utf-8')
33+ dtype_len = np.array([len(dtype_bytes)], dtype=np.uint32).tobytes()
34+
35+ # Compress the array data
36+ data_bytes = x.tobytes()
37+ compressed_data = zlib.compress(data_bytes, level=level)
38+
39+ # Combine metadata and compressed data
40+ return dtype_len + dtype_bytes + shape_bytes + compressed_data
41+
42+
43+def zlib_decode(x: bytes, dtype: str, shape: tuple) -> np.ndarray:
44+ """Decode zlib compressed bytes back to numpy array.
45+
46+ Args:
47+ x: Compressed bytes
48+ dtype: Expected numpy dtype
49+ shape: Expected array shape
50+
51+ Returns:
52+ Decompressed numpy array
53+ """
54+ # Read dtype length
55+ dtype_len = np.frombuffer(x[:4], dtype=np.uint32)[0]
56+ offset = 4
57+
58+ # Read dtype string (not used but stored for completeness)
59+ # dtype_str = x[offset:offset + dtype_len].decode('utf-8')
60+ offset += dtype_len
61+
62+ # Read shape (not used but stored for completeness)
63+ # Determine number of dimensions from shape parameter
64+ num_dims = len(shape)
65+ shape_size = num_dims * 8 # int64
66+ # stored_shape = np.frombuffer(x[offset:offset + shape_size], dtype=np.int64)
67+ offset += shape_size
68+
69+ # Decompress the data
70+ compressed_data = x[offset:]
71+ decompressed_data = zlib.decompress(compressed_data)
72+
73+ # Reconstruct array
74+ arr = np.frombuffer(decompressed_data, dtype=np.dtype(dtype))
75+ return arr.reshape(shape)
76+
77+
78+algorithm_dicts_base = [
79+ {
80+ "name": "zlib",
81+ "version": "1",
82+ "encode": lambda x: zlib_encode(x, level=9),
83+ "decode": lambda x, dtype, shape: zlib_decode(x, dtype, shape),
84+ "description": "zlib compression at level 9 (maximum compression)",
85+ "tags": ["zlib"],
86+ "source_file": SOURCE_FILE,
87+ "long_description": LONG_DESCRIPTION,
88+ }
89+]
90+
91+algorithm_dicts = []
92+for a in algorithm_dicts_base:
93+ algorithm_dicts.append(a)
94+
95+# add delta encoding
96+for a in algorithm_dicts_base:
97+ def encode0(x: np.ndarray, a=a) -> bytes:
98+ assert x.ndim == 2 and x.shape[0] > 1, "Input array must be 2D with more than one timepoint"
99+ x_diff = np.diff(x, axis=0)
100+ first_timepoint = x[0:1, :].flatten()
101+ encoded_diff = a["encode"](x_diff)
102+ # Store the first value at the start
103+ first_timepoint_bytes = first_timepoint.tobytes()
104+ return first_timepoint_bytes + encoded_diff
105+ def decode0(x: bytes, dtype: str, shape: tuple, a=a) -> np.ndarray:
106+ dtype_np = np.dtype(dtype)
107+ num_bytes_first_timepoint = dtype_np.itemsize * shape[1]
108+ first_timepoint_bytes = x[:num_bytes_first_timepoint]
109+ first_timepoint = np.frombuffer(first_timepoint_bytes, dtype=dtype_np)
110+ encoded_diff = x[num_bytes_first_timepoint:]
111+ x_diff = a["decode"](encoded_diff, dtype, (shape[0]-1, shape[1]))
112+ x_reconstructed = np.empty(shape, dtype=dtype_np)
113+ x_reconstructed[0] = first_timepoint
114+ x_reconstructed[1:] = first_timepoint + np.cumsum(x_diff, axis=0)
115+ return x_reconstructed
116+ algorithm_dicts.append({
117+ "name": a["name"] + "-delta",
118+ "version": a["version"],
119+ "encode": encode0,
120+ "decode": decode0,
121+ "description": a["description"] + " with delta encoding",
122+ "tags": a["tags"] + ["delta"],
123+ "source_file": a["source_file"],
124+ "long_description": a["long_description"]
125+ })
126+
127+algorithms = [
128+ Algorithm(**a)
129+ for a in algorithm_dicts
130+]
python/ephys_compression_tests/algorithms/zlib/zlib.mdadded+5−0View file
@@ -0,0 +1,5 @@
1+# zlib Algorithm
2+
3+zlib is a widely-used, general-purpose data compression library that implements the DEFLATE compression algorithm. DEFLATE is a combination of LZ77 (Lempel-Ziv 1977) and Huffman coding. The zlib library is extremely portable, efficient, and free for use in any application.
4+
5+zlib compression is commonly used in many applications including PNG image compression, HTTP compression, and git repositories. It provides a good balance between compression ratio and speed, making it suitable for a wide range of compression tasks. Python includes zlib as a built-in module, making it readily available without external dependencies.
python/ephys_compression_tests/cli.pyadded+148−0View file
@@ -0,0 +1,148 @@
1+#!/usr/bin/env python3
2+
3+import click
4+from typing import List, Optional
5+from .run_benchmarks.run_benchmarks import run_benchmarks
6+from .algorithms import algorithms
7+from .datasets import datasets
8+
9+
10+def get_available_algorithms() -> List[str]:
11+ """Get list of available algorithm names"""
12+ return [alg.name for alg in algorithms]
13+
14+
15+def get_available_datasets() -> List[str]:
16+ """Get list of available dataset names"""
17+ return [ds.name for ds in datasets]
18+
19+
20+def filter_algorithms(selected: Optional[List[str]] = None) -> List[dict]:
21+ """Filter algorithms based on selected names"""
22+ if not selected:
23+ return algorithms
24+ return [alg for alg in algorithms if alg.name in selected]
25+
26+
27+def filter_datasets(selected: Optional[List[str]] = None) -> List[dict]:
28+ """Filter datasets based on selected names"""
29+ if not selected:
30+ return datasets
31+ return [ds for ds in datasets if ds.name in selected]
32+
33+
34+def validate_algorithms(ctx, param, value):
35+ if not value:
36+ return None
37+ available = get_available_algorithms()
38+ invalid = [alg for alg in value if alg not in available]
39+ if invalid:
40+ raise click.BadParameter(
41+ f"Invalid algorithm(s): {', '.join(invalid)}. "
42+ f"Available algorithms: {', '.join(available)}"
43+ )
44+ return value
45+
46+
47+def validate_datasets(ctx, param, value):
48+ if not value:
49+ return None
50+ available = get_available_datasets()
51+ invalid = [ds for ds in value if ds not in available]
52+ if invalid:
53+ raise click.BadParameter(
54+ f"Invalid dataset(s): {', '.join(invalid)}. "
55+ f"Available datasets: {', '.join(available)}"
56+ )
57+ return value
58+
59+
60+@click.group()
61+def cli():
62+ """Benchmark compression algorithms for electrophysiology data"""
63+ pass
64+
65+
66+@cli.command()
67+def list():
68+ """List available algorithms and datasets"""
69+ click.echo("\nAvailable Algorithms:")
70+ for alg in algorithms:
71+ desc = alg.description if alg.description else "No description"
72+ click.echo(f" {alg.name:<20} - {desc}")
73+
74+ click.echo("\nAvailable Datasets:")
75+ for ds in datasets:
76+ desc = ds.description if ds.description else "No description"
77+ click.echo(f" {ds.name:<20} - {desc}")
78+
79+
80+@cli.command()
81+@click.option(
82+ "--algorithm",
83+ "-a",
84+ multiple=True,
85+ callback=validate_algorithms,
86+ help="Algorithm(s) to benchmark (can be specified multiple times)",
87+)
88+@click.option(
89+ "--dataset",
90+ "-d",
91+ multiple=True,
92+ callback=validate_datasets,
93+ help="Dataset(s) to benchmark (can be specified multiple times)",
94+)
95+@click.option(
96+ "--cache-dir",
97+ default=".benchmark_cache",
98+ help="Directory to store cached results",
99+ type=click.Path(),
100+)
101+@click.option("--quiet", "-q", is_flag=True, help="Reduce output verbosity")
102+@click.option("--force", "-f", is_flag=True, help="Force re-run without using cache")
103+def run(algorithm, dataset, cache_dir, quiet, force):
104+ """Run benchmarks with specified options"""
105+ # Filter algorithms and datasets
106+ filtered_algorithms = filter_algorithms(algorithm)
107+ filtered_datasets = filter_datasets(dataset)
108+
109+ if not filtered_algorithms:
110+ click.echo("Error: No matching algorithms found", err=True)
111+ ctx = click.get_current_context()
112+ ctx.exit(1)
113+ if not filtered_datasets:
114+ click.echo("Error: No matching datasets found", err=True)
115+ ctx = click.get_current_context()
116+ ctx.exit(1)
117+
118+ # Run benchmarks with filtered options
119+ results = run_benchmarks(
120+ cache_dir=cache_dir,
121+ verbose=not quiet,
122+ selected_algorithms=filtered_algorithms,
123+ selected_datasets=filtered_datasets,
124+ force=force,
125+ )
126+
127+ # Print summary
128+ click.echo("\nBenchmark Summary:")
129+ for result in results["results"]:
130+ click.echo(
131+ f"\n{result['dataset']} + {result['algorithm']}:"
132+ f"\n Compression ratio: {result['compression_ratio']:.2f}x"
133+ f"\n Encode speed: {result['encode_mb_per_sec']:.2f} MB/s"
134+ f"\n Decode speed: {result['decode_mb_per_sec']:.2f} MB/s"
135+ )
136+ if result["rmse"] != 0.0 or result["max_error"] != 0.0:
137+ click.echo(
138+ f" RMSE: {result['rmse']:.4f}"
139+ f"\n Max error: {result['max_error']:.4f}"
140+ )
141+
142+
143+def main():
144+ cli()
145+
146+
147+if __name__ == "__main__":
148+ main()
python/ephys_compression_tests/datasets/__init__.pyadded+12−0View file
@@ -0,0 +1,12 @@
1+from .retina512 import datasets as retina512_datasets
2+from .aind_compression import datasets as aind_compression_datasets
3+from ..types import Dataset
4+
5+datasets_list = [
6+ retina512_datasets,
7+ aind_compression_datasets,
8+]
9+
10+datasets: list[Dataset] = []
11+for d in datasets_list:
12+ datasets.extend(d)
python/ephys_compression_tests/datasets/aind_compression/__init__.pyadded+188−0View file
@@ -0,0 +1,188 @@
1+import numpy as np
2+import os
3+import requests
4+import io
5+from ...types import Dataset
6+
7+from ..._filters import bandpass_filter
8+
9+
10+SOURCE_FILE = "aind-compression/__init__.py"
11+
12+
13+def _load_long_description():
14+ current_dir = os.path.dirname(os.path.abspath(__file__))
15+ md_path = os.path.join(current_dir, "aind-compression.md")
16+ with open(md_path, "r", encoding="utf-8") as f:
17+ return f.read()
18+
19+
20+LONG_DESCRIPTION = _load_long_description()
21+
22+tags = ["real", "ecephys", "timeseries", "integer", "correlated"]
23+
24+
25+# It's important to correct the quantization levels before using these datasets
26+# Wavpack in particular will do a lot worse if the data is not properly quantized
27+def correct_quantization_for_channel(data: np.ndarray) -> np.ndarray:
28+ closed_to_zero_val = np.argmin(np.abs(data))
29+ print(f'Value closest to zero: {data[closed_to_zero_val]} at index {closed_to_zero_val}')
30+ data = data - data[closed_to_zero_val]
31+ unique_vals = np.unique(data)
32+ # differences between unique values
33+ diffs = np.diff(unique_vals)
34+ # minimum diff is the quantization step size
35+ diff0 = np.min(diffs[diffs > 0])
36+ print(f'Identified quantization step size: {diff0}')
37+ # divide by quantization step size
38+ data = data / diff0
39+ data = data.astype(np.int16)
40+ return data
41+
42+def correct_quantization(data: np.ndarray) -> np.ndarray:
43+ if data.ndim == 1:
44+ return correct_quantization_for_channel(data)
45+ elif data.ndim == 2:
46+ corrected_channels = []
47+ for ch in range(data.shape[1]):
48+ print(f'Correcting quantization for channel {ch}...')
49+ corrected_ch = correct_quantization_for_channel(data[:, ch])
50+ corrected_channels.append(corrected_ch)
51+ return np.stack(corrected_channels, axis=1)
52+ else:
53+ raise ValueError(f'Unsupported data ndim: {data.ndim}')
54+
55+def load_aind_np2_probeB_ch101() -> np.ndarray:
56+ """Load AIND CH101 dataset from external URL.
57+
58+ Returns:
59+ Array containing the loaded data
60+ """
61+ url = "https://tempory.net/ephys-compression-tests/aind_CH101.raw.npy"
62+ print(f'Loading AIND dataset from {url}...')
63+ response = requests.get(url)
64+ response.raise_for_status()
65+ data = np.load(io.BytesIO(response.content)).flatten()
66+ data = correct_quantization(data)
67+ return data
68+
69+def load_aind_np2_probeB_ch101_110() -> np.ndarray:
70+ url = "https://tempory.net/ephys-compression-tests/aind/aind_compression_np2_probeB_ch101-110.raw.npy"
71+ print(f'Loading AIND dataset from {url}...')
72+ response = requests.get(url)
73+ response.raise_for_status()
74+ data = np.load(io.BytesIO(response.content))
75+ data = correct_quantization(data)
76+ return data
77+
78+def load_aind_np1_probeA_101_110() -> np.ndarray:
79+ url = "https://tempory.net/ephys-compression-tests/aind/aind-np1-probeA-ch101-110.raw.npy"
80+ print(f'Loading AIND dataset from {url}...')
81+ response = requests.get(url)
82+ response.raise_for_status()
83+ data = np.load(io.BytesIO(response.content))
84+ data = correct_quantization(data)
85+ return data
86+
87+# ibl-np1-probe00
88+def load_ibl_np1_probe00_101_110() -> np.ndarray:
89+ url = "https://tempory.net/ephys-compression-tests/aind/ibl-np1-probe00-ch101-110.raw.npy"
90+ print(f'Loading IBL dataset from {url}...')
91+ response = requests.get(url)
92+ response.raise_for_status()
93+ data = np.load(io.BytesIO(response.content))
94+ data = correct_quantization(data)
95+ return data
96+
97+dataset_dicts_base = [
98+ {
99+ "name": "aind-compression-np2-ProbeB-ch101",
100+ "version": "2",
101+ "description": "AIND CH101 dataset",
102+ "create": load_aind_np2_probeB_ch101,
103+ "tags": tags + ["single-channel"],
104+ "source_file": SOURCE_FILE,
105+ "long_description": LONG_DESCRIPTION,
106+ },
107+ {
108+ "name": "aind-compression-np2-ProbeB-ch101-110",
109+ "version": "2",
110+ "description": "AIND CH101-110 dataset",
111+ "create": load_aind_np2_probeB_ch101_110,
112+ "tags": tags + ["multi-channel"],
113+ "source_file": SOURCE_FILE,
114+ "long_description": LONG_DESCRIPTION,
115+ },
116+ {
117+ "name": "aind-compression-np1-ProbeA-ch101-110",
118+ "version": "2",
119+ "description": "AIND NP1 ProbeA CH101-110 dataset",
120+ "create": load_aind_np1_probeA_101_110,
121+ "tags": tags + ["multi-channel"],
122+ "source_file": SOURCE_FILE,
123+ "long_description": LONG_DESCRIPTION,
124+ },
125+ {
126+ "name": "ibl-compression-np1-Probe00-ch101-110",
127+ "version": "2",
128+ "description": "IBL NP1 Probe00 CH101-110 dataset",
129+ "create": load_ibl_np1_probe00_101_110,
130+ "tags": tags + ["multi-channel"],
131+ "source_file": SOURCE_FILE,
132+ "long_description": LONG_DESCRIPTION,
133+ }
134+]
135+
136+dataset_dicts = []
137+for d in dataset_dicts_base:
138+ dataset_dicts.append(d)
139+
140+# Add filtered versions
141+for d in dataset_dicts_base:
142+ def create0(d=d) -> np.ndarray:
143+ data = d["create"]()
144+ filtered = bandpass_filter(data, sampling_frequency=30000, lowcut=300, highcut=4000)
145+ filtered = filtered.astype(data.dtype)
146+ return filtered
147+
148+ dataset_dicts.append(
149+ {
150+ "name": f'{d["name"]}-filtered',
151+ "version": "2",
152+ "description": f'{d["description"]} (bandpass filtered 300-4000 Hz)',
153+ "create": create0,
154+ "tags": d["tags"] + ["filtered", "bandpass"],
155+ "source_file": SOURCE_FILE,
156+ "long_description": LONG_DESCRIPTION,
157+ }
158+ )
159+
160+# Add common-mode corrected versions
161+# Oddly enough, this didn't seem to help compression much, so leave it commented out for now
162+# for d in dataset_dicts_base:
163+# if "multi-channel" in d["tags"]:
164+# def create1(d=d) -> np.ndarray:
165+# data = d["create"]()
166+# median_signal = np.median(data, axis=1)
167+# # now create a new array where the first channel is the median
168+# # and the rest are the original channels minus the median
169+# # but we exclude the last channel because it can be recovered
170+# new_data = np.zeros_like(data)
171+# new_data[:, 0] = median_signal
172+# for ch in range(1, data.shape[1]):
173+# new_data[:, ch] = data[:, ch] - median_signal
174+# return new_data
175+# dataset_dicts.append(
176+# {
177+# "name": f'{d["name"]}-cmc',
178+# "version": "1",
179+# "description": f'{d["description"]} (common-mode corrected)',
180+# "create": create1,
181+# "tags": d["tags"] + ["common-mode-corrected"],
182+# "source_file": SOURCE_FILE,
183+# "long_description": LONG_DESCRIPTION,
184+# }
185+# )
186+
187+
188+datasets = [Dataset(**a) for a in dataset_dicts]
python/ephys_compression_tests/datasets/aind_compression/aind-compression.mdadded+3−0View file
@@ -0,0 +1,3 @@
1+Examples from AIND Compression Benchmark
2+
3+See: https://allenneuraldynamics.github.io/data.html#aind-benchmark-dataephys-compression
python/ephys_compression_tests/datasets/retina512/__init__.pyadded+77−0View file
@@ -0,0 +1,77 @@
1+import numpy as np
2+import os
3+import requests
4+import io
5+from ...types import Dataset
6+
7+from ..._filters import bandpass_filter
8+
9+
10+SOURCE_FILE = "retina512/__init__.py"
11+
12+
13+def _load_long_description():
14+ current_dir = os.path.dirname(os.path.abspath(__file__))
15+ md_path = os.path.join(current_dir, "retina512.md")
16+ with open(md_path, "r", encoding="utf-8") as f:
17+ return f.read()
18+
19+
20+LONG_DESCRIPTION = _load_long_description()
21+
22+tags = ["real", "ecephys", "timeseries", "single-channel", "integer", "correlated"]
23+
24+
25+def load_retina512_example_ch0_seg2_6() -> np.ndarray:
26+ """Load Retina512 example dataset from external URL.
27+
28+ Returns:
29+ Array containing the loaded data
30+ """
31+ url = "https://tempory.net/ephys-compression-tests/vyom_example_ch0_seg2-6.npy"
32+ print(f'Loading Retina512 example dataset from {url}...')
33+ response = requests.get(url)
34+ response.raise_for_status()
35+ data = np.load(io.BytesIO(response.content))
36+ return data
37+
38+
39+
40+dataset_dicts_base = [
41+ {
42+ "name": "retina512-ch0-seg2-6",
43+ "version": "1",
44+ "description": "Retina512 example dataset",
45+ "create": load_retina512_example_ch0_seg2_6,
46+ "tags": tags,
47+ "source_file": SOURCE_FILE,
48+ "long_description": LONG_DESCRIPTION,
49+ }
50+]
51+
52+dataset_dicts = []
53+for d in dataset_dicts_base:
54+ dataset_dicts.append(d)
55+
56+# Add filtered versions
57+for d in dataset_dicts_base:
58+ def create0(d=d) -> np.ndarray:
59+ data = d["create"]()
60+ filtered = bandpass_filter(data, sampling_frequency=20000, lowcut=300, highcut=4000)
61+ filtered = filtered.astype(data.dtype)
62+ return filtered
63+
64+ dataset_dicts.append(
65+ {
66+ "name": f'{d["name"]}-filtered',
67+ "version": "1",
68+ "description": f'{d["description"]} (bandpass filtered 300-4000 Hz)',
69+ "create": create0,
70+ "tags": d["tags"] + ["filtered", "bandpass"],
71+ "source_file": SOURCE_FILE,
72+ "long_description": LONG_DESCRIPTION,
73+ }
74+ )
75+
76+
77+datasets = [Dataset(**a) for a in dataset_dicts]
python/ephys_compression_tests/datasets/retina512/retina512.mdadded+1−0View file
@@ -0,0 +1 @@
1+Examples from Vyom Raval
python/ephys_compression_tests/run_benchmarks/__init__.pyadded+1−0View file
@@ -0,0 +1 @@
1+from .run_benchmarks import run_benchmarks
python/ephys_compression_tests/run_benchmarks/_memobin.pyadded+255−0View file
@@ -0,0 +1,255 @@
1+import json
2+import requests
3+import time
4+from typing import Optional, TypeVar, Callable
5+
6+T = TypeVar("T")
7+
8+
9+def _retry_with_backoff(
10+ func: Callable[..., T], num_retries: int = 4, *args, **kwargs
11+) -> T:
12+ """Execute a function with exponential backoff retry logic.
13+
14+ Args:
15+ func: Function to execute
16+ num_retries: Maximum number of retries
17+ args: Positional arguments for the function
18+ kwargs: Keyword arguments for the function
19+
20+ Returns:
21+ The function's return value
22+
23+ Raises:
24+ The last exception encountered after all retries are exhausted
25+ """
26+ last_exception = None
27+ for attempt in range(num_retries):
28+ try:
29+ return func(*args, **kwargs)
30+ except Exception as e:
31+ print(f" Attempt {attempt + 1} failed with error: {str(e)}")
32+ last_exception = e
33+ if attempt < num_retries - 1:
34+ sleep_time = 2**attempt # 1, 2, 4, 8 seconds
35+ time.sleep(sleep_time)
36+ else:
37+ raise last_exception
38+ raise RuntimeError("Unexpected: retry loop completed without return or raise")
39+
40+
41+def create_signed_upload_url(
42+ url: str, size: int, user_id: str, memobin_api_key: str, num_retries: int = 4
43+) -> str:
44+ """Create a signed upload URL for memobin.
45+
46+ Args:
47+ url: The target URL for the file
48+ size: Size of the file in bytes
49+ user_id: User ID for memobin
50+ memobin_api_key: API key for memobin authentication
51+
52+ Returns:
53+ The signed upload URL
54+
55+ Raises:
56+ ValueError: If the URL prefix is invalid
57+ requests.RequestException: If the API request fails
58+ """
59+
60+ def _create_url() -> str:
61+ prefix = "https://tempory.net/f/memobin/"
62+ if not url.startswith(prefix):
63+ raise ValueError("Invalid url. Does not have proper prefix")
64+
65+ file_path = url[len(prefix) :]
66+ tempory_api_url = "https://hub.tempory.net/api/uploadFile"
67+
68+ response = requests.post(
69+ tempory_api_url,
70+ headers={
71+ "Content-Type": "application/json",
72+ "Authorization": f"Bearer {memobin_api_key}",
73+ },
74+ json={
75+ "appName": "memobin",
76+ "filePath": file_path,
77+ "size": size,
78+ "userId": user_id,
79+ },
80+ )
81+
82+ if not response.ok:
83+ raise requests.RequestException("Failed to get signed url")
84+
85+ result = response.json()
86+ upload_url = result["uploadUrl"]
87+ download_url = result["downloadUrl"]
88+
89+ if download_url != url:
90+ raise ValueError(f"Mismatch between download url and url: {download_url} != {url}")
91+
92+ return upload_url
93+
94+ return _retry_with_backoff(_create_url, num_retries)
95+
96+
97+def construct_memobin_url(
98+ alg_name: str,
99+ dataset_name: str,
100+ alg_version: str,
101+ dataset_version: str,
102+ system_version: str,
103+ file_type: str = "metadata.json",
104+) -> str:
105+ """Construct the memobin URL for a specific benchmark result or dataset.
106+
107+ Args:
108+ alg_name: Name of the algorithm
109+ dataset_name: Name of the dataset
110+ alg_version: Version of the algorithm
111+ dataset_version: Version of the dataset
112+ system_version: Version of the system
113+ file_type: Type of file (metadata.json or data.bin)
114+
115+ Returns:
116+ The constructed memobin URL
117+ """
118+ path = f"{alg_name}/{dataset_name}/{alg_version}/{dataset_version}/{system_version}/{file_type}"
119+ return f"https://tempory.net/f/memobin/ephys_compression_tests/{path}"
120+
121+
122+def construct_dataset_url(
123+ dataset_name: str, dataset_version: str, format: str = "dat"
124+) -> str:
125+ """Construct the memobin URL for a dataset.
126+
127+ Args:
128+ dataset_name: Name of the dataset
129+ dataset_version: Version of the dataset
130+ format: File format ("dat", "npy", or "json")
131+
132+ Returns:
133+ The constructed memobin URL for the dataset
134+ """
135+ path = f"datasets/{dataset_name}/{dataset_version}/{dataset_name}-{dataset_version}.{format}"
136+ return f"https://tempory.net/f/memobin/ephys_compression_tests/{path}"
137+
138+
139+def construct_reconstructed_url(
140+ algorithm_name: str,
141+ dataset_name: str,
142+ algorithm_version: str,
143+ dataset_version: str,
144+ system_version: str,
145+ format: str = "dat",
146+) -> str:
147+ """Construct the memobin URL for a reconstructed array.
148+
149+ Args:
150+ algorithm_name: Name of the algorithm
151+ dataset_name: Name of the dataset
152+ algorithm_version: Version of the algorithm
153+ dataset_version: Version of the dataset
154+ system_version: Version of the system
155+ format: File format ("dat", "npy", or "json")
156+
157+ Returns:
158+ The constructed memobin URL for the reconstructed array
159+ """
160+ version_str = f"v{algorithm_version}-{dataset_version}-{system_version}"
161+ path = f"reconstructed/{algorithm_name}/{dataset_name}/{version_str}/reconstructed.{format}"
162+ return f"https://tempory.net/f/memobin/ephys_compression_tests/{path}"
163+
164+
165+def upload_to_memobin(
166+ data: dict | bytes,
167+ url: str,
168+ memobin_api_key: str,
169+ content_type: str = "application/json",
170+ num_retries: int = 4,
171+) -> None:
172+ """Upload data to memobin.
173+
174+ Args:
175+ data: The data to upload (dict for JSON or bytes for binary)
176+ url: The target URL for the file
177+ memobin_api_key: API key for memobin authentication
178+ content_type: Content type of the data
179+
180+ Raises:
181+ requests.RequestException: If the upload fails
182+ """
183+ if isinstance(data, dict):
184+ data_bytes = json.dumps(data).encode("utf-8")
185+ else:
186+ data_bytes = data
187+ size = len(data_bytes)
188+
189+ def _do_upload() -> None:
190+ upload_url = create_signed_upload_url(
191+ url, size, "ephys_compression_tests", memobin_api_key, num_retries
192+ )
193+
194+ response = requests.put(
195+ upload_url, data=data_bytes, headers={"Content-Type": content_type}
196+ )
197+
198+ if not response.ok:
199+ raise requests.RequestException("Failed to upload data to memobin")
200+
201+ _retry_with_backoff(_do_upload, num_retries)
202+
203+
204+def exists_in_memobin(url: str, num_retries: int = 4) -> bool:
205+ """Check if a file exists in memobin using a HEAD request.
206+
207+ Args:
208+ url: The URL to check
209+
210+ Returns:
211+ True if the file exists, False otherwise
212+ """
213+
214+ def _check_exists() -> bool:
215+ try:
216+ response = requests.head(url)
217+ return (
218+ 200 <= response.status_code < 300
219+ ) # Any 2xx status code indicates success
220+ except requests.RequestException:
221+ return False
222+
223+ return _retry_with_backoff(_check_exists, num_retries)
224+
225+
226+def download_from_memobin(
227+ url: str, as_json: bool = True, num_retries: int = 4
228+) -> Optional[dict | bytes]:
229+ """Download data from memobin.
230+
231+ Args:
232+ url: The URL to download from
233+ as_json: Whether to parse the response as JSON
234+
235+ Returns:
236+ The downloaded data as a dictionary or bytes, or None if not found
237+
238+ Raises:
239+ requests.RequestException: If the download fails for a reason other than 404
240+ """
241+
242+ def _do_download() -> Optional[dict | bytes]:
243+ response = None
244+ try:
245+ response = requests.get(url)
246+ if response.status_code == 404:
247+ return None
248+ response.raise_for_status()
249+ return response.json() if as_json else response.content
250+ except requests.RequestException as e:
251+ if response and response.status_code == 404:
252+ return None
253+ raise e
254+
255+ return _retry_with_backoff(_do_download, num_retries)
python/ephys_compression_tests/run_benchmarks/benchmark_timing.pyadded+137−0View file
@@ -0,0 +1,137 @@
1+from typing import Any, Tuple, Callable, Dict
2+from statistics import median
3+import time
4+import numpy as np
5+
6+
7+def run_timed_trials(
8+ data: np.ndarray, operation: Callable, *args
9+) -> Tuple[float, float, Any]:
10+ """Run multiple trials of an operation until total time exceeds 1 second.
11+
12+ Args:
13+ data: Input numpy array for calculating throughput
14+ operation: Function to benchmark
15+ *args: Arguments to pass to the operation
16+
17+ Returns:
18+ Tuple containing:
19+ - median_time: Median execution time across trials
20+ - mb_per_sec: Throughput in MB/s
21+ - result: Result from the last trial execution
22+ """
23+ times = []
24+ total_time = 0
25+ array_size_mb = data.nbytes / (1024 * 1024) # Convert to MB
26+
27+ operation(
28+ *args
29+ ) # execute once prior to timing in case there's any initial overhead
30+
31+ ret = None
32+ while total_time < 1.0:
33+ start_time = time.perf_counter()
34+ ret = operation(*args) # Execute operation
35+ trial_time = time.perf_counter() - start_time
36+ times.append(trial_time)
37+ total_time += trial_time
38+
39+ median_time = median(times)
40+ mb_per_sec = array_size_mb / median_time
41+ return median_time, mb_per_sec, ret
42+
43+
44+def run_compression_benchmark(
45+ data: np.ndarray,
46+ algorithm_name: str,
47+ encode_fn: Callable,
48+ decode_fn: Callable,
49+ verbose: bool = True,
50+ lossy: bool = False,
51+) -> Tuple[Dict[str, Any], bytes, np.ndarray]:
52+ """Run compression and decompression benchmarks for an algorithm.
53+
54+ Args:
55+ data: Input numpy array to compress
56+ algorithm_name: Name of the algorithm being benchmarked
57+ encode_fn: Compression function
58+ decode_fn: Decompression function
59+ verbose: Whether to print progress messages
60+ lossy: Whether the algorithm is lossy
61+
62+ Returns:
63+ Tuple containing:
64+ - result: Dictionary with benchmark metrics
65+ - encoded: Compressed data bytes
66+ - decoded: Decompressed data array
67+ """
68+ if data.ndim == 1:
69+ data = data[:, np.newaxis]
70+ original_size = len(data.tobytes())
71+ dtype = str(data.dtype)
72+
73+ if verbose:
74+ print(" Encoding...")
75+ encode_time, encode_mb_per_sec, encoded = run_timed_trials(data, encode_fn, data)
76+ compressed_size = len(encoded)
77+ compression_ratio = original_size / compressed_size
78+
79+ if verbose:
80+ print(" Compression complete:")
81+ print(f" Compressed size: {compressed_size:,} bytes")
82+ print(f" Compression ratio: {compression_ratio:.2f}x")
83+ print(f" Encode time: {encode_time*1000:.2f}ms")
84+ print(f" Encode throughput: {encode_mb_per_sec:.2f} MB/s")
85+ print(" Decoding...")
86+
87+ decode_time, decode_mb_per_sec, decoded = run_timed_trials(
88+ data, decode_fn, encoded, dtype, data.shape
89+ )
90+
91+ if verbose:
92+ print(f" Decode time: {decode_time*1000:.2f}ms")
93+ print(f" Decode throughput: {decode_mb_per_sec:.2f} MB/s")
94+
95+ # Verify correctness
96+ if len(data) != len(decoded):
97+ raise ValueError(
98+ f"Decompression failed: decoded length {len(decoded)} != original length {len(data)}"
99+ )
100+
101+ if not lossy:
102+ if not np.array_equal(data, decoded):
103+ print(data[:100])
104+ print(decoded[:100])
105+ for j in range(len(data)):
106+ if data[j] != decoded[j]:
107+ print(f"Error at index {j}: {data[j]} != {decoded[j]}")
108+ break
109+ raise ValueError(f"Decompression verification failed for {algorithm_name}")
110+ rmse = 0.0
111+ max_error = 0.0
112+ else:
113+ # compute RMSE and max error
114+ rmse = float(np.sqrt(np.mean((data - decoded) ** 2)))
115+ max_error = float(np.max(np.abs(data - decoded)))
116+ print(f" RMSE: {rmse:.4f}, Max error: {max_error:.4f}")
117+
118+ if verbose:
119+ print(" Verification successful!")
120+
121+ result = {
122+ "compression_ratio": compression_ratio,
123+ "encode_time": encode_time,
124+ "decode_time": decode_time,
125+ "encode_mb_per_sec": encode_mb_per_sec,
126+ "decode_mb_per_sec": decode_mb_per_sec,
127+ "original_size": original_size,
128+ "compressed_size": compressed_size,
129+ "array_shape": data.shape,
130+ "array_dtype": dtype,
131+ "timestamp": time.time(),
132+ "cache_status": "new",
133+ "rmse": rmse,
134+ "max_error": max_error,
135+ }
136+
137+ return result, encoded, decoded
python/ephys_compression_tests/run_benchmarks/cache_management.pyadded+127−0View file
@@ -0,0 +1,127 @@
1+import os
2+import json
3+from typing import Optional, Dict, Any
4+import numpy as np
5+from ._memobin import (
6+ construct_memobin_url,
7+ download_from_memobin,
8+)
9+
10+
11+def check_cached_result(
12+ cache_dir: str,
13+ dataset_name: str,
14+ algorithm_name: str,
15+ algorithm_version: str,
16+ dataset_version: str,
17+ system_version: str,
18+ force: bool = False,
19+ verbose: bool = True,
20+) -> Optional[Dict[str, Any]]:
21+ """Check for cached benchmark results locally and in memobin.
22+
23+ Args:
24+ cache_dir: Directory containing cached results
25+ dataset_name: Name of the dataset
26+ algorithm_name: Name of the algorithm
27+ algorithm_version: Version of the algorithm
28+ dataset_version: Version of the dataset
29+ system_version: Version of the system
30+ force: If True, ignore cached results
31+ verbose: Whether to print progress messages
32+
33+ Returns:
34+ Cached result dictionary if found and valid, None otherwise
35+ """
36+ test_dir = os.path.join(cache_dir, dataset_name, algorithm_name)
37+ metadata_file = os.path.join(test_dir, "metadata.json")
38+
39+ # First try local cache (unless force flag is set)
40+ cached_data = None
41+ if not force and os.path.exists(metadata_file):
42+ with open(metadata_file, "r") as f:
43+ cached_data = json.load(f)
44+ # if versions do not match, then set to None
45+ if isinstance(cached_data, dict) and "result" in cached_data:
46+ result = cached_data["result"]
47+ if (
48+ result["algorithm_version"] != algorithm_version
49+ or result["dataset_version"] != dataset_version
50+ or result.get("system_version", "") != system_version
51+ ):
52+ cached_data = None
53+
54+ # If not in local cache, try memobin (unless force flag is set)
55+ if cached_data is None and not force:
56+ memobin_url = construct_memobin_url(
57+ algorithm_name,
58+ dataset_name,
59+ algorithm_version,
60+ dataset_version,
61+ system_version,
62+ "metadata.json",
63+ )
64+ if verbose:
65+ print(" Looking for cached result in memobin...")
66+ cached_data = download_from_memobin(memobin_url)
67+ if cached_data is not None:
68+ if verbose:
69+ print(" Found result in memobin, saving locally...")
70+ # Save to local cache
71+ os.makedirs(test_dir, exist_ok=True)
72+ with open(metadata_file, "w") as f:
73+ json.dump(cached_data, f, indent=2)
74+
75+ if (
76+ cached_data is not None
77+ and isinstance(cached_data, dict)
78+ and "result" in cached_data
79+ ):
80+ result = cached_data["result"]
81+ if (
82+ isinstance(result, dict)
83+ and result.get("algorithm_version") == algorithm_version
84+ and result.get("dataset_version") == dataset_version
85+ and result.get("system_version", "") == system_version
86+ ):
87+ result["cache_status"] = "cached"
88+ return result
89+
90+ return None
91+
92+
93+def save_result_to_cache(
94+ result: Dict[str, Any],
95+ encoded_data: bytes,
96+ cache_dir: str,
97+ dataset_name: str,
98+ algorithm_name: str,
99+ reconstructed_data: Optional[np.ndarray] = None,
100+) -> None:
101+ """Save benchmark result and compressed data to cache.
102+
103+ Args:
104+ result: Benchmark result dictionary
105+ encoded_data: Compressed data bytes
106+ cache_dir: Directory to store cached results
107+ dataset_name: Name of the dataset
108+ algorithm_name: Name of the algorithm
109+ reconstructed_data: Optional reconstructed array for lossy algorithms
110+ """
111+ test_dir = os.path.join(cache_dir, dataset_name, algorithm_name)
112+ metadata_file = os.path.join(test_dir, "metadata.json")
113+ compressed_file = os.path.join(test_dir, "compressed.dat")
114+ reconstructed_file = os.path.join(test_dir, "reconstructed.dat")
115+
116+ os.makedirs(test_dir, exist_ok=True)
117+ cache_data = {"result": result}
118+
119+ with open(metadata_file, "w") as f:
120+ json.dump(cache_data, f, indent=2)
121+ with open(compressed_file, "wb") as f:
122+ f.write(encoded_data)
123+
124+ # Save reconstructed data for lossy algorithms
125+ if reconstructed_data is not None:
126+ with open(reconstructed_file, "wb") as f:
127+ f.write(reconstructed_data.tobytes())
python/ephys_compression_tests/run_benchmarks/collect_info.pyadded+92−0View file
@@ -0,0 +1,92 @@
1+from typing import List, Dict, Any
2+from ._memobin import construct_dataset_url, construct_reconstructed_url
3+from ..types import Algorithm
4+
5+GITHUB_ALGORITHMS_PREFIX = "https://github.com/concept-collection/ephys_compression_tests/blob/main/python/ephys_compression_tests/algorithms/"
6+GITHUB_DATASETS_PREFIX = "https://github.com/concept-collection/ephys_compression_tests/blob/main/python/ephys_compression_tests/datasets/"
7+
8+
9+def collect_algorithm_info(algorithms: List[Dict[str, Algorithm]]) -> List[Dict[str, Any]]:
10+ """Collect information about compression algorithms.
11+
12+ Args:
13+ algorithms: List of algorithm dictionaries
14+
15+ Returns:
16+ List of algorithm information dictionaries
17+ """
18+ algorithm_info = []
19+ for algorithm in algorithms:
20+ info = {
21+ "name": algorithm.name,
22+ "description": algorithm.description if algorithm.description else "",
23+ "long_description": algorithm.long_description if algorithm.long_description else "",
24+ "version": algorithm.version,
25+ "tags": algorithm.tags if algorithm.tags else [],
26+ }
27+ if algorithm.source_file:
28+ info["source_file"] = GITHUB_ALGORITHMS_PREFIX + algorithm.source_file
29+ algorithm_info.append(info)
30+ return algorithm_info
31+
32+
33+def collect_dataset_info(datasets: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
34+ """Collect information about benchmark datasets.
35+
36+ Args:
37+ datasets: List of dataset dictionaries
38+
39+ Returns:
40+ List of dataset information dictionaries
41+ """
42+ dataset_info = []
43+ for dataset in datasets:
44+ info = {
45+ "name": dataset.name,
46+ "description": dataset.description if dataset.description else "",
47+ "long_description": dataset.long_description if dataset.long_description else "",
48+ "version": dataset.version,
49+ "tags": dataset.tags if dataset.tags else [],
50+ "data_url_raw": construct_dataset_url(
51+ dataset.name, dataset.version, "dat"
52+ ),
53+ "data_url_npy": construct_dataset_url(
54+ dataset.name, dataset.version, "npy"
55+ ),
56+ "data_url_json": construct_dataset_url(
57+ dataset.name, dataset.version, "json"
58+ ),
59+ }
60+ if dataset.source_file:
61+ info["source_file"] = GITHUB_DATASETS_PREFIX + dataset.source_file
62+ dataset_info.append(info)
63+ return dataset_info
64+
65+
66+def add_reconstructed_urls_to_results(results: List[Dict[str, Any]], algorithms: List[Algorithm]) -> None:
67+ """Add reconstructed data URL to results for lossy algorithms.
68+
69+ Args:
70+ results: List of benchmark result dictionaries (modified in-place)
71+ algorithms: List of algorithm objects
72+ """
73+ # Create a lookup dict for algorithm tags
74+ alg_tags_map = {alg.name: alg.tags for alg in algorithms}
75+
76+ for result in results:
77+ alg_name = result.get("algorithm")
78+ if not alg_name:
79+ continue
80+
81+ alg_tags = alg_tags_map.get(alg_name, [])
82+
83+ # Only add reconstructed URL for lossy algorithms (just .dat format)
84+ if "lossy" in alg_tags:
85+ result["reconstructed_url_raw"] = construct_reconstructed_url(
86+ alg_name,
87+ result["dataset"],
88+ result["algorithm_version"],
89+ result["dataset_version"],
90+ result["system_version"],
91+ "dat",
92+ )
python/ephys_compression_tests/run_benchmarks/is_compatible.pyadded+43−0View file
@@ -0,0 +1,43 @@
1+from typing import List
2+
3+
4+def is_compatible(algorithm_tags: List[str], dataset_tags: List[str]) -> bool:
5+ """Check if an algorithm is compatible with a dataset based on their tags.
6+
7+ Args:
8+ algorithm_tags: List of tags for the algorithm
9+ dataset_tags: List of tags for the dataset
10+
11+ Returns:
12+ True if the algorithm should be applied to the dataset
13+ """
14+ # If algorithm has delta_encoding or lpc_prediction, dataset must have continuous, timeseries, 1d, integer
15+ if "delta_encoding" in algorithm_tags or "lpc_prediction" in algorithm_tags:
16+ if (
17+ "correlated" not in dataset_tags
18+ or "timeseries" not in dataset_tags
19+ or "1d" not in dataset_tags
20+ or "integer" not in dataset_tags
21+ ):
22+ return False
23+
24+ # If algorithm has zero_rle, dataset must have sparse, timeseries, 1d
25+ if "zero_rle" in algorithm_tags:
26+ if (
27+ "sparse" not in dataset_tags
28+ or "timeseries" not in dataset_tags
29+ or "1d" not in dataset_tags
30+ ):
31+ return False
32+
33+ # If algorithm has integer, dataset must have integer
34+ if "integer" in algorithm_tags:
35+ if "integer" not in dataset_tags:
36+ return False
37+
38+ # If algorithm has "no_bernoulli", dataset must not have "bernoulli"
39+ if "no_bernoulli" in algorithm_tags:
40+ if "bernoulli" in dataset_tags:
41+ return False
42+
43+ return True
python/ephys_compression_tests/run_benchmarks/run_benchmarks.pyadded+250−0View file
@@ -0,0 +1,250 @@
1+import os
2+import time
3+from typing import Dict, Any, List, Optional
4+import numpy as np
5+
6+from ..algorithms import algorithms
7+from ..datasets import datasets
8+from ._memobin import construct_memobin_url, upload_to_memobin
9+from .upload_dataset import upload_dataset_to_memobin
10+from .upload_reconstructed import upload_reconstructed_to_memobin
11+from .cache_management import check_cached_result, save_result_to_cache
12+from .benchmark_timing import run_compression_benchmark
13+from .collect_info import collect_algorithm_info, collect_dataset_info
14+from .is_compatible import is_compatible
15+from .upload_benchmark_status import upload_benchmark_status
16+from ..types import Algorithm, Dataset
17+
18+system_version = "v6"
19+
20+
21+def run_benchmarks(
22+ cache_dir: str = ".benchmark_cache",
23+ verbose: bool = True,
24+ selected_algorithms: Optional[List[Algorithm]] = None,
25+ selected_datasets: Optional[List[Dataset]] = None,
26+ force: bool = False,
27+) -> Dict[str, Any]:
28+ """Run all benchmarks, with caching based on algorithm and dataset versions.
29+
30+ Results are stored in separate directories for each dataset/algorithm combination:
31+ cache_dir/
32+ dataset_name/
33+ algorithm_name/
34+ metadata.json # Contains algorithm version, dataset version, and results
35+ compressed.dat # The actual compressed data
36+
37+ Args:
38+ cache_dir: Directory to store cached results
39+ verbose: Whether to print progress messages
40+ selected_algorithms: Optional list of specific algorithms to run
41+ selected_datasets: Optional list of specific datasets to run
42+ force: If True, ignore cached results
43+
44+ Returns:
45+ Dictionary containing benchmark results and metadata
46+ """
47+ print("\n=== Starting Benchmark Run ===")
48+ print(f"Cache directory: {cache_dir}")
49+
50+ os.makedirs(cache_dir, exist_ok=True)
51+
52+ start_time = time.time()
53+ last_status_upload = 0 # Track last status upload time
54+ results = []
55+ print("\nRunning benchmarks for all dataset-algorithm combinations...")
56+
57+ # Use selected datasets/algorithms or fall back to all
58+ datasets_to_run = selected_datasets if selected_datasets is not None else datasets
59+ algorithms_to_run = (
60+ selected_algorithms if selected_algorithms is not None else algorithms
61+ )
62+
63+ # Calculate total number of benchmarks
64+ total_benchmarks = sum(
65+ 1
66+ for dataset in datasets_to_run
67+ for algorithm in algorithms_to_run
68+ if is_compatible(algorithm.tags, dataset.tags)
69+ )
70+
71+ # Run benchmarks for each dataset and algorithm combination
72+ memobin_api_key = os.environ.get("MEMOBIN_API_KEY")
73+ upload_enabled = os.environ.get("UPLOAD_TO_MEMOBIN") == "1"
74+
75+ for dataset in datasets_to_run:
76+ dataset_tags = dataset.tags
77+ print(f"\n*** Dataset: {dataset.name} (tags: {dataset_tags}) ***")
78+
79+ # only create the dataset if it is needed
80+ data = None
81+
82+ for algorithm in algorithms_to_run:
83+ alg_name = algorithm.name
84+ alg_tags = algorithm.tags
85+
86+ # Skip if algorithm and dataset are not compatible based on tags
87+ if not is_compatible(alg_tags, dataset_tags):
88+ if verbose:
89+ print(
90+ f"\nSkipping algorithm {alg_name} (tags: {alg_tags}) - incompatible with dataset tags"
91+ )
92+ continue
93+
94+ print(f"\nTesting algorithm: {alg_name} on dataset: {dataset.name}")
95+
96+ # Upload current status to memobin if enabled (once per minute)
97+ current_time = time.time()
98+ if (
99+ memobin_api_key
100+ and upload_enabled
101+ and (current_time - last_status_upload >= 60)
102+ ): # Check if 60 seconds have passed
103+ try:
104+ upload_benchmark_status(
105+ memobin_api_key,
106+ dataset.name,
107+ alg_name,
108+ results,
109+ total_benchmarks,
110+ start_time,
111+ )
112+ last_status_upload = current_time # Update last upload time
113+ except Exception as e:
114+ print(f" Warning: Failed to upload status to memobin: {str(e)}")
115+
116+ # Check if we can use cached result
117+ cached_result = check_cached_result(
118+ cache_dir,
119+ dataset.name,
120+ alg_name,
121+ algorithm.version,
122+ dataset.version,
123+ system_version,
124+ force,
125+ verbose,
126+ )
127+
128+ if cached_result is not None:
129+ print(" Using cached result")
130+ results.append(cached_result)
131+ continue
132+
133+ print(f" Running benchmark for {alg_name} on {dataset.name}...")
134+ if data is None:
135+ data = dataset.create()
136+ print(f"Created dataset: shape={data.shape}, dtype={data.dtype}")
137+ else:
138+ print("Dataset already created")
139+
140+ # Upload dataset to memobin if enabled
141+ if memobin_api_key and upload_enabled:
142+ try:
143+ upload_dataset_to_memobin(
144+ data,
145+ dataset.name,
146+ dataset.version,
147+ memobin_api_key,
148+ cache_dir,
149+ verbose,
150+ )
151+ except Exception as e:
152+ print(f" Warning: Failed to upload dataset to memobin: {str(e)}")
153+
154+ # Run the benchmark
155+ lossy = "lossy" in alg_tags
156+ result, encoded, decoded = run_compression_benchmark(
157+ data,
158+ alg_name,
159+ algorithm.encode,
160+ algorithm.decode,
161+ verbose,
162+ lossy=lossy
163+ )
164+
165+ # Add metadata to result
166+ result.update(
167+ {
168+ "dataset": dataset.name,
169+ "algorithm": alg_name,
170+ "algorithm_version": algorithm.version,
171+ "dataset_version": dataset.version,
172+ "system_version": system_version,
173+ }
174+ )
175+ results.append(result)
176+
177+ # Save result, compressed data, and reconstructed data (for lossy algorithms)
178+ save_result_to_cache(
179+ result,
180+ encoded,
181+ cache_dir,
182+ dataset.name,
183+ alg_name,
184+ reconstructed_data=decoded if lossy else None,
185+ )
186+ print(
187+ f" Results saved to: {os.path.join(cache_dir, dataset.name, alg_name)}"
188+ )
189+
190+ # Upload to memobin if enabled
191+ if memobin_api_key and upload_enabled:
192+ try:
193+ memobin_url = construct_memobin_url(
194+ alg_name,
195+ dataset.name,
196+ algorithm.version,
197+ dataset.version,
198+ system_version,
199+ )
200+ upload_to_memobin(
201+ {"result": result},
202+ memobin_url,
203+ memobin_api_key,
204+ )
205+ if verbose:
206+ print(" Successfully uploaded benchmark result to memobin")
207+ except Exception as e:
208+ print(f" Warning: Failed to upload to memobin: {str(e)}")
209+
210+ # Upload reconstructed array for lossy algorithms
211+ if lossy:
212+ try:
213+ upload_reconstructed_to_memobin(
214+ decoded,
215+ alg_name,
216+ dataset.name,
217+ algorithm.version,
218+ dataset.version,
219+ system_version,
220+ memobin_api_key,
221+ verbose,
222+ )
223+ except Exception as e:
224+ print(f" Warning: Failed to upload reconstructed data to memobin: {str(e)}")
225+
226+ print("\n=== Benchmark Run Complete ===\n")
227+
228+ # Collect algorithm and dataset information
229+ algorithm_info = collect_algorithm_info(algorithms)
230+ dataset_info = collect_dataset_info(datasets)
231+
232+ # Add reconstructed URLs to results for lossy algorithms
233+ from .collect_info import add_reconstructed_urls_to_results
234+ add_reconstructed_urls_to_results(results, algorithms_to_run)
235+
236+ # Upload final benchmark status
237+ if memobin_api_key and upload_enabled:
238+ try:
239+ upload_benchmark_status(
240+ memobin_api_key,
241+ "All datasets",
242+ "All algorithms",
243+ results,
244+ total_benchmarks,
245+ start_time,
246+ )
247+ except Exception as e:
248+ print(f" Warning: Failed to upload final status to memobin: {str(e)}")
249+
250+ return {"results": results, "algorithms": algorithm_info, "datasets": dataset_info}
python/ephys_compression_tests/run_benchmarks/upload_benchmark_status.pyadded+39−0View file
@@ -0,0 +1,39 @@
1+from typing import Any, Dict, List
2+import time
3+from datetime import datetime
4+from ._memobin import (
5+ upload_to_memobin,
6+)
7+
8+
9+def upload_benchmark_status(
10+ memobin_api_key: str,
11+ current_dataset: str,
12+ current_algorithm: str,
13+ completed_benchmarks: List[Dict[str, Any]],
14+ total_benchmarks: int,
15+ start_time: float,
16+) -> None:
17+ """Upload current benchmark status to memobin.
18+
19+ Args:
20+ memobin_api_key: API key for memobin authentication
21+ current_dataset: Name of the current dataset being processed
22+ current_algorithm: Name of the current algorithm being tested
23+ completed_benchmarks: List of completed benchmark results
24+ total_benchmarks: Total number of benchmarks to run
25+ start_time: Timestamp when the benchmark run started
26+ """
27+ status = {
28+ "current_dataset": current_dataset,
29+ "current_algorithm": current_algorithm,
30+ "completed_count": len(completed_benchmarks),
31+ "total_count": total_benchmarks,
32+ "progress_percentage": (len(completed_benchmarks) / total_benchmarks) * 100,
33+ "elapsed_time": time.time() - start_time,
34+ "last_update": datetime.now().isoformat(),
35+ "completed_benchmarks": completed_benchmarks,
36+ }
37+
38+ status_url = "https://tempory.net/f/memobin/ephys_compression_tests/benchmark_status/current.json"
39+ upload_to_memobin(status, status_url, memobin_api_key)
python/ephys_compression_tests/run_benchmarks/upload_dataset.pyadded+79−0View file
@@ -0,0 +1,79 @@
1+import os
2+import numpy as np
3+from ._memobin import (
4+ construct_dataset_url,
5+ exists_in_memobin,
6+ upload_to_memobin,
7+)
8+
9+
10+def upload_dataset_to_memobin(
11+ data: np.ndarray,
12+ dataset_name: str,
13+ dataset_version: str,
14+ memobin_api_key: str,
15+ cache_dir: str,
16+ verbose: bool = True,
17+) -> None:
18+ """Upload dataset to memobin in multiple formats.
19+
20+ Args:
21+ data: The numpy array dataset to upload
22+ dataset_name: Name of the dataset
23+ dataset_version: Version of the dataset
24+ memobin_api_key: API key for memobin
25+ cache_dir: Directory for temporary files
26+ verbose: Whether to print progress messages
27+ """
28+ try:
29+ # Upload array metadata as JSON
30+ dataset_url_json = construct_dataset_url(dataset_name, dataset_version, "json")
31+ if not exists_in_memobin(dataset_url_json):
32+ if verbose:
33+ print(" Uploading dataset metadata to memobin...")
34+ metadata = {"dtype": str(data.dtype), "shape": data.shape}
35+ upload_to_memobin(
36+ metadata,
37+ dataset_url_json,
38+ memobin_api_key,
39+ content_type="application/json",
40+ )
41+ if verbose:
42+ print(" Successfully uploaded metadata")
43+
44+ # Upload raw .dat format
45+ dataset_url_raw = construct_dataset_url(dataset_name, dataset_version, "dat")
46+ if not exists_in_memobin(dataset_url_raw):
47+ if verbose:
48+ print(" Uploading dataset (raw) to memobin...")
49+ upload_to_memobin(
50+ data.tobytes(),
51+ dataset_url_raw,
52+ memobin_api_key,
53+ content_type="application/octet-stream",
54+ )
55+ if verbose:
56+ print(" Successfully uploaded raw dataset")
57+
58+ # Upload .npy format
59+ dataset_url_npy = construct_dataset_url(dataset_name, dataset_version, "npy")
60+ if not exists_in_memobin(dataset_url_npy):
61+ if verbose:
62+ print(" Uploading dataset (npy) to memobin...")
63+ # Save array to a temporary .npy file
64+ temp_npy = os.path.join(cache_dir, "temp.npy")
65+ np.save(temp_npy, data)
66+ with open(temp_npy, "rb") as f:
67+ npy_bytes = f.read()
68+ os.remove(temp_npy) # Clean up temp file
69+
70+ upload_to_memobin(
71+ npy_bytes,
72+ dataset_url_npy,
73+ memobin_api_key,
74+ content_type="application/octet-stream",
75+ )
76+ if verbose:
77+ print(" Successfully uploaded npy dataset")
78+ except Exception as e:
79+ print(f" Warning: Failed to upload dataset to memobin: {str(e)}")
python/ephys_compression_tests/run_benchmarks/upload_reconstructed.pyadded+48−0View file
@@ -0,0 +1,48 @@
1+import numpy as np
2+from ._memobin import (
3+ construct_reconstructed_url,
4+ exists_in_memobin,
5+ upload_to_memobin,
6+)
7+
8+
9+def upload_reconstructed_to_memobin(
10+ data: np.ndarray,
11+ algorithm_name: str,
12+ dataset_name: str,
13+ algorithm_version: str,
14+ dataset_version: str,
15+ system_version: str,
16+ memobin_api_key: str,
17+ verbose: bool = True,
18+) -> None:
19+ """Upload reconstructed array to memobin as raw .dat format.
20+
21+ Args:
22+ data: The reconstructed numpy array to upload
23+ algorithm_name: Name of the algorithm
24+ dataset_name: Name of the dataset
25+ algorithm_version: Version of the algorithm
26+ dataset_version: Version of the dataset
27+ system_version: Version of the system
28+ memobin_api_key: API key for memobin
29+ verbose: Whether to print progress messages
30+ """
31+ try:
32+ # Upload raw .dat format
33+ reconstructed_url_raw = construct_reconstructed_url(
34+ algorithm_name, dataset_name, algorithm_version, dataset_version, system_version, "dat"
35+ )
36+ if not exists_in_memobin(reconstructed_url_raw):
37+ if verbose:
38+ print(" Uploading reconstructed array to memobin...")
39+ upload_to_memobin(
40+ data.tobytes(),
41+ reconstructed_url_raw,
42+ memobin_api_key,
43+ content_type="application/octet-stream",
44+ )
45+ if verbose:
46+ print(" Successfully uploaded reconstructed data")
47+ except Exception as e:
48+ print(f" Warning: Failed to upload reconstructed data to memobin: {str(e)}")
python/ephys_compression_tests/types.pyadded+42−0View file
@@ -0,0 +1,42 @@
1+from typing import Callable
2+import numpy as np
3+
4+class Algorithm:
5+ def __init__(self, *,
6+ name: str,
7+ version: str,
8+ encode: Callable[[np.ndarray], bytes],
9+ decode: Callable[[bytes, np.dtype, tuple], np.ndarray],
10+ description: str,
11+ tags: list[str],
12+ source_file: str,
13+ long_description: str
14+ ):
15+ self.name = name
16+ self.version = version
17+ self.encode = encode
18+ self.decode = decode
19+ self.description = description
20+ self.tags = tags
21+ self.source_file = source_file
22+ self.long_description = long_description
23+
24+class Dataset:
25+ def __init__(self, *,
26+ name: str,
27+ version: str,
28+ create: Callable[[], np.ndarray],
29+ description: str,
30+ tags: list[str],
31+ source_file: str,
32+ long_description: str,
33+ ideal_compression_ratio: float = 0
34+ ):
35+ self.name = name
36+ self.version = version
37+ self.create = create
38+ self.description = description
39+ self.tags = tags
40+ self.source_file = source_file
41+ self.long_description = long_description
42+ self.ideal_compression_ratio = ideal_compression_ratio
python/pyproject.tomladded+37−0View file
@@ -0,0 +1,37 @@
1+[build-system]
2+requires = ["setuptools>=61.0", "wheel"]
3+build-backend = "setuptools.build_meta"
4+
5+[project]
6+name = "ephys_compression_tests"
7+version = "0.1.0"
8+description = "Benchmarking compression methods for electrophysiology data"
9+readme = "README.md"
10+requires-python = ">=3.8"
11+authors = [
12+ { name = "Jeremy Magland" }
13+]
14+dependencies = [
15+ "numpy",
16+ "scipy",
17+ "zstandard",
18+ "simple_ans",
19+ "requests",
20+ "lindi",
21+ "brotli",
22+ "click",
23+ "numba",
24+ "segyio",
25+ "lz4",
26+ "pyedflib",
27+ "nibabel",
28+ "blosc2",
29+ "wavpack-numcodecs"
30+]
31+
32+[tool.setuptools.packages.find]
33+where = ["."]
34+include = ["ephys_compression_tests*"]
35+
36+[project.scripts]
37+ephys_compression_tests = "ephys_compression_tests.cli:main"
scripts/run_benchmarks.pyadded+38−0View file
@@ -0,0 +1,38 @@
1+#!/usr/bin/env python3
2+
3+import json
4+import os
5+from pathlib import Path
6+from ephys_compression_tests import run_benchmarks
7+from ephys_compression_tests.run_benchmarks._memobin import upload_to_memobin, construct_memobin_url
8+
9+def main():
10+ # Run benchmarks
11+ print("Running benchmarks...")
12+ results = run_benchmarks()
13+
14+ # Save detailed results to JSON
15+ output_dir = Path("benchmark_results")
16+ output_dir.mkdir(exist_ok=True)
17+ output_file = output_dir / "results.json"
18+
19+ with open(output_file, "w") as f:
20+ json.dump(results, f, indent=2)
21+
22+ print(f"\nDetailed results saved to {output_file}")
23+
24+ # Upload results to memobin if enabled
25+ memobin_api_key = os.environ.get("MEMOBIN_API_KEY")
26+ upload_enabled = os.environ.get("UPLOAD_TO_MEMOBIN") == "1"
27+
28+ if memobin_api_key and upload_enabled:
29+ try:
30+ # Construct URL for the global results file
31+ url = "https://tempory.net/f/memobin/ephys_compression_tests/global/results.json"
32+ upload_to_memobin(results, url, memobin_api_key)
33+ print("Successfully uploaded results to memobin")
34+ except Exception as e:
35+ print(f"Warning: Failed to upload results to memobin: {str(e)}")
36+
37+if __name__ == "__main__":
38+ main()
web-ui/.gitignoreadded+24−0View file
@@ -0,0 +1,24 @@
1+# Logs
2+logs
3+*.log
4+npm-debug.log*
5+yarn-debug.log*
6+yarn-error.log*
7+pnpm-debug.log*
8+lerna-debug.log*
9+
10+node_modules
11+dist
12+dist-ssr
13+*.local
14+
15+# Editor directories and files
16+.vscode/*
17+!.vscode/extensions.json
18+.idea
19+.DS_Store
20+*.suo
21+*.ntvs*
22+*.njsproj
23+*.sln
24+*.sw?
web-ui/README.mdadded+50−0View file
@@ -0,0 +1,50 @@
1+# React + TypeScript + Vite
2+
3+This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
4+
5+Currently, two official plugins are available:
6+
7+- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
8+- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
9+
10+## Expanding the ESLint configuration
11+
12+If you are developing a production application, we recommend updating the configuration to enable type aware lint rules:
13+
14+- Configure the top-level `parserOptions` property like this:
15+
16+```js
17+export default tseslint.config({
18+ languageOptions: {
19+ // other options...
20+ parserOptions: {
21+ project: ['./tsconfig.node.json', './tsconfig.app.json'],
22+ tsconfigRootDir: import.meta.dirname,
23+ },
24+ },
25+})
26+```
27+
28+- Replace `tseslint.configs.recommended` to `tseslint.configs.recommendedTypeChecked` or `tseslint.configs.strictTypeChecked`
29+- Optionally add `...tseslint.configs.stylisticTypeChecked`
30+- Install [eslint-plugin-react](https://github.com/jsx-eslint/eslint-plugin-react) and update the config:
31+
32+```js
33+// eslint.config.js
34+import react from 'eslint-plugin-react'
35+
36+export default tseslint.config({
37+ // Set the react version
38+ settings: { react: { version: '18.3' } },
39+ plugins: {
40+ // Add the react plugin
41+ react,
42+ },
43+ rules: {
44+ // other rules...
45+ // Enable its recommended rules
46+ ...react.configs.recommended.rules,
47+ ...react.configs['jsx-runtime'].rules,
48+ },
49+})
50+```
web-ui/eslint.config.jsadded+28−0View file
@@ -0,0 +1,28 @@
1+import js from '@eslint/js'
2+import globals from 'globals'
3+import reactHooks from 'eslint-plugin-react-hooks'
4+import reactRefresh from 'eslint-plugin-react-refresh'
5+import tseslint from 'typescript-eslint'
6+
7+export default tseslint.config(
8+ { ignores: ['dist'] },
9+ {
10+ extends: [js.configs.recommended, ...tseslint.configs.recommended],
11+ files: ['**/*.{ts,tsx}'],
12+ languageOptions: {
13+ ecmaVersion: 2020,
14+ globals: globals.browser,
15+ },
16+ plugins: {
17+ 'react-hooks': reactHooks,
18+ 'react-refresh': reactRefresh,
19+ },
20+ rules: {
21+ ...reactHooks.configs.recommended.rules,
22+ 'react-refresh/only-export-components': [
23+ 'warn',
24+ { allowConstantExport: true },
25+ ],
26+ },
27+ },
28+)
web-ui/index.htmladded+26−0View file
@@ -0,0 +1,26 @@
1+<!doctype html>
2+<html lang="en">
3+ <head>
4+ <meta charset="UTF-8" />
5+ <script>
6+ // Single Page Apps for GitHub Pages
7+ // MIT License
8+ // https://github.com/rafgraph/spa-github-pages
9+ (function(l) {
10+ if (l.search[1] === '/' ) {
11+ var decoded = l.search.slice(1).split('&')[0].split('=')[0].replace(/~and~/g, '&');
12+ window.history.replaceState(null, null,
13+ l.pathname.slice(0, -1) + decoded + l.hash
14+ );
15+ }
16+ }(window.location))
17+ </script>
18+ <link rel="icon" type="image/svg+xml" href="./favicon.svg" />
19+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
20+ <title>Ephys Compression Tests</title>
21+ </head>
22+ <body>
23+ <div id="root"></div>
24+ <script type="module" src="./src/main.tsx"></script>
25+ </body>
26+</html>
web-ui/package-lock.jsonadded+7968−0View file
This diff is 7,973 lines long and is not shown.
web-ui/package.jsonadded+47−0View file
@@ -0,0 +1,47 @@
1+{
2+ "name": "web-ui",
3+ "private": true,
4+ "version": "0.0.0",
5+ "type": "module",
6+ "scripts": {
7+ "dev": "vite",
8+ "build": "tsc -b && vite build",
9+ "lint": "eslint .",
10+ "preview": "vite preview",
11+ "format": "prettier --write \"src/**/*.{ts,tsx,js,jsx}\"",
12+ "format:check": "prettier --check \"src/**/*.{ts,tsx,js,jsx}\""
13+ },
14+ "dependencies": {
15+ "@tanstack/react-table": "^8.20.6",
16+ "@types/plotly.js": "^2.35.2",
17+ "axios": "^1.7.9",
18+ "katex": "^0.16.21",
19+ "plotly.js-dist-min": "^2.35.3",
20+ "react": "^18.3.1",
21+ "react-dom": "^18.3.1",
22+ "react-markdown": "^9.0.3",
23+ "react-plotly.js": "^2.6.0",
24+ "react-router-dom": "^7.1.3",
25+ "rehype-katex": "^7.0.1",
26+ "remark-math": "^6.0.0",
27+ "yaml": "^2.7.0"
28+ },
29+ "devDependencies": {
30+ "@eslint/js": "^9.17.0",
31+ "@types/react": "^18.3.18",
32+ "@types/react-dom": "^18.3.5",
33+ "@types/react-plotly.js": "^2.6.3",
34+ "@vitejs/plugin-react": "^4.3.4",
35+ "autoprefixer": "^10.4.20",
36+ "eslint": "^9.17.0",
37+ "eslint-plugin-react-hooks": "^5.0.0",
38+ "eslint-plugin-react-refresh": "^0.4.16",
39+ "globals": "^15.14.0",
40+ "postcss": "^8.5.1",
41+ "prettier": "^3.4.2",
42+ "tailwindcss": "^4.0.0",
43+ "typescript": "~5.6.2",
44+ "typescript-eslint": "^8.18.2",
45+ "vite": "^6.0.5"
46+ }
47+}
web-ui/public/.gitignoreadded+1−0View file
@@ -0,0 +1 @@
1+*.pdf
web-ui/public/404.htmladded+26−0View file
@@ -0,0 +1,26 @@
1+<!DOCTYPE html>
2+<html>
3+ <head>
4+ <meta charset="utf-8">
5+ <title>Ephys Compression Tests</title>
6+ <script>
7+ // Single Page Apps for GitHub Pages
8+ // MIT License
9+ // https://github.com/rafgraph/spa-github-pages
10+ (function(){
11+ var pathSegmentsToKeep = 1;
12+
13+ var l = window.location;
14+ l.replace(
15+ l.protocol + '//' + l.hostname + (l.port ? ':' + l.port : '') +
16+ l.pathname.split('/').slice(0, 1 + pathSegmentsToKeep).join('/') + '/?/' +
17+ l.pathname.slice(1).split('/').slice(pathSegmentsToKeep).join('/').replace(/&/g, '~and~') +
18+ (l.search ? '&' + l.search.slice(1).replace(/&/g, '~and~') : '') +
19+ l.hash
20+ );
21+ }());
22+ </script>
23+ </head>
24+ <body>
25+ </body>
26+</html>
web-ui/public/favicon.svgadded+31−0View file
@@ -0,0 +1,31 @@
1+<?xml version="1.0" encoding="UTF-8"?>
2+<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
3+ <!-- Background shape -->
4+ <rect width="32" height="32" rx="6" fill="#f8f9fa"/>
5+
6+ <!-- Scientific waveform representing data -->
7+ <path d="M6 16 L10 16 L12 12 L14 20 L16 12 L18 20 L20 16 L24 16"
8+ stroke="#2563eb"
9+ stroke-width="2"
10+ fill="none"
11+ stroke-linecap="round"
12+ stroke-linejoin="round"/>
13+
14+ <!-- Compression brackets -->
15+ <path d="M4 8 L4 24 L8 20 M8 12 L4 8"
16+ stroke="#1e40af"
17+ stroke-width="2"
18+ fill="none"
19+ stroke-linecap="round"
20+ stroke-linejoin="round"/>
21+
22+ <path d="M28 8 L28 24 L24 20 M24 12 L28 8"
23+ stroke="#1e40af"
24+ stroke-width="2"
25+ fill="none"
26+ stroke-linecap="round"
27+ stroke-linejoin="round"/>
28+
29+ <!-- Central dot -->
30+ <circle cx="16" cy="16" r="1.5" fill="#1e40af"/>
31+</svg>
web-ui/public/logo.svgadded+31−0View file
@@ -0,0 +1,31 @@
1+<?xml version="1.0" encoding="UTF-8"?>
2+<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
3+ <!-- Background shape -->
4+ <rect width="32" height="32" rx="6" fill="#f8f9fa"/>
5+
6+ <!-- Scientific waveform representing data -->
7+ <path d="M6 16 L10 16 L12 12 L14 20 L16 12 L18 20 L20 16 L24 16"
8+ stroke="#2563eb"
9+ stroke-width="2"
10+ fill="none"
11+ stroke-linecap="round"
12+ stroke-linejoin="round"/>
13+
14+ <!-- Compression brackets -->
15+ <path d="M4 8 L4 24 L8 20 M8 12 L4 8"
16+ stroke="#1e40af"
17+ stroke-width="2"
18+ fill="none"
19+ stroke-linecap="round"
20+ stroke-linejoin="round"/>
21+
22+ <path d="M28 8 L28 24 L24 20 M24 12 L28 8"
23+ stroke="#1e40af"
24+ stroke-width="2"
25+ fill="none"
26+ stroke-linecap="round"
27+ stroke-linejoin="round"/>
28+
29+ <!-- Central dot -->
30+ <circle cx="16" cy="16" r="1.5" fill="#1e40af"/>
31+</svg>
web-ui/public/vite.svgadded+1−0View file
@@ -0,0 +1 @@
1+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>
web-ui/src/App.cssadded+35−0View file
@@ -0,0 +1,35 @@
1+/* Existing styles */
2+
3+/* Markdown content styles */
4+.markdown-content {
5+ font-family: system-ui, -apple-system, sans-serif;
6+ line-height: 1.6;
7+ color: #333;
8+}
9+
10+.markdown-content h1 {
11+ font-size: 2.2rem;
12+ margin-bottom: 1.5rem;
13+ color: #1a1a1a;
14+}
15+
16+.markdown-content h2 {
17+ font-size: 1.8rem;
18+ margin: 2rem 0 1rem;
19+ color: #1a1a1a;
20+}
21+
22+.markdown-content p {
23+ margin-bottom: 1.2rem;
24+ font-size: 1.1rem;
25+}
26+
27+.markdown-content ul {
28+ margin: 1rem 0;
29+ padding-left: 2rem;
30+}
31+
32+.markdown-content li {
33+ margin: 0.5rem 0;
34+ font-size: 1.1rem;
35+}
web-ui/src/App.tsxadded+173−0View file
@@ -0,0 +1,173 @@
1+import { BrowserRouter, Routes, Route, Link } from "react-router-dom";
2+import { useEffect, useState } from "react";
3+import axios from "axios";
4+import { ScrollToTop } from "./components/ScrollToTop";
5+import "./components/AppHeader.css";
6+import Home from "./pages/Home";
7+import BenchmarkView from "./pages/BenchmarkView";
8+import Monitor from "./pages/Monitor";
9+import Submit from "./pages/Submit";
10+import { BenchmarkData } from "./types";
11+
12+function App() {
13+ const [benchmarkData, setBenchmarkData] = useState<BenchmarkData | null>(
14+ null,
15+ );
16+ const [isLoading, setIsLoading] = useState(true);
17+ const [error, setError] = useState<string | null>(null);
18+
19+ useEffect(() => {
20+ const fetchData = async () => {
21+ try {
22+ setIsLoading(true);
23+ setError(null);
24+ const cacheBust = Math.random().toString(36).substring(2, 15);
25+ const response = await axios.get(
26+ `https://tempory.net/f/memobin/ephys_compression_tests/global/results.json?cachebust=${cacheBust}`,
27+ );
28+ setBenchmarkData(response.data);
29+ } catch (error) {
30+ const message =
31+ error instanceof Error ? error.message : "Failed to fetch data";
32+ setError(message);
33+ console.error("Error fetching benchmark data:", error);
34+ } finally {
35+ setIsLoading(false);
36+ }
37+ };
38+
39+ fetchData();
40+ }, []);
41+
42+ return (
43+ <BrowserRouter basename="/ephys_compression_tests/">
44+ <ScrollToTop />
45+ <div
46+ style={{
47+ paddingTop: "3rem",
48+ padding: "3rem 2rem 2rem 2rem",
49+ }}
50+ >
51+ <nav
52+ style={{
53+ position: "fixed",
54+ top: 0,
55+ left: 0,
56+ right: 0,
57+ padding: "0.35rem min(2rem, 4%)",
58+ backgroundColor: "white",
59+ zIndex: 1000,
60+ }}
61+ >
62+ <div
63+ style={{
64+ display: "flex",
65+ justifyContent: "space-between",
66+ alignItems: "center",
67+ minHeight: "32px",
68+ }}
69+ >
70+ <Link
71+ to="/"
72+ style={{
73+ display: "flex",
74+ alignItems: "center",
75+ textDecoration: "none",
76+ minWidth: 0,
77+ maxWidth: "calc(100% - 80px)",
78+ }}
79+ >
80+ <img
81+ src="/ephys_compression_tests/logo.svg"
82+ alt="Ephys Compression Tests Logo"
83+ style={{
84+ width: "28px",
85+ height: "28px",
86+ marginRight: "10px",
87+ flexShrink: 0,
88+ }}
89+ />
90+ <span
91+ style={{
92+ minWidth: 0,
93+ whiteSpace: "nowrap",
94+ overflow: "hidden",
95+ textOverflow: "ellipsis",
96+ }}
97+ >
98+ <span
99+ style={{
100+ fontSize: "1rem",
101+ fontWeight: "500",
102+ color: "#2c2c2c",
103+ }}
104+ >
105+ Ephys Compression Tests
106+ </span>
107+ <span className="app-header-subtitle">
108+ {" · "}
109+ <span style={{ fontSize: "1rem", color: "#777" }}>
110+ Comparing compression algorithms for ephys data
111+ </span>
112+ </span>
113+ </span>
114+ </Link>
115+ <div style={{ display: "flex", gap: "1.5rem" }}>
116+ <Link
117+ to="/datasets"
118+ style={{
119+ color: "#0066cc",
120+ textDecoration: "none",
121+ fontWeight: "500",
122+ }}
123+ >
124+ Datasets
125+ </Link>
126+ <Link
127+ to="/algorithms"
128+ style={{
129+ color: "#0066cc",
130+ textDecoration: "none",
131+ fontWeight: "500",
132+ }}
133+ >
134+ Algorithms
135+ </Link>
136+ </div>
137+ </div>
138+ </nav>
139+ <main>
140+ {isLoading ? (
141+ <div>Loading benchmark data...</div>
142+ ) : error ? (
143+ <div>Error: {error}</div>
144+ ) : (
145+ <Routes>
146+ <Route path="/" element={<Home />} />
147+ <Route
148+ path="/datasets"
149+ element={<BenchmarkView benchmarkData={benchmarkData} />}
150+ />
151+ <Route
152+ path="/algorithms"
153+ element={<BenchmarkView benchmarkData={benchmarkData} />}
154+ />
155+ <Route
156+ path="/dataset/:datasetName"
157+ element={<BenchmarkView benchmarkData={benchmarkData} />}
158+ />
159+ <Route
160+ path="/algorithm/:algorithmName"
161+ element={<BenchmarkView benchmarkData={benchmarkData} />}
162+ />
163+ <Route path="/monitor" element={<Monitor />} />
164+ <Route path="/submit" element={<Submit />} />
165+ </Routes>
166+ )}
167+ </main>
168+ </div>
169+ </BrowserRouter>
170+ );
171+}
172+
173+export default App;
web-ui/src/assets/react.svgadded+1−0View file
@@ -0,0 +1 @@
1+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>
web-ui/src/components/AppHeader.cssadded+9−0View file
@@ -0,0 +1,9 @@
1+.app-header-subtitle {
2+ display: none;
3+}
4+
5+@media (min-width: 640px) {
6+ .app-header-subtitle {
7+ display: inline;
8+ }
9+}
web-ui/src/components/BenchmarkTable.tsxadded+1−0View file
@@ -0,0 +1 @@
1+export { BenchmarkTable } from "./benchmark/table/BenchmarkTable";
web-ui/src/components/Button.cssadded+18−0View file
@@ -0,0 +1,18 @@
1+.soft-button {
2+ display: inline-block;
3+ padding: 0.6rem 1.2rem;
4+ background-color: #2b7de9;
5+ color: white;
6+ text-decoration: none;
7+ border-radius: 20px;
8+ font-weight: 500;
9+ box-shadow: 0 2px 4px rgba(0,0,0,0.1);
10+ transition: all 0.2s ease;
11+ cursor: pointer;
12+}
13+
14+.soft-button:hover {
15+ background-color: #1a68d4;
16+ transform: translateY(-1px);
17+ box-shadow: 0 4px 8px rgba(0,0,0,0.15);
18+}
web-ui/src/components/ScrollToTop.tsxadded+12−0View file
@@ -0,0 +1,12 @@
1+import { useEffect } from "react";
2+import { useLocation } from "react-router-dom";
3+
4+export function ScrollToTop() {
5+ const { pathname } = useLocation();
6+
7+ useEffect(() => {
8+ window.scrollTo(0, 0);
9+ }, [pathname]);
10+
11+ return null;
12+}
web-ui/src/components/TagFilter.tsxadded+45−0View file
@@ -0,0 +1,45 @@
1+interface TagFilterProps {
2+ availableTags: string[];
3+ selectedTags: string[];
4+ onTagToggle: (tag: string) => void;
5+ label: string;
6+}
7+
8+export function TagFilter({
9+ availableTags,
10+ selectedTags,
11+ onTagToggle,
12+ label,
13+}: TagFilterProps) {
14+ return (
15+ <div style={{ marginTop: "1rem" }}>
16+ <div style={{ display: "flex", alignItems: "center", gap: "1rem" }}>
17+ <div
18+ style={{ fontSize: "0.9rem", color: "#666", whiteSpace: "nowrap" }}
19+ >
20+ {label}:
21+ </div>
22+ <div style={{ display: "flex", flexWrap: "wrap", gap: "0.5rem" }}>
23+ {availableTags.map((tag) => (
24+ <button
25+ key={tag}
26+ onClick={() => onTagToggle(tag)}
27+ style={{
28+ padding: "4px 8px",
29+ border: "1px solid #ddd",
30+ borderRadius: "4px",
31+ background: selectedTags.includes(tag) ? "#0066cc" : "white",
32+ color: selectedTags.includes(tag) ? "white" : "#333",
33+ cursor: "pointer",
34+ fontSize: "0.8rem",
35+ transition: "all 0.2s ease",
36+ }}
37+ >
38+ {tag}
39+ </button>
40+ ))}
41+ </div>
42+ </div>
43+ </div>
44+ );
45+}
web-ui/src/components/algorithm/AlgorithmContent.tsxadded+35−0View file
@@ -0,0 +1,35 @@
1+import { Algorithm, BenchmarkData } from "../../types";
2+import { BaseContent } from "../shared/BaseContent";
3+import "../shared/ContentStyles.css";
4+
5+interface AlgorithmContentProps {
6+ algorithm: Algorithm;
7+ benchmarkData: BenchmarkData | null;
8+ chartData: Array<{
9+ algorithmOrDataset: string;
10+ compression_ratio: number;
11+ reference_compression_ratio: number | null;
12+ encode_speed: number;
13+ decode_speed: number;
14+ rmse?: number;
15+ tags: string[];
16+ }>;
17+}
18+
19+export const AlgorithmContent = ({
20+ algorithm,
21+ benchmarkData,
22+ chartData,
23+}: AlgorithmContentProps) => {
24+ return (
25+ <BaseContent
26+ item={algorithm}
27+ benchmarkData={benchmarkData}
28+ chartData={chartData}
29+ tagNavigationPrefix="/algorithms"
30+ filterKey="algorithm"
31+ showSortByCompressionRatio={false}
32+ showNormalizeByReference={true}
33+ />
34+ );
35+};
web-ui/src/components/benchmark/charts/BenchmarkCharts.tsxadded+253−0View file
@@ -0,0 +1,253 @@
1+import { useState } from "react";
2+import Plot from "react-plotly.js";
3+
4+interface BenchmarkBarChartProps {
5+ title: string;
6+ data: ChartData[];
7+ dataKey: keyof Pick<
8+ ChartData,
9+ "compression_ratio" | "encode_speed" | "decode_speed" | "rmse" | "max_error"
10+ >;
11+ color: string;
12+ xAxisTitle: string;
13+ normalize?: boolean;
14+}
15+
16+function BenchmarkBarChart({
17+ title,
18+ data,
19+ dataKey,
20+ color,
21+ xAxisTitle,
22+ normalize,
23+}: BenchmarkBarChartProps) {
24+ const normalizedData =
25+ normalize && dataKey === "compression_ratio"
26+ ? data.map((d) => ({
27+ ...d,
28+ compression_ratio: d.reference_compression_ratio
29+ ? d.compression_ratio / d.reference_compression_ratio
30+ : d.compression_ratio,
31+ reference_compression_ratio: d.reference_compression_ratio ? 1 : null,
32+ }))
33+ : data;
34+
35+ return (
36+ <div style={{ margin: "0 20px 20px 0" }}>
37+ <h3 style={{ marginBottom: "10px" }}>{title}</h3>
38+ <Plot
39+ data={[
40+ {
41+ type: "bar",
42+ orientation: "h",
43+ y: normalizedData.map((d) => d.algorithmOrDataset),
44+ x: normalizedData.map((d) => {
45+ const value = d[dataKey];
46+ return value !== undefined ? value : 0;
47+ }),
48+ marker: { color },
49+ name: title,
50+ hovertemplate:
51+ normalize && dataKey === "compression_ratio"
52+ ? "%{x:.3f}×<extra></extra>"
53+ : "%{x:.2f}<extra></extra>",
54+ },
55+ ...(dataKey === "compression_ratio" &&
56+ normalizedData.some((d) => d.reference_compression_ratio !== null)
57+ ? [
58+ ...normalizedData
59+ .filter((d) => d.reference_compression_ratio !== null)
60+ .flatMap((d) => [
61+ {
62+ type: "scatter" as const,
63+ mode: "lines" as const,
64+ y: [d.algorithmOrDataset, d.algorithmOrDataset],
65+ x: [0, d.reference_compression_ratio],
66+ line: { color, width: 1 },
67+ showlegend: false,
68+ hoverinfo: "skip" as const,
69+ },
70+ {
71+ type: "scatter" as const,
72+ mode: "markers" as const,
73+ y: [d.algorithmOrDataset],
74+ x: [d.reference_compression_ratio],
75+ marker: { color: "#aaaaaa", size: 8 },
76+ name: "Best Compression",
77+ hovertemplate: normalize
78+ ? "Best: 1.000×<extra></extra>"
79+ : "Best: %{x:.2f}<extra></extra>",
80+ showlegend:
81+ d.algorithmOrDataset ===
82+ normalizedData[0].algorithmOrDataset,
83+ },
84+ ]),
85+ ]
86+ : []),
87+ ]}
88+ layout={{
89+ width: 700,
90+ height: Math.max(300, data.length * 23 + 40),
91+ margin: { t: 5, r: 30, l: 200, b: 30 },
92+ xaxis: { title: xAxisTitle },
93+ yaxis: {
94+ automargin: true,
95+ ticksuffix: " ",
96+ tickmode: "array",
97+ tickvals: normalizedData.map((d) => d.algorithmOrDataset),
98+ ticktext: normalizedData.map((d) =>
99+ d.tags.includes("lossy")
100+ ? `<span style="color: red;">${d.algorithmOrDataset}*</span>`
101+ : d.algorithmOrDataset
102+ ),
103+ },
104+ dragmode: false,
105+ }}
106+ config={{ displayModeBar: false }}
107+ />
108+ </div>
109+ );
110+}
111+
112+interface ChartData {
113+ algorithmOrDataset: string;
114+ compression_ratio: number;
115+ reference_compression_ratio: number | null; // the highest compression ratio for the dataset (if algorithmOrDataset is a dataset)
116+ encode_speed: number;
117+ decode_speed: number;
118+ rmse?: number;
119+ max_error?: number;
120+ tags: string[];
121+}
122+
123+interface BenchmarkChartsProps {
124+ chartData: ChartData[];
125+ showSortByCompressionRatio?: boolean;
126+ showNormalizeByReference?: boolean;
127+}
128+
129+export function BenchmarkCharts({
130+ chartData,
131+ showSortByCompressionRatio,
132+ showNormalizeByReference,
133+}: BenchmarkChartsProps) {
134+ const [sortByRatio, setSortByRatio] = useState(
135+ showSortByCompressionRatio ? true : false,
136+ );
137+ const [normalize, setNormalize] = useState(false);
138+ const [showLossyAlgs, setShowLossyAlgs] = useState(true);
139+ const [errorMetric, setErrorMetric] = useState<"rmse" | "max_error">("rmse");
140+
141+ if (!chartData.length) return null;
142+
143+ // Filter data based on showLossyAlgs
144+ // If showLossyAlgs is true, show all algorithms (both lossy and lossless)
145+ // If showLossyAlgs is false, only show lossless algorithms
146+ const filteredData = showLossyAlgs
147+ ? chartData
148+ : chartData.filter((d) => !d.tags.includes("lossy"));
149+
150+ const sortedData = sortByRatio
151+ ? [...filteredData].sort((a, b) => a.compression_ratio - b.compression_ratio)
152+ : filteredData;
153+
154+ // For Error chart, only show lossy algorithms with error values
155+ const lossyData = chartData.filter(
156+ (d) => d.tags.includes("lossy") &&
157+ (errorMetric === "rmse" ? d.rmse !== undefined : d.max_error !== undefined)
158+ );
159+ const sortedLossyData = sortByRatio
160+ ? [...lossyData].sort((a, b) => a.compression_ratio - b.compression_ratio)
161+ : lossyData;
162+
163+ return (
164+ <div>
165+ {showSortByCompressionRatio && (
166+ <div style={{ marginBottom: "10px", display: "flex", gap: "16px" }}>
167+ <label style={{ display: "flex", alignItems: "center", gap: "8px" }}>
168+ <input
169+ type="checkbox"
170+ checked={sortByRatio}
171+ onChange={(e) => setSortByRatio(e.target.checked)}
172+ />
173+ Sort by compression ratio
174+ </label>
175+ <label style={{ display: "flex", alignItems: "center", gap: "8px" }}>
176+ <input
177+ type="checkbox"
178+ checked={showLossyAlgs}
179+ onChange={(e) => setShowLossyAlgs(e.target.checked)}
180+ />
181+ Show lossy algs
182+ </label>
183+ </div>
184+ )}
185+ {showNormalizeByReference && (
186+ <div style={{ marginBottom: "10px" }}>
187+ <label style={{ display: "flex", alignItems: "center", gap: "8px" }}>
188+ <input
189+ type="checkbox"
190+ checked={normalize}
191+ onChange={(e) => setNormalize(e.target.checked)}
192+ />
193+ Normalize to best compression
194+ </label>
195+ </div>
196+ )}
197+ <div
198+ style={{
199+ display: "flex",
200+ flexWrap: "wrap",
201+ gap: "20px",
202+ }}
203+ >
204+ <BenchmarkBarChart
205+ title="Compression Ratio"
206+ data={sortedData}
207+ dataKey="compression_ratio"
208+ color="#8884d8"
209+ xAxisTitle={normalize ? "Fraction of Best Compression" : "Ratio"}
210+ normalize={normalize}
211+ />
212+ <BenchmarkBarChart
213+ title="Encode Speed (MB/s)"
214+ data={sortedData}
215+ dataKey="encode_speed"
216+ color="#82ca9d"
217+ xAxisTitle="MB/s"
218+ />
219+ <BenchmarkBarChart
220+ title="Decode Speed (MB/s)"
221+ data={sortedData}
222+ dataKey="decode_speed"
223+ color="#ff7300"
224+ xAxisTitle="MB/s"
225+ />
226+ {sortedLossyData.length > 0 && (
227+ <div>
228+ <div style={{ marginBottom: "10px" }}>
229+ <label style={{ display: "flex", alignItems: "center", gap: "8px" }}>
230+ Error Metric:
231+ <select
232+ value={errorMetric}
233+ onChange={(e) => setErrorMetric(e.target.value as "rmse" | "max_error")}
234+ style={{ marginLeft: "8px", padding: "4px 8px" }}
235+ >
236+ <option value="rmse">RMSE</option>
237+ <option value="max_error">Maximum</option>
238+ </select>
239+ </label>
240+ </div>
241+ <BenchmarkBarChart
242+ title="Error (Lossy Algs)"
243+ data={sortedLossyData}
244+ dataKey={errorMetric}
245+ color="#d62728"
246+ xAxisTitle={errorMetric === "rmse" ? "RMSE" : "Maximum Error"}
247+ />
248+ </div>
249+ )}
250+ </div>
251+ </div>
252+ );
253+}
web-ui/src/components/benchmark/charts/BenchmarkScatterPlots.tsxadded+181−0View file
@@ -0,0 +1,181 @@
1+import Plot from "react-plotly.js";
2+import { useState } from "react";
3+
4+interface ChartData {
5+ algorithmOrDataset: string;
6+ compression_ratio: number;
7+ encode_speed: number;
8+ decode_speed: number;
9+ tags: string[];
10+}
11+
12+interface BenchmarkScatterPlotsProps {
13+ chartData: ChartData[];
14+}
15+
16+export function BenchmarkScatterPlots({
17+ chartData,
18+}: BenchmarkScatterPlotsProps) {
19+ const [showLabels, setShowLabels] = useState(false);
20+ const [showLossyAlgs, setShowLossyAlgs] = useState(true);
21+
22+ if (!chartData.length) return null;
23+
24+ // Filter data based on showLossyAlgs
25+ const filteredData = showLossyAlgs
26+ ? chartData
27+ : chartData.filter((d) => !d.tags.includes("lossy"));
28+
29+ const uniqueAlgorithms = Array.from(
30+ new Set(filteredData.map((d) => d.algorithmOrDataset)),
31+ );
32+
33+ const colors = [
34+ "#1f77b4", // blue
35+ "#ff7f0e", // orange
36+ "#2ca02c", // green
37+ "#d62728", // red
38+ "#9467bd", // purple
39+ "#8c564b", // brown
40+ "#e377c2", // pink
41+ "#7f7f7f", // gray
42+ ];
43+
44+ const markers = ["circle", "square", "diamond", "triangle-up", "star"];
45+
46+ // Create traces for each algorithm
47+ const traces = uniqueAlgorithms.flatMap((algo, i) => {
48+ const algoData = filteredData.filter((d) => d.algorithmOrDataset === algo);
49+ const isLossy = algoData.length > 0 && algoData[0].tags.includes("lossy");
50+ const displayName = isLossy ? `${algo}*` : algo;
51+ const baseTrace = {
52+ name: displayName,
53+ mode: showLabels ? ("markers+text" as const) : ("markers" as const),
54+ marker: {
55+ color: isLossy ? "red" : colors[i % colors.length],
56+ symbol: markers[Math.floor(i / colors.length) % markers.length],
57+ size: 10,
58+ },
59+ text: showLabels ? algoData.map(() => displayName) : [],
60+ textposition: "top center" as const,
61+ textfont: isLossy ? { color: "red" } : undefined,
62+ showlegend: true,
63+ legendgroup: algo,
64+ };
65+
66+ return [
67+ // Compression Ratio vs Decode Speed (upper left)
68+ {
69+ ...baseTrace,
70+ x: algoData.map((d) => d.compression_ratio),
71+ y: algoData.map((d) => d.decode_speed),
72+ xaxis: "x" as const,
73+ yaxis: "y" as const,
74+ showlegend: true,
75+ },
76+ // Compression Ratio vs Encode Speed (lower left)
77+ {
78+ ...baseTrace,
79+ x: algoData.map((d) => d.compression_ratio),
80+ y: algoData.map((d) => d.encode_speed),
81+ xaxis: "x2" as const,
82+ yaxis: "y2" as const,
83+ showlegend: false,
84+ },
85+ // Decode Speed vs Encode Speed (lower right)
86+ {
87+ ...baseTrace,
88+ x: algoData.map((d) => d.decode_speed),
89+ y: algoData.map((d) => d.encode_speed),
90+ xaxis: "x3" as const,
91+ yaxis: "y3" as const,
92+ showlegend: false,
93+ },
94+ ];
95+ });
96+
97+ return (
98+ <div style={{ margin: "20px 0" }}>
99+ <div style={{ marginBottom: "10px" }}>
100+ <h2 style={{ marginBottom: "10px" }}>Performance Relationships</h2>
101+ <div style={{ display: "flex", gap: "16px" }}>
102+ <label style={{ display: "flex", alignItems: "center", gap: "8px" }}>
103+ <input
104+ type="checkbox"
105+ checked={showLabels}
106+ onChange={(e) => setShowLabels(e.target.checked)}
107+ />
108+ Show point labels
109+ </label>
110+ <label style={{ display: "flex", alignItems: "center", gap: "8px" }}>
111+ <input
112+ type="checkbox"
113+ checked={showLossyAlgs}
114+ onChange={(e) => setShowLossyAlgs(e.target.checked)}
115+ />
116+ Show lossy algs
117+ </label>
118+ </div>
119+ </div>
120+ <Plot
121+ data={traces}
122+ layout={{
123+ width: 1000,
124+ height: 670,
125+ grid: {
126+ rows: 2,
127+ columns: 2,
128+ pattern: "independent",
129+ },
130+ xaxis: {
131+ title: "Compression Ratio",
132+ domain: [0, 0.45],
133+ },
134+ yaxis: {
135+ title: "Decode Speed (MB/s)",
136+ domain: [0.55, 1],
137+ },
138+ xaxis2: {
139+ title: "Compression Ratio",
140+ domain: [0, 0.45],
141+ },
142+ yaxis2: {
143+ title: "Encode Speed (MB/s)",
144+ domain: [0, 0.45],
145+ },
146+ xaxis3: {
147+ title: "Decode Speed (MB/s)",
148+ domain: [0.55, 1],
149+ },
150+ yaxis3: {
151+ title: "Encode Speed (MB/s)",
152+ domain: [0, 0.45],
153+ },
154+ showlegend: true,
155+ legend: {
156+ x: 1.08,
157+ y: 1,
158+ xanchor: "left" as const,
159+ yanchor: "top" as const,
160+ },
161+ margin: {
162+ l: 60,
163+ r: 40,
164+ t: 20,
165+ b: 60,
166+ },
167+ }}
168+ config={{
169+ displayModeBar: true,
170+ displaylogo: false,
171+ modeBarButtonsToRemove: [
172+ "lasso2d",
173+ "select2d",
174+ "hoverClosestCartesian",
175+ "hoverCompareCartesian",
176+ ],
177+ }}
178+ />
179+ </div>
180+ );
181+}
web-ui/src/components/benchmark/export/csvExport.tsadded+49−0View file
@@ -0,0 +1,49 @@
1+import { BenchmarkResult } from "../../../types";
2+import { formatNumber, formatSize } from "../utils/formatters";
3+import { columns } from "../table/columns";
4+
5+export const exportToCsv = (
6+ data: BenchmarkResult[],
7+ selectedDataset: string,
8+) => {
9+ // Convert data to CSV
10+ const headers = columns.map((col) => col.header).join(",");
11+ const rows = data
12+ .map((row) =>
13+ columns
14+ .map((col) => {
15+ const value = row[col.accessorKey as keyof BenchmarkResult];
16+ // Format numbers according to their display format
17+ if (col.accessorKey === "compression_ratio") {
18+ return `${formatNumber(value as number)}x`;
19+ } else if (
20+ col.accessorKey === "encode_time" ||
21+ col.accessorKey === "decode_time"
22+ ) {
23+ return formatNumber(value as number, 4);
24+ } else if (
25+ col.accessorKey === "original_size" ||
26+ col.accessorKey === "compressed_size"
27+ ) {
28+ return formatSize(value as number);
29+ } else if (typeof value === "number") {
30+ return formatNumber(value);
31+ }
32+ return value;
33+ })
34+ .join(","),
35+ )
36+ .join("\n");
37+ const csv = `${headers}\n${rows}`;
38+
39+ // Create and trigger download
40+ const blob = new Blob([csv], { type: "text/csv" });
41+ const url = window.URL.createObjectURL(blob);
42+ const a = document.createElement("a");
43+ a.href = url;
44+ a.download = `benchmark-results${selectedDataset ? `-${selectedDataset}` : ""}.csv`;
45+ document.body.appendChild(a);
46+ a.click();
47+ document.body.removeChild(a);
48+ window.URL.revokeObjectURL(url);
49+};
web-ui/src/components/benchmark/table/BenchmarkTable.tsxadded+98−0View file
@@ -0,0 +1,98 @@
1+import {
2+ flexRender,
3+ getCoreRowModel,
4+ getSortedRowModel,
5+ useReactTable,
6+} from "@tanstack/react-table";
7+import { BenchmarkResult } from "../../../types";
8+import { exportToCsv } from "../export/csvExport";
9+import { columns } from "./columns";
10+
11+interface BenchmarkTableProps {
12+ results: BenchmarkResult[];
13+}
14+
15+export function BenchmarkTable({ results }: BenchmarkTableProps) {
16+ const table = useReactTable({
17+ data: results,
18+ columns,
19+ getCoreRowModel: getCoreRowModel(),
20+ getSortedRowModel: getSortedRowModel(),
21+ });
22+
23+ return (
24+ <div className="table-container">
25+ <table>
26+ <thead>
27+ {table.getHeaderGroups().map((headerGroup) => (
28+ <tr key={headerGroup.id}>
29+ {headerGroup.headers.map((header) => (
30+ <th
31+ key={header.id}
32+ onClick={header.column.getToggleSortingHandler()}
33+ style={{ cursor: "pointer" }}
34+ >
35+ {flexRender(
36+ header.column.columnDef.header,
37+ header.getContext(),
38+ )}
39+ {header.column.getIsSorted() && (
40+ <span style={{ marginLeft: "4px" }}>
41+ {header.column.getIsSorted() === "asc" ? "↑" : "↓"}
42+ </span>
43+ )}
44+ </th>
45+ ))}
46+ </tr>
47+ ))}
48+ </thead>
49+ <tbody>
50+ {table.getRowModel().rows.map((row) => (
51+ <tr key={row.id}>
52+ {row.getVisibleCells().map((cell) => (
53+ <td key={cell.id}>
54+ {flexRender(cell.column.columnDef.cell, cell.getContext())}
55+ </td>
56+ ))}
57+ </tr>
58+ ))}
59+ </tbody>
60+ </table>
61+
62+ <div
63+ style={{
64+ marginTop: "12px",
65+ display: "flex",
66+ justifyContent: "flex-end",
67+ }}
68+ >
69+ <button
70+ onClick={() => exportToCsv(results, "benchmark_results")}
71+ style={{
72+ padding: "8px 16px",
73+ backgroundColor: "#4CAF50",
74+ color: "white",
75+ border: "none",
76+ borderRadius: "4px",
77+ cursor: "pointer",
78+ display: "flex",
79+ alignItems: "center",
80+ gap: "8px",
81+ }}
82+ >
83+ <svg
84+ width="16"
85+ height="16"
86+ viewBox="0 0 16 16"
87+ fill="none"
88+ xmlns="http://www.w3.org/2000/svg"
89+ >
90+ <path d="M8 12L3 7H6V1H10V7H13L8 12Z" fill="currentColor" />
91+ <path d="M2 14V15H14V14H2Z" fill="currentColor" />
92+ </svg>
93+ Download CSV
94+ </button>
95+ </div>
96+ </div>
97+ );
98+}
web-ui/src/components/benchmark/table/columns.tsxadded+102−0View file
@@ -0,0 +1,102 @@
1+import { createColumnHelper } from "@tanstack/react-table";
2+import { BenchmarkResult } from "../../../types";
3+import { formatNumber, formatSize } from "../utils/formatters";
4+import { Link } from "react-router-dom";
5+
6+const columnHelper = createColumnHelper<BenchmarkResult>();
7+
8+export const columns = [
9+ columnHelper.accessor("dataset", {
10+ header: "Dataset",
11+ cell: (info) => (
12+ <Link
13+ to={`/dataset/${info.getValue()}`}
14+ style={{ color: "#2563eb", textDecoration: "none" }}
15+ onMouseEnter={(e) =>
16+ (e.currentTarget.style.textDecoration = "underline")
17+ }
18+ onMouseLeave={(e) => (e.currentTarget.style.textDecoration = "none")}
19+ >
20+ {info.getValue()}
21+ </Link>
22+ ),
23+ }),
24+ columnHelper.accessor("algorithm", {
25+ header: "Algorithm",
26+ cell: (info) => (
27+ <Link
28+ to={`/algorithm/${info.getValue()}`}
29+ style={{ color: "#2563eb", textDecoration: "none" }}
30+ onMouseEnter={(e) =>
31+ (e.currentTarget.style.textDecoration = "underline")
32+ }
33+ onMouseLeave={(e) => (e.currentTarget.style.textDecoration = "none")}
34+ >
35+ {info.getValue()}
36+ </Link>
37+ ),
38+ }),
39+ columnHelper.accessor("compression_ratio", {
40+ header: "Compression Ratio",
41+ cell: (info) => `${formatNumber(info.getValue())}`,
42+ sortingFn: (rowA, rowB) => {
43+ const a = rowA.original.compression_ratio;
44+ const b = rowB.original.compression_ratio;
45+ return a - b;
46+ },
47+ }),
48+ columnHelper.accessor("encode_time", {
49+ header: "Encode Time (s)",
50+ cell: (info) => formatNumber(info.getValue(), 4),
51+ sortingFn: (rowA, rowB) => {
52+ const a = rowA.original.encode_time;
53+ const b = rowB.original.encode_time;
54+ return a - b;
55+ },
56+ }),
57+ columnHelper.accessor("decode_time", {
58+ header: "Decode Time (s)",
59+ cell: (info) => formatNumber(info.getValue(), 4),
60+ sortingFn: (rowA, rowB) => {
61+ const a = rowA.original.decode_time;
62+ const b = rowB.original.decode_time;
63+ return a - b;
64+ },
65+ }),
66+ columnHelper.accessor("encode_mb_per_sec", {
67+ header: "Encode Speed (MB/s)",
68+ cell: (info) => formatNumber(info.getValue()),
69+ sortingFn: (rowA, rowB) => {
70+ const a = rowA.original.encode_mb_per_sec;
71+ const b = rowB.original.encode_mb_per_sec;
72+ return a - b;
73+ },
74+ }),
75+ columnHelper.accessor("decode_mb_per_sec", {
76+ header: "Decode Speed (MB/s)",
77+ cell: (info) => formatNumber(info.getValue()),
78+ sortingFn: (rowA, rowB) => {
79+ const a = rowA.original.decode_mb_per_sec;
80+ const b = rowB.original.decode_mb_per_sec;
81+ return a - b;
82+ },
83+ }),
84+ columnHelper.accessor("original_size", {
85+ header: "Original Size",
86+ cell: (info) => formatSize(info.getValue()),
87+ sortingFn: (rowA, rowB) => {
88+ const a = rowA.original.original_size;
89+ const b = rowB.original.original_size;
90+ return a - b;
91+ },
92+ }),
93+ columnHelper.accessor("compressed_size", {
94+ header: "Compressed Size",
95+ cell: (info) => formatSize(info.getValue()),
96+ sortingFn: (rowA, rowB) => {
97+ const a = rowA.original.compressed_size;
98+ const b = rowB.original.compressed_size;
99+ return a - b;
100+ },
101+ }),
102+];
web-ui/src/components/benchmark/utils/formatters.tsadded+11−0View file
@@ -0,0 +1,11 @@
1+export const formatNumber = (num: number, decimals = 2) => {
2+ return new Intl.NumberFormat("en-US", {
3+ minimumFractionDigits: decimals,
4+ maximumFractionDigits: decimals,
5+ }).format(num);
6+};
7+
8+export const formatSize = (bytes: number) => {
9+ const mb = bytes / (1024 * 1024);
10+ return `${formatNumber(mb)} MB`;
11+};
web-ui/src/components/dataset/ComparisonModeSelector.tsxadded+57−0View file
@@ -0,0 +1,57 @@
1+import { ComparisonMode } from "../../types/comparison";
2+
3+interface ComparisonModeSelectorProps {
4+ mode: ComparisonMode;
5+ onModeChange: (mode: ComparisonMode) => void;
6+}
7+
8+export const ComparisonModeSelector = ({
9+ mode,
10+ onModeChange,
11+}: ComparisonModeSelectorProps) => {
12+ const modes: { value: ComparisonMode; label: string }[] = [
13+ { value: "original", label: "Original Only" },
14+ { value: "overlay", label: "Overlay (Original + Reconstructed)" },
15+ { value: "residuals", label: "Residuals (Original - Reconstructed)" },
16+ { value: "side-by-side", label: "Side-by-Side" },
17+ ];
18+
19+ return (
20+ <div style={{ marginBottom: "16px" }}>
21+ <label
22+ style={{
23+ display: "block",
24+ marginBottom: "8px",
25+ fontSize: "14px",
26+ fontWeight: "500",
27+ color: "#333",
28+ }}
29+ >
30+ View mode:
31+ </label>
32+ <div style={{ display: "flex", gap: "12px", flexWrap: "wrap" }}>
33+ {modes.map((m) => (
34+ <label
35+ key={m.value}
36+ style={{
37+ display: "flex",
38+ alignItems: "center",
39+ cursor: "pointer",
40+ fontSize: "14px",
41+ }}
42+ >
43+ <input
44+ type="radio"
45+ name="comparison-mode"
46+ value={m.value}
47+ checked={mode === m.value}
48+ onChange={() => onModeChange(m.value)}
49+ style={{ marginRight: "6px" }}
50+ />
51+ {m.label}
52+ </label>
53+ ))}
54+ </div>
55+ </div>
56+ );
57+};
web-ui/src/components/dataset/DatasetContent.tsxadded+133−0View file
@@ -0,0 +1,133 @@
1+import { Dataset, BenchmarkData } from "../../types";
2+import { useEffect, useRef, useState } from "react";
3+import TimeseriesView from "./TimeseriesView";
4+import { BaseContent } from "../shared/BaseContent";
5+import { LossyAlgorithmSelector } from "./LossyAlgorithmSelector";
6+import { ComparisonModeSelector } from "./ComparisonModeSelector";
7+import { ReconstructedDataInfo, ComparisonMode } from "../../types/comparison";
8+import "../shared/ContentStyles.css";
9+
10+interface DatasetContentProps {
11+ dataset: Dataset;
12+ benchmarkData: BenchmarkData | null;
13+ chartData: Array<{
14+ algorithmOrDataset: string;
15+ compression_ratio: number;
16+ reference_compression_ratio: number | null;
17+ encode_speed: number;
18+ decode_speed: number;
19+ rmse?: number;
20+ tags: string[];
21+ }>;
22+}
23+
24+export const DatasetContent = ({
25+ dataset,
26+ benchmarkData,
27+ chartData,
28+}: DatasetContentProps) => {
29+ const containerRef = useRef<HTMLDivElement>(null);
30+ const [containerWidth, setContainerWidth] = useState(1200);
31+ const [reconstructedInfo, setReconstructedInfo] = useState<ReconstructedDataInfo | null>(null);
32+ const [comparisonMode, setComparisonMode] = useState<ComparisonMode>("original");
33+
34+ useEffect(() => {
35+ if (!containerRef.current) return;
36+
37+ const resizeObserver = new ResizeObserver((entries) => {
38+ for (const entry of entries) {
39+ setContainerWidth(entry.contentRect.width - 32);
40+ }
41+ });
42+
43+ resizeObserver.observe(containerRef.current);
44+
45+ return () => {
46+ resizeObserver.disconnect();
47+ };
48+ }, []);
49+
50+ const downloadSection =
51+ dataset.data_url_npy || dataset.data_url_raw ? (
52+ <div>
53+ <span className="metadata-label">Download: </span>
54+ <span style={{ display: "inline-flex", gap: "0.5rem" }}>
55+ {dataset.data_url_npy && (
56+ <a
57+ href={dataset.data_url_npy}
58+ download={`${dataset.name}-${dataset.version}.npy`}
59+ className="download-link"
60+ >
61+ NPY
62+ </a>
63+ )}
64+ {dataset.data_url_raw && (
65+ <a
66+ href={dataset.data_url_raw}
67+ download={`${dataset.name}-${dataset.version}.dat`}
68+ className="download-link"
69+ >
70+ RAW
71+ </a>
72+ )}
73+ </span>
74+ </div>
75+ ) : null;
76+
77+ const timeseriesSection = (
78+ <div className="content-container">
79+ {benchmarkData && (
80+ <>
81+ <LossyAlgorithmSelector
82+ dataset={dataset}
83+ benchmarkResults={benchmarkData.results}
84+ selectedAlgorithm={reconstructedInfo?.algorithm || null}
85+ onSelectAlgorithm={(info) => {
86+ setReconstructedInfo(info);
87+ if (info === null) {
88+ setComparisonMode("original");
89+ }
90+ }}
91+ />
92+ {reconstructedInfo && (
93+ <ComparisonModeSelector
94+ mode={comparisonMode}
95+ onModeChange={setComparisonMode}
96+ />
97+ )}
98+ </>
99+ )}
100+ <div
101+ ref={containerRef}
102+ style={{
103+ width: "100%",
104+ height: "300px",
105+ backgroundColor: "#f5f5f5",
106+ borderRadius: "4px",
107+ padding: "1rem",
108+ }}
109+ >
110+ <TimeseriesView
111+ width={containerWidth}
112+ height={250}
113+ dataset={dataset}
114+ comparisonMode={comparisonMode}
115+ reconstructedInfo={reconstructedInfo}
116+ />
117+ </div>
118+ </div>
119+ );
120+
121+ return (
122+ <BaseContent
123+ item={dataset}
124+ benchmarkData={benchmarkData}
125+ chartData={chartData}
126+ tagNavigationPrefix="/datasets"
127+ filterKey="dataset"
128+ downloadSection={downloadSection}
129+ additionalContent={timeseriesSection}
130+ showSortByCompressionRatio={true}
131+ />
132+ );
133+};
web-ui/src/components/dataset/LossyAlgorithmSelector.tsxadded+88−0View file
@@ -0,0 +1,88 @@
1+import { BenchmarkResult, Dataset } from "../../types";
2+import { ReconstructedDataInfo } from "../../types/comparison";
3+
4+interface LossyAlgorithmSelectorProps {
5+ dataset: Dataset;
6+ benchmarkResults: BenchmarkResult[];
7+ selectedAlgorithm: string | null;
8+ onSelectAlgorithm: (info: ReconstructedDataInfo | null) => void;
9+}
10+
11+export const LossyAlgorithmSelector = ({
12+ dataset,
13+ benchmarkResults,
14+ selectedAlgorithm,
15+ onSelectAlgorithm,
16+}: LossyAlgorithmSelectorProps) => {
17+ // Filter for lossy algorithms with results for this dataset
18+ const lossyResults = benchmarkResults.filter(
19+ (result) =>
20+ result.dataset === dataset.name &&
21+ result.reconstructed_url_raw != null &&
22+ result.rmse != null
23+ );
24+
25+ if (lossyResults.length === 0) {
26+ return null;
27+ }
28+
29+ const handleChange = (event: React.ChangeEvent<HTMLSelectElement>) => {
30+ const value = event.target.value;
31+ if (value === "") {
32+ onSelectAlgorithm(null);
33+ return;
34+ }
35+
36+ const result = lossyResults.find((r) => r.algorithm === value);
37+ if (result && result.reconstructed_url_raw) {
38+ onSelectAlgorithm({
39+ algorithm: result.algorithm,
40+ rmse: result.rmse || 0,
41+ max_error: result.max_error || 0,
42+ reconstructedUrl: result.reconstructed_url_raw,
43+ datasetUrl: dataset.data_url_raw || "",
44+ datasetJsonUrl: dataset.data_url_json || "",
45+ });
46+ }
47+ };
48+
49+ return (
50+ <div style={{ marginBottom: "16px" }}>
51+ <label
52+ htmlFor="lossy-algorithm-select"
53+ style={{
54+ display: "block",
55+ marginBottom: "8px",
56+ fontSize: "14px",
57+ fontWeight: "500",
58+ color: "#333",
59+ }}
60+ >
61+ Compare with lossy reconstruction:
62+ </label>
63+ <select
64+ id="lossy-algorithm-select"
65+ value={selectedAlgorithm || ""}
66+ onChange={handleChange}
67+ style={{
68+ width: "100%",
69+ padding: "8px 12px",
70+ fontSize: "14px",
71+ borderRadius: "4px",
72+ border: "1px solid #ccc",
73+ backgroundColor: "white",
74+ cursor: "pointer",
75+ }}
76+ >
77+ <option value="">Original data only</option>
78+ {lossyResults.map((result) => (
79+ <option key={result.algorithm} value={result.algorithm}>
80+ {result.algorithm} (RMSE: {result.rmse?.toFixed(3)}, Max Error:{" "}
81+ {result.max_error?.toFixed(3)}, Ratio:{" "}
82+ {result.compression_ratio?.toFixed(2)})
83+ </option>
84+ ))}
85+ </select>
86+ </div>
87+ );
88+};
web-ui/src/components/dataset/TimeseriesNavigationBar.tsxadded+98−0View file
@@ -0,0 +1,98 @@
1+import React, { useRef } from "react";
2+import { Range } from "./WorkerTypes";
3+
4+interface TimeseriesNavigationBarProps {
5+ width: number;
6+ height: number;
7+ totalRange: Range;
8+ viewRange: Range;
9+ onViewRangeChange: (range: Range) => void;
10+}
11+
12+const TimeseriesNavigationBar: React.FC<TimeseriesNavigationBarProps> = ({
13+ width,
14+ height,
15+ totalRange,
16+ viewRange,
17+ onViewRangeChange,
18+}) => {
19+ const containerRef = useRef<HTMLDivElement>(null);
20+
21+ // Constants
22+ const minMarkerWidth = 25; // Minimum width of the marker in pixels
23+ const padding = 10; // Padding on left and right
24+ const barWidth = width - 2 * padding;
25+
26+ // Convert data range to pixel coordinates
27+ const rangeToPixel = (value: number): number => {
28+ const ratio = (value - totalRange.min) / (totalRange.max - totalRange.min);
29+ return padding + ratio * barWidth;
30+ };
31+
32+ // Convert pixel coordinates to data range
33+ const pixelToRange = (pixel: number): number => {
34+ const ratio = (pixel - padding) / barWidth;
35+ return totalRange.min + ratio * (totalRange.max - totalRange.min);
36+ };
37+
38+ // Calculate marker position and width
39+ const markerLeft = rangeToPixel(viewRange.min);
40+ const rawMarkerWidth = rangeToPixel(viewRange.max) - markerLeft;
41+ const markerWidth = Math.max(rawMarkerWidth, minMarkerWidth);
42+
43+ const handleClick = (e: React.MouseEvent) => {
44+ if (!containerRef.current) return;
45+
46+ const rect = containerRef.current.getBoundingClientRect();
47+ const clickX = e.clientX - rect.left;
48+
49+ // Click on the bar - center the view on click position
50+ const clickedValue = pixelToRange(clickX);
51+ const currentSize = viewRange.max - viewRange.min;
52+ const halfSize = currentSize / 2;
53+
54+ let newMin = clickedValue - halfSize;
55+ let newMax = clickedValue + halfSize;
56+
57+ // Clamp to total range bounds
58+ if (newMin < totalRange.min) {
59+ newMin = totalRange.min;
60+ newMax = newMin + currentSize;
61+ }
62+ if (newMax > totalRange.max) {
63+ newMax = totalRange.max;
64+ newMin = newMax - currentSize;
65+ }
66+
67+ onViewRangeChange({ min: newMin, max: newMax });
68+ };
69+
70+ return (
71+ <div
72+ ref={containerRef}
73+ style={{
74+ width,
75+ height,
76+ position: "relative",
77+ backgroundColor: "#f0f0f0",
78+ borderRadius: 4,
79+ cursor: "pointer",
80+ }}
81+ onClick={handleClick}
82+ >
83+ <div
84+ style={{
85+ position: "absolute",
86+ left: markerLeft,
87+ width: markerWidth,
88+ height: "100%",
89+ backgroundColor: "#007bff",
90+ borderRadius: 4,
91+ pointerEvents: "none",
92+ }}
93+ />
94+ </div>
95+ );
96+};
97+
98+export default TimeseriesNavigationBar;
web-ui/src/components/dataset/TimeseriesView.tsxadded+715−0View file
@@ -0,0 +1,715 @@
1+import { useEffect, useMemo, useReducer, useState, useCallback } from "react";
2+import TimeseriesNavigationBar from "./TimeseriesNavigationBar";
3+import { SupportedTypedArray } from "../../hooks/TimeseriesDataClient";
4+import { useTimeseriesDataClient } from "../../hooks/useTimeseriesDataClient";
5+import { Dataset } from "../../types";
6+import { Margins, Range, WorkerMessage } from "./WorkerTypes";
7+import { initialState, timeseriesViewReducer } from "./timeseriesViewReducer";
8+import { ReconstructedDataInfo, ComparisonMode } from "../../types/comparison";
9+import { TimeseriesDataClient } from "../../hooks/TimeseriesDataClient";
10+
11+interface TimeseriesViewProps {
12+ width: number;
13+ height: number;
14+ dataset: Dataset;
15+ comparisonMode?: ComparisonMode;
16+ reconstructedInfo?: ReconstructedDataInfo | null;
17+}
18+
19+const TimeseriesView: React.FC<TimeseriesViewProps> = ({
20+ width,
21+ height,
22+ dataset,
23+ comparisonMode = "original",
24+ reconstructedInfo = null,
25+}) => {
26+ const { client, error: clientError } = useTimeseriesDataClient(dataset);
27+ const [dataT, setDataT] = useState<number[] | null>(null);
28+ const [dataY, setDataY] = useState<SupportedTypedArray | null>(null);
29+ const [dataYAll, setDataYAll] = useState<SupportedTypedArray[] | null>(null);
30+ const [dataYReconstructed, setDataYReconstructed] = useState<SupportedTypedArray | null>(null);
31+ const [dataYResiduals, setDataYResiduals] = useState<SupportedTypedArray | null>(null);
32+ const [reconstructedClient, setReconstructedClient] = useState<TimeseriesDataClient | null>(null);
33+ const [error, setError] = useState<string | null>(clientError);
34+ const [isLoading, setIsLoading] = useState(false);
35+ const [selectedChannel, setSelectedChannel] = useState<number | "all">(0);
36+ const [numChannels, setNumChannels] = useState<number>(1);
37+
38+ const [canvasElement, setCanvasElement] = useState<HTMLCanvasElement | null>(
39+ null,
40+ );
41+ const [overlayCanvasElement, setOverlayCanvasElement] =
42+ useState<HTMLCanvasElement | null>(null);
43+ const [state, dispatch] = useReducer(timeseriesViewReducer, initialState);
44+ const { selectedIndex, isDragging, lastDragX, xRange } = state;
45+ // Wheel zooming is disabled because it causes the page to scroll when the user
46+ // tries to scroll the timeseries view, creating a poor user experience.
47+ // Instead, we provide explicit zoom control buttons.
48+ const [isWheelEnabled] = useState(false); // Keep state for potential future use, but always false
49+ const [showHint, setShowHint] = useState(true);
50+
51+ // Hide hint when user interacts with the graph
52+ const hideHint = useCallback(() => {
53+ setShowHint(false);
54+ }, []);
55+
56+ // Auto-hide hint after 4 seconds
57+ useEffect(() => {
58+ if (showHint) {
59+ const timer = setTimeout(() => {
60+ setShowHint(false);
61+ }, 5000);
62+ return () => clearTimeout(timer);
63+ }
64+ }, [showHint]);
65+
66+ const [container, setContainer] = useState<HTMLDivElement | null>(null);
67+ const [worker, setWorker] = useState<Worker | null>(null);
68+ const [margins] = useState<Margins>({
69+ left: 50,
70+ right: 20,
71+ top: 20,
72+ bottom: 50,
73+ });
74+
75+ // Load data for current range
76+ useEffect(() => {
77+ if (!client || !xRange) return;
78+
79+ const loadRangeData = async () => {
80+ try {
81+ setIsLoading(true);
82+ const start = Math.floor(xRange.min);
83+ const end = Math.ceil(xRange.max) + 1;
84+
85+ if (selectedChannel === "all") {
86+ // Load all channels
87+ const allChannelData = await Promise.all(
88+ Array.from({ length: numChannels }, (_, ch) =>
89+ client.fetchRange(start, end, ch)
90+ )
91+ );
92+ setDataYAll(allChannelData);
93+ setDataY(null);
94+ } else {
95+ // Load single channel
96+ const rangeData = await client.fetchRange(start, end, selectedChannel);
97+ setDataY(rangeData);
98+ setDataYAll(null);
99+ }
100+
101+ const dT = Array.from(
102+ { length: end - start },
103+ (_, i) => i + start,
104+ );
105+ setDataT(dT);
106+ setError(null);
107+ } catch (err) {
108+ setError(
109+ err instanceof Error ? err.message : "Failed to load data range",
110+ );
111+ } finally {
112+ setIsLoading(false);
113+ }
114+ };
115+
116+ loadRangeData();
117+ }, [client, xRange, selectedChannel, numChannels]);
118+
119+ // Initialize reconstructed data client when reconstructedInfo changes
120+ useEffect(() => {
121+ if (!reconstructedInfo) {
122+ setReconstructedClient(null);
123+ setDataYReconstructed(null);
124+ setDataYResiduals(null);
125+ return;
126+ }
127+
128+ // When a reconstruction is selected for comparison, switch from "all" to channel 0
129+ if (selectedChannel === "all") {
130+ setSelectedChannel(0);
131+ }
132+
133+ const initClient = async () => {
134+ try {
135+ const client = await TimeseriesDataClient.create(
136+ reconstructedInfo.datasetJsonUrl,
137+ reconstructedInfo.reconstructedUrl,
138+ 1000
139+ );
140+ setReconstructedClient(client);
141+ } catch (err) {
142+ console.error("Failed to initialize reconstructed data client:", err);
143+ setError("Failed to load reconstructed data");
144+ }
145+ };
146+
147+ initClient();
148+ }, [reconstructedInfo, selectedChannel]);
149+
150+ // Load reconstructed data for current range
151+ useEffect(() => {
152+ if (!reconstructedClient || !xRange || selectedChannel === "all" || comparisonMode === "original") {
153+ setDataYReconstructed(null);
154+ setDataYResiduals(null);
155+ return;
156+ }
157+
158+ const loadReconstructedData = async () => {
159+ try {
160+ const start = Math.floor(xRange.min);
161+ const end = Math.ceil(xRange.max) + 1;
162+ const channel = typeof selectedChannel === "number" ? selectedChannel : 0;
163+
164+ const reconstructedData = await reconstructedClient.fetchRange(start, end, channel);
165+ setDataYReconstructed(reconstructedData);
166+
167+ // Compute residuals if we have both original and reconstructed
168+ if (dataY && reconstructedData.length === dataY.length) {
169+ const residuals = new Float32Array(dataY.length);
170+ for (let i = 0; i < dataY.length; i++) {
171+ residuals[i] = dataY[i] - reconstructedData[i];
172+ }
173+ setDataYResiduals(residuals);
174+ }
175+ } catch (err) {
176+ console.error("Failed to load reconstructed data:", err);
177+ }
178+ };
179+
180+ loadReconstructedData();
181+ }, [reconstructedClient, xRange, selectedChannel, dataY, comparisonMode]);
182+
183+ // Update xRange when client is initialized
184+ useEffect(() => {
185+ if (client) {
186+ const shape = client.getShape();
187+ const channels = client.getNumChannels();
188+ setNumChannels(channels);
189+ // Default to "all" if 20 or fewer channels, otherwise default to channel 0
190+ setSelectedChannel(channels > 1 && channels <= 20 ? "all" : 0);
191+ dispatch({
192+ type: "SET_X_RANGE",
193+ range: { min: 0, max: Math.min(999, shape - 1) },
194+ });
195+ }
196+ }, [client]);
197+
198+ // Set up wheel event listener
199+ useEffect(() => {
200+ if (!container || !client) return;
201+
202+ const handleWheel = (e: WheelEvent) => {
203+ if (!isWheelEnabled) {
204+ return; // Allow page scrolling if wheel zoom not enabled
205+ }
206+ e.preventDefault();
207+ e.stopPropagation();
208+
209+ const rect = container.getBoundingClientRect();
210+ const x = e.clientX - rect.left;
211+ const xRatio =
212+ (x - margins.left) / (width - margins.left - margins.right);
213+
214+ // Calculate zoom center in data coordinates
215+ const zoomCenter = xRange.min + (xRange.max - xRange.min) * xRatio;
216+
217+ // Calculate new range
218+ const zoomFactor = e.deltaY > 0 ? 1.1 : 1 / 1.1;
219+ const shape = client.getShape();
220+
221+ // Ensure we don't zoom out beyond data bounds
222+ const newMin = Math.max(
223+ 0,
224+ zoomCenter - (zoomCenter - xRange.min) * zoomFactor,
225+ );
226+ const newMax = Math.min(
227+ shape - 1,
228+ zoomCenter + (xRange.max - zoomCenter) * zoomFactor,
229+ );
230+
231+ dispatch({ type: "SET_X_RANGE", range: { min: newMin, max: newMax } });
232+ };
233+
234+ container.addEventListener("wheel", handleWheel, { passive: false });
235+ return () => {
236+ container.removeEventListener("wheel", handleWheel);
237+ };
238+ }, [container, client, width, margins, isWheelEnabled]);
239+
240+ // Set up mouse event listeners for panning
241+ useEffect(() => {
242+ if (!container || !client) return;
243+
244+ const handleMouseDown = (e: MouseEvent) => {
245+ dispatch({ type: "SET_IS_DRAGGING", isDragging: true });
246+ dispatch({ type: "SET_LAST_DRAG_X", x: e.clientX });
247+ };
248+
249+ const handleMouseMove = (e: MouseEvent) => {
250+ if (!isDragging || lastDragX === 0) return;
251+
252+ const deltaX = e.clientX - lastDragX;
253+ const xRatio = deltaX / (width - margins.left - margins.right);
254+ const dataDelta = (xRange.max - xRange.min) * xRatio;
255+ const shape = client.getShape();
256+
257+ if (xRange.min - dataDelta < 0) return;
258+ if (xRange.max - dataDelta > shape - 1) return;
259+
260+ const newMin = xRange.min - dataDelta;
261+ const newMax = xRange.max - dataDelta;
262+
263+ // Only update if we're still within bounds
264+ if (newMin >= 0 && newMax <= shape - 1) {
265+ dispatch({ type: "SET_X_RANGE", range: { min: newMin, max: newMax } });
266+ }
267+
268+ dispatch({ type: "SET_LAST_DRAG_X", x: e.clientX });
269+ };
270+
271+ const handleMouseUp = () => {
272+ dispatch({ type: "SET_IS_DRAGGING", isDragging: false });
273+ dispatch({ type: "SET_LAST_DRAG_X", x: 0 });
274+ };
275+
276+ container.addEventListener("mousedown", handleMouseDown);
277+ window.addEventListener("mousemove", handleMouseMove);
278+ window.addEventListener("mouseup", handleMouseUp);
279+
280+ return () => {
281+ container.removeEventListener("mousedown", handleMouseDown);
282+ window.removeEventListener("mousemove", handleMouseMove);
283+ window.removeEventListener("mouseup", handleMouseUp);
284+ };
285+ }, [container, client, width, margins, xRange, isDragging, lastDragX]);
286+
287+ // Set worker
288+ useEffect(() => {
289+ if (!canvasElement) return;
290+ const worker = new Worker(
291+ new URL("./TimeseriesViewWorker", import.meta.url),
292+ {
293+ type: "module",
294+ },
295+ );
296+ let offscreenCanvas: OffscreenCanvas;
297+ try {
298+ offscreenCanvas = canvasElement.transferControlToOffscreen();
299+ } catch (err) {
300+ console.warn(err);
301+ console.warn(
302+ "Unable to transfer control to offscreen canvas (expected during dev)",
303+ );
304+ return;
305+ }
306+ const msg: WorkerMessage = {
307+ type: "initialize",
308+ canvas: offscreenCanvas,
309+ };
310+ worker.postMessage(msg, [offscreenCanvas]);
311+
312+ setWorker(worker);
313+
314+ return () => {
315+ worker.terminate();
316+ };
317+ }, [canvasElement]);
318+
319+ // Calculate yRange from data
320+ const yRange = useMemo<Range>(() => {
321+ if (dataYAll) {
322+ // Calculate range across all channels
323+ let min = Infinity;
324+ let max = -Infinity;
325+ for (const channelData of dataYAll) {
326+ const channelMin = computeMin(channelData);
327+ const channelMax = computeMax(channelData);
328+ if (channelMin < min) min = channelMin;
329+ if (channelMax > max) max = channelMax;
330+ }
331+ return { min, max };
332+ } else if (dataY) {
333+ let min = computeMin(dataY);
334+ let max = computeMax(dataY);
335+
336+ // When comparing with reconstructed data, include that data in the range calculation
337+ if (comparisonMode === "overlay" || comparisonMode === "side-by-side") {
338+ if (dataYReconstructed) {
339+ const reconstructedMin = computeMin(dataYReconstructed);
340+ const reconstructedMax = computeMax(dataYReconstructed);
341+ if (reconstructedMin < min) min = reconstructedMin;
342+ if (reconstructedMax > max) max = reconstructedMax;
343+ }
344+ } else if (comparisonMode === "residuals") {
345+ // For residuals mode, use only the residuals range
346+ if (dataYResiduals) {
347+ min = computeMin(dataYResiduals);
348+ max = computeMax(dataYResiduals);
349+ }
350+ }
351+
352+ return { min, max };
353+ }
354+ return { min: 0, max: 1 };
355+ }, [dataY, dataYAll, dataYReconstructed, dataYResiduals, comparisonMode]);
356+
357+ // Handle dimension changes
358+ useEffect(() => {
359+ if (!worker) return;
360+ if (!dataT) return;
361+ if (!dataY && !dataYAll) return;
362+
363+ const msg: WorkerMessage = {
364+ type: "render",
365+ timeseriesT: dataT,
366+ timeseriesY: dataY ? Array.from(dataY) : [],
367+ timeseriesYAll: dataYAll ? dataYAll.map(ch => Array.from(ch)) : undefined,
368+ timeseriesYReconstructed: dataYReconstructed ? Array.from(dataYReconstructed) : undefined,
369+ timeseriesYResiduals: dataYResiduals ? Array.from(dataYResiduals) : undefined,
370+ comparisonMode: comparisonMode,
371+ width,
372+ height,
373+ margins,
374+ xRange,
375+ yRange,
376+ };
377+ worker.postMessage(msg);
378+ }, [width, height, dataT, dataY, dataYAll, dataYReconstructed, dataYResiduals, comparisonMode, worker, margins, xRange, yRange]);
379+
380+ // Render cursor on overlay canvas
381+ useEffect(() => {
382+ if (!overlayCanvasElement || selectedIndex === null || (!dataY && !dataYAll)) return;
383+ const ctx = overlayCanvasElement.getContext("2d");
384+ if (!ctx) return;
385+
386+ // Clear overlay canvas
387+ ctx.clearRect(0, 0, width, height);
388+
389+ // Draw cursor line
390+ const xRatio = (selectedIndex - xRange.min) / (xRange.max - xRange.min);
391+ const x = margins.left + xRatio * (width - margins.left - margins.right);
392+ ctx.beginPath();
393+ ctx.strokeStyle = "#ff0000";
394+ ctx.lineWidth = 1;
395+ ctx.setLineDash([4, 4]);
396+ ctx.moveTo(x, margins.top);
397+ ctx.lineTo(x, height - margins.bottom);
398+ ctx.stroke();
399+ }, [
400+ selectedIndex,
401+ overlayCanvasElement,
402+ width,
403+ height,
404+ margins,
405+ dataT,
406+ dataY,
407+ dataYAll,
408+ xRange,
409+ ]);
410+
411+ const selectedValue = useMemo(() => {
412+ if (selectedIndex === -1 || !dataT) return null;
413+
414+ if (dataYAll) {
415+ // Return all channel values
416+ const values: number[] = [];
417+ for (let ch = 0; ch < dataYAll.length; ch++) {
418+ for (let i = 0; i < dataT.length; i++) {
419+ if (dataT[i] === selectedIndex) {
420+ values.push(dataYAll[ch][i]);
421+ break;
422+ }
423+ }
424+ }
425+ return values.length > 0 ? values : null;
426+ } else if (dataY) {
427+ // Return single channel value
428+ for (let i = 0; i < dataT.length; i++) {
429+ if (dataT[i] === selectedIndex) {
430+ return dataY[i];
431+ }
432+ }
433+ }
434+ return null;
435+ }, [selectedIndex, dataT, dataY, dataYAll]);
436+
437+ if (error || clientError) {
438+ return <div>Error loading data: {error || clientError}</div>;
439+ }
440+
441+ if (isLoading && !dataY) {
442+ return <div>Loading...</div>;
443+ }
444+
445+ const handleCanvasClick = (e: React.MouseEvent<HTMLDivElement>) => {
446+ if (!overlayCanvasElement || (!dataY && !dataYAll) || isDragging) return;
447+
448+ const rect = overlayCanvasElement.getBoundingClientRect();
449+ const x = e.clientX - rect.left;
450+ const xRatio = (x - margins.left) / (width - margins.left - margins.right);
451+ const index = Math.round(xRange.min + xRatio * (xRange.max - xRange.min));
452+ if (index >= 0) {
453+ dispatch({ type: "SET_SELECTED_INDEX", index });
454+ }
455+ };
456+
457+ // Zoom control functions - zoom centered on the selected index (current timepoint)
458+ const handleZoomIn = () => {
459+ if (!client) return;
460+ // Use selectedIndex as center if set, otherwise use view center
461+ const center = selectedIndex !== -1 ? selectedIndex : (xRange.min + xRange.max) / 2;
462+ const currentRange = xRange.max - xRange.min;
463+ const newRange = currentRange / 1.5; // Zoom in by 1.5x
464+ const newMin = Math.max(0, center - newRange / 2);
465+ const newMax = Math.min(client.getShape() - 1, center + newRange / 2);
466+ dispatch({ type: "SET_X_RANGE", range: { min: newMin, max: newMax } });
467+ };
468+
469+ const handleZoomOut = () => {
470+ if (!client) return;
471+ // Use selectedIndex as center if set, otherwise use view center
472+ const center = selectedIndex !== -1 ? selectedIndex : (xRange.min + xRange.max) / 2;
473+ const currentRange = xRange.max - xRange.min;
474+ const newRange = currentRange * 1.5; // Zoom out by 1.5x
475+ const shape = client.getShape();
476+ const newMin = Math.max(0, center - newRange / 2);
477+ const newMax = Math.min(shape - 1, center + newRange / 2);
478+ dispatch({ type: "SET_X_RANGE", range: { min: newMin, max: newMax } });
479+ };
480+
481+ const handleZoomReset = () => {
482+ if (!client) return;
483+ const shape = client.getShape();
484+ dispatch({
485+ type: "SET_X_RANGE",
486+ range: { min: 0, max: Math.min(999, shape - 1) },
487+ });
488+ };
489+
490+ return (
491+ <div style={{ position: "relative", width, height: height + 50 }}>
492+ <div style={{ marginBottom: 10, height: 20 }}>
493+ <TimeseriesNavigationBar
494+ width={width}
495+ height={20}
496+ totalRange={{ min: 0, max: client ? client.getShape() - 1 : 999 }}
497+ viewRange={xRange}
498+ onViewRangeChange={(range) =>
499+ dispatch({ type: "SET_X_RANGE", range })
500+ }
501+ />
502+ </div>
503+ {numChannels > 1 && (
504+ <div style={{ marginBottom: 10, display: "flex", alignItems: "center", gap: 8 }}>
505+ <label htmlFor="channel-select" style={{ fontSize: "14px", color: "#666" }}>
506+ Channel:
507+ </label>
508+ <select
509+ id="channel-select"
510+ value={selectedChannel}
511+ onChange={(e) => {
512+ const value = e.target.value;
513+ setSelectedChannel(value === "all" ? "all" : Number(value));
514+ }}
515+ style={{
516+ padding: "4px 8px",
517+ fontSize: "14px",
518+ borderRadius: "4px",
519+ border: "1px solid #ccc",
520+ backgroundColor: "white",
521+ cursor: "pointer",
522+ }}
523+ >
524+ <option value="all">All (overlay)</option>
525+ {Array.from({ length: numChannels }, (_, i) => (
526+ <option key={i} value={i}>
527+ {i}
528+ </option>
529+ ))}
530+ </select>
531+ </div>
532+ )}
533+ {showHint && (
534+ <div
535+ style={{
536+ position: "absolute",
537+ top: margins.top + 10,
538+ right: margins.right + 10,
539+ display: "flex",
540+ flexDirection: "column",
541+ alignItems: "flex-end",
542+ gap: "8px",
543+ zIndex: 10,
544+ opacity: showHint ? 0.8 : 0,
545+ transition: "opacity 0.5s ease-out",
546+ pointerEvents: "none",
547+ fontSize: "12px",
548+ color: "#666",
549+ }}
550+ >
551+ <div
552+ style={{
553+ display: "flex",
554+ alignItems: "center",
555+ gap: "4px",
556+ backgroundColor: "rgba(255, 255, 255, 0.9)",
557+ padding: "2px 6px",
558+ borderRadius: "4px",
559+ }}
560+ >
561+ <span>Drag to pan</span>
562+ <svg width="14" height="14" viewBox="0 0 24 24" fill="#666">
563+ <path d="M15 3h2v5h-2V3zm4 0h2v5h-2V3zm-4 7h2v5h-2v-5zm4 0h2v5h-2v-5zm-4 7h2v5h-2v-5zm4 0h2v5h-2v-5z" />
564+ </svg>
565+ </div>
566+ </div>
567+ )}
568+ {/* Zoom control buttons - positioned at bottom right to avoid blocking channel selector */}
569+ <div
570+ style={{
571+ position: "absolute",
572+ bottom: margins.bottom + 10,
573+ right: margins.right + 10,
574+ display: "flex",
575+ gap: "6px",
576+ zIndex: 10,
577+ }}
578+ >
579+ <button
580+ onClick={handleZoomIn}
581+ disabled={!client}
582+ style={{
583+ padding: "6px 8px",
584+ fontSize: "14px",
585+ borderRadius: "4px",
586+ border: "1px solid #ccc",
587+ backgroundColor: "white",
588+ cursor: client ? "pointer" : "not-allowed",
589+ opacity: client ? 1 : 0.5,
590+ display: "flex",
591+ alignItems: "center",
592+ justifyContent: "center",
593+ }}
594+ title="Zoom in"
595+ >
596+ <svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2">
597+ <circle cx="11" cy="11" r="8" />
598+ <path d="M21 21l-4.35-4.35" />
599+ <line x1="8" y1="11" x2="14" y2="11" />
600+ <line x1="11" y1="8" x2="11" y2="14" />
601+ </svg>
602+ </button>
603+ <button
604+ onClick={handleZoomOut}
605+ disabled={!client}
606+ style={{
607+ padding: "6px 8px",
608+ fontSize: "14px",
609+ borderRadius: "4px",
610+ border: "1px solid #ccc",
611+ backgroundColor: "white",
612+ cursor: client ? "pointer" : "not-allowed",
613+ opacity: client ? 1 : 0.5,
614+ display: "flex",
615+ alignItems: "center",
616+ justifyContent: "center",
617+ }}
618+ title="Zoom out"
619+ >
620+ <svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2">
621+ <circle cx="11" cy="11" r="8" />
622+ <path d="M21 21l-4.35-4.35" />
623+ <line x1="8" y1="11" x2="14" y2="11" />
624+ </svg>
625+ </button>
626+ <button
627+ onClick={handleZoomReset}
628+ disabled={!client}
629+ style={{
630+ padding: "6px 8px",
631+ fontSize: "14px",
632+ borderRadius: "4px",
633+ border: "1px solid #ccc",
634+ backgroundColor: "white",
635+ cursor: client ? "pointer" : "not-allowed",
636+ opacity: client ? 1 : 0.5,
637+ display: "flex",
638+ alignItems: "center",
639+ justifyContent: "center",
640+ }}
641+ title="Reset zoom"
642+ >
643+ <svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2">
644+ <path d="M3 12a9 9 0 0 1 9-9 9.75 9.75 0 0 1 6.74 2.74L21 8" />
645+ <path d="M21 3v5h-5" />
646+ <path d="M21 12a9 9 0 0 1-9 9 9.75 9.75 0 0 1-6.74-2.74L3 16" />
647+ <path d="M3 21v-5h5" />
648+ </svg>
649+ </button>
650+ </div>
651+ <div
652+ ref={setContainer}
653+ style={{ position: "relative", width, height }}
654+ onClick={(e) => {
655+ handleCanvasClick(e);
656+ hideHint();
657+ }}
658+ onMouseDown={hideHint}
659+ >
660+ <canvas
661+ ref={setCanvasElement}
662+ key={`canvas-${width}-${height}`}
663+ width={width}
664+ height={height}
665+ style={{
666+ position: "absolute",
667+ width: "100%",
668+ height: "100%",
669+ }}
670+ />
671+ <canvas
672+ ref={setOverlayCanvasElement}
673+ width={width}
674+ height={height}
675+ style={{
676+ position: "absolute",
677+ width: "100%",
678+ height: "100%",
679+ pointerEvents: "none",
680+ }}
681+ />
682+ </div>
683+ {selectedIndex !== -1 && selectedValue && (
684+ <div style={{ height: 30, padding: "5px 0", color: "#666" }}>
685+ Index: {selectedIndex},{" "}
686+ {Array.isArray(selectedValue)
687+ ? `Values: [${selectedValue.slice(0, 5).map(v => v.toFixed(3)).join(", ")}${selectedValue.length > 5 ? ", ..." : ""}]`
688+ : `Value: ${selectedValue.toFixed(3)}`}
689+ </div>
690+ )}
691+ </div>
692+ );
693+};
694+
695+const computeMin = (data: SupportedTypedArray) => {
696+ let min = Infinity;
697+ for (let i = 0; i < data.length; i++) {
698+ if (data[i] < min) {
699+ min = data[i];
700+ }
701+ }
702+ return min;
703+};
704+
705+const computeMax = (data: SupportedTypedArray) => {
706+ let max = -Infinity;
707+ for (let i = 0; i < data.length; i++) {
708+ if (data[i] > max) {
709+ max = data[i];
710+ }
711+ }
712+ return max;
713+};
714+
715+export default TimeseriesView;
web-ui/src/components/dataset/TimeseriesViewWorker.tsadded+359−0View file
@@ -0,0 +1,359 @@
1+// Web worker for rendering timeseries data to canvas
2+
3+import { Margins, Range, WorkerMessage } from "./WorkerTypes";
4+
5+// Helper function to find a nice integer tick interval
6+function getNiceTickInterval(range: number, maxTicks: number): number {
7+ const minInterval = Math.ceil(range / maxTicks);
8+ if (minInterval <= 1) return 1;
9+
10+ const magnitude = Math.pow(10, Math.floor(Math.log10(minInterval)));
11+ const niceIntervals = [1, 2, 5, 10];
12+
13+ for (const interval of niceIntervals) {
14+ const tickInterval = interval * magnitude;
15+ if (tickInterval >= minInterval) {
16+ return Math.ceil(tickInterval);
17+ }
18+ }
19+ return Math.ceil(niceIntervals[niceIntervals.length - 1] * magnitude * 10);
20+}
21+
22+// Helper function to estimate the width of a number in pixels
23+// This is an approximation since we can't measure text width directly in a worker
24+function estimateNumberWidth(num: number): number {
25+ const numStr = Math.abs(num).toString();
26+ const digitWidth = 8; // Approximate width of a digit in pixels
27+ const padding = 4; // Padding between numbers
28+ return (numStr.length + (num < 0 ? 1 : 0)) * digitWidth + padding;
29+}
30+
31+// Helper function to get tick positions
32+function getTickPositions(
33+ range: Range,
34+ width: number,
35+ considerNumberWidth = false, // Only true for x-axis where we need to handle large integers
36+): { value: number; x: number }[] {
37+ let pixelsPerTick = 20; // Default minimum pixels between ticks
38+
39+ if (considerNumberWidth) {
40+ // For x-axis, calculate spacing based on largest number width
41+ const maxAbsValue = Math.max(Math.abs(range.min), Math.abs(range.max));
42+ const maxNumberWidth = estimateNumberWidth(maxAbsValue);
43+ pixelsPerTick = Math.max(maxNumberWidth, 20); // Use the larger of estimated width or minimum spacing
44+ }
45+ const maxTicks = Math.floor(width / pixelsPerTick);
46+ const tickInterval = getNiceTickInterval(range.max - range.min, maxTicks);
47+
48+ const firstTick = Math.ceil(range.min / tickInterval) * tickInterval;
49+ const lastTick = Math.floor(range.max);
50+
51+ const ticks: { value: number; x: number }[] = [];
52+ for (let value = firstTick; value <= lastTick; value += tickInterval) {
53+ const x = (value - range.min) / (range.max - range.min);
54+ if (Number.isInteger(value)) {
55+ ticks.push({ value, x });
56+ }
57+ }
58+
59+ return ticks;
60+}
61+
62+let canvas: OffscreenCanvas | null = null;
63+let ctx: OffscreenCanvasRenderingContext2D | null = null;
64+
65+function renderTimeseries(
66+ timeseriesT: number[],
67+ timeseriesY: number[],
68+ timeseriesYAll: number[][] | undefined,
69+ timeseriesYReconstructed: number[] | undefined,
70+ timeseriesYResiduals: number[] | undefined,
71+ comparisonMode: string | undefined,
72+ width: number,
73+ height: number,
74+ margins: Margins,
75+ xRange: Range,
76+ yRange: Range,
77+) {
78+ if (!ctx || !canvas) return;
79+
80+ const context = ctx; // Create a stable reference to satisfy TypeScript
81+
82+ // Clear canvas
83+ context.clearRect(0, 0, width, height);
84+
85+ // Draw axes
86+ context.strokeStyle = "#666666";
87+ context.lineWidth = 1;
88+ context.beginPath();
89+
90+ // Y axis
91+ context.moveTo(margins.left, margins.top);
92+ context.lineTo(margins.left, height - margins.bottom);
93+
94+ // X axis
95+ context.moveTo(margins.left, height - margins.bottom);
96+ context.lineTo(width - margins.right, height - margins.bottom);
97+
98+ context.stroke();
99+
100+ // Calculate the drawing area dimensions
101+ const drawingWidth = width - margins.left - margins.right;
102+ const drawingHeight = height - margins.top - margins.bottom;
103+
104+ // Set up clipping region for timeseries
105+ context.save();
106+ context.beginPath();
107+ context.rect(margins.left, margins.top, drawingWidth, drawingHeight);
108+ context.clip();
109+
110+ // Calculate scaling factors
111+ const xScale = drawingWidth / (xRange.max - xRange.min);
112+ const yScale = drawingHeight / (yRange.max - yRange.min);
113+
114+ // Draw timeseries based on comparison mode
115+ const mode = comparisonMode || "original";
116+
117+ if (mode === "side-by-side" && timeseriesYReconstructed) {
118+ // Split canvas vertically
119+ const halfWidth = drawingWidth / 2;
120+
121+ // Draw original on left
122+ context.strokeStyle = "#2196f3";
123+ context.lineWidth = 2;
124+ context.beginPath();
125+ for (let i = 0; i < timeseriesT.length; i++) {
126+ const x = margins.left + ((timeseriesT[i] - xRange.min) * halfWidth) / (xRange.max - xRange.min);
127+ const y = margins.top + drawingHeight - (timeseriesY[i] - yRange.min) * yScale;
128+ if (i === 0) context.moveTo(x, y);
129+ else context.lineTo(x, y);
130+ }
131+ context.stroke();
132+
133+ // Draw reconstructed on right
134+ context.strokeStyle = "#ff9800"; // orange
135+ context.lineWidth = 2;
136+ context.beginPath();
137+ for (let i = 0; i < timeseriesT.length; i++) {
138+ const x = margins.left + halfWidth + ((timeseriesT[i] - xRange.min) * halfWidth) / (xRange.max - xRange.min);
139+ const y = margins.top + drawingHeight - (timeseriesYReconstructed[i] - yRange.min) * yScale;
140+ if (i === 0) context.moveTo(x, y);
141+ else context.lineTo(x, y);
142+ }
143+ context.stroke();
144+
145+ // Draw divider line
146+ context.strokeStyle = "#999";
147+ context.lineWidth = 1;
148+ context.beginPath();
149+ context.moveTo(margins.left + halfWidth, margins.top);
150+ context.lineTo(margins.left + halfWidth, height - margins.bottom);
151+ context.stroke();
152+ } else if (mode === "overlay" && timeseriesYReconstructed) {
153+ // Draw original in blue
154+ context.strokeStyle = "#2196f3";
155+ context.lineWidth = 2;
156+ context.beginPath();
157+ for (let i = 0; i < timeseriesT.length; i++) {
158+ const x = margins.left + (timeseriesT[i] - xRange.min) * xScale;
159+ const y = margins.top + drawingHeight - (timeseriesY[i] - yRange.min) * yScale;
160+ if (i === 0) context.moveTo(x, y);
161+ else context.lineTo(x, y);
162+ }
163+ context.stroke();
164+
165+ // Draw reconstructed in orange
166+ context.strokeStyle = "#ff9800";
167+ context.lineWidth = 2;
168+ context.beginPath();
169+ for (let i = 0; i < timeseriesT.length; i++) {
170+ const x = margins.left + (timeseriesT[i] - xRange.min) * xScale;
171+ const y = margins.top + drawingHeight - (timeseriesYReconstructed[i] - yRange.min) * yScale;
172+ if (i === 0) context.moveTo(x, y);
173+ else context.lineTo(x, y);
174+ }
175+ context.stroke();
176+ } else if (mode === "residuals" && timeseriesYResiduals) {
177+ // Draw residuals with diverging colors
178+ context.lineWidth = 2;
179+ context.beginPath();
180+
181+ // Draw zero line
182+ const zeroY = margins.top + drawingHeight - (0 - yRange.min) * yScale;
183+ context.strokeStyle = "#999";
184+ context.lineWidth = 1;
185+ context.setLineDash([4, 4]);
186+ context.moveTo(margins.left, zeroY);
187+ context.lineTo(width - margins.right, zeroY);
188+ context.stroke();
189+ context.setLineDash([]);
190+
191+ // Draw residuals
192+ context.strokeStyle = "#9c27b0"; // purple for residuals
193+ context.lineWidth = 2;
194+ context.beginPath();
195+ for (let i = 0; i < timeseriesT.length; i++) {
196+ const x = margins.left + (timeseriesT[i] - xRange.min) * xScale;
197+ const y = margins.top + drawingHeight - (timeseriesYResiduals[i] - yRange.min) * yScale;
198+ if (i === 0) context.moveTo(x, y);
199+ else context.lineTo(x, y);
200+ }
201+ context.stroke();
202+ } else if (timeseriesYAll && timeseriesYAll.length > 0) {
203+ // Draw all channels with different colors
204+ const colors = [
205+ "#2196f3", // blue
206+ "#f44336", // red
207+ "#4caf50", // green
208+ "#ff9800", // orange
209+ "#9c27b0", // purple
210+ "#00bcd4", // cyan
211+ "#ffeb3b", // yellow
212+ "#795548", // brown
213+ ];
214+
215+ timeseriesYAll.forEach((channelY, channelIdx) => {
216+ context.strokeStyle = colors[channelIdx % colors.length];
217+ context.lineWidth = 1.5;
218+ context.beginPath();
219+
220+ for (let i = 0; i < timeseriesT.length; i++) {
221+ const x = margins.left + (timeseriesT[i] - xRange.min) * xScale;
222+ const y = margins.top + drawingHeight - (channelY[i] - yRange.min) * yScale;
223+ if (i === 0) context.moveTo(x, y);
224+ else context.lineTo(x, y);
225+ }
226+ context.stroke();
227+ });
228+ } else {
229+ // Draw single channel - original only
230+ context.strokeStyle = "#2196f3";
231+ context.lineWidth = 2;
232+ context.beginPath();
233+
234+ for (let i = 0; i < timeseriesT.length; i++) {
235+ const x = margins.left + (timeseriesT[i] - xRange.min) * xScale;
236+ const y = margins.top + drawingHeight - (timeseriesY[i] - yRange.min) * yScale;
237+ if (i === 0) context.moveTo(x, y);
238+ else context.lineTo(x, y);
239+ }
240+ context.stroke();
241+ }
242+
243+ // Remove clipping before drawing ticks
244+ context.restore();
245+
246+ // Draw Y-axis ticks and labels
247+ const yTicks = getTickPositions(yRange, drawingHeight);
248+
249+ context.textAlign = "right";
250+ context.textBaseline = "middle";
251+ context.fillStyle = "#666666";
252+ context.font = "12px Arial";
253+
254+ yTicks.forEach((tick) => {
255+ const y = margins.top + drawingHeight - tick.x * drawingHeight;
256+
257+ // Draw tick mark
258+ context.beginPath();
259+ context.moveTo(margins.left - 6, y);
260+ context.lineTo(margins.left, y);
261+ context.stroke();
262+
263+ // Draw label
264+ context.fillText(tick.value.toString(), margins.left - 8, y);
265+ });
266+
267+ // Draw X-axis ticks and labels
268+ const ticks = getTickPositions(xRange, drawingWidth, true); // Consider number width for x-axis
269+
270+ context.textAlign = "center";
271+ context.textBaseline = "top";
272+ context.fillStyle = "#666666";
273+ context.font = "12px Arial";
274+
275+ ticks.forEach((tick) => {
276+ const x = margins.left + tick.x * drawingWidth;
277+
278+ // Draw tick mark
279+ context.beginPath();
280+ context.moveTo(x, height - margins.bottom);
281+ context.lineTo(x, height - margins.bottom + 6);
282+ context.stroke();
283+
284+ // Draw label
285+ context.fillText(tick.value.toString(), x, height - margins.bottom + 8);
286+ });
287+}
288+
289+self.onmessage = (evt: MessageEvent) => {
290+ const message = evt.data as WorkerMessage;
291+
292+ if (message.type === "initialize") {
293+ canvas = message.canvas;
294+ ctx = canvas.getContext("2d");
295+ if (!ctx) {
296+ self.postMessage({
297+ type: "error",
298+ error: "Failed to get canvas context",
299+ });
300+ return;
301+ }
302+ self.postMessage({ type: "initialized" });
303+ return;
304+ }
305+
306+ if (message.type === "render") {
307+ throttleRender(() => {
308+ const {
309+ timeseriesT,
310+ timeseriesY,
311+ timeseriesYAll,
312+ timeseriesYReconstructed,
313+ timeseriesYResiduals,
314+ comparisonMode,
315+ width,
316+ height,
317+ margins,
318+ xRange,
319+ yRange,
320+ } = message;
321+ renderTimeseries(
322+ timeseriesT,
323+ timeseriesY,
324+ timeseriesYAll,
325+ timeseriesYReconstructed,
326+ timeseriesYResiduals,
327+ comparisonMode,
328+ width,
329+ height,
330+ margins,
331+ xRange,
332+ yRange,
333+ );
334+ self.postMessage({ type: "render_complete" });
335+ });
336+ return;
337+ }
338+};
339+
340+let renderStack: (() => void)[] = [];
341+let lastRenderTime = 0;
342+
343+const throttleRender = (callback: () => void) => {
344+ renderStack.push(callback);
345+ const checkRender = () => {
346+ if (renderStack.length === 0) return;
347+ const elapsed = Date.now() - lastRenderTime;
348+ if (elapsed > 100) {
349+ lastRenderTime = Date.now();
350+ renderStack[renderStack.length - 1]();
351+ renderStack = [];
352+ } else {
353+ setTimeout(checkRender, 150);
354+ }
355+ };
356+ checkRender();
357+};
358+
359+export {}; // Needed for TypeScript modules
web-ui/src/components/dataset/WorkerTypes.tsadded+30−0View file
@@ -0,0 +1,30 @@
1+import { ComparisonMode } from "../../types/comparison";
2+
3+export interface Range {
4+ min: number;
5+ max: number;
6+}
7+
8+export interface Margins {
9+ left: number;
10+ right: number;
11+ top: number;
12+ bottom: number;
13+}
14+
15+export type WorkerMessage =
16+ | { type: "initialize"; canvas: OffscreenCanvas }
17+ | {
18+ type: "render";
19+ timeseriesT: number[];
20+ timeseriesY: number[];
21+ timeseriesYAll?: number[][]; // For multi-channel overlay
22+ timeseriesYReconstructed?: number[]; // For comparison with lossy reconstruction
23+ timeseriesYResiduals?: number[]; // For showing residuals
24+ comparisonMode?: ComparisonMode;
25+ width: number;
26+ height: number;
27+ margins: Margins;
28+ xRange: Range;
29+ yRange: Range;
30+ };
web-ui/src/components/dataset/timeseriesViewReducer.tsadded+55−0View file
@@ -0,0 +1,55 @@
1+import { Range } from "./WorkerTypes";
2+
3+// State Type
4+export interface TimeseriesViewState {
5+ selectedIndex: number;
6+ isDragging: boolean;
7+ lastDragX: number;
8+ xRange: Range;
9+}
10+
11+// Initial State
12+export const initialState: TimeseriesViewState = {
13+ selectedIndex: -1,
14+ isDragging: false,
15+ lastDragX: 0,
16+ xRange: { min: 0, max: 999 },
17+};
18+
19+// Action Types Union
20+type TimeseriesViewAction =
21+ | { type: "SET_SELECTED_INDEX"; index: number }
22+ | { type: "SET_IS_DRAGGING"; isDragging: boolean }
23+ | { type: "SET_LAST_DRAG_X"; x: number }
24+ | { type: "SET_X_RANGE"; range: Range };
25+
26+// Reducer
27+export const timeseriesViewReducer = (
28+ state: TimeseriesViewState = initialState,
29+ action: TimeseriesViewAction,
30+): TimeseriesViewState => {
31+ switch (action.type) {
32+ case "SET_SELECTED_INDEX":
33+ return {
34+ ...state,
35+ selectedIndex: action.index,
36+ };
37+ case "SET_IS_DRAGGING":
38+ return {
39+ ...state,
40+ isDragging: action.isDragging,
41+ };
42+ case "SET_LAST_DRAG_X":
43+ return {
44+ ...state,
45+ lastDragX: action.x,
46+ };
47+ case "SET_X_RANGE":
48+ return {
49+ ...state,
50+ xRange: action.range,
51+ };
52+ default:
53+ return state;
54+ }
55+};
web-ui/src/components/shared/BaseContent.tsxadded+139−0View file
@@ -0,0 +1,139 @@
1+import { useNavigate } from "react-router-dom";
2+import { useState } from "react";
3+import ReactMarkdown from "react-markdown";
4+import remarkMath from "remark-math";
5+import rehypeKatex from "rehype-katex";
6+import { BenchmarkData } from "../../types";
7+import { BenchmarkCharts } from "../benchmark/charts/BenchmarkCharts";
8+import { BenchmarkScatterPlots } from "../benchmark/charts/BenchmarkScatterPlots";
9+import { BenchmarkTable } from "../benchmark/table/BenchmarkTable";
10+import "./ContentStyles.css";
11+
12+export interface BaseItem {
13+ name: string;
14+ description: string;
15+ long_description?: string;
16+ version: string;
17+ tags: string[];
18+ source_file?: string;
19+}
20+
21+interface BaseContentProps {
22+ item: BaseItem;
23+ benchmarkData: BenchmarkData | null;
24+ chartData: Array<{
25+ algorithmOrDataset: string;
26+ compression_ratio: number;
27+ reference_compression_ratio: number | null;
28+ encode_speed: number;
29+ decode_speed: number;
30+ rmse?: number;
31+ tags: string[];
32+ }>;
33+ tagNavigationPrefix: string;
34+ filterKey: "dataset" | "algorithm";
35+ downloadSection?: React.ReactNode;
36+ additionalContent?: React.ReactNode;
37+ showSortByCompressionRatio?: boolean;
38+ showNormalizeByReference?: boolean;
39+}
40+
41+export const BaseContent = ({
42+ item,
43+ benchmarkData,
44+ chartData,
45+ tagNavigationPrefix,
46+ filterKey,
47+ downloadSection,
48+ additionalContent,
49+ showSortByCompressionRatio,
50+ showNormalizeByReference,
51+}: BaseContentProps) => {
52+ const navigate = useNavigate();
53+ const [isExpanded, setIsExpanded] = useState(false);
54+
55+ return (
56+ <div>
57+ <div className="content-container">
58+ <p className="content-header">
59+ <strong>{item.name}</strong> | {item.description}
60+ </p>
61+ {item.long_description && (
62+ <>
63+ <button
64+ className="description-toggle"
65+ onClick={() => setIsExpanded(!isExpanded)}
66+ >
67+ {isExpanded ? "View less" : "Read more"}
68+ </button>
69+ {isExpanded && (
70+ <div className="long-description">
71+ <ReactMarkdown
72+ remarkPlugins={[remarkMath]}
73+ rehypePlugins={[rehypeKatex]}
74+ >
75+ {item.long_description}
76+ </ReactMarkdown>
77+ </div>
78+ )}
79+ </>
80+ )}
81+ </div>
82+ <div className="metadata-section">
83+ <div>
84+ <span className="metadata-label">Version: </span>
85+ <span className="metadata-value">{item.version}</span>
86+ </div>
87+ <div>
88+ <span className="metadata-label">Tags: </span>
89+ {item.tags.map((tag) => (
90+ <span
91+ key={tag}
92+ className="tag"
93+ onClick={() => navigate(`${tagNavigationPrefix}?tag=${tag}`)}
94+ >
95+ {tag}
96+ </span>
97+ ))}
98+ </div>
99+ {downloadSection}
100+ {item.source_file && (
101+ <div>
102+ <span className="metadata-label">Source: </span>
103+ <a
104+ href={item.source_file}
105+ target="_blank"
106+ rel="noopener noreferrer"
107+ className="source-link"
108+ >
109+ View
110+ </a>
111+ </div>
112+ )}
113+ </div>
114+ {additionalContent}
115+ {benchmarkData && (
116+ <>
117+ <div className="benchmark-section">
118+ <h2 className="benchmark-title">Benchmark Results</h2>
119+ <BenchmarkCharts
120+ chartData={chartData}
121+ showSortByCompressionRatio={showSortByCompressionRatio}
122+ showNormalizeByReference={showNormalizeByReference}
123+ />
124+ {filterKey === "dataset" && (
125+ <BenchmarkScatterPlots chartData={chartData} />
126+ )}
127+ </div>
128+ <div className="benchmark-section">
129+ <BenchmarkTable
130+ results={benchmarkData.results.filter(
131+ (result) => result[filterKey] === item.name,
132+ )}
133+ />
134+ </div>
135+ </>
136+ )}
137+ </div>
138+ );
139+};
web-ui/src/components/shared/ContentStyles.cssadded+178−0View file
@@ -0,0 +1,178 @@
1+.content-container {
2+ margin-bottom: 1.5rem;
3+}
4+
5+.content-header {
6+ font-size: 0.9rem;
7+ line-height: 1.5;
8+ margin-bottom: 0.5rem;
9+}
10+
11+.description-toggle {
12+ color: #0066cc;
13+ text-decoration: none;
14+ font-size: 0.8rem;
15+ cursor: pointer;
16+ background: none;
17+ border: none;
18+ padding: 0;
19+ margin-top: 0.25rem;
20+ display: block;
21+}
22+
23+.description-toggle:hover {
24+ text-decoration: underline;
25+}
26+
27+.long-description {
28+ font-size: 0.9rem;
29+ line-height: 1.5;
30+ margin-top: 0.5rem;
31+ padding: 0.5rem;
32+ background-color: #f8f8f8;
33+ border-radius: 4px;
34+}
35+
36+/* Markdown styles */
37+.long-description h1,
38+.long-description h2,
39+.long-description h3,
40+.long-description h4,
41+.long-description h5,
42+.long-description h6 {
43+ margin-top: 1.5em;
44+ margin-bottom: 0.5em;
45+ font-weight: 600;
46+}
47+
48+.long-description h1 { font-size: 1.5em; }
49+.long-description h2 { font-size: 1.3em; }
50+.long-description h3 { font-size: 1.1em; }
51+
52+.long-description p {
53+ margin-bottom: 1em;
54+}
55+
56+.long-description ul,
57+.long-description ol {
58+ margin: 1em 0;
59+ padding-left: 2em;
60+}
61+
62+.long-description li {
63+ margin: 0.5em 0;
64+}
65+
66+.long-description code {
67+ background-color: #eee;
68+ padding: 0.2em 0.4em;
69+ border-radius: 3px;
70+ font-family: monospace;
71+ font-size: 0.9em;
72+}
73+
74+.long-description pre {
75+ background-color: #eee;
76+ padding: 1em;
77+ border-radius: 4px;
78+ overflow-x: auto;
79+ margin: 1em 0;
80+}
81+
82+.long-description pre code {
83+ background-color: transparent;
84+ padding: 0;
85+}
86+
87+.long-description blockquote {
88+ border-left: 4px solid #ddd;
89+ margin: 1em 0;
90+ padding-left: 1em;
91+ color: #666;
92+}
93+
94+.long-description a {
95+ color: #0066cc;
96+ text-decoration: none;
97+}
98+
99+.long-description a:hover {
100+ text-decoration: underline;
101+}
102+
103+.long-description img {
104+ max-width: 100%;
105+ height: auto;
106+ margin: 1em 0;
107+}
108+
109+.long-description table {
110+ border-collapse: collapse;
111+ width: 100%;
112+ margin: 1em 0;
113+}
114+
115+.long-description th,
116+.long-description td {
117+ border: 1px solid #ddd;
118+ padding: 0.5em;
119+ text-align: left;
120+}
121+
122+.long-description th {
123+ background-color: #f0f0f0;
124+}
125+
126+.metadata-section {
127+ margin-bottom: 1.5rem;
128+ display: flex;
129+ gap: 2rem;
130+ flex-wrap: wrap;
131+}
132+
133+.metadata-label {
134+ font-weight: bold;
135+ font-size: 0.9rem;
136+}
137+
138+.metadata-value {
139+ font-size: 0.9rem;
140+}
141+
142+.tag {
143+ color: #0066cc;
144+ text-decoration: none;
145+ padding: 2px 6px;
146+ background-color: #f0f0f0;
147+ border-radius: 4px;
148+ font-size: 0.9rem;
149+ cursor: pointer;
150+}
151+
152+.source-link {
153+ color: #0066cc;
154+ text-decoration: none;
155+ padding: 2px 6px;
156+ background-color: #f0f0f0;
157+ border-radius: 4px;
158+ font-size: 0.9rem;
159+}
160+
161+.benchmark-section {
162+ margin-bottom: 1.5rem;
163+}
164+
165+.benchmark-title {
166+ font-size: 1.2rem;
167+ font-weight: bold;
168+ margin-bottom: 0.5rem;
169+}
170+
171+.download-link {
172+ color: #0066cc;
173+ text-decoration: none;
174+ padding: 2px 6px;
175+ background-color: #f0f0f0;
176+ border-radius: 4px;
177+ font-size: 0.9rem;
178+}
web-ui/src/components/tables/DatasetAlgorithmTables.tsxadded+432−0View file
@@ -0,0 +1,432 @@
1+import { Link } from "react-router-dom";
2+import { TagFilter } from "../TagFilter";
3+import { Dataset, Algorithm, BenchmarkResult } from "../../types";
4+
5+interface DatasetTableProps {
6+ filteredDatasets: Dataset[];
7+ availableDatasetTags: string[];
8+ selectedTags: string[];
9+ toggleTag: (tag: string) => void;
10+ benchmarkResults: BenchmarkResult[];
11+}
12+
13+interface AlgorithmTableProps {
14+ filteredAlgorithms: Algorithm[];
15+ availableAlgorithmTags: string[];
16+ selectedTags: string[];
17+ toggleTag: (tag: string) => void;
18+}
19+
20+export const DatasetTable = ({
21+ filteredDatasets,
22+ availableDatasetTags,
23+ selectedTags,
24+ toggleTag,
25+ benchmarkResults,
26+}: DatasetTableProps) => {
27+ const getBestCompressionResult = (datasetName: string) => {
28+ const datasetResults = benchmarkResults.filter(
29+ (result) => result.dataset === datasetName,
30+ );
31+ if (datasetResults.length === 0) return { algorithm: "N/A", ratio: 0 };
32+
33+ const bestResult = datasetResults.reduce((best, current) =>
34+ current.compression_ratio > best.compression_ratio ? current : best,
35+ );
36+ return {
37+ algorithm: bestResult.algorithm,
38+ ratio: bestResult.compression_ratio,
39+ };
40+ };
41+
42+ return (
43+ <div style={{ overflowX: "auto" }}>
44+ <div style={{ marginBottom: "1rem" }}>
45+ <TagFilter
46+ availableTags={availableDatasetTags}
47+ selectedTags={selectedTags}
48+ onTagToggle={toggleTag}
49+ label="Filter datasets"
50+ />
51+ </div>
52+ <table style={{ width: "100%", borderCollapse: "collapse" }}>
53+ <thead>
54+ <tr style={{ backgroundColor: "#f5f5f5" }}>
55+ <th
56+ style={{
57+ padding: "8px 12px",
58+ textAlign: "left",
59+ borderBottom: "1px solid #ddd",
60+ fontSize: "0.9rem",
61+ whiteSpace: "nowrap",
62+ }}
63+ >
64+ Name
65+ </th>
66+ <th
67+ style={{
68+ padding: "8px 12px",
69+ textAlign: "left",
70+ borderBottom: "1px solid #ddd",
71+ fontSize: "0.9rem",
72+ whiteSpace: "nowrap",
73+ }}
74+ >
75+ Version
76+ </th>
77+ <th
78+ style={{
79+ padding: "8px 12px",
80+ textAlign: "left",
81+ borderBottom: "1px solid #ddd",
82+ fontSize: "0.9rem",
83+ whiteSpace: "nowrap",
84+ }}
85+ >
86+ Description
87+ </th>
88+ <th
89+ style={{
90+ padding: "8px 12px",
91+ textAlign: "left",
92+ borderBottom: "1px solid #ddd",
93+ fontSize: "0.9rem",
94+ whiteSpace: "nowrap",
95+ }}
96+ >
97+ Tags
98+ </th>
99+ <th
100+ style={{
101+ padding: "8px 12px",
102+ textAlign: "left",
103+ borderBottom: "1px solid #ddd",
104+ fontSize: "0.9rem",
105+ whiteSpace: "nowrap",
106+ }}
107+ >
108+ Source
109+ </th>
110+ <th
111+ style={{
112+ padding: "8px 12px",
113+ textAlign: "left",
114+ borderBottom: "1px solid #ddd",
115+ fontSize: "0.9rem",
116+ whiteSpace: "nowrap",
117+ }}
118+ >
119+ Data
120+ </th>
121+ <th
122+ style={{
123+ padding: "8px 12px",
124+ textAlign: "left",
125+ borderBottom: "1px solid #ddd",
126+ fontSize: "0.9rem",
127+ whiteSpace: "nowrap",
128+ }}
129+ >
130+ Best Compression
131+ </th>
132+ </tr>
133+ </thead>
134+ <tbody>
135+ {filteredDatasets.map((dataset, index) => (
136+ <tr
137+ key={`${dataset.name}-${dataset.version}`}
138+ style={{ backgroundColor: index % 2 === 0 ? "white" : "#fafafa" }}
139+ >
140+ <td
141+ style={{
142+ padding: "6px 12px",
143+ borderBottom: "1px solid #ddd",
144+ fontSize: "0.9rem",
145+ }}
146+ >
147+ <Link
148+ to={`/dataset/${encodeURIComponent(dataset.name)}`}
149+ style={{
150+ color: "#0066cc",
151+ textDecoration: "none",
152+ fontWeight: "500",
153+ }}
154+ >
155+ {dataset.name}
156+ </Link>
157+ </td>
158+ <td
159+ style={{
160+ padding: "6px 12px",
161+ borderBottom: "1px solid #ddd",
162+ fontSize: "0.9rem",
163+ }}
164+ >
165+ {dataset.version}
166+ </td>
167+ <td
168+ style={{
169+ padding: "6px 12px",
170+ borderBottom: "1px solid #ddd",
171+ fontSize: "0.9rem",
172+ }}
173+ >
174+ {dataset.description}
175+ </td>
176+ <td
177+ style={{
178+ padding: "6px 12px",
179+ borderBottom: "1px solid #ddd",
180+ fontSize: "0.9rem",
181+ }}
182+ >
183+ {dataset.tags.map((tag) => (
184+ <span
185+ key={tag}
186+ style={{
187+ display: "inline-block",
188+ backgroundColor: "#e1e1e1",
189+ padding: "2px 6px",
190+ borderRadius: "3px",
191+ margin: "1px",
192+ fontSize: "0.8rem",
193+ }}
194+ >
195+ {tag}
196+ </span>
197+ ))}
198+ </td>
199+ <td
200+ style={{
201+ padding: "6px 12px",
202+ borderBottom: "1px solid #ddd",
203+ fontSize: "0.9rem",
204+ }}
205+ >
206+ {dataset.source_file && (
207+ <a
208+ href={dataset.source_file}
209+ target="_blank"
210+ rel="noopener noreferrer"
211+ style={{ color: "#0066cc", textDecoration: "none" }}
212+ >
213+ View Source
214+ </a>
215+ )}
216+ </td>
217+ <td
218+ style={{
219+ padding: "6px 12px",
220+ borderBottom: "1px solid #ddd",
221+ fontSize: "0.9rem",
222+ }}
223+ >
224+ {dataset.data_url_npy && (
225+ <a
226+ href={dataset.data_url_npy}
227+ download={`${dataset.name}-${dataset.version}.npy`}
228+ style={{ color: "#0066cc", textDecoration: "none" }}
229+ >
230+ Download
231+ </a>
232+ )}
233+ </td>
234+ <td
235+ style={{
236+ padding: "6px 12px",
237+ borderBottom: "1px solid #ddd",
238+ fontSize: "0.9rem",
239+ }}
240+ >
241+ {(() => {
242+ const result = getBestCompressionResult(dataset.name);
243+ return (
244+ <>
245+ <Link
246+ to={`/algorithm/${encodeURIComponent(result.algorithm)}`}
247+ style={{
248+ color: "#0066cc",
249+ textDecoration: "none",
250+ fontWeight: "500",
251+ }}
252+ >
253+ {result.algorithm}
254+ </Link>
255+ {result.algorithm !== "N/A" &&
256+ ` (${result.ratio.toFixed(1)})`}
257+ </>
258+ );
259+ })()}
260+ </td>
261+ </tr>
262+ ))}
263+ </tbody>
264+ </table>
265+ </div>
266+ );
267+};
268+
269+export const AlgorithmTable = ({
270+ filteredAlgorithms,
271+ availableAlgorithmTags,
272+ selectedTags,
273+ toggleTag,
274+}: AlgorithmTableProps) => (
275+ <div style={{ overflowX: "auto" }}>
276+ <div style={{ marginBottom: "1rem" }}>
277+ <TagFilter
278+ availableTags={availableAlgorithmTags}
279+ selectedTags={selectedTags}
280+ onTagToggle={toggleTag}
281+ label="Filter algorithms"
282+ />
283+ </div>
284+ <table style={{ width: "100%", borderCollapse: "collapse" }}>
285+ <thead>
286+ <tr style={{ backgroundColor: "#f5f5f5" }}>
287+ <th
288+ style={{
289+ padding: "8px 12px",
290+ textAlign: "left",
291+ borderBottom: "1px solid #ddd",
292+ fontSize: "0.9rem",
293+ whiteSpace: "nowrap",
294+ }}
295+ >
296+ Name
297+ </th>
298+ <th
299+ style={{
300+ padding: "8px 12px",
301+ textAlign: "left",
302+ borderBottom: "1px solid #ddd",
303+ fontSize: "0.9rem",
304+ whiteSpace: "nowrap",
305+ }}
306+ >
307+ Version
308+ </th>
309+ <th
310+ style={{
311+ padding: "8px 12px",
312+ textAlign: "left",
313+ borderBottom: "1px solid #ddd",
314+ fontSize: "0.9rem",
315+ whiteSpace: "nowrap",
316+ }}
317+ >
318+ Description
319+ </th>
320+ <th
321+ style={{
322+ padding: "8px 12px",
323+ textAlign: "left",
324+ borderBottom: "1px solid #ddd",
325+ fontSize: "0.9rem",
326+ whiteSpace: "nowrap",
327+ }}
328+ >
329+ Tags
330+ </th>
331+ <th
332+ style={{
333+ padding: "8px 12px",
334+ textAlign: "left",
335+ borderBottom: "1px solid #ddd",
336+ fontSize: "0.9rem",
337+ whiteSpace: "nowrap",
338+ }}
339+ >
340+ Source
341+ </th>
342+ </tr>
343+ </thead>
344+ <tbody>
345+ {filteredAlgorithms.map((algorithm, index) => (
346+ <tr
347+ key={`${algorithm.name}-${algorithm.version}`}
348+ style={{ backgroundColor: index % 2 === 0 ? "white" : "#fafafa" }}
349+ >
350+ <td
351+ style={{
352+ padding: "6px 12px",
353+ borderBottom: "1px solid #ddd",
354+ fontSize: "0.9rem",
355+ }}
356+ >
357+ <Link
358+ to={`/algorithm/${encodeURIComponent(algorithm.name)}`}
359+ style={{
360+ color: "#0066cc",
361+ textDecoration: "none",
362+ fontWeight: "500",
363+ }}
364+ >
365+ {algorithm.name}
366+ </Link>
367+ </td>
368+ <td
369+ style={{
370+ padding: "6px 12px",
371+ borderBottom: "1px solid #ddd",
372+ fontSize: "0.9rem",
373+ }}
374+ >
375+ {algorithm.version}
376+ </td>
377+ <td
378+ style={{
379+ padding: "6px 12px",
380+ borderBottom: "1px solid #ddd",
381+ fontSize: "0.9rem",
382+ }}
383+ >
384+ {algorithm.description}
385+ </td>
386+ <td
387+ style={{
388+ padding: "6px 12px",
389+ borderBottom: "1px solid #ddd",
390+ fontSize: "0.9rem",
391+ }}
392+ >
393+ {algorithm.tags.map((tag) => (
394+ <span
395+ key={tag}
396+ style={{
397+ display: "inline-block",
398+ backgroundColor: "#e1e1e1",
399+ padding: "2px 6px",
400+ borderRadius: "3px",
401+ margin: "1px",
402+ fontSize: "0.8rem",
403+ }}
404+ >
405+ {tag}
406+ </span>
407+ ))}
408+ </td>
409+ <td
410+ style={{
411+ padding: "6px 12px",
412+ borderBottom: "1px solid #ddd",
413+ fontSize: "0.9rem",
414+ }}
415+ >
416+ {algorithm.source_file && (
417+ <a
418+ href={algorithm.source_file}
419+ target="_blank"
420+ rel="noopener noreferrer"
421+ style={{ color: "#0066cc", textDecoration: "none" }}
422+ >
423+ View Source
424+ </a>
425+ )}
426+ </td>
427+ </tr>
428+ ))}
429+ </tbody>
430+ </table>
431+ </div>
432+);
web-ui/src/content/home-content.ymladded+48−0View file
@@ -0,0 +1,48 @@
1+title: Welcome to Ephys Compression Tests
2+description:
3+ Ephys Compression Tests is an open-source project for comparing compression algorithms
4+ on electrophysiology data.
5+
6+sections:
7+ algorithms:
8+ title: Algorithms
9+ description:
10+ Explore the supported compression algorithms, including traditional methods
11+ like zlib and modern approaches like ANS.
12+ link: /algorithms
13+ linkText: View Algorithms
14+
15+ datasets:
16+ title: Datasets
17+ description:
18+ Explore the curated collection of ephys datasets.
19+ See the performance of the various compression algorithms on each dataset.
20+ link: /datasets
21+ linkText: View Datasets
22+
23+ monitor:
24+ title: Live Monitor
25+ description:
26+ The benchmarking runs automatically in a GitHub Actions workflow. Track
27+ current progress, view completed benchmarks, and monitor performance
28+ metrics as they are generated.
29+ link: /monitor
30+ linkText: View Monitor
31+
32+ source:
33+ title: Source Code
34+ description:
35+ Explore the GitHub repository to view the source code, contribute to the
36+ project, or run your own benchmarks locally.
37+ link: https://github.com/concept-collection/ephys_compression_tests
38+ linkText: View on GitHub
39+ external: true
40+
41+ submit:
42+ title: Submit an Algorithm or Dataset
43+ description:
44+ Want to contribute? Learn how to add your own compression algorithm
45+ or ephys dataset to the benchmark suite. Follow the step-by-step
46+ guide for preparing and submitting your contribution.
47+ link: /submit
48+ linkText: Submission Guide
web-ui/src/env.d.tsadded+1−0View file
@@ -0,0 +1 @@
1+declare const __BUILD_DATE__: string;
web-ui/src/hooks/TimeseriesDataClient.tsadded+196−0View file
@@ -0,0 +1,196 @@
1+export type SupportedTypedArray =
2+ | Uint8Array
3+ | Uint16Array
4+ | Uint32Array
5+ | Int16Array
6+ | Int32Array
7+ | Float32Array;
8+
9+interface ChunkCache {
10+ [key: number]: SupportedTypedArray;
11+}
12+
13+type DType = "uint8" | "uint16" | "uint32" | "int16" | "int32" | "float32";
14+
15+const TypedArrayConstructors = {
16+ uint8: Uint8Array,
17+ uint16: Uint16Array,
18+ uint32: Uint32Array,
19+ int16: Int16Array,
20+ int32: Int32Array,
21+ float32: Float32Array,
22+} as const;
23+
24+export class TimeseriesDataClient {
25+ private shape: number = 0;
26+ private numChannels: number = 1;
27+ private dtype: DType | null = null;
28+ private chunkSize: number;
29+ private cache: ChunkCache = {};
30+ private inProgressFetches: { [key: number]: Promise<SupportedTypedArray> } =
31+ {};
32+ private datasetJsonUrl: string;
33+ private datasetDataUrl: string;
34+
35+ constructor(
36+ datasetJsonUrl: string,
37+ datasetDataUrl: string,
38+ chunkSize: number = 100000,
39+ ) {
40+ this.datasetJsonUrl = datasetJsonUrl;
41+ this.datasetDataUrl = datasetDataUrl;
42+ this.chunkSize = chunkSize;
43+ }
44+
45+ static async create(
46+ datasetJsonUrl: string,
47+ datasetDataUrl: string,
48+ chunkSize: number = 1000,
49+ ): Promise<TimeseriesDataClient> {
50+ const client = new TimeseriesDataClient(
51+ datasetJsonUrl,
52+ datasetDataUrl,
53+ chunkSize,
54+ );
55+ await client.initialize();
56+ return client;
57+ }
58+
59+ private async initialize() {
60+ const infoUrl = this.datasetJsonUrl;
61+ const response = await fetch(infoUrl);
62+ if (!response.ok) {
63+ throw new Error(`Failed to fetch dataset info: ${response.statusText}`);
64+ }
65+ const info = await response.json();
66+
67+ // Handle multi-dimensional shape: [num_timepoints, num_channels]
68+ if (Array.isArray(info.shape)) {
69+ this.shape = info.shape[0];
70+ this.numChannels = info.shape.length > 1 ? info.shape[1] : 1;
71+ } else {
72+ this.shape = info.shape;
73+ this.numChannels = 1;
74+ }
75+
76+ if (!this.isValidDType(info.dtype)) {
77+ throw new Error(`Unsupported data type: ${info.dtype}`);
78+ }
79+ this.dtype = info.dtype;
80+ }
81+
82+ private isValidDType(dtype: string): dtype is DType {
83+ return dtype in TypedArrayConstructors;
84+ }
85+
86+ private getChunkIndices(start: number, end: number): number[] {
87+ const startChunk = Math.floor(start / this.chunkSize);
88+ const endChunk = Math.floor(end / this.chunkSize);
89+ const chunks: number[] = [];
90+ for (let i = startChunk; i <= endChunk; i++) {
91+ chunks.push(i);
92+ }
93+ return chunks;
94+ }
95+
96+ private async fetchChunk(chunkIndex: number): Promise<SupportedTypedArray> {
97+ // Return cached chunk if available
98+ if (this.cache[chunkIndex]) {
99+ return this.cache[chunkIndex];
100+ }
101+
102+ // If this chunk is already being fetched, wait for it to complete
103+ const inProgressFetch = this.inProgressFetches[chunkIndex];
104+ if (inProgressFetch !== undefined) {
105+ return inProgressFetch;
106+ }
107+
108+ // Start new fetch and track it
109+ const fetchPromise = (async () => {
110+ if (!this.dtype) {
111+ throw new Error("Data type not initialized");
112+ }
113+
114+ const start = chunkIndex * this.chunkSize;
115+ const end = Math.min(start + this.chunkSize, this.shape);
116+ const url = this.datasetDataUrl;
117+ const itemSize = TypedArrayConstructors[this.dtype].BYTES_PER_ELEMENT;
118+ // Account for multi-channel data: each timepoint has numChannels values
119+ const byteStart = start * this.numChannels * itemSize;
120+ const byteEnd = end * this.numChannels * itemSize;
121+
122+ try {
123+ const response = await fetch(url, {
124+ headers: {
125+ Range: `bytes=${byteStart}-${byteEnd - 1}`,
126+ },
127+ });
128+ if (!response.ok) {
129+ throw new Error(`Failed to fetch chunk: ${response.statusText}`);
130+ }
131+
132+ const buffer = await response.arrayBuffer();
133+ const ArrayConstructor = TypedArrayConstructors[this.dtype];
134+ const data = new ArrayConstructor(buffer);
135+ this.cache[chunkIndex] = data;
136+ return data;
137+ } finally {
138+ // Clean up the in-progress fetch regardless of success/failure
139+ delete this.inProgressFetches[chunkIndex];
140+ }
141+ })();
142+
143+ // Store the promise for other requests to wait on
144+ this.inProgressFetches[chunkIndex] = fetchPromise;
145+ return fetchPromise;
146+ }
147+
148+ async fetchRange(start: number, end: number, channel: number = 0): Promise<SupportedTypedArray> {
149+ if (!this.dtype) {
150+ throw new Error("Data type not initialized");
151+ }
152+
153+ if (channel < 0 || channel >= this.numChannels) {
154+ throw new Error(`Invalid channel ${channel}. Must be between 0 and ${this.numChannels - 1}`);
155+ }
156+
157+ const chunkIndices = this.getChunkIndices(start, end);
158+ const chunks = await Promise.all(
159+ chunkIndices.map((idx) => this.fetchChunk(idx)),
160+ );
161+
162+ // Calculate total length needed
163+ const length = end - start;
164+ const ArrayConstructor = TypedArrayConstructors[this.dtype];
165+ const result = new ArrayConstructor(length);
166+
167+ // Copy data from chunks into result array
168+ let resultOffset = 0;
169+ for (let i = 0; i < chunks.length; i++) {
170+ const chunk = chunks[i];
171+ const chunkStart = chunkIndices[i] * this.chunkSize;
172+ const copyStart = Math.max(0, start - chunkStart);
173+ const copyEnd = Math.min(chunk.length / this.numChannels, end - chunkStart);
174+
175+ // Extract the selected channel from interleaved data
176+ for (let t = copyStart; t < copyEnd; t++) {
177+ const sourceIdx = t * this.numChannels + channel;
178+ result[resultOffset++] = chunk[sourceIdx];
179+ }
180+ }
181+
182+ return result;
183+ }
184+
185+ getShape(): number {
186+ return this.shape;
187+ }
188+
189+ getDType(): DType | null {
190+ return this.dtype;
191+ }
192+
193+ getNumChannels(): number {
194+ return this.numChannels;
195+ }
196+}
web-ui/src/hooks/useBenchmarkChartData.tsadded+47−0View file
@@ -0,0 +1,47 @@
1+import { useMemo } from "react";
2+import { Algorithm, BenchmarkResult } from "../types";
3+
4+export function useBenchmarkChartData(
5+ results: BenchmarkResult[],
6+ algorithms: Algorithm[],
7+ selectedDataset?: string | null,
8+ selectedAlgorithm?: string | null,
9+) {
10+ return useMemo(() => {
11+ if (selectedDataset) {
12+ return results
13+ .filter((row) => row.dataset === selectedDataset)
14+ .map((row) => {
15+ const algorithm = algorithms.find((a) => a.name === row.algorithm);
16+ return {
17+ algorithmOrDataset: row.algorithm,
18+ compression_ratio: row.compression_ratio,
19+ reference_compression_ratio: null,
20+ encode_speed: row.encode_mb_per_sec,
21+ decode_speed: row.decode_mb_per_sec,
22+ rmse: row.rmse,
23+ max_error: row.max_error,
24+ tags: algorithm?.tags || [],
25+ };
26+ });
27+ } else if (selectedAlgorithm) {
28+ return results
29+ .filter((row) => row.algorithm === selectedAlgorithm)
30+ .map((row) => ({
31+ algorithmOrDataset: row.dataset,
32+ compression_ratio: row.compression_ratio,
33+ reference_compression_ratio: Math.max(
34+ ...results
35+ .filter((r) => r.dataset === row.dataset)
36+ .map((r) => r.compression_ratio),
37+ ),
38+ encode_speed: row.encode_mb_per_sec,
39+ decode_speed: row.decode_mb_per_sec,
40+ rmse: row.rmse,
41+ max_error: row.max_error,
42+ tags: [],
43+ }));
44+ }
45+ return [];
46+ }, [results, algorithms, selectedDataset, selectedAlgorithm]);
47+}
web-ui/src/hooks/useMarkdownContent.tsadded+31−0View file
@@ -0,0 +1,31 @@
1+import { useState, useEffect } from "react";
2+
3+export const useMarkdownContent = (path: string) => {
4+ const [content, setContent] = useState<string>("");
5+ const [error, setError] = useState<string | null>(null);
6+
7+ useEffect(() => {
8+ const fetchContent = async () => {
9+ try {
10+ const response = await fetch(path);
11+ if (!response.ok) {
12+ throw new Error(
13+ `Failed to load markdown content: ${response.statusText}`,
14+ );
15+ }
16+ const text = await response.text();
17+ setContent(text);
18+ } catch (err) {
19+ setError(
20+ err instanceof Error
21+ ? err.message
22+ : "Failed to load markdown content",
23+ );
24+ }
25+ };
26+
27+ fetchContent();
28+ }, [path]);
29+
30+ return { content, error };
31+};
web-ui/src/hooks/useMarkdownPosts.tsadded+62−0View file
@@ -0,0 +1,62 @@
1+import { useState, useEffect } from "react";
2+
3+interface Post {
4+ path: string;
5+ content: string;
6+ date: Date;
7+}
8+
9+export const useMarkdownPosts = (directory: string) => {
10+ const [posts, setPosts] = useState<Post[]>([]);
11+ const [error, setError] = useState<string | null>(null);
12+ const [loading, setLoading] = useState(true);
13+
14+ useEffect(() => {
15+ const fetchPosts = async () => {
16+ try {
17+ // First fetch the index
18+ const indexResponse = await fetch(`${directory}/index.txt`);
19+ if (!indexResponse.ok) {
20+ throw new Error(
21+ `Failed to load post index: ${indexResponse.statusText}`,
22+ );
23+ }
24+ const indexContent = await indexResponse.text();
25+ const paths = indexContent.trim().split("\n");
26+
27+ // Then fetch all posts in parallel
28+ const postPromises = paths.map(async (path) => {
29+ const response = await fetch(`${directory}/${path}`);
30+ if (!response.ok) {
31+ throw new Error(
32+ `Failed to load post ${path}: ${response.statusText}`,
33+ );
34+ }
35+ const content = await response.text();
36+
37+ // Parse date from filename (format: YYYY-MM-DD-title.md)
38+ const dateMatch = path.match(/^(\d{4}-\d{2}-\d{2})/);
39+ if (!dateMatch) {
40+ throw new Error(`Invalid post filename format: ${path}`);
41+ }
42+ const date = new Date(dateMatch[1]);
43+
44+ return { path, content, date };
45+ });
46+
47+ const loadedPosts = await Promise.all(postPromises);
48+ // Sort posts by date, newest first
49+ loadedPosts.sort((a, b) => b.date.getTime() - a.date.getTime());
50+ setPosts(loadedPosts);
51+ setLoading(false);
52+ } catch (err) {
53+ setError(err instanceof Error ? err.message : "Failed to load posts");
54+ setLoading(false);
55+ }
56+ };
57+
58+ fetchPosts();
59+ }, [directory]);
60+
61+ return { posts, error, loading };
62+};
web-ui/src/hooks/useTagFilter.tsadded+30−0View file
@@ -0,0 +1,30 @@
1+import { useMemo } from "react";
2+
3+interface TaggableItem {
4+ tags: string[];
5+}
6+
7+export function useTagFilter<T extends TaggableItem>(
8+ items: T[],
9+ selectedTags: string[],
10+) {
11+ const availableTags = useMemo(() => {
12+ const tagSet = new Set<string>();
13+ items.forEach((item) => {
14+ item.tags.forEach((tag) => tagSet.add(tag));
15+ });
16+ return Array.from(tagSet).sort();
17+ }, [items]);
18+
19+ const filteredItems = useMemo(() => {
20+ if (selectedTags.length === 0) return items;
21+ return items.filter((item) =>
22+ selectedTags.every((tag) => item.tags.includes(tag)),
23+ );
24+ }, [items, selectedTags]);
25+
26+ return {
27+ availableTags,
28+ filteredItems,
29+ };
30+}
web-ui/src/hooks/useTimeseriesData.tsadded+89−0View file
@@ -0,0 +1,89 @@
1+import { useEffect, useState } from "react";
2+import { Dataset } from "../types";
3+
4+const getDtypeSize = (dtype: string): number => {
5+ switch (dtype) {
6+ case "uint8":
7+ return 1;
8+ case "uint16":
9+ return 2;
10+ case "uint32":
11+ return 4;
12+ case "int16":
13+ return 2;
14+ case "int32":
15+ return 4;
16+ default:
17+ throw new Error(`Unsupported dtype: ${dtype}`);
18+ }
19+};
20+
21+const createTypedArray = (buffer: ArrayBuffer, dtype: string): number[] => {
22+ switch (dtype) {
23+ case "uint8":
24+ return Array.from(new Uint8Array(buffer));
25+ case "uint16":
26+ return Array.from(new Uint16Array(buffer));
27+ case "uint32":
28+ return Array.from(new Uint32Array(buffer));
29+ case "int16":
30+ return Array.from(new Int16Array(buffer));
31+ case "int32":
32+ return Array.from(new Int32Array(buffer));
33+ default:
34+ throw new Error(`Unsupported dtype: ${dtype}`);
35+ }
36+};
37+
38+export const useTimeseriesData = (dataset: Dataset) => {
39+ const [data, setData] = useState<number[] | null>(null);
40+ const [error, setError] = useState<string | null>(null);
41+
42+ useEffect(() => {
43+ const fetchData = async () => {
44+ if (!dataset.data_url_raw) {
45+ setError("No raw data URL available");
46+ return;
47+ }
48+
49+ const metaJsonUrl = dataset.data_url_json;
50+ if (!metaJsonUrl) {
51+ setError("No JSON metadata URL available");
52+ return;
53+ }
54+
55+ try {
56+ const metaResponse = await fetch(metaJsonUrl);
57+ if (!metaResponse.ok) {
58+ throw new Error(`HTTP error! status: ${metaResponse.status}`);
59+ }
60+
61+ const metaJson = await metaResponse.json();
62+
63+ const dtype = metaJson.dtype;
64+ const bytesPerElement = getDtypeSize(dtype);
65+
66+ const numBytes = bytesPerElement * 1000;
67+
68+ const response = await fetch(dataset.data_url_raw, {
69+ headers: {
70+ Range: `bytes=0-${numBytes - 1}`, // First 1000 elements
71+ },
72+ });
73+
74+ if (!response.ok) {
75+ throw new Error(`HTTP error! status: ${response.status}`);
76+ }
77+
78+ const buffer = await response.arrayBuffer();
79+ const data = createTypedArray(buffer, dtype);
80+ setData(data);
81+ } catch (err) {
82+ setError(err instanceof Error ? err.message : "Failed to fetch data");
83+ }
84+ };
85+ fetchData();
86+ }, [dataset]);
87+
88+ return { data, error };
89+};
web-ui/src/hooks/useTimeseriesDataClient.tsadded+39−0View file
@@ -0,0 +1,39 @@
1+import { useEffect, useState } from "react";
2+import { Dataset } from "../types";
3+import { TimeseriesDataClient } from "./TimeseriesDataClient";
4+
5+interface UseTimeseriesDataClientResult {
6+ client: TimeseriesDataClient | null;
7+ error: string | null;
8+}
9+
10+export const useTimeseriesDataClient = (
11+ dataset: Dataset,
12+ chunkSize: number = 1000,
13+): UseTimeseriesDataClientResult => {
14+ const [client, setClient] = useState<TimeseriesDataClient | null>(null);
15+ const [error, setError] = useState<string | null>(null);
16+
17+ useEffect(() => {
18+ const initClient = async () => {
19+ try {
20+ const newClient = await TimeseriesDataClient.create(
21+ dataset.data_url_json || "",
22+ dataset.data_url_raw || "",
23+ chunkSize,
24+ );
25+ setClient(newClient);
26+ setError(null);
27+ } catch (err) {
28+ setError(
29+ err instanceof Error ? err.message : "Failed to initialize client",
30+ );
31+ setClient(null);
32+ }
33+ };
34+
35+ initClient();
36+ }, [dataset.data_url_json, dataset.data_url_raw, chunkSize]);
37+
38+ return { client, error };
39+};
web-ui/src/index.cssadded+68−0View file
@@ -0,0 +1,68 @@
1+:root {
2+ font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
3+ line-height: 1.5;
4+ font-weight: 400;
5+}
6+
7+body {
8+ margin: 0;
9+ min-width: 320px;
10+ min-height: 100vh;
11+ background-color: #f5f5f5;
12+}
13+
14+table {
15+ width: 100%;
16+ border-collapse: collapse;
17+ background: white;
18+ box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
19+ border-radius: 4px;
20+ overflow: hidden;
21+ font-size: 0.9rem;
22+ line-height: 1.3;
23+}
24+
25+th {
26+ background-color: #f8f9fa;
27+ padding: 8px 12px;
28+ text-align: left;
29+ font-weight: 600;
30+ color: #333;
31+ border-bottom: 1px solid #dee2e6;
32+ cursor: pointer;
33+ white-space: nowrap;
34+}
35+
36+th:hover {
37+ background-color: #e9ecef;
38+}
39+
40+td {
41+ padding: 6px 12px;
42+ border-bottom: 1px solid #dee2e6;
43+ color: #444;
44+}
45+
46+tr:hover {
47+ background-color: #f8f9fa;
48+}
49+
50+tr:last-child td {
51+ border-bottom: none;
52+}
53+
54+.table-container {
55+ margin: 12px 0;
56+ overflow-x: auto;
57+}
58+
59+/* Make filter controls more compact */
60+select {
61+ padding: 4px 8px !important;
62+ font-size: 0.9rem !important;
63+ min-width: 150px !important;
64+}
65+
66+.table-container label {
67+ font-size: 0.9rem;
68+}
web-ui/src/main.tsxadded+10−0View file
@@ -0,0 +1,10 @@
1+import { StrictMode } from "react";
2+import { createRoot } from "react-dom/client";
3+import "./index.css";
4+import App from "./App.tsx";
5+
6+createRoot(document.getElementById("root")!).render(
7+ <StrictMode>
8+ <App />
9+ </StrictMode>,
10+);
web-ui/src/pages/BenchmarkView.tsxadded+252−0View file
@@ -0,0 +1,252 @@
1+import { useLocation, useNavigate, useParams } from "react-router-dom";
2+import { useReducer, useEffect } from "react";
3+import { tabsReducer } from "../reducers/tabsReducer";
4+import { AlgorithmContent } from "../components/algorithm/AlgorithmContent";
5+import { DatasetContent } from "../components/dataset/DatasetContent";
6+import {
7+ AlgorithmTable,
8+ DatasetTable,
9+} from "../components/tables/DatasetAlgorithmTables";
10+import { useBenchmarkChartData } from "../hooks/useBenchmarkChartData";
11+import { useTagFilter } from "../hooks/useTagFilter";
12+import { BenchmarkData } from "../types";
13+
14+interface BenchmarkViewProps {
15+ benchmarkData: BenchmarkData | null;
16+}
17+
18+export default function BenchmarkView({ benchmarkData }: BenchmarkViewProps) {
19+ const location = useLocation();
20+ const navigate = useNavigate();
21+ const { datasetName, algorithmName } = useParams<{
22+ datasetName?: string;
23+ algorithmName?: string;
24+ }>();
25+
26+ const [tabsState, dispatch] = useReducer(tabsReducer, {
27+ tabs: [
28+ { id: "datasets", label: "Datasets", route: "/datasets" },
29+ { id: "algorithms", label: "Algorithms", route: "/algorithms" },
30+ ],
31+ activeTabId: "datasets",
32+ });
33+
34+ // Effect to handle URL changes and update tabs
35+ useEffect(() => {
36+ if (datasetName) {
37+ dispatch({
38+ type: "ADD_TAB",
39+ payload: {
40+ id: `dataset-${datasetName}`,
41+ label: datasetName,
42+ route: `/dataset/${datasetName}`,
43+ },
44+ });
45+ } else if (algorithmName) {
46+ dispatch({
47+ type: "ADD_TAB",
48+ payload: {
49+ id: `algorithm-${algorithmName}`,
50+ label: algorithmName,
51+ route: `/algorithm/${algorithmName}`,
52+ },
53+ });
54+ } else if (location.pathname.includes("/algorithms")) {
55+ dispatch({ type: "SET_ACTIVE_TAB", payload: "algorithms" });
56+ } else if (location.pathname.includes("/datasets")) {
57+ dispatch({ type: "SET_ACTIVE_TAB", payload: "datasets" });
58+ }
59+ }, [datasetName, algorithmName, location.pathname]);
60+
61+ // Handle tab click
62+ const handleTabClick = (tabId: string, route: string) => {
63+ dispatch({ type: "SET_ACTIVE_TAB", payload: tabId });
64+ navigate(route);
65+ };
66+
67+ // Get specific dataset or algorithm if viewing one
68+ const dataset = datasetName
69+ ? benchmarkData?.datasets.find((d) => d.name === datasetName)
70+ : undefined;
71+ const algorithm = algorithmName
72+ ? benchmarkData?.algorithms.find((a) => a.name === algorithmName)
73+ : undefined;
74+
75+ // Get chart data for specific dataset or algorithm view
76+ const chartData = useBenchmarkChartData(
77+ benchmarkData?.results || [],
78+ benchmarkData?.algorithms || [],
79+ dataset?.name || null,
80+ algorithm?.name || null,
81+ );
82+
83+ // Get selected tags from URL
84+ const searchParams = new URLSearchParams(location.search);
85+ const selectedTags = searchParams.get("tag")?.split(",") || [];
86+
87+ // Set up tag filtering for datasets and algorithms
88+ const {
89+ availableTags: availableDatasetTags,
90+ filteredItems: filteredDatasets,
91+ } = useTagFilter(
92+ benchmarkData?.datasets || [],
93+ location.pathname.includes("/datasets") ? selectedTags : [],
94+ );
95+
96+ const {
97+ availableTags: availableAlgorithmTags,
98+ filteredItems: filteredAlgorithms,
99+ } = useTagFilter(
100+ benchmarkData?.algorithms || [],
101+ location.pathname.includes("/algorithms") ? selectedTags : [],
102+ );
103+
104+ // Handle tag toggling by updating URL
105+ const handleTagToggle = (tag: string) => {
106+ const newTags = selectedTags.includes(tag)
107+ ? selectedTags.filter((t) => t !== tag)
108+ : [...selectedTags, tag];
109+
110+ const params = new URLSearchParams();
111+ if (newTags.length > 0) {
112+ params.set("tag", newTags.join(","));
113+ }
114+ navigate({ search: params.toString() });
115+ };
116+
117+ return (
118+ <div>
119+ <main>
120+ <div
121+ style={{
122+ position: "fixed",
123+ top: "3rem",
124+ left: 0,
125+ right: 0,
126+ backgroundColor: "white",
127+ zIndex: 999,
128+ padding: "0 2rem 0 2rem",
129+ marginTop: "-4px",
130+ boxShadow: "0 2px 4px rgba(0,0,0,0.1)",
131+ borderBottom: "1px solid #eaeaea",
132+ }}
133+ >
134+ <div
135+ style={{
136+ paddingBottom: "2px",
137+ display: "flex",
138+ gap: "4px",
139+ overflowX: "auto",
140+ width: "100%",
141+ backgroundColor: "white",
142+ }}
143+ >
144+ {tabsState.tabs.map((tab) => (
145+ <div
146+ key={tab.id}
147+ style={{
148+ display: "flex",
149+ alignItems: "center",
150+ gap: "4px",
151+ }}
152+ >
153+ <button
154+ onClick={() => handleTabClick(tab.id, tab.route)}
155+ style={{
156+ padding: "8px 16px",
157+ border: "none",
158+ background: "none",
159+ borderBottom:
160+ tabsState.activeTabId === tab.id
161+ ? "2px solid #0066cc"
162+ : "none",
163+ color:
164+ tabsState.activeTabId === tab.id ? "#0066cc" : "#666",
165+ fontWeight:
166+ tabsState.activeTabId === tab.id ? "600" : "normal",
167+ cursor: "pointer",
168+ textDecoration: "none",
169+ whiteSpace: "nowrap",
170+ }}
171+ >
172+ {tab.label}
173+ </button>
174+ {tab.id !== "datasets" && tab.id !== "algorithms" && (
175+ <button
176+ onClick={(e) => {
177+ e.stopPropagation();
178+ const newActiveTab =
179+ tab.id === tabsState.activeTabId
180+ ? tabsState.tabs[0].id // Default to first tab if closing active
181+ : tabsState.activeTabId;
182+ dispatch({ type: "CLOSE_TAB", payload: tab.id });
183+ // Navigate if closing active tab
184+ if (tab.id === tabsState.activeTabId) {
185+ const defaultTab = tabsState.tabs.find(
186+ (t) => t.id === newActiveTab,
187+ );
188+ if (defaultTab) {
189+ navigate(defaultTab.route);
190+ }
191+ }
192+ }}
193+ style={{
194+ padding: "4px",
195+ border: "none",
196+ background: "none",
197+ color: "#666",
198+ cursor: "pointer",
199+ fontSize: "12px",
200+ display: "flex",
201+ alignItems: "center",
202+ justifyContent: "center",
203+ width: "20px",
204+ height: "20px",
205+ borderRadius: "50%",
206+ marginRight: "4px",
207+ marginLeft: "-4px",
208+ }}
209+ aria-label="Close tab"
210+ >
211+ ×
212+ </button>
213+ )}
214+ </div>
215+ ))}
216+ </div>
217+ </div>
218+
219+ <div style={{ padding: "3rem 0 1rem 0" }}>
220+ {dataset ? (
221+ <DatasetContent
222+ dataset={dataset}
223+ benchmarkData={benchmarkData}
224+ chartData={chartData}
225+ />
226+ ) : algorithm ? (
227+ <AlgorithmContent
228+ algorithm={algorithm}
229+ benchmarkData={benchmarkData}
230+ chartData={chartData}
231+ />
232+ ) : tabsState.activeTabId === "datasets" ? (
233+ <DatasetTable
234+ filteredDatasets={filteredDatasets}
235+ availableDatasetTags={availableDatasetTags}
236+ selectedTags={selectedTags}
237+ toggleTag={handleTagToggle}
238+ benchmarkResults={benchmarkData?.results || []}
239+ />
240+ ) : (
241+ <AlgorithmTable
242+ filteredAlgorithms={filteredAlgorithms}
243+ availableAlgorithmTags={availableAlgorithmTags}
244+ selectedTags={selectedTags}
245+ toggleTag={handleTagToggle}
246+ />
247+ )}
248+ </div>
249+ </main>
250+ </div>
251+ );
252+}
web-ui/src/pages/Home.tsxadded+192−0View file
@@ -0,0 +1,192 @@
1+import React, { useEffect, useState } from "react";
2+import axios from "axios";
3+import { Link } from "react-router-dom";
4+import yaml from "yaml";
5+import contentYaml from "../content/home-content.yml?raw";
6+import { HomeContent, HomeSection } from "../types/home-content";
7+import "../components/Button.css";
8+
9+const content = yaml.parse(contentYaml) as HomeContent;
10+
11+const SectionCard: React.FC<{ section: HomeSection }> = ({ section }) => {
12+ if (section.external) {
13+ return (
14+ <div
15+ style={{
16+ padding: "0.75rem",
17+ border: "1px solid #eaeaea",
18+ borderRadius: "8px",
19+ backgroundColor: "#f9f9f9",
20+ display: "flex",
21+ flexDirection: "column",
22+ height: "100%",
23+ }}
24+ >
25+ <h2 style={{ fontSize: "1.25rem", marginBottom: "0.5rem" }}>
26+ {section.title}
27+ </h2>
28+ <p style={{ marginBottom: "0.5rem" }}>{section.description}</p>
29+ <a
30+ href={section.link}
31+ target="_blank"
32+ rel="noopener noreferrer"
33+ className="soft-button"
34+ style={{ marginTop: "auto", alignSelf: "flex-start" }}
35+ >
36+ {section.linkText}
37+ </a>
38+ </div>
39+ );
40+ }
41+
42+ return (
43+ <div
44+ style={{
45+ padding: "0.75rem",
46+ border: "1px solid #eaeaea",
47+ borderRadius: "8px",
48+ backgroundColor: "#f9f9f9",
49+ display: "flex",
50+ flexDirection: "column",
51+ height: "100%",
52+ }}
53+ >
54+ <h2 style={{ fontSize: "1.25rem", marginBottom: "0.5rem" }}>
55+ {section.title}
56+ </h2>
57+ <p style={{ marginBottom: "0.5rem" }}>{section.description}</p>
58+ <Link
59+ to={section.link}
60+ className="soft-button"
61+ style={{ marginTop: "auto", alignSelf: "flex-start" }}
62+ >
63+ {section.linkText}
64+ </Link>
65+ </div>
66+ );
67+};
68+
69+interface BenchmarkStatus {
70+ current_dataset: string;
71+ current_algorithm: string;
72+ completed_count: number;
73+ total_count: number;
74+ progress_percentage: number;
75+ elapsed_time: number;
76+ last_update: string;
77+ completed_benchmarks: Array<{
78+ dataset: string;
79+ algorithm: string;
80+ compression_ratio: number;
81+ encode_time: number;
82+ decode_time: number;
83+ cache_status: string;
84+ }>;
85+}
86+
87+export default function Home() {
88+ const [status, setStatus] = useState<BenchmarkStatus | null>(null);
89+
90+ useEffect(() => {
91+ const fetchStatus = async () => {
92+ try {
93+ const cacheBust = Math.random().toString(36).substring(2, 15);
94+ const response = await axios.get(
95+ `https://tempory.net/f/memobin/ephys_compression_tests/benchmark_status/current.json?cachebust=${cacheBust}`,
96+ );
97+ setStatus(response.data);
98+ } catch (error) {
99+ console.error("Error fetching benchmark status:", error);
100+ }
101+ };
102+
103+ fetchStatus();
104+ }, []);
105+
106+ return (
107+ <div style={{ maxWidth: "1200px", margin: "0 auto", padding: "2rem" }}>
108+ <h1
109+ style={{
110+ marginBottom: "2rem",
111+ display: "flex",
112+ alignItems: "center",
113+ gap: "1rem",
114+ }}
115+ >
116+ <img
117+ src={`${import.meta.env.BASE_URL}logo.svg`}
118+ alt="Ephys Compression Tests Logo"
119+ style={{ width: "40px", height: "auto" }}
120+ />
121+ {content.title}
122+ </h1>
123+
124+ <p
125+ style={{ fontSize: "1.1rem", lineHeight: "1.6", marginBottom: "2rem" }}
126+ >
127+ {content.description}
128+ </p>
129+
130+ <div
131+ style={{
132+ display: "grid",
133+ gap: "2rem",
134+ gridTemplateColumns: "repeat(auto-fit, minmax(300px, 1fr))",
135+ }}
136+ >
137+ {Object.entries(content.sections).map(([key, section]) => (
138+ <SectionCard key={key} section={section} />
139+ ))}
140+ </div>
141+
142+ <hr
143+ style={{
144+ margin: "3rem 0",
145+ border: "none",
146+ borderTop: "1px solid #eaeaea",
147+ }}
148+ />
149+
150+ <footer
151+ style={{ textAlign: "center", color: "#666", fontSize: "0.9rem" }}
152+ >
153+ <p>
154+ Last UI update: {__BUILD_DATE__}
155+ <br />
156+ {status && (
157+ <>
158+ <div
159+ style={{
160+ margin: "1rem 0",
161+ padding: "0.5rem",
162+ border: "1px solid #eaeaea",
163+ borderRadius: "4px",
164+ display: "inline-block",
165+ }}
166+ >
167+ Last benchmark run:{" "}
168+ {new Date(status.last_update).toLocaleString()}
169+ <br />
170+ Status:{" "}
171+ {status.progress_percentage === 100
172+ ? "Completed"
173+ : "In Progress"}{" "}
174+ ({status.completed_count}/{status.total_count} benchmarks)
175+ </div>
176+ <br />
177+ </>
178+ )}
179+ Released under{" "}
180+ <a
181+ href="https://github.com/concept-collection/ephys_compression_tests/blob/main/LICENSE"
182+ target="_blank"
183+ rel="noopener noreferrer"
184+ style={{ color: "#666", textDecoration: "underline" }}
185+ >
186+ Apache License 2.0
187+ </a>
188+ </p>
189+ </footer>
190+ </div>
191+ );
192+}
web-ui/src/pages/Monitor.tsxadded+287−0View file
@@ -0,0 +1,287 @@
1+import { useEffect, useState } from "react";
2+import axios from "axios";
3+
4+interface BenchmarkStatus {
5+ current_dataset: string;
6+ current_algorithm: string;
7+ completed_count: number;
8+ total_count: number;
9+ progress_percentage: number;
10+ elapsed_time: number;
11+ last_update: string;
12+ completed_benchmarks: Array<{
13+ dataset: string;
14+ algorithm: string;
15+ compression_ratio: number;
16+ encode_time: number;
17+ decode_time: number;
18+ cache_status: string;
19+ }>;
20+}
21+
22+export default function Monitor() {
23+ const [status, setStatus] = useState<BenchmarkStatus | null>(null);
24+ const [error, setError] = useState<string | null>(null);
25+ const [loading, setLoading] = useState(true);
26+
27+ const fetchStatus = async () => {
28+ setLoading(true);
29+ try {
30+ const cacheBust = Math.random().toString(36).substring(2, 15);
31+ const response = await axios.get(
32+ `https://tempory.net/f/memobin/ephys_compression_tests/benchmark_status/current.json?cachebust=${cacheBust}`,
33+ );
34+ setStatus(response.data);
35+ setError(null);
36+ } catch (error) {
37+ const message =
38+ error instanceof Error ? error.message : "Failed to fetch status";
39+ setError(message);
40+ console.error("Error fetching benchmark status:", error);
41+ } finally {
42+ setLoading(false);
43+ }
44+ };
45+
46+ useEffect(() => {
47+ fetchStatus();
48+ }, []);
49+
50+ if (loading) {
51+ return <div>Loading benchmark status...</div>;
52+ }
53+
54+ if (error) {
55+ return <div>Error: {error}</div>;
56+ }
57+
58+ if (!status) {
59+ return <div>No active benchmark run found.</div>;
60+ }
61+
62+ const formatTime = (seconds: number) => {
63+ const hours = Math.floor(seconds / 3600);
64+ const minutes = Math.floor((seconds % 3600) / 60);
65+ const remainingSeconds = Math.floor(seconds % 60);
66+ return `${hours}h ${minutes}m ${remainingSeconds}s`;
67+ };
68+
69+ return (
70+ <div style={{ padding: "20px" }}>
71+ <div
72+ style={{
73+ display: "flex",
74+ alignItems: "center",
75+ gap: "20px",
76+ marginBottom: "20px",
77+ }}
78+ >
79+ <h1 style={{ margin: 0 }}>Benchmark Progress</h1>
80+ <button
81+ onClick={fetchStatus}
82+ style={{
83+ padding: "8px 16px",
84+ backgroundColor: "#4CAF50",
85+ color: "white",
86+ border: "none",
87+ borderRadius: "4px",
88+ cursor: "pointer",
89+ display: "flex",
90+ alignItems: "center",
91+ gap: "8px",
92+ }}
93+ disabled={loading}
94+ >
95+ {loading ? "Refreshing..." : "Refresh"}
96+ </button>
97+ </div>
98+
99+ <div style={{ marginBottom: "20px" }}>
100+ <h2>Current Status</h2>
101+ <div
102+ style={{
103+ border: "1px solid #eee",
104+ padding: "20px",
105+ borderRadius: "8px",
106+ backgroundColor: "#f9f9f9",
107+ }}
108+ >
109+ <p>
110+ <strong>Current Dataset:</strong> {status.current_dataset}
111+ </p>
112+ <p>
113+ <strong>Current Algorithm:</strong> {status.current_algorithm}
114+ </p>
115+ <p>
116+ <strong>Progress:</strong> {status.completed_count} /{" "}
117+ {status.total_count} ({status.progress_percentage.toFixed(1)}%)
118+ </p>
119+ <p>
120+ <strong>Elapsed Time:</strong> {formatTime(status.elapsed_time)}
121+ </p>
122+ <p>
123+ <strong>Last Update:</strong>{" "}
124+ {new Date(status.last_update).toLocaleString()}
125+ </p>
126+
127+ <div style={{ marginTop: "10px" }}>
128+ <div
129+ style={{
130+ width: "100%",
131+ height: "20px",
132+ backgroundColor: "#eee",
133+ borderRadius: "10px",
134+ overflow: "hidden",
135+ }}
136+ >
137+ <div
138+ style={{
139+ width: `${status.progress_percentage}%`,
140+ height: "100%",
141+ backgroundColor: "#4CAF50",
142+ transition: "width 0.5s ease-in-out",
143+ }}
144+ />
145+ </div>
146+ </div>
147+ </div>
148+ </div>
149+
150+ <div>
151+ <h2>Completed Benchmarks</h2>
152+ <div style={{ overflowX: "auto" }}>
153+ <table
154+ style={{
155+ width: "100%",
156+ borderCollapse: "collapse",
157+ marginTop: "10px",
158+ }}
159+ >
160+ <thead>
161+ <tr style={{ backgroundColor: "#f5f5f5" }}>
162+ <th
163+ style={{
164+ padding: "12px",
165+ textAlign: "left",
166+ borderBottom: "2px solid #ddd",
167+ }}
168+ >
169+ Dataset
170+ </th>
171+ <th
172+ style={{
173+ padding: "12px",
174+ textAlign: "left",
175+ borderBottom: "2px solid #ddd",
176+ }}
177+ >
178+ Algorithm
179+ </th>
180+ <th
181+ style={{
182+ padding: "12px",
183+ textAlign: "right",
184+ borderBottom: "2px solid #ddd",
185+ }}
186+ >
187+ Compression Ratio
188+ </th>
189+ <th
190+ style={{
191+ padding: "12px",
192+ textAlign: "right",
193+ borderBottom: "2px solid #ddd",
194+ }}
195+ >
196+ Encode Time (ms)
197+ </th>
198+ <th
199+ style={{
200+ padding: "12px",
201+ textAlign: "right",
202+ borderBottom: "2px solid #ddd",
203+ }}
204+ >
205+ Decode Time (ms)
206+ </th>
207+ <th
208+ style={{
209+ padding: "12px",
210+ textAlign: "center",
211+ borderBottom: "2px solid #ddd",
212+ }}
213+ >
214+ Cache Status
215+ </th>
216+ </tr>
217+ </thead>
218+ <tbody>
219+ {status.completed_benchmarks.map((benchmark, index) => (
220+ <tr
221+ key={index}
222+ style={{
223+ backgroundColor:
224+ benchmark.cache_status === "cached"
225+ ? "#f5f5f5"
226+ : index % 2 === 0
227+ ? "white"
228+ : "#fafafa",
229+ }}
230+ >
231+ <td
232+ style={{ padding: "12px", borderBottom: "1px solid #ddd" }}
233+ >
234+ {benchmark.dataset}
235+ </td>
236+ <td
237+ style={{ padding: "12px", borderBottom: "1px solid #ddd" }}
238+ >
239+ {benchmark.algorithm}
240+ </td>
241+ <td
242+ style={{
243+ padding: "12px",
244+ textAlign: "right",
245+ borderBottom: "1px solid #ddd",
246+ }}
247+ >
248+ {benchmark.compression_ratio.toFixed(2)}x
249+ </td>
250+ <td
251+ style={{
252+ padding: "12px",
253+ textAlign: "right",
254+ borderBottom: "1px solid #ddd",
255+ }}
256+ >
257+ {(benchmark.encode_time * 1000).toFixed(2)}
258+ </td>
259+ <td
260+ style={{
261+ padding: "12px",
262+ textAlign: "right",
263+ borderBottom: "1px solid #ddd",
264+ }}
265+ >
266+ {(benchmark.decode_time * 1000).toFixed(2)}
267+ </td>
268+ <td
269+ style={{
270+ padding: "12px",
271+ textAlign: "center",
272+ borderBottom: "1px solid #ddd",
273+ color:
274+ benchmark.cache_status === "cached" ? "#888" : "#000",
275+ }}
276+ >
277+ {benchmark.cache_status}
278+ </td>
279+ </tr>
280+ ))}
281+ </tbody>
282+ </table>
283+ </div>
284+ </div>
285+ </div>
286+ );
287+}
web-ui/src/pages/Submit.tsxadded+14−0View file
@@ -0,0 +1,14 @@
1+import submitContent from "./submit.md?raw";
2+import ReactMarkdown from "react-markdown";
3+import remarkMath from "remark-math";
4+import rehypeKatex from "rehype-katex";
5+
6+export default function Submit() {
7+ return (
8+ <div className="content-container">
9+ <ReactMarkdown remarkPlugins={[remarkMath]} rehypePlugins={[rehypeKatex]}>
10+ {submitContent}
11+ </ReactMarkdown>
12+ </div>
13+ );
14+}
web-ui/src/pages/submit.mdadded+143−0View file
@@ -0,0 +1,143 @@
1+# Contributing to ephys_compression_tests
2+
3+This guide explains how to contribute new algorithms or datasets to ephys_compression_tests.
4+
5+## Overview
6+
7+ephys_compression_tests welcomes contributions of new compression algorithms and ephys datasets. The framework is designed to make it easy to add new components while ensuring consistent benchmarking and evaluation.
8+
9+## Getting Started
10+
11+1. Fork and clone the repository:
12+```bash
13+git clone https://github.com/[your-username]/ephys_compression_tests.git
14+cd ephys_compression_tests
15+```
16+
17+2. Install dependencies:
18+```bash
19+# Install Python package
20+cd ephys_compression_tests
21+pip install -e .
22+
23+# Install pre-commit hooks for code compliance checks
24+pip install pre-commit
25+pre-commit install
26+```
27+
28+## Adding a New Algorithm
29+
30+New algorithms are added in `ephys_compression_tests/src/ephys_compression_tests/algorithms/`. Each algorithm should:
31+
32+1. Create a new directory with:
33+ - `__init__.py`: Algorithm implementation
34+ - `algorithm-name.md`: Documentation and description
35+
36+2. In `__init__.py`:
37+ - Implement compression/decompression functions
38+ - Define metadata (version, tags, compatibility)
39+ - Follow existing algorithms as examples
40+
41+Example structure:
42+```
43+algorithms/
44+└── my_algorithm/
45+ ├── __init__.py
46+ └── my_algorithm.md
47+```
48+
49+## Adding a New Dataset
50+
51+New datasets are added in `ephys_compression_tests/src/ephys_compression_tests/datasets/`. Each dataset should:
52+
53+1. Create a new directory with:
54+ - `__init__.py`: Dataset generation/loading code
55+ - `dataset-name.md`: Documentation and description
56+
57+2. In `__init__.py`:
58+ - Implement data generation/loading
59+ - Define metadata (version, tags)
60+ - Follow existing datasets as examples
61+
62+Example structure:
63+```
64+datasets/
65+└── my_dataset/
66+ ├── __init__.py
67+ └── my_dataset.md
68+```
69+
70+## Testing Locally
71+
72+Run benchmarks for your new component:
73+```bash
74+ephys_compression_tests run --algorithm my_algorithm --dataset my_dataset
75+```
76+
77+The framework will automatically:
78+- Run the benchmarks
79+- Verify results by decompressing and comparing with original data
80+- Measure compression ratios and throughput
81+
82+## Code Formatting
83+
84+The project uses specific formatters for each language:
85+- Python: black formatter
86+- TypeScript/JavaScript: ESLint + Prettier
87+- C++: clang-format
88+
89+To format your code before committing:
90+```bash
91+# From project root
92+./devel/format_code.sh
93+```
94+
95+This will format all code according to project standards.
96+
97+## Code Compliance
98+
99+Pre-commit hooks will check code compliance when you commit changes. They verify:
100+- Code formatting
101+- Import ordering
102+- Type checking
103+- Other project-specific rules
104+
105+If checks fail, format your code using the format script and try again.
106+
107+## Creating a Pull Request
108+
109+1. Create a new branch:
110+```bash
111+git checkout -b add-my-component
112+```
113+
114+2. Format code and ensure it passes compliance checks:
115+```bash
116+./devel/format_code.sh
117+```
118+
119+3. Commit your changes:
120+```bash
121+git add .
122+git commit -m "Add new algorithm/dataset: [name]"
123+```
124+
125+4. Push to your fork:
126+```bash
127+git push origin add-my-component
128+```
129+
130+5. Open a pull request on GitHub with:
131+ - Clear description of the new component
132+ - Any relevant background or references
133+ - Local benchmark results
134+ - Confirmation that code is formatted and passes checks
135+
136+## Guidelines
137+
138+- Follow existing code structure and patterns
139+- Include thorough documentation
140+- Add appropriate tags for filtering
141+- Test compatibility with existing components
142+- Format code using provided script
143+- Ensure all pre-commit checks pass
web-ui/src/reducers/tabsReducer.tsadded+85−0View file
@@ -0,0 +1,85 @@
1+export type TabAction =
2+ | { type: "ADD_TAB"; payload: { id: string; label: string; route: string } }
3+ | { type: "SET_ACTIVE_TAB"; payload: string }
4+ | { type: "CLOSE_TAB"; payload: string }
5+ | { type: "REORDER_TABS"; payload: { fromIndex: number; toIndex: number } };
6+
7+export interface TabItem {
8+ id: string;
9+ label: string;
10+ route: string;
11+}
12+
13+export interface TabsState {
14+ tabs: TabItem[];
15+ activeTabId: string;
16+}
17+
18+const initialState: TabsState = {
19+ tabs: [
20+ { id: "datasets", label: "Datasets", route: "/datasets" },
21+ { id: "algorithms", label: "Algorithms", route: "/algorithms" },
22+ ],
23+ activeTabId: "datasets",
24+};
25+
26+export function tabsReducer(
27+ state: TabsState = initialState,
28+ action: TabAction,
29+): TabsState {
30+ switch (action.type) {
31+ case "ADD_TAB": {
32+ // If tab already exists, just set it as active
33+ if (state.tabs.some((tab) => tab.id === action.payload.id)) {
34+ return {
35+ ...state,
36+ activeTabId: action.payload.id,
37+ };
38+ }
39+
40+ // Otherwise add new tab
41+ return {
42+ ...state,
43+ tabs: [...state.tabs, action.payload],
44+ activeTabId: action.payload.id,
45+ };
46+ }
47+ case "SET_ACTIVE_TAB":
48+ return {
49+ ...state,
50+ activeTabId: action.payload,
51+ };
52+ case "REORDER_TABS": {
53+ const newTabs = [...state.tabs];
54+ const [movedTab] = newTabs.splice(action.payload.fromIndex, 1);
55+ newTabs.splice(action.payload.toIndex, 0, movedTab);
56+ return {
57+ ...state,
58+ tabs: newTabs,
59+ };
60+ }
61+ case "CLOSE_TAB": {
62+ // Don't allow closing of datasets or algorithms tabs
63+ if (action.payload === "datasets" || action.payload === "algorithms") {
64+ return state;
65+ }
66+
67+ const newTabs = state.tabs.filter((tab) => tab.id !== action.payload);
68+
69+ // If we're closing the active tab, activate the last tab in the list
70+ if (state.activeTabId === action.payload) {
71+ return {
72+ tabs: newTabs,
73+ activeTabId: newTabs[newTabs.length - 1].id,
74+ };
75+ }
76+
77+ return {
78+ ...state,
79+ tabs: newTabs,
80+ };
81+ }
82+ default:
83+ return state;
84+ }
85+}
web-ui/src/types.tsadded+63−0View file
@@ -0,0 +1,63 @@
1+export interface BenchmarkResult {
2+ dataset: string;
3+ algorithm: string;
4+ algorithm_version: string;
5+ dataset_version: string;
6+ system_version: string;
7+ compression_ratio: number;
8+ encode_time: number;
9+ decode_time: number;
10+ encode_mb_per_sec: number;
11+ decode_mb_per_sec: number;
12+ original_size: number;
13+ compressed_size: number;
14+ array_shape: number[];
15+ array_dtype: string;
16+ timestamp: number;
17+ rmse?: number;
18+ max_error?: number;
19+ reconstructed_url_raw?: string;
20+}
21+
22+export interface Algorithm {
23+ name: string;
24+ description: string;
25+ long_description?: string;
26+ version: string;
27+ tags: string[];
28+ source_file?: string;
29+}
30+
31+export interface Dataset {
32+ name: string;
33+ description: string;
34+ long_description?: string;
35+ version: string;
36+ tags: string[];
37+ source_file?: string;
38+ data_url_npy?: string; // URL to download the dataset as .npy
39+ data_url_raw?: string; // URL to download the raw dataset as .dat
40+ data_url_json?: string; // URL to download the dataset info as .json (dtype and shape)
41+}
42+
43+export interface BenchmarkData {
44+ results: BenchmarkResult[];
45+ algorithms: Algorithm[];
46+ datasets: Dataset[];
47+}
48+
49+export interface TabItem {
50+ id: string;
51+ label: string;
52+ route: string;
53+}
54+
55+export interface TabsState {
56+ tabs: TabItem[];
57+ activeTabId: string;
58+}
59+
60+export type TabAction =
61+ | { type: "ADD_TAB"; payload: { id: string; label: string; route: string } }
62+ | { type: "SET_ACTIVE_TAB"; payload: string }
63+ | { type: "REORDER_TABS"; payload: { fromIndex: number; toIndex: number } };
web-ui/src/types/comparison.tsadded+10−0View file
@@ -0,0 +1,10 @@
1+export type ComparisonMode = "original" | "overlay" | "residuals" | "side-by-side";
2+
3+export interface ReconstructedDataInfo {
4+ algorithm: string;
5+ rmse: number;
6+ max_error: number;
7+ reconstructedUrl: string;
8+ datasetUrl: string;
9+ datasetJsonUrl: string;
10+}
web-ui/src/types/home-content.tsadded+15−0View file
@@ -0,0 +1,15 @@
1+export interface HomeSection {
2+ title: string;
3+ description: string;
4+ link: string;
5+ linkText: string;
6+ external?: boolean;
7+}
8+
9+export interface HomeContent {
10+ title: string;
11+ description: string;
12+ sections: {
13+ [key: string]: HomeSection;
14+ };
15+}
web-ui/src/vite-env.d.tsadded+1−0View file
@@ -0,0 +1 @@
1+/// <reference types="vite/client" />
web-ui/tsconfig.app.jsonadded+24−0View file
@@ -0,0 +1,24 @@
1+{
2+ "compilerOptions": {
3+ "target": "ES2020",
4+ "useDefineForClassFields": true,
5+ "lib": ["ES2020", "DOM", "DOM.Iterable"],
6+ "module": "ESNext",
7+ "skipLibCheck": true,
8+
9+ /* Bundler mode */
10+ "moduleResolution": "bundler",
11+ "allowImportingTsExtensions": true,
12+ "isolatedModules": true,
13+ "moduleDetection": "force",
14+ "noEmit": true,
15+ "jsx": "react-jsx",
16+
17+ /* Linting */
18+ "strict": true,
19+ "noUnusedLocals": true,
20+ "noUnusedParameters": true,
21+ "noFallthroughCasesInSwitch": true
22+ },
23+ "include": ["src"]
24+}
web-ui/tsconfig.app.tsbuildinfoadded+1−0View file
@@ -0,0 +1 @@
1+{"root":["./src/App.tsx","./src/env.d.ts","./src/main.tsx","./src/types.ts","./src/vite-env.d.ts","./src/components/BenchmarkTable.tsx","./src/components/ScrollToTop.tsx","./src/components/TagFilter.tsx","./src/components/algorithm/AlgorithmContent.tsx","./src/components/benchmark/charts/BenchmarkCharts.tsx","./src/components/benchmark/charts/BenchmarkScatterPlots.tsx","./src/components/benchmark/export/csvExport.ts","./src/components/benchmark/table/BenchmarkTable.tsx","./src/components/benchmark/table/columns.tsx","./src/components/benchmark/utils/formatters.ts","./src/components/dataset/DatasetContent.tsx","./src/components/dataset/TimeseriesNavigationBar.tsx","./src/components/dataset/TimeseriesView.tsx","./src/components/dataset/TimeseriesViewWorker.ts","./src/components/dataset/WorkerTypes.ts","./src/components/dataset/timeseriesViewReducer.ts","./src/components/shared/BaseContent.tsx","./src/components/tables/DatasetAlgorithmTables.tsx","./src/hooks/TimeseriesDataClient.ts","./src/hooks/useBenchmarkChartData.ts","./src/hooks/useMarkdownContent.ts","./src/hooks/useMarkdownPosts.ts","./src/hooks/useTagFilter.ts","./src/hooks/useTimeseriesData.ts","./src/hooks/useTimeseriesDataClient.ts","./src/pages/BenchmarkView.tsx","./src/pages/Home.tsx","./src/pages/Monitor.tsx","./src/pages/Submit.tsx","./src/reducers/tabsReducer.ts","./src/types/home-content.ts"],"version":"5.6.3"}
web-ui/tsconfig.jsonadded+7−0View file
@@ -0,0 +1,7 @@
1+{
2+ "files": [],
3+ "references": [
4+ { "path": "./tsconfig.app.json" },
5+ { "path": "./tsconfig.node.json" }
6+ ]
7+}
web-ui/tsconfig.node.jsonadded+24−0View file
@@ -0,0 +1,24 @@
1+{
2+ "compilerOptions": {
3+ "incremental": true,
4+ "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.node.tsbuildinfo",
5+ "target": "ES2022",
6+ "lib": ["ES2023"],
7+ "module": "ESNext",
8+ "skipLibCheck": true,
9+
10+ /* Bundler mode */
11+ "moduleResolution": "bundler",
12+ "allowImportingTsExtensions": true,
13+ "isolatedModules": true,
14+ "moduleDetection": "force",
15+ "noEmit": true,
16+
17+ /* Linting */
18+ "strict": true,
19+ "noUnusedLocals": true,
20+ "noUnusedParameters": true,
21+ "noFallthroughCasesInSwitch": true
22+ },
23+ "include": ["vite.config.ts"]
24+}
web-ui/vite.config.tsadded+11−0View file
@@ -0,0 +1,11 @@
1+import { defineConfig } from 'vite'
2+import react from '@vitejs/plugin-react'
3+
4+// https://vite.dev/config/
5+export default defineConfig({
6+ plugins: [react()],
7+ base: '/ephys_compression_tests/', // Base URL for GitHub Pages deployment
8+ define: {
9+ __BUILD_DATE__: JSON.stringify(new Date().toLocaleDateString())
10+ }
11+})