long descriptions of datasets
6 changed files+136−0
benchcompress/src/benchcompress/datasets/electrophysiology/__init__.pymodified+20−0View file
@@ -1,5 +1,6 @@
11 import numpy as np
22 import lindi
3+import os
34 from typing import cast
45 from ..._filters import bandpass_filter
56 from ..._analysis import estimate_noise_level
@@ -7,6 +8,16 @@ from ..._analysis import estimate_noise_level
78
89 SOURCE_FILE = "electrophysiology/__init__.py"
910
11+
12+def _load_long_description():
13+ current_dir = os.path.dirname(os.path.abspath(__file__))
14+ md_path = os.path.join(current_dir, "electrophysiology.md")
15+ with open(md_path, "r", encoding="utf-8") as f:
16+ return f.read()
17+
18+
19+LONG_DESCRIPTION = _load_long_description()
20+
1021 tags = ["real", "electrophysiology", "timeseries", "1d", "integer", "continuous"]
1122
1223
@@ -178,6 +189,7 @@ datasets = [
178189 ).flatten(),
179190 "tags": tags,
180191 "source_file": SOURCE_FILE,
192+ "long_description": LONG_DESCRIPTION,
181193 },
182194 {
183195 "name": "ephys-000409-ch101",
@@ -188,6 +200,7 @@ datasets = [
188200 ).flatten(),
189201 "tags": tags,
190202 "source_file": SOURCE_FILE,
203+ "long_description": LONG_DESCRIPTION,
191204 },
192205 {
193206 "name": "ephys-001290-ch0",
@@ -198,6 +211,7 @@ datasets = [
198211 ).flatten(),
199212 "tags": tags,
200213 "source_file": SOURCE_FILE,
214+ "long_description": LONG_DESCRIPTION,
201215 },
202216 {
203217 "name": "ephys-000876-ch45-filtered",
@@ -210,6 +224,7 @@ datasets = [
210224 ),
211225 "tags": tags + ["filtered"],
212226 "source_file": SOURCE_FILE,
227+ "long_description": LONG_DESCRIPTION,
213228 },
214229 {
215230 "name": "ephys-000409-ch101-filtered",
@@ -222,6 +237,7 @@ datasets = [
222237 ),
223238 "tags": tags + ["filtered"],
224239 "source_file": SOURCE_FILE,
240+ "long_description": LONG_DESCRIPTION,
225241 },
226242 {
227243 "name": "ephys-001290-ch0-filtered",
@@ -234,6 +250,7 @@ datasets = [
234250 ),
235251 "tags": tags + ["filtered"],
236252 "source_file": SOURCE_FILE,
253+ "long_description": LONG_DESCRIPTION,
237254 },
238255 {
239256 "name": "ephys-000876-ch45-sparse",
@@ -246,6 +263,7 @@ datasets = [
246263 ),
247264 "tags": tags + ["filtered", "sparse"],
248265 "source_file": SOURCE_FILE,
266+ "long_description": LONG_DESCRIPTION,
249267 },
250268 {
251269 "name": "ephys-000409-ch101-sparse",
@@ -258,6 +276,7 @@ datasets = [
258276 ),
259277 "tags": tags + ["filtered", "sparse"],
260278 "source_file": SOURCE_FILE,
279+ "long_description": LONG_DESCRIPTION,
261280 },
262281 {
263282 "name": "ephys-001290-ch0-sparse",
@@ -270,5 +289,6 @@ datasets = [
270289 ),
271290 "tags": tags + ["filtered", "sparse"],
272291 "source_file": SOURCE_FILE,
292+ "long_description": LONG_DESCRIPTION,
273293 },
274294 ]
benchcompress/src/benchcompress/datasets/electrophysiology/electrophysiology.mdadded+43−0View file
@@ -0,0 +1,43 @@
1+# Electrophysiology Dataset
2+
3+Electrophysiology datasets contain recordings of electrical activity from neurons in the brain. These recordings are made using microelectrodes that detect the small voltage changes produced by neurons when they fire action potentials (spikes).
4+
5+## Data Sources
6+
7+The datasets are samples from single channels within larger electrophysiology recordings available on the DANDI Archive:
8+
9+- Channel 45 from a session in Dandiset 000876 (https://dandiarchive.org/dandiset/000876)
10+- Channel 101 from a session in Dandiset 000409 (https://dandiarchive.org/dandiset/000409)
11+- Channel 0 from a session in Dandiset 001290 (https://dandiarchive.org/dandiset/001290)
12+
13+## Dataset Variants
14+
15+We provide three types of variants for each sampled channel:
16+
17+### 1. Raw Data
18+- Direct recordings from the electrodes
19+- Contains both neural signals and background noise
20+- Integer-valued samples representing voltage
21+- Sampling rate: 30 kHz
22+
23+### 2. Filtered Data
24+- Bandpass filtered between 300-6000 Hz to isolate spike activity
25+- Normalized by estimated noise level
26+- Quantized with step size 0.25
27+- Stored as 16-bit integers
28+
29+### 3. Sparse Data
30+- Applies activity-based suppression to focus on regions with neural firing
31+- Uses adaptive thresholding to detect active regions
32+- Suppresses low-activity regions while preserving spike waveforms
33+- Also quantized and stored as 16-bit integers
34+
35+## Compression Considerations
36+
37+These datasets present different challenges for compression algorithms:
38+
39+1. Raw data contains broadband noise and slow drifts
40+2. Filtered data emphasizes spike waveforms but maintains continuous values
41+3. Sparse data has many near-zero regions interspersed with spike events
42+
43+The sparse variants are particularly interesting as they represent a common preprocessing step in neuroscience analysis, where only time periods containing neural activity are retained.
benchcompress/src/benchcompress/datasets/gaussian/__init__.pymodified+17−0View file
@@ -1,9 +1,20 @@
11 import numpy as np
2+import os
23
34
45 SOURCE_FILE = "gaussian/__init__.py"
56
67
8+def _load_long_description():
9+ current_dir = os.path.dirname(os.path.abspath(__file__))
10+ md_path = os.path.join(current_dir, "gaussian.md")
11+ with open(md_path, "r", encoding="utf-8") as f:
12+ return f.read()
13+
14+
15+LONG_DESCRIPTION = _load_long_description()
16+
17+
718 def create_gaussian_quantized(
819 *, n_samples: int, stddev: float, seed: int
920 ) -> np.ndarray:
@@ -39,6 +50,7 @@ datasets = [
3950 "description": "Rounded Gaussian integers with σ=1.",
4051 "tags": tags_quantized,
4152 "source_file": SOURCE_FILE,
53+ "long_description": LONG_DESCRIPTION,
4254 },
4355 {
4456 "name": "gaussian-q2",
@@ -49,6 +61,7 @@ datasets = [
4961 "description": "Rounded Gaussian integers with σ=2.",
5062 "tags": tags_quantized,
5163 "source_file": SOURCE_FILE,
64+ "long_description": LONG_DESCRIPTION,
5265 },
5366 {
5467 "name": "gaussian-q3",
@@ -59,6 +72,7 @@ datasets = [
5972 "description": "Rounded Gaussian integers with σ=3.",
6073 "tags": tags_quantized,
6174 "source_file": SOURCE_FILE,
75+ "long_description": LONG_DESCRIPTION,
6276 },
6377 {
6478 "name": "gaussian-q5",
@@ -69,6 +83,7 @@ datasets = [
6983 "description": "Rounded Gaussian integers with σ=5.",
7084 "tags": tags_quantized,
7185 "source_file": SOURCE_FILE,
86+ "long_description": LONG_DESCRIPTION,
7287 },
7388 {
7489 "name": "gaussian-q8",
@@ -79,6 +94,7 @@ datasets = [
7994 "description": "Rounded Gaussian integers with σ=8.",
8095 "tags": tags_quantized,
8196 "source_file": SOURCE_FILE,
97+ "long_description": LONG_DESCRIPTION,
8298 },
8399 {
84100 "name": "gaussian-flt1",
@@ -87,5 +103,6 @@ datasets = [
87103 "description": "Floating point Gaussian numbers with σ=1.",
88104 "tags": tags_float,
89105 "source_file": SOURCE_FILE,
106+ "long_description": LONG_DESCRIPTION,
90107 },
91108 ]
benchcompress/src/benchcompress/datasets/gaussian/gaussian.mdadded+22−0View file
@@ -0,0 +1,22 @@
1+# Gaussian Dataset
2+
3+These datasets contain sequences of numbers drawn from Gaussian distributions with μ=0 and various values of σ. All values are independently and identically distributed (i.i.d.).
4+
5+## Variants
6+
7+We provide two types of Gaussian datasets:
8+
9+### Quantized Integer Variants
10+Numbers are drawn from a Gaussian distribution then rounded to the nearest integer:
11+- gaussian-q1 (σ=1)
12+- gaussian-q2 (σ=2)
13+- gaussian-q3 (σ=3)
14+- gaussian-q5 (σ=5)
15+- gaussian-q8 (σ=8)
16+
17+The quantization creates discrete integer values, with larger σ values producing a wider spread of integers.
18+
19+### Floating Point Variant
20+- gaussian-flt1 (σ=1)
21+
22+This variant preserves the continuous nature of the Gaussian distribution using 32-bit floating point numbers.
benchcompress/src/benchcompress/datasets/seismic/__init__.pymodified+13−0View file
@@ -3,8 +3,19 @@ import segyio
33 import os
44 import requests
55
6+
67 SOURCE_FILE = "seismic/__init__.py"
78
9+
10+def _load_long_description():
11+ current_dir = os.path.dirname(os.path.abspath(__file__))
12+ md_path = os.path.join(current_dir, "seismic.md")
13+ with open(md_path, "r", encoding="utf-8") as f:
14+ return f.read()
15+
16+
17+LONG_DESCRIPTION = _load_long_description()
18+
819 tags = ["real", "seismic", "continuous", "timeseries", "1d"]
920 tags_float = tags + ["float"]
1021 tags_integer = tags + ["integer"]
@@ -62,6 +73,7 @@ datasets = [
6273 "create": lambda: _load_seismic_data(),
6374 "tags": tags_float,
6475 "source_file": SOURCE_FILE,
76+ "long_description": LONG_DESCRIPTION,
6577 },
6678 {
6779 "name": "seismic-04A-04B-quantized",
@@ -70,5 +82,6 @@ datasets = [
7082 "create": lambda: _load_quantized_seismic_data(),
7183 "tags": tags_integer,
7284 "source_file": SOURCE_FILE,
85+ "long_description": LONG_DESCRIPTION,
7386 },
7487 ]
benchcompress/src/benchcompress/datasets/seismic/seismic.mdadded+21−0View file
@@ -0,0 +1,21 @@
1+# Seismic Dataset
2+
3+This dataset contains marine seismic reflection data. Processed seismic reflection data and associated files from Roger Revelle voyage RR1508: 16 May - 18 June 2015. The research voyage aimed to characterise the thermal regime of the gas hydrate systems on the southern Hikurangi margin east of New Zealand. Data were processed using the Globe Claritas processing software.
4+
5+## Data Source
6+
7+The data comes from a SEG-Y file available on Zenodo (https://zenodo.org/records/8152964). SEG-Y is a standard format for storing seismic data that includes both the recorded waveforms and metadata about the survey.
8+
9+## Variants
10+
11+We provide two versions of the seismic data:
12+
13+### 1. Raw Data (seismic-04A-04B)
14+- Original floating point values from the SEG-Y file
15+- Contains the natural amplitude variations of the seismic waves
16+- Stored as 32-bit floating point numbers
17+
18+### 2. Quantized Data (seismic-04A-04B-quantized)
19+- Values are scaled and rounded to integers
20+- Uses a quantization step of 10000
21+- Stored as 32-bit integers