add filtered datasets
1 changed file+59−3
zia_benchmark/src/zia_benchmark/datasets/real/__init__.pymodified+59−3View file
@@ -1,6 +1,8 @@
11 import numpy as np
22 import lindi
33 from typing import cast
4+from ..._filters import bandpass_filter
5+from ..._analysis import estimate_noise_level
46
57
68 def _load_real_000876(
@@ -20,7 +22,7 @@ def _load_real_000876(
2022 h5f = lindi.LindiH5pyFile.from_hdf5_file(nwb_url)
2123 ds = h5f["/acquisition/ElectricalSeriesAP/data"]
2224 assert isinstance(ds, lindi.LindiH5pyDataset)
23- ret = ds[:num_samples, start_channel : start_channel + num_channels]
25+ ret = ds[:num_samples, start_channel:start_channel + num_channels]
2426 return cast(np.ndarray, ret)
2527
2628
@@ -41,7 +43,7 @@ def _load_real_000409(
4143 h5f = lindi.LindiH5pyFile.from_hdf5_file(nwb_url)
4244 ds = h5f["/acquisition/ElectricalSeriesAp/data"]
4345 assert isinstance(ds, lindi.LindiH5pyDataset)
44- ret = ds[:num_samples, start_channel : start_channel + num_channels]
46+ ret = ds[:num_samples, start_channel:start_channel + num_channels]
4547 return cast(np.ndarray, ret)
4648
4749
@@ -62,10 +64,37 @@ def _load_real_001290(
6264 h5f = lindi.LindiH5pyFile.from_hdf5_file(nwb_url)
6365 ds = h5f["/acquisition/ElectricalSeriesAPImec/data"]
6466 assert isinstance(ds, lindi.LindiH5pyDataset)
65- ret = ds[:num_samples, start_channel : start_channel + num_channels]
67+ ret = ds[:num_samples, start_channel:start_channel + num_channels]
6668 return cast(np.ndarray, ret)
6769
6870
71+def _create_filtered_version(X: np.ndarray) -> np.ndarray:
72+ """Create filtered version of a dataset using bandpass filtering and quantization.
73+
74+ Args:
75+ X: Input signal array
76+
77+ Returns:
78+ Filtered and quantized signal array
79+ """
80+ v = 0.25 # step size for quantization
81+ lowcut = 300
82+ highcut = 6000
83+ sampling_frequency = 30000
84+
85+ # Bandpass filter
86+ X_filt = bandpass_filter(X - np.median(X), sampling_frequency=sampling_frequency, lowcut=lowcut, highcut=highcut)
87+
88+ # Normalize by noise level
89+ noise_level = estimate_noise_level(X_filt, sampling_frequency=sampling_frequency)
90+ X_filt_normalized = X_filt / noise_level
91+
92+ # Quantize
93+ X2b = X_filt_normalized / v
94+ X2 = np.round(X2b).astype(np.int16)
95+
96+ return X2
97+
6998 datasets = [
7099 {
71100 "name": "real-000876-ch45",
@@ -94,4 +123,31 @@ datasets = [
94123 ).flatten(),
95124 "tags": ["continuous", "neurophysiology"],
96125 },
126+ {
127+ "name": "real-000876-ch45-filtered",
128+ "version": "1",
129+ "description": "Filtered version of real-000876-ch45 with bandpass filtering (300-6000 Hz) and quantization",
130+ "create": lambda: _create_filtered_version(_load_real_000876(
131+ num_samples=500_000, num_channels=1, start_channel=45
132+ ).flatten()),
133+ "tags": ["continuous", "neurophysiology", "filtered"],
134+ },
135+ {
136+ "name": "real-000409-ch101-filtered",
137+ "version": "1",
138+ "description": "Filtered version of real-000409-ch101 with bandpass filtering (300-6000 Hz) and quantization",
139+ "create": lambda: _create_filtered_version(_load_real_000409(
140+ num_samples=500_000, num_channels=1, start_channel=101
141+ ).flatten()),
142+ "tags": ["continuous", "neurophysiology", "filtered"],
143+ },
144+ {
145+ "name": "real-001290-ch0-filtered",
146+ "version": "1",
147+ "description": "Filtered version of real-001290-ch0 with bandpass filtering (300-6000 Hz) and quantization",
148+ "create": lambda: _create_filtered_version(_load_real_001290(
149+ num_samples=500_000, num_channels=1, start_channel=0
150+ ).flatten()),
151+ "tags": ["continuous", "neurophysiology", "filtered"],
152+ },
97153 ]