1import numpy as np
2import lindi
3import os
4from typing import cast
5from ..._filters import bandpass_filter
6from ..._analysis import estimate_noise_level
9SOURCE_FILE = "ecephys/__init__.py"
12def _load_long_description():
13 current_dir = os.path.dirname(os.path.abspath(__file__))
14 md_path = os.path.join(current_dir, "ecephys.md")
15 with open(md_path, "r", encoding="utf-8") as f:
16 return f.read()
19LONG_DESCRIPTION = _load_long_description()
21tags = ["real", "ecephys", "timeseries", "1d", "integer", "continuous"]
24def _load_real_000876(
25 *, num_samples: int, num_channels: int, start_channel: int
26) -> np.ndarray:
27 """Load data from DANDI dataset 000876.
29 Args:
30 num_samples: Number of samples to load
31 num_channels: Number of channels to load
32 start_channel: Starting channel index
34 Returns:
35 Array of shape (num_samples, num_channels) containing the loaded data
36 """
37 nwb_url = "https://api.dandiarchive.org/api/assets/7e1de06d-d478-40e2-9b64-9dd04eafaa4c/download/"
38 h5f = lindi.LindiH5pyFile.from_hdf5_file(nwb_url)
39 ds = h5f["/acquisition/ElectricalSeriesAP/data"]
40 assert isinstance(ds, lindi.LindiH5pyDataset)
41 ret = ds[:num_samples, start_channel : start_channel + num_channels]
42 return cast(np.ndarray, ret)
45def _load_real_000409(
46 *, num_samples: int, num_channels: int, start_channel: int
47) -> np.ndarray:
48 """Load data from DANDI dataset 000409.
50 Args:
51 num_samples: Number of samples to load
52 num_channels: Number of channels to load
53 start_channel: Starting channel index
55 Returns:
56 Array of shape (num_samples, num_channels) containing the loaded data
57 """
58 nwb_url = "https://api.dandiarchive.org/api/assets/c04f6b30-82bf-40e1-9210-34f0bcd8be24/download/"
59 h5f = lindi.LindiH5pyFile.from_hdf5_file(nwb_url)
60 ds = h5f["/acquisition/ElectricalSeriesAp/data"]
61 assert isinstance(ds, lindi.LindiH5pyDataset)
62 ret = ds[:num_samples, start_channel : start_channel + num_channels]
63 return cast(np.ndarray, ret)
66def _load_real_001290(
67 *, num_samples: int, num_channels: int, start_channel: int
68) -> np.ndarray:
69 """Load data from DANDI dataset 001290.
71 Args:
72 num_samples: Number of samples to load
73 num_channels: Number of channels to load
74 start_channel: Starting channel index
76 Returns:
77 Array of shape (num_samples, num_channels) containing the loaded data
78 """
79 nwb_url = "https://api.dandiarchive.org/api/assets/78c99d23-da88-4ecd-9086-c488a126eac5/download/"
80 h5f = lindi.LindiH5pyFile.from_hdf5_file(nwb_url)
81 ds = h5f["/acquisition/ElectricalSeriesAPImec/data"]
82 assert isinstance(ds, lindi.LindiH5pyDataset)
83 ret = ds[:num_samples, start_channel : start_channel + num_channels]
84 return cast(np.ndarray, ret)
87# embargoed
88# def _load_real_001259(*, num_samples: int) -> np.ndarray:
89# """Load data from DANDI dataset 001259.
91# Args:
92# num_samples: Number of samples to load
94# Returns:
95# Array of shape (num_samples, 1) containing the loaded data
96# """
97# # this one only has one channel
98# # https://neurosift.app/?p=/nwb&url=https://api.dandiarchive.org/api/assets/6fa78218-f4d4-45f3-a627-5afe4aa88c3e/download/&dandisetId=001259&dandisetVersion=draft
99# nwb_url = "https://api.dandiarchive.org/api/assets/6fa78218-f4d4-45f3-a627-5afe4aa88c3e/download/"
100# h5f = lindi.LindiH5pyFile.from_hdf5_file(nwb_url)
101# ds = h5f["/acquisition/ElectricalSeries"]
102# assert isinstance(ds, lindi.LindiH5pyDataset)
103# # this one only has one channel
104# ret = ds[:num_samples, 0]
105# return cast(np.ndarray, ret)
108def _create_filtered_version(X: np.ndarray) -> np.ndarray:
109 """Create filtered version of a dataset using bandpass filtering and quantization.
111 Args:
112 X: Input signal array
114 Returns:
115 Filtered and quantized signal array
116 """
117 v = 0.25 # step size for quantization
118 lowcut = 300
119 highcut = 6000
120 sampling_frequency = 30000
122 # Bandpass filter
123 X_filt = bandpass_filter(
124 X - np.median(X),
125 sampling_frequency=sampling_frequency,
126 lowcut=lowcut,
127 highcut=highcut,
128 )
130 # Normalize by noise level
131 noise_level = estimate_noise_level(X_filt, sampling_frequency=sampling_frequency)
132 X_filt_normalized = X_filt / noise_level
134 # Quantize
135 X2b = X_filt_normalized / v
136 X2 = np.round(X2b).astype(np.int16)
138 return X2
141def _create_sparse_version(X: np.ndarray) -> np.ndarray:
142 """Create sparse version of a dataset using activity-based suppression.
144 Args:
145 X: Input signal array
147 Returns:
148 Sparse signal array with suppressed low-activity regions
149 """
150 v = 0.25 # step size for quantization
151 lowcut = 300
152 highcut = 6000
153 sampling_frequency = 30000
155 # Bandpass filter
156 X_filt = bandpass_filter(
157 X - np.median(X),
158 sampling_frequency=sampling_frequency,
159 lowcut=lowcut,
160 highcut=highcut,
161 )
163 # Normalize by noise level
164 noise_level = estimate_noise_level(X_filt, sampling_frequency=sampling_frequency)
165 X_filt_normalized = X_filt / noise_level
167 # Activity detection
168 def sliding_max(x: np.ndarray, delta: int) -> np.ndarray:
169 y = np.zeros_like(x)
170 for i in range(len(x)):
171 y[i] = np.max(x[max(0, i - delta) : min(len(x), i + delta + 1)])
172 return y
174 def smoothed(x: np.ndarray, delta: int) -> np.ndarray:
175 y = np.zeros_like(x)
176 for i in range(len(x)):
177 y[i] = np.mean(x[max(0, i - delta) : min(len(x), i + delta + 1)])
178 return y
180 # Activity detection parameters
181 activity_threshold = [3, 6] # [min, max] thresholds for activity detection
183 # Detect activity regions
184 Y = sliding_max(np.abs(X_filt_normalized), 50)
185 Y = smoothed(Y, 20)
186 Y = np.minimum(
187 1,
188 np.maximum(
189 0,
190 (Y - activity_threshold[0])
191 / (activity_threshold[1] - activity_threshold[0]),
192 ),
193 )
195 # Apply suppression
196 Y_scaled = X_filt_normalized * Y
197 X3b = Y_scaled / v
198 X3 = np.round(X3b).astype(np.int16)
200 return X3
203datasets = [
204 {
205 "name": "ecephys-000876-ch45",
206 "version": "1",
207 "description": "Raw extracellular electrophysiology recording from DANDI:000876.",
208 "create": lambda: _load_real_000876(
209 num_samples=500_000, num_channels=1, start_channel=45
210 ).flatten(),
211 "tags": tags,
212 "source_file": SOURCE_FILE,
213 "long_description": LONG_DESCRIPTION,
214 },
215 {
216 "name": "ecephys-000409-ch101",
217 "version": "1",
218 "description": "Raw extracellular electrophysiology recording from DANDI:000409.",
219 "create": lambda: _load_real_000409(
220 num_samples=500_000, num_channels=1, start_channel=101
221 ).flatten(),
222 "tags": tags,
223 "source_file": SOURCE_FILE,
224 "long_description": LONG_DESCRIPTION,
225 },
226 {
227 "name": "ecephys-001290-ch0",
228 "version": "1",
229 "description": "Raw extracellular electrophysiology recording from DANDI:001290.",
230 "create": lambda: _load_real_001290(
231 num_samples=500_000, num_channels=1, start_channel=0
232 ).flatten(),
233 "tags": tags,
234 "source_file": SOURCE_FILE,
235 "long_description": LONG_DESCRIPTION,
236 },
237 # embargoed
238 # {
239 # "name": "ecephys-001259",
240 # "version": "1",
241 # "description": "Raw extracellular electrophysiology recording from DANDI:001259.",
242 # "create": lambda: _load_real_001259(num_samples=500_000).flatten(),
243 # "tags": tags,
244 # "source_file": SOURCE_FILE,
245 # "long_description": LONG_DESCRIPTION,
246 # },
247 {
248 "name": "ecephys-000876-ch45-filtered",
249 "version": "1",
250 "description": "Preprocessed version of real-000876-ch45. Bandpass filtered (300-6000 Hz).",
251 "create": lambda: _create_filtered_version(
252 _load_real_000876(
253 num_samples=500_000, num_channels=1, start_channel=45
254 ).flatten()
255 ),
256 "tags": tags + ["filtered"],
257 "source_file": SOURCE_FILE,
258 "long_description": LONG_DESCRIPTION,
259 },
260 {
261 "name": "ecephys-000409-ch101-filtered",
262 "version": "1",
263 "description": "Preprocessed version of real-000409-ch101. Bandpass filtered (300-6000 Hz).",
264 "create": lambda: _create_filtered_version(
265 _load_real_000409(
266 num_samples=500_000, num_channels=1, start_channel=101
267 ).flatten()
268 ),
269 "tags": tags + ["filtered"],
270 "source_file": SOURCE_FILE,
271 "long_description": LONG_DESCRIPTION,
272 },
273 {
274 "name": "ecephys-001290-ch0-filtered",
275 "version": "1",
276 "description": "Preprocessed version of real-001290-ch0. Bandpass filtered (300-6000 Hz).",
277 "create": lambda: _create_filtered_version(
278 _load_real_001290(
279 num_samples=500_000, num_channels=1, start_channel=0
280 ).flatten()
281 ),
282 "tags": tags + ["filtered"],
283 "source_file": SOURCE_FILE,
284 "long_description": LONG_DESCRIPTION,
285 },
286 # embargoed
287 # {
288 # "name": "ecephys-001259-filtered",
289 # "version": "1",
290 # "description": "Preprocessed version of real-001259. Bandpass filtered (300-6000 Hz).",
291 # "create": lambda: _create_filtered_version(
292 # _load_real_001259(num_samples=500_000).flatten()
293 # ),
294 # "tags": tags + ["filtered"],
295 # "source_file": SOURCE_FILE,
296 # "long_description": LONG_DESCRIPTION,
297 # },
298 {
299 "name": "ecephys-000876-ch45-sparse",
300 "version": "1",
301 "description": "Sparse version of real-000876-ch45. Activity-based suppression applied.",
302 "create": lambda: _create_sparse_version(
303 _load_real_000876(
304 num_samples=500_000, num_channels=1, start_channel=45
305 ).flatten()
306 ),
307 "tags": tags + ["filtered", "sparse"],
308 "source_file": SOURCE_FILE,
309 "long_description": LONG_DESCRIPTION,
310 },
311 {
312 "name": "ecephys-000409-ch101-sparse",
313 "version": "1",
314 "description": "Sparse version of real-000409-ch101. Activity-based suppression applied.",
315 "create": lambda: _create_sparse_version(
316 _load_real_000409(
317 num_samples=500_000, num_channels=1, start_channel=101
318 ).flatten()
319 ),
320 "tags": tags + ["filtered", "sparse"],
321 "source_file": SOURCE_FILE,
322 "long_description": LONG_DESCRIPTION,
323 },
324 {
325 "name": "ecephys-001290-ch0-sparse",
326 "version": "1",
327 "description": "Sparse version of real-001290-ch0. Activity-based suppression applied.",
328 "create": lambda: _create_sparse_version(
329 _load_real_001290(
330 num_samples=500_000, num_channels=1, start_channel=0
331 ).flatten()
332 ),
333 "tags": tags + ["filtered", "sparse"],
334 "source_file": SOURCE_FILE,
335 "long_description": LONG_DESCRIPTION,
336 },
337 # embargoed
338 # {
339 # "name": "ecephys-001259-sparse",
340 # "version": "1",
341 # "description": "Sparse version of real-001259. Activity-based suppression applied.",
342 # "create": lambda: _create_sparse_version(
343 # _load_real_001259(num_samples=500_000).flatten()
344 # ),
345 # "tags": tags + ["filtered", "sparse"],
346 # "source_file": SOURCE_FILE,
347 # "long_description": LONG_DESCRIPTION,
348 # },
349]