1import numpy as np
2import lindi
3import os
4from typing import cast
5from ..._filters import bandpass_filter
6from ..._analysis import estimate_noise_level
7from ...types import Dataset
10SOURCE_FILE = "ecephys/__init__.py"
13def _load_long_description():
14 current_dir = os.path.dirname(os.path.abspath(__file__))
15 md_path = os.path.join(current_dir, "ecephys.md")
16 with open(md_path, "r", encoding="utf-8") as f:
17 return f.read()
20LONG_DESCRIPTION = _load_long_description()
22tags = ["real", "ecephys", "timeseries", "1d", "integer", "correlated"]
25def _load_real_000876(
26 *, num_samples: int, num_channels: int, start_channel: int
27) -> np.ndarray:
28 """Load data from DANDI dataset 000876.
30 Args:
31 num_samples: Number of samples to load
32 num_channels: Number of channels to load
33 start_channel: Starting channel index
35 Returns:
36 Array of shape (num_samples, num_channels) containing the loaded data
37 """
38 nwb_url = "https://api.dandiarchive.org/api/assets/7e1de06d-d478-40e2-9b64-9dd04eafaa4c/download/"
39 h5f = lindi.LindiH5pyFile.from_hdf5_file(nwb_url)
40 ds = h5f["/acquisition/ElectricalSeriesAP/data"]
41 assert isinstance(ds, lindi.LindiH5pyDataset)
42 ret = ds[:num_samples, start_channel : start_channel + num_channels]
43 return cast(np.ndarray, ret)
46def _load_real_000409(
47 *, num_samples: int, num_channels: int, start_channel: int
48) -> np.ndarray:
49 """Load data from DANDI dataset 000409.
51 Args:
52 num_samples: Number of samples to load
53 num_channels: Number of channels to load
54 start_channel: Starting channel index
56 Returns:
57 Array of shape (num_samples, num_channels) containing the loaded data
58 """
59 nwb_url = "https://api.dandiarchive.org/api/assets/c04f6b30-82bf-40e1-9210-34f0bcd8be24/download/"
60 h5f = lindi.LindiH5pyFile.from_hdf5_file(nwb_url)
61 ds = h5f["/acquisition/ElectricalSeriesAp/data"]
62 assert isinstance(ds, lindi.LindiH5pyDataset)
63 ret = ds[:num_samples, start_channel : start_channel + num_channels]
64 return cast(np.ndarray, ret)
67def _load_real_001290(
68 *, num_samples: int, num_channels: int, start_channel: int
69) -> np.ndarray:
70 """Load data from DANDI dataset 001290.
72 Args:
73 num_samples: Number of samples to load
74 num_channels: Number of channels to load
75 start_channel: Starting channel index
77 Returns:
78 Array of shape (num_samples, num_channels) containing the loaded data
79 """
80 nwb_url = "https://api.dandiarchive.org/api/assets/78c99d23-da88-4ecd-9086-c488a126eac5/download/"
81 h5f = lindi.LindiH5pyFile.from_hdf5_file(nwb_url)
82 ds = h5f["/acquisition/ElectricalSeriesAPImec/data"]
83 assert isinstance(ds, lindi.LindiH5pyDataset)
84 ret = ds[:num_samples, start_channel : start_channel + num_channels]
85 return cast(np.ndarray, ret)
88# embargoed
89# def _load_real_001259(*, num_samples: int) -> np.ndarray:
90# """Load data from DANDI dataset 001259.
92# Args:
93# num_samples: Number of samples to load
95# Returns:
96# Array of shape (num_samples, 1) containing the loaded data
97# """
98# # this one only has one channel
99# # https://neurosift.app/?p=/nwb&url=https://api.dandiarchive.org/api/assets/6fa78218-f4d4-45f3-a627-5afe4aa88c3e/download/&dandisetId=001259&dandisetVersion=draft
100# nwb_url = "https://api.dandiarchive.org/api/assets/6fa78218-f4d4-45f3-a627-5afe4aa88c3e/download/"
101# h5f = lindi.LindiH5pyFile.from_hdf5_file(nwb_url)
102# ds = h5f["/acquisition/ElectricalSeries"]
103# assert isinstance(ds, lindi.LindiH5pyDataset)
104# # this one only has one channel
105# ret = ds[:num_samples, 0]
106# return cast(np.ndarray, ret)
109def _create_filtered_version(X: np.ndarray) -> np.ndarray:
110 """Create filtered version of a dataset using bandpass filtering and quantization.
112 Args:
113 X: Input signal array
115 Returns:
116 Filtered and quantized signal array
117 """
118 v = 0.25 # step size for quantization
119 lowcut = 300
120 highcut = 6000
121 sampling_frequency = 30000
123 # Bandpass filter
124 X_filt = bandpass_filter(
125 X - np.median(X),
126 sampling_frequency=sampling_frequency,
127 lowcut=lowcut,
128 highcut=highcut,
129 )
131 # Normalize by noise level
132 noise_level = estimate_noise_level(X_filt, sampling_frequency=sampling_frequency)
133 X_filt_normalized = X_filt / noise_level
135 # Quantize
136 X2b = X_filt_normalized / v
137 X2 = np.round(X2b).astype(np.int16)
139 return X2
142def _create_sparse_version(X: np.ndarray) -> np.ndarray:
143 """Create sparse version of a dataset using activity-based suppression.
145 Args:
146 X: Input signal array
148 Returns:
149 Sparse signal array with suppressed low-activity regions
150 """
151 v = 0.25 # step size for quantization
152 lowcut = 300
153 highcut = 6000
154 sampling_frequency = 30000
156 # Bandpass filter
157 X_filt = bandpass_filter(
158 X - np.median(X),
159 sampling_frequency=sampling_frequency,
160 lowcut=lowcut,
161 highcut=highcut,
162 )
164 # Normalize by noise level
165 noise_level = estimate_noise_level(X_filt, sampling_frequency=sampling_frequency)
166 X_filt_normalized = X_filt / noise_level
168 # Activity detection
169 def sliding_max(x: np.ndarray, delta: int) -> np.ndarray:
170 y = np.zeros_like(x)
171 for i in range(len(x)):
172 y[i] = np.max(x[max(0, i - delta) : min(len(x), i + delta + 1)])
173 return y
175 def smoothed(x: np.ndarray, delta: int) -> np.ndarray:
176 y = np.zeros_like(x)
177 for i in range(len(x)):
178 y[i] = np.mean(x[max(0, i - delta) : min(len(x), i + delta + 1)])
179 return y
181 # Activity detection parameters
182 activity_threshold = [3, 6] # [min, max] thresholds for activity detection
184 # Detect activity regions
185 Y = sliding_max(np.abs(X_filt_normalized), 50)
186 Y = smoothed(Y, 20)
187 Y = np.minimum(
188 1,
189 np.maximum(
190 0,
191 (Y - activity_threshold[0])
192 / (activity_threshold[1] - activity_threshold[0]),
193 ),
194 )
196 # Apply suppression
197 Y_scaled = X_filt_normalized * Y
198 X3b = Y_scaled / v
199 X3 = np.round(X3b).astype(np.int16)
201 return X3
204datasets_dicts = [
205 {
206 "name": "ecephys-000876-ch45",
207 "version": "1",
208 "description": "Raw extracellular electrophysiology recording from DANDI:000876.",
209 "create": lambda: _load_real_000876(
210 num_samples=500_000, num_channels=1, start_channel=45
211 ).flatten(),
212 "tags": tags,
213 "source_file": SOURCE_FILE,
214 "long_description": LONG_DESCRIPTION,
215 },
216 {
217 "name": "ecephys-000409-ch101",
218 "version": "1",
219 "description": "Raw extracellular electrophysiology recording from DANDI:000409.",
220 "create": lambda: _load_real_000409(
221 num_samples=500_000, num_channels=1, start_channel=101
222 ).flatten(),
223 "tags": tags,
224 "source_file": SOURCE_FILE,
225 "long_description": LONG_DESCRIPTION,
226 },
227 {
228 "name": "ecephys-001290-ch0",
229 "version": "1",
230 "description": "Raw extracellular electrophysiology recording from DANDI:001290.",
231 "create": lambda: _load_real_001290(
232 num_samples=500_000, num_channels=1, start_channel=0
233 ).flatten(),
234 "tags": tags,
235 "source_file": SOURCE_FILE,
236 "long_description": LONG_DESCRIPTION,
237 },
238 # embargoed
239 # {
240 # "name": "ecephys-001259",
241 # "version": "1",
242 # "description": "Raw extracellular electrophysiology recording from DANDI:001259.",
243 # "create": lambda: _load_real_001259(num_samples=500_000).flatten(),
244 # "tags": tags,
245 # "source_file": SOURCE_FILE,
246 # "long_description": LONG_DESCRIPTION,
247 # },
248 {
249 "name": "ecephys-000876-ch45-filtered",
250 "version": "1",
251 "description": "Preprocessed version of real-000876-ch45. Bandpass filtered (300-6000 Hz).",
252 "create": lambda: _create_filtered_version(
253 _load_real_000876(
254 num_samples=500_000, num_channels=1, start_channel=45
255 ).flatten()
256 ),
257 "tags": tags + ["filtered"],
258 "source_file": SOURCE_FILE,
259 "long_description": LONG_DESCRIPTION,
260 },
261 {
262 "name": "ecephys-000409-ch101-filtered",
263 "version": "1",
264 "description": "Preprocessed version of real-000409-ch101. Bandpass filtered (300-6000 Hz).",
265 "create": lambda: _create_filtered_version(
266 _load_real_000409(
267 num_samples=500_000, num_channels=1, start_channel=101
268 ).flatten()
269 ),
270 "tags": tags + ["filtered"],
271 "source_file": SOURCE_FILE,
272 "long_description": LONG_DESCRIPTION,
273 },
274 {
275 "name": "ecephys-001290-ch0-filtered",
276 "version": "1",
277 "description": "Preprocessed version of real-001290-ch0. Bandpass filtered (300-6000 Hz).",
278 "create": lambda: _create_filtered_version(
279 _load_real_001290(
280 num_samples=500_000, num_channels=1, start_channel=0
281 ).flatten()
282 ),
283 "tags": tags + ["filtered"],
284 "source_file": SOURCE_FILE,
285 "long_description": LONG_DESCRIPTION,
286 },
287 # embargoed
288 # {
289 # "name": "ecephys-001259-filtered",
290 # "version": "1",
291 # "description": "Preprocessed version of real-001259. Bandpass filtered (300-6000 Hz).",
292 # "create": lambda: _create_filtered_version(
293 # _load_real_001259(num_samples=500_000).flatten()
294 # ),
295 # "tags": tags + ["filtered"],
296 # "source_file": SOURCE_FILE,
297 # "long_description": LONG_DESCRIPTION,
298 # },
299 {
300 "name": "ecephys-000876-ch45-sparse",
301 "version": "1",
302 "description": "Sparse version of real-000876-ch45. Activity-based suppression applied.",
303 "create": lambda: _create_sparse_version(
304 _load_real_000876(
305 num_samples=500_000, num_channels=1, start_channel=45
306 ).flatten()
307 ),
308 "tags": tags + ["filtered", "sparse"],
309 "source_file": SOURCE_FILE,
310 "long_description": LONG_DESCRIPTION,
311 },
312 {
313 "name": "ecephys-000409-ch101-sparse",
314 "version": "1",
315 "description": "Sparse version of real-000409-ch101. Activity-based suppression applied.",
316 "create": lambda: _create_sparse_version(
317 _load_real_000409(
318 num_samples=500_000, num_channels=1, start_channel=101
319 ).flatten()
320 ),
321 "tags": tags + ["filtered", "sparse"],
322 "source_file": SOURCE_FILE,
323 "long_description": LONG_DESCRIPTION,
324 },
325 {
326 "name": "ecephys-001290-ch0-sparse",
327 "version": "1",
328 "description": "Sparse version of real-001290-ch0. Activity-based suppression applied.",
329 "create": lambda: _create_sparse_version(
330 _load_real_001290(
331 num_samples=500_000, num_channels=1, start_channel=0
332 ).flatten()
333 ),
334 "tags": tags + ["filtered", "sparse"],
335 "source_file": SOURCE_FILE,
336 "long_description": LONG_DESCRIPTION,
337 },
338 # embargoed
339 # {
340 # "name": "ecephys-001259-sparse",
341 # "version": "1",
342 # "description": "Sparse version of real-001259. Activity-based suppression applied.",
343 # "create": lambda: _create_sparse_version(
344 # _load_real_001259(num_samples=500_000).flatten()
345 # ),
346 # "tags": tags + ["filtered", "sparse"],
347 # "source_file": SOURCE_FILE,
348 # "long_description": LONG_DESCRIPTION,
349 # },
350]
353datasets = [
354 Dataset(**a)
355 for a in datasets_dicts
356]