sparse versions of real datasets
1 changed file+98−0
zia_benchmark/src/zia_benchmark/datasets/real/__init__.pymodified+98−0View file
@@ -106,6 +106,68 @@ def _create_filtered_version(X: np.ndarray) -> np.ndarray:
106106 return X2
107107
108108
109+def _create_sparse_version(X: np.ndarray) -> np.ndarray:
110+ """Create sparse version of a dataset using activity-based suppression.
111+
112+ Args:
113+ X: Input signal array
114+
115+ Returns:
116+ Sparse signal array with suppressed low-activity regions
117+ """
118+ v = 0.25 # step size for quantization
119+ lowcut = 300
120+ highcut = 6000
121+ sampling_frequency = 30000
122+
123+ # Bandpass filter
124+ X_filt = bandpass_filter(
125+ X - np.median(X),
126+ sampling_frequency=sampling_frequency,
127+ lowcut=lowcut,
128+ highcut=highcut,
129+ )
130+
131+ # Normalize by noise level
132+ noise_level = estimate_noise_level(X_filt, sampling_frequency=sampling_frequency)
133+ X_filt_normalized = X_filt / noise_level
134+
135+ # Activity detection
136+ def sliding_max(x: np.ndarray, delta: int) -> np.ndarray:
137+ y = np.zeros_like(x)
138+ for i in range(len(x)):
139+ y[i] = np.max(x[max(0, i - delta) : min(len(x), i + delta + 1)])
140+ return y
141+
142+ def smoothed(x: np.ndarray, delta: int) -> np.ndarray:
143+ y = np.zeros_like(x)
144+ for i in range(len(x)):
145+ y[i] = np.mean(x[max(0, i - delta) : min(len(x), i + delta + 1)])
146+ return y
147+
148+ # Activity detection parameters
149+ activity_threshold = [3, 6] # [min, max] thresholds for activity detection
150+
151+ # Detect activity regions
152+ Y = sliding_max(np.abs(X_filt_normalized), 50)
153+ Y = smoothed(Y, 20)
154+ Y = np.minimum(
155+ 1,
156+ np.maximum(
157+ 0,
158+ (Y - activity_threshold[0])
159+ / (activity_threshold[1] - activity_threshold[0]),
160+ ),
161+ )
162+
163+ # Apply suppression
164+ Y_scaled = X_filt_normalized * Y
165+ X3b = Y_scaled / v
166+ X3 = np.round(X3b).astype(np.int16)
167+
168+ return X3
169+
170+
109171 datasets = [
110172 {
111173 "name": "real-000876-ch45",
@@ -173,4 +235,40 @@ datasets = [
173235 "tags": ["continuous", "neurophysiology", "filtered"],
174236 "source_file": SOURCE_FILE,
175237 },
238+ {
239+ "name": "real-000876-ch45-sparse",
240+ "version": "1",
241+ "description": "Sparse version of real-000876-ch45. Activity-based suppression applied.",
242+ "create": lambda: _create_sparse_version(
243+ _load_real_000876(
244+ num_samples=500_000, num_channels=1, start_channel=45
245+ ).flatten()
246+ ),
247+ "tags": ["continuous", "neurophysiology", "filtered", "sparse"],
248+ "source_file": SOURCE_FILE,
249+ },
250+ {
251+ "name": "real-000409-ch101-sparse",
252+ "version": "1",
253+ "description": "Sparse version of real-000409-ch101. Activity-based suppression applied.",
254+ "create": lambda: _create_sparse_version(
255+ _load_real_000409(
256+ num_samples=500_000, num_channels=1, start_channel=101
257+ ).flatten()
258+ ),
259+ "tags": ["continuous", "neurophysiology", "filtered", "sparse"],
260+ "source_file": SOURCE_FILE,
261+ },
262+ {
263+ "name": "real-001290-ch0-sparse",
264+ "version": "1",
265+ "description": "Sparse version of real-001290-ch0. Activity-based suppression applied.",
266+ "create": lambda: _create_sparse_version(
267+ _load_real_001290(
268+ num_samples=500_000, num_channels=1, start_channel=0
269+ ).flatten()
270+ ),
271+ "tags": ["continuous", "neurophysiology", "filtered", "sparse"],
272+ "source_file": SOURCE_FILE,
273+ },
176274 ]