add seismic dataset
3 changed files+195−8
benchcompress/src/benchcompress/datasets/seismic/__init__.pymodified+52−5View file
@@ -21,7 +21,7 @@ tags_float = tags + ["float"]
2121 tags_integer = tags + ["integer"]
2222
2323
24-def _load_seismic_data() -> np.ndarray:
24+def _load_04A_04B_seismic_data() -> np.ndarray:
2525 """Load seismic data from the SEG-Y file.
2626
2727 Returns:
@@ -58,8 +58,37 @@ def _load_seismic_data() -> np.ndarray:
5858 return X.ravel()
5959
6060
61-def _load_quantized_seismic_data():
62- X = _load_seismic_data()
61+def _load_quantized_04A_04B_seismic_data():
62+ X = _load_04A_04B_seismic_data()
63+ step = 10000
64+ X = np.round(X / step).astype(np.int32)
65+ return X
66+
67+
68+def _load_lim_2024_seismic_data() -> np.ndarray:
69+ """Load seismic data from the 2022 Goesan earthquake.
70+
71+ Returns:
72+ Array containing the loaded seismic data
73+ """
74+ file_path = "lim_et_al_2024.01.concat.npy"
75+ if not os.path.exists(file_path):
76+ # Download the numpy array file
77+ url = "https://zenodo.org/records/14774624/files/lim_et_al_2024.01.concat.npy?download=1"
78+ response = requests.get(url)
79+ with open(file_path, "wb") as f:
80+ f.write(response.content)
81+ print(f"Downloaded {file_path}")
82+ else:
83+ print(f"{file_path} already exists locally.")
84+
85+ # Load the numpy array
86+ X = np.load(file_path)
87+ return X
88+
89+
90+def _load_quantized_lim_2024_seismic_data():
91+ X = _load_lim_2024_seismic_data()
6392 step = 10000
6493 X = np.round(X / step).astype(np.int32)
6594 return X
@@ -70,7 +99,7 @@ datasets = [
7099 "name": "seismic-04A-04B",
71100 "version": "1",
72101 "description": "Seismic data from Roger Revelle voyage RR1508.",
73- "create": lambda: _load_seismic_data(),
102+ "create": lambda: _load_04A_04B_seismic_data(),
74103 "tags": tags_float,
75104 "source_file": SOURCE_FILE,
76105 "long_description": LONG_DESCRIPTION,
@@ -79,7 +108,25 @@ datasets = [
79108 "name": "seismic-04A-04B-quantized",
80109 "version": "1",
81110 "description": "Seismic data from Roger Revelle voyage RR1508, quantized.",
82- "create": lambda: _load_quantized_seismic_data(),
111+ "create": lambda: _load_quantized_04A_04B_seismic_data(),
112+ "tags": tags_integer,
113+ "source_file": SOURCE_FILE,
114+ "long_description": LONG_DESCRIPTION,
115+ },
116+ {
117+ "name": "seismic-lim-2024-01",
118+ "version": "1",
119+ "description": "Seismic data from the 2022 Mw 3.8 Goesan earthquake in South Korea.",
120+ "create": lambda: _load_lim_2024_seismic_data(),
121+ "tags": tags_float,
122+ "source_file": SOURCE_FILE,
123+ "long_description": LONG_DESCRIPTION,
124+ },
125+ {
126+ "name": "seismic-lim-2024-01-quantized",
127+ "version": "1",
128+ "description": "Seismic data from the 2022 Mw 3.8 Goesan earthquake in South Korea, quantized.",
129+ "create": lambda: _load_quantized_lim_2024_seismic_data(),
83130 "tags": tags_integer,
84131 "source_file": SOURCE_FILE,
85132 "long_description": LONG_DESCRIPTION,
benchcompress/src/benchcompress/datasets/seismic/seismic.mdmodified+23−3View file
@@ -8,14 +8,34 @@ The data comes from a SEG-Y file available on Zenodo (https://zenodo.org/records
88
99 ## Variants
1010
11-We provide two versions of the seismic data:
11+We provide two datasets with two versions each:
1212
13-### 1. Raw Data (seismic-04A-04B)
13+### Revelle RR1508 Data (seismic-04A-04B)
1414 - Original floating point values from the SEG-Y file
1515 - Contains the natural amplitude variations of the seismic waves
1616 - Stored as 32-bit floating point numbers
1717
18-### 2. Quantized Data (seismic-04A-04B-quantized)
18+### Revelle RR1508 Quantized Data (seismic-04A-04B-quantized)
1919 - Values are scaled and rounded to integers
2020 - Uses a quantization step of 10000
2121 - Stored as 32-bit integers
22+
23+### Goesan Earthquake Data (seismic-lim-2024-01)
24+- Original floating point values from the continuous seismic recording
25+- Data from the 2022 Mw 3.8 earthquake in Goesan, South Korea
26+- Part of a study analyzing 42 earthquakes including foreshocks and aftershocks
27+- Records from permanent seismic networks with closest station at 8.3 km from epicenter
28+- Stored as 32-bit floating point numbers
29+
30+### Goesan Earthquake Quantized Data (seismic-lim-2024-01-quantized)
31+- Values are scaled and rounded to integers
32+- Uses a quantization step of 10000
33+- Stored as 32-bit integers
34+
35+## Source Details
36+
37+### Roger Revelle RR1508
38+The data comes from a SEG-Y file available on Zenodo (https://zenodo.org/records/8152964). SEG-Y is a standard format for storing seismic data that includes both the recorded waveforms and metadata about the survey.
39+
40+### Goesan Earthquake 2022
41+This dataset contains seismic recordings from the 2022 Mw 3.8 Goesan earthquake in South Korea. The earthquake occurred on October 28, 2022, and was preceded by a Mw 3.3 foreshock 17 seconds before the mainshock. The study analyzed 42 earthquakes in total, including the mainshock, foreshock, and aftershocks, to understand the interactions between seismic events. The data revealed that the mainshock occurred at the southeastern tip of the hypocenter distribution of three foreshocks, with aftershocks showing a diffused pattern propagating toward both ends of the inferred lineament. The data is available on Zenodo (https://zenodo.org/records/14774624).
devel/prepare_lim_et_al_2024.pyadded+120−0View file
@@ -0,0 +1,120 @@
1+#!/usr/bin/env python
2+
3+"""
4+This script prepares a seismic dataset from Lim et al. 2024 for use in the benchcompress project.
5+It downloads seismic waveform data from Zenodo, extracts .sac files from a specific folder,
6+and concatenates them into a single numpy array. The resulting dataset will be used to evaluate
7+various compression algorithms in benchcompress to determine their effectiveness on real-world
8+seismic waveform data.
9+
10+The original data is from:
11+Lim, H., & Zhang, M. (2024). Machine Learning Phase Picker Training Dataset in Shanghai.
12+Zenodo. https://doi.org/10.5281/zenodo.10457508
13+"""
14+
15+import os
16+import sys
17+from urllib.request import urlretrieve
18+import tarfile
19+import glob
20+from obspy import read
21+import numpy as np
22+
23+def download_file(url, local_filename):
24+ if os.path.exists(local_filename):
25+ print(f"File {local_filename} already exists locally")
26+ return True
27+
28+ print(f"Downloading {url} to {local_filename}...")
29+ try:
30+ urlretrieve(url, local_filename)
31+ print("Download completed successfully")
32+ return True
33+ except Exception as e:
34+ print(f"Error downloading file: {e}")
35+ return False
36+
37+def extract_data_folder(archive_path):
38+ if not os.path.exists(archive_path):
39+ print(f"Archive file {archive_path} not found")
40+ return False
41+
42+ print(f"Extracting /data/01 folder from {archive_path}...")
43+ try:
44+ with tarfile.open(archive_path, 'r:gz') as tar:
45+ # Extract only files in the /data/01 directory
46+ members = [m for m in tar.getmembers() if m.name.startswith('data/01/')]
47+ for member in members:
48+ tar.extract(member)
49+ print("Extraction completed successfully")
50+ return True
51+ except Exception as e:
52+ print(f"Error extracting archive: {e}")
53+ return False
54+
55+def load_and_concatenate_sac_files(directory):
56+ # Find all .sac files in the directory
57+ sac_files = glob.glob(f"{directory}/*.sac")
58+ if not sac_files:
59+ print(f"No .sac files found in {directory}")
60+ return None
61+
62+ print(f"Found {len(sac_files)} .sac files")
63+
64+ all_data = []
65+ for i, filename in enumerate(sac_files, 1):
66+ try:
67+ print(f"Loading file {i}/{len(sac_files)}: {filename}")
68+ st = read(filename)
69+ tr = st[0] # Get the first trace
70+ all_data.append(tr.data)
71+ print(f" Shape: {tr.data.shape}, dtype: {tr.data.dtype}")
72+ except Exception as e:
73+ print(f"Error loading {filename}: {e}")
74+ continue
75+
76+ if not all_data:
77+ print("No data was successfully loaded")
78+ return None
79+
80+ # Concatenate all data arrays
81+ concatenated_data = np.concatenate(all_data)
82+ print(f"\nConcatenation complete!")
83+ print(f"Final array shape: {concatenated_data.shape}")
84+ print(f"Final array dtype: {concatenated_data.dtype}")
85+
86+ return concatenated_data
87+
88+def main():
89+ url = "https://zenodo.org/records/10457508/files/data.tar.gz?download=1"
90+ local_file = "lim_et_al_2024.zip"
91+ sac_directory = "data/01"
92+ output_file = "lim_et_al_2024.01.concat.npy"
93+
94+ # Download if needed
95+ if not download_file(url, local_file):
96+ print("Failed to download file")
97+ sys.exit(1)
98+
99+ # Extract data folder
100+ if not extract_data_folder(local_file):
101+ print("Failed to extract data folder")
102+ sys.exit(1)
103+
104+ # Load and concatenate SAC files
105+ concatenated_data = load_and_concatenate_sac_files(sac_directory)
106+ if concatenated_data is None:
107+ print("Failed to process SAC files")
108+ sys.exit(1)
109+
110+ # Save concatenated data
111+ print(f"\nSaving concatenated data to {output_file}...")
112+ try:
113+ np.save(output_file, concatenated_data)
114+ print("Data saved successfully")
115+ except Exception as e:
116+ print(f"Error saving data: {e}")
117+ sys.exit(1)
118+
119+if __name__ == "__main__":
120+ main()