upload .npy file
5 changed files+52−18
web-ui/src/pages/Datasets.tsxmodified+3−3View file
@@ -185,10 +185,10 @@ function Datasets({ datasets }: DatasetsProps) {
185185 fontSize: "0.9rem",
186186 }}
187187 >
188- {dataset.data_url && (
188+ {dataset.data_url_npy && (
189189 <a
190- href={dataset.data_url}
191- download={`${dataset.name}-${dataset.version}.bin`}
190+ href={dataset.data_url_npy}
191+ download={`${dataset.name}-${dataset.version}.npy`}
192192 style={{
193193 color: "#0066cc",
194194 textDecoration: "none",
web-ui/src/types.tsmodified+2−1View file
@@ -30,7 +30,8 @@ export interface Dataset {
3030 version: string;
3131 tags: string[];
3232 source_file?: string;
33- data_url?: string; // URL to download the raw dataset
33+ data_url_npy?: string; // URL to download the dataset as .npy
34+ data_url_raw?: string; // URL to download the raw dataset as .dat
3435 }
3536
3637 export interface BenchmarkData {
zia_benchmark/src/zia_benchmark/_memobin.pymodified+5−2View file
@@ -80,17 +80,20 @@ def construct_memobin_url(
8080 return f"https://tempory.net/f/memobin/{path}"
8181
8282
83-def construct_dataset_url(dataset_name: str, dataset_version: str) -> str:
83+def construct_dataset_url(
84+ dataset_name: str, dataset_version: str, format: str = "dat"
85+) -> str:
8486 """Construct the memobin URL for a dataset.
8587
8688 Args:
8789 dataset_name: Name of the dataset
8890 dataset_version: Version of the dataset
91+ format: File format ("dat" or "npy")
8992
9093 Returns:
9194 The constructed memobin URL for the dataset
9295 """
93- path = f"datasets/{dataset_name}/{dataset_version}/data.bin"
96+ path = f"datasets/{dataset_name}/{dataset_version}/data.{format}"
9497 return f"https://tempory.net/f/memobin/{path}"
9598
9699
zia_benchmark/src/zia_benchmark/datasets/bernoulli/__init__.pymodified+5−5View file
@@ -13,7 +13,7 @@ def create_bernoulli(*, n_samples: int, p: float, seed: int) -> np.ndarray:
1313 datasets = [
1414 {
1515 "name": "bernoulli-0.1",
16- "version": "2",
16+ "version": "3",
1717 "create": lambda: create_bernoulli(n_samples=1_000_000, p=0.1, seed=0),
1818 "description": "Binary sequence with 10% probability of ones.",
1919 "tags": ["binary"],
@@ -21,7 +21,7 @@ datasets = [
2121 },
2222 {
2323 "name": "bernoulli-0.2",
24- "version": "1",
24+ "version": "3",
2525 "create": lambda: create_bernoulli(n_samples=1_000_000, p=0.2, seed=0),
2626 "description": "Binary sequence with 20% probability of ones.",
2727 "tags": ["binary"],
@@ -29,7 +29,7 @@ datasets = [
2929 },
3030 {
3131 "name": "bernoulli-0.3",
32- "version": "1",
32+ "version": "3",
3333 "create": lambda: create_bernoulli(n_samples=1_000_000, p=0.3, seed=0),
3434 "description": "Binary sequence with 30% probability of ones.",
3535 "tags": ["binary"],
@@ -37,7 +37,7 @@ datasets = [
3737 },
3838 {
3939 "name": "bernoulli-0.4",
40- "version": "1",
40+ "version": "3",
4141 "create": lambda: create_bernoulli(n_samples=1_000_000, p=0.4, seed=0),
4242 "description": "Binary sequence with 40% probability of ones.",
4343 "tags": ["binary"],
@@ -45,7 +45,7 @@ datasets = [
4545 },
4646 {
4747 "name": "bernoulli-0.5",
48- "version": "1",
48+ "version": "3",
4949 "create": lambda: create_bernoulli(n_samples=1_000_000, p=0.5, seed=0),
5050 "description": "Binary sequence with 50% probability of ones and 50% probability of zeros.",
5151 "tags": ["binary"],
zia_benchmark/src/zia_benchmark/run_benchmarks.pymodified+37−7View file
@@ -161,21 +161,46 @@ def run_benchmarks(
161161 upload_enabled = os.environ.get("UPLOAD_TO_MEMOBIN") == "1"
162162 if memobin_api_key and upload_enabled:
163163 try:
164- dataset_url = construct_dataset_url(
165- dataset["name"], dataset["version"]
164+ # Upload raw .dat format
165+ dataset_url_raw = construct_dataset_url(
166+ dataset["name"], dataset["version"], "dat"
166167 )
167- if not exists_in_memobin(dataset_url):
168+ if not exists_in_memobin(dataset_url_raw):
168169 if verbose:
169- print(" Uploading dataset to memobin...")
170+ print(" Uploading dataset (raw) to memobin...")
170171 upload_to_memobin(
171172 data.tobytes(),
172- dataset_url,
173+ dataset_url_raw,
173174 os.environ.get("MEMOBIN_USER_ID", "default"),
174175 memobin_api_key,
175176 content_type="application/octet-stream",
176177 )
177178 if verbose:
178- print(" Successfully uploaded dataset")
179+ print(" Successfully uploaded raw dataset")
180+
181+ # Upload .npy format
182+ dataset_url_npy = construct_dataset_url(
183+ dataset["name"], dataset["version"], "npy"
184+ )
185+ if not exists_in_memobin(dataset_url_npy):
186+ if verbose:
187+ print(" Uploading dataset (npy) to memobin...")
188+ # Save array to a temporary .npy file
189+ temp_npy = os.path.join(cache_dir, "temp.npy")
190+ np.save(temp_npy, data)
191+ with open(temp_npy, "rb") as f:
192+ npy_bytes = f.read()
193+ os.remove(temp_npy) # Clean up temp file
194+
195+ upload_to_memobin(
196+ npy_bytes,
197+ dataset_url_npy,
198+ os.environ.get("MEMOBIN_USER_ID", "default"),
199+ memobin_api_key,
200+ content_type="application/octet-stream",
201+ )
202+ if verbose:
203+ print(" Successfully uploaded npy dataset")
179204 except Exception as e:
180205 print(
181206 f" Warning: Failed to upload dataset to memobin: {str(e)}"
@@ -319,7 +344,12 @@ def run_benchmarks(
319344 "description": dataset.get("description", ""),
320345 "version": dataset["version"],
321346 "tags": dataset.get("tags", []),
322- "data_url": construct_dataset_url(dataset["name"], dataset["version"]),
347+ "data_url_raw": construct_dataset_url(
348+ dataset["name"], dataset["version"], "dat"
349+ ),
350+ "data_url_npy": construct_dataset_url(
351+ dataset["name"], dataset["version"], "npy"
352+ ),
323353 }
324354 if "source_file" in dataset:
325355 info["source_file"] = GITHUB_DATASETS_PREFIX + dataset["source_file"]