upload dataset .json
6 changed files+153−11
web-ui/src/App.tsxmodified+5−0View file
@@ -3,6 +3,7 @@ import { useEffect, useState } from "react";
33 import axios from "axios";
44 import Home from "./pages/Home";
55 import Datasets from "./pages/Datasets";
6+import Dataset from "./pages/Dataset";
67 import Algorithms from "./pages/Algorithms";
78 import About from "./pages/About";
89 import { BenchmarkData } from "./types";
@@ -143,6 +144,10 @@ function App() {
143144 <Algorithms algorithms={benchmarkData?.algorithms || []} />
144145 }
145146 />
147+ <Route
148+ path="/dataset/:datasetName"
149+ element={<Dataset datasets={benchmarkData?.datasets || []} />}
150+ />
146151 <Route path="/about" element={<About />} />
147152 </Routes>
148153 )}
web-ui/src/pages/Dataset.tsxadded+119−0View file
@@ -0,0 +1,119 @@
1+import { useParams } from 'react-router-dom';
2+import { Dataset as DatasetType } from '../types';
3+
4+interface DatasetProps {
5+ datasets: DatasetType[];
6+}
7+
8+function Dataset({ datasets }: DatasetProps) {
9+ const { datasetName } = useParams<{ datasetName: string }>();
10+ const dataset = datasets.find(d => d.name === datasetName);
11+
12+ if (!dataset) {
13+ return <div>Dataset not found</div>;
14+ }
15+
16+ return (
17+ <div>
18+ <h1
19+ style={{
20+ fontSize: "2rem",
21+ fontWeight: "bold",
22+ color: "#333",
23+ marginBottom: "1rem",
24+ }}
25+ >
26+ {dataset.name}
27+ </h1>
28+ <div style={{ maxWidth: "800px", margin: "0 auto" }}>
29+ <div style={{ marginBottom: "1.5rem" }}>
30+ <h2 style={{ fontSize: "1.2rem", fontWeight: "bold", marginBottom: "0.5rem" }}>Description</h2>
31+ <p style={{ fontSize: "0.9rem", lineHeight: "1.5" }}>{dataset.description}</p>
32+ </div>
33+ <div style={{ marginBottom: "1.5rem" }}>
34+ <h2 style={{ fontSize: "1.2rem", fontWeight: "bold", marginBottom: "0.5rem" }}>Version</h2>
35+ <p style={{ fontSize: "0.9rem" }}>{dataset.version}</p>
36+ </div>
37+ <div style={{ marginBottom: "1.5rem" }}>
38+ <h2 style={{ fontSize: "1.2rem", fontWeight: "bold", marginBottom: "0.5rem" }}>Tags</h2>
39+ <div>
40+ {dataset.tags.map((tag) => (
41+ <span
42+ key={tag}
43+ style={{
44+ display: "inline-block",
45+ backgroundColor: "#e1e1e1",
46+ padding: "2px 6px",
47+ borderRadius: "3px",
48+ margin: "2px",
49+ fontSize: "0.8rem",
50+ }}
51+ >
52+ {tag}
53+ </span>
54+ ))}
55+ </div>
56+ </div>
57+ <div style={{ marginBottom: "1.5rem" }}>
58+ <h2 style={{ fontSize: "1.2rem", fontWeight: "bold", marginBottom: "0.5rem" }}>Downloads</h2>
59+ <div style={{ display: "flex", gap: "1rem" }}>
60+ {dataset.data_url_npy && (
61+ <a
62+ href={dataset.data_url_npy}
63+ download={`${dataset.name}-${dataset.version}.npy`}
64+ style={{
65+ color: "#0066cc",
66+ textDecoration: "none",
67+ padding: "4px 8px",
68+ backgroundColor: "#f0f0f0",
69+ borderRadius: "4px",
70+ fontSize: "0.9rem",
71+ }}
72+ >
73+ Download NPY
74+ </a>
75+ )}
76+ {dataset.data_url_raw && (
77+ <a
78+ href={dataset.data_url_raw}
79+ download={`${dataset.name}-${dataset.version}.dat`}
80+ style={{
81+ color: "#0066cc",
82+ textDecoration: "none",
83+ padding: "4px 8px",
84+ backgroundColor: "#f0f0f0",
85+ borderRadius: "4px",
86+ fontSize: "0.9rem",
87+ }}
88+ >
89+ Download RAW
90+ </a>
91+ )}
92+ </div>
93+ </div>
94+ {dataset.source_file && (
95+ <div style={{ marginBottom: "1.5rem" }}>
96+ <h2 style={{ fontSize: "1.2rem", fontWeight: "bold", marginBottom: "0.5rem" }}>Source</h2>
97+ <a
98+ href={dataset.source_file}
99+ target="_blank"
100+ rel="noopener noreferrer"
101+ style={{
102+ color: "#0066cc",
103+ textDecoration: "none",
104+ padding: "4px 8px",
105+ backgroundColor: "#f0f0f0",
106+ borderRadius: "4px",
107+ fontSize: "0.9rem",
108+ }}
109+ >
110+ View Source
111+ </a>
112+ </div>
113+ )}
114+ </div>
115+ </div>
116+ );
117+}
118+
119+export default Dataset;
web-ui/src/pages/Datasets.tsxmodified+1−1View file
@@ -106,7 +106,7 @@ function Datasets({ datasets }: DatasetsProps) {
106106 }}
107107 >
108108 <Link
109- to={`/home?dataset=${encodeURIComponent(dataset.name)}`}
109+ to={`/dataset/${encodeURIComponent(dataset.name)}`}
110110 style={{
111111 color: "#0066cc",
112112 textDecoration: "none",
zia_benchmark/src/zia_benchmark/_memobin.pymodified+2−4View file
@@ -88,7 +88,7 @@ def construct_dataset_url(
8888 Args:
8989 dataset_name: Name of the dataset
9090 dataset_version: Version of the dataset
91- format: File format ("dat" or "npy")
91+ format: File format ("dat", "npy", or "json")
9292
9393 Returns:
9494 The constructed memobin URL for the dataset
@@ -100,7 +100,6 @@ def construct_dataset_url(
100100 def upload_to_memobin(
101101 data: dict | bytes,
102102 url: str,
103- user_id: str,
104103 memobin_api_key: str,
105104 content_type: str = "application/json",
106105 ) -> None:
@@ -109,7 +108,6 @@ def upload_to_memobin(
109108 Args:
110109 data: The data to upload (dict for JSON or bytes for binary)
111110 url: The target URL for the file
112- user_id: User ID for memobin
113111 memobin_api_key: API key for memobin authentication
114112 content_type: Content type of the data
115113
@@ -122,7 +120,7 @@ def upload_to_memobin(
122120 data_bytes = data
123121 size = len(data_bytes)
124122
125- upload_url = create_signed_upload_url(url, size, user_id, memobin_api_key)
123+ upload_url = create_signed_upload_url(url, size, 'zia', memobin_api_key)
126124
127125 response = requests.put(
128126 upload_url, data=data_bytes, headers={"Content-Type": content_type}
zia_benchmark/src/zia_benchmark/algorithms/simple_ans/__init__.pymodified+3−3View file
@@ -240,14 +240,14 @@ def simple_ans_markov_decode(x: bytes, dtype: str) -> np.ndarray:
240240 algorithms = [
241241 {
242242 "name": "simple-ans",
243- "version": "2",
243+ "version": "3",
244244 "encode": lambda x: simple_ans_encode(x),
245245 "decode": lambda x, dtype: simple_ans_decode(x, dtype),
246246 "source_file": SOURCE_FILE,
247247 },
248248 {
249249 "name": "simple-ans-delta",
250- "version": "2",
250+ "version": "3",
251251 "encode": lambda x: simple_ans_delta_encode(x),
252252 "decode": lambda x, dtype: simple_ans_delta_decode(x, dtype),
253253 "tags": ["delta_encoding"],
@@ -255,7 +255,7 @@ algorithms = [
255255 },
256256 {
257257 "name": "simple-ans-markov",
258- "version": "2",
258+ "version": "3",
259259 "encode": lambda x: simple_ans_markov_encode(x),
260260 "decode": lambda x, dtype: simple_ans_markov_decode(x, dtype),
261261 "tags": ["markov_prediction"],
zia_benchmark/src/zia_benchmark/run_benchmarks.pymodified+23−3View file
@@ -161,6 +161,26 @@ def run_benchmarks(
161161 upload_enabled = os.environ.get("UPLOAD_TO_MEMOBIN") == "1"
162162 if memobin_api_key and upload_enabled:
163163 try:
164+ # Upload array metadata as JSON
165+ dataset_url_json = construct_dataset_url(
166+ dataset["name"], dataset["version"], "json"
167+ )
168+ if not exists_in_memobin(dataset_url_json):
169+ if verbose:
170+ print(" Uploading dataset metadata to memobin...")
171+ metadata = {
172+ "dtype": str(data.dtype),
173+ "shape": data.shape
174+ }
175+ upload_to_memobin(
176+ metadata,
177+ dataset_url_json,
178+ memobin_api_key,
179+ content_type="application/json",
180+ )
181+ if verbose:
182+ print(" Successfully uploaded metadata")
183+
164184 # Upload raw .dat format
165185 dataset_url_raw = construct_dataset_url(
166186 dataset["name"], dataset["version"], "dat"
@@ -171,7 +191,6 @@ def run_benchmarks(
171191 upload_to_memobin(
172192 data.tobytes(),
173193 dataset_url_raw,
174- os.environ.get("MEMOBIN_USER_ID", "default"),
175194 memobin_api_key,
176195 content_type="application/octet-stream",
177196 )
@@ -195,7 +214,6 @@ def run_benchmarks(
195214 upload_to_memobin(
196215 npy_bytes,
197216 dataset_url_npy,
198- os.environ.get("MEMOBIN_USER_ID", "default"),
199217 memobin_api_key,
200218 content_type="application/octet-stream",
201219 )
@@ -314,7 +332,6 @@ def run_benchmarks(
314332 upload_to_memobin(
315333 cache_data,
316334 memobin_url,
317- os.environ.get("MEMOBIN_USER_ID", "default"),
318335 memobin_api_key,
319336 )
320337 if verbose:
@@ -350,6 +367,9 @@ def run_benchmarks(
350367 "data_url_npy": construct_dataset_url(
351368 dataset["name"], dataset["version"], "npy"
352369 ),
370+ "data_url_json": construct_dataset_url(
371+ dataset["name"], dataset["version"], "json"
372+ ),
353373 }
354374 if "source_file" in dataset:
355375 info["source_file"] = GITHUB_DATASETS_PREFIX + dataset["source_file"]