1/* eslint-disable @typescript-eslint/no-explicit-any */
2import { Canceler, postRemoteH5WorkerRequest } from "./helpers";
3import RemoteH5FileLindi, {
4 getRemoteH5FileLindi,
5} from "./lindi/RemoteH5FileLindi";
7export type RemoteH5FileX =
8 | RemoteH5File
9 | MergedRemoteH5File
10 | RemoteH5FileLindi;
12export type RemoteH5Group = {
13 path: string;
14 subgroups: RemoteH5Subgroup[];
15 datasets: RemoteH5Subdataset[];
16 attrs: { [key: string]: any };
17};
19export type RemoteH5Subgroup = {
20 name: string;
21 path: string;
22 attrs: { [key: string]: any };
23};
25export type RemoteH5Subdataset = {
26 name: string;
27 path: string;
28 shape: number[];
29 dtype: string;
30 attrs: { [key: string]: any };
31 chunks?: number[];
32 compressor?: string;
33 filters?: string[];
34};
36export type RemoteH5Dataset = {
37 name: string;
38 path: string;
39 shape: number[];
40 dtype: string;
41 attrs: { [key: string]: any };
42 chunks?: number[];
43 compressor?: string;
44 filters?: string[];
45};
47export type DatasetDataType =
48 | Float32Array
49 | Float64Array
50 | Int8Array
51 | Int16Array
52 | Int32Array
53 | Uint8Array
54 | Uint16Array
55 | Uint32Array;
57const defaultChunkSize = 1024 * 100;
58// const defaultChunkSize = 1024 * 1024 * 2
60export const globalRemoteH5FileStats = {
61 getGroupCount: 0,
62 getDatasetCount: 0,
63 getDatasetDataCount: 0,
64 numPendingRequests: 0,
65};
67type GetGroupResponse = {
68 success: boolean;
69 group?: RemoteH5Group;
70};
72type GetDatasetResponse = {
73 success: boolean;
74 dataset?: RemoteH5Dataset;
75};
77export class RemoteH5File {
78 #groupCache: { [path: string]: GetGroupResponse | null } = {}; // null means in progress
79 #datasetCache: { [path: string]: GetDatasetResponse | null } = {}; // null means in progress
80 #sourceUrls: string[] | undefined = undefined;
81 constructor(
82 public url: string,
83 private o: { chunkSize?: number },
84 ) {}
85 get dataIsRemote() {
86 return !this.url.startsWith("http://localhost");
87 }
88 async getGroup(path: string): Promise<RemoteH5Group | undefined> {
89 const cc = this.#groupCache[path];
90 if (cc) return cc.group;
91 if (cc === null) {
92 // in progress
93 while (this.#groupCache[path] === null) {
94 await new Promise((resolve) => setTimeout(resolve, 100));
95 }
96 const cc2 = this.#groupCache[path];
97 if (cc2) return cc2.group;
98 else throw Error("Unexpected");
99 }
100 this.#groupCache[path] = null;
101 const dummyCanceler = { onCancel: [] };
102 let resp;
103 try {
104 resp = await postRemoteH5WorkerRequest(
105 {
106 type: "getGroup",
107 url: this.url,
108 path,
109 chunkSize: this.o.chunkSize || defaultChunkSize,
110 },
111 dummyCanceler,
112 );
113 } catch {
114 this.#groupCache[path] = { success: false };
115 return undefined;
116 }
117 this.#groupCache[path] = resp;
118 globalRemoteH5FileStats.getGroupCount++;
119 return resp.group;
120 }
121 async getDataset(path: string): Promise<RemoteH5Dataset | undefined> {
122 const cc = this.#datasetCache[path];
123 if (cc) return cc.dataset;
124 if (cc === null) {
125 // in progress
126 while (this.#datasetCache[path] === null) {
127 await new Promise((resolve) => setTimeout(resolve, 100));
128 }
129 const cc2 = this.#datasetCache[path];
130 if (cc2) return cc2.dataset;
131 else throw Error("Unexpected");
132 }
133 this.#datasetCache[path] = null;
134 const dummyCanceler = { onCancel: [] };
135 let resp;
136 try {
137 resp = await postRemoteH5WorkerRequest(
138 {
139 type: "getDataset",
140 url: this.url,
141 path,
142 chunkSize: this.o.chunkSize || defaultChunkSize,
143 },
144 dummyCanceler,
145 );
146 } catch {
147 this.#datasetCache[path] = { success: false };
148 return undefined;
149 }
150 this.#datasetCache[path] = resp;
151 globalRemoteH5FileStats.getDatasetCount++;
152 return resp.dataset;
153 }
154 async getDatasetData(
155 path: string,
156 o: {
157 slice?: [number, number][];
158 allowBigInt?: boolean;
159 canceler?: Canceler;
160 },
161 ): Promise<DatasetDataType | undefined> {
162 if (o.slice) {
163 for (const ss of o.slice) {
164 if (isNaN(ss[0]) || isNaN(ss[1])) {
165 console.warn("Invalid slice", path, o.slice);
166 throw Error("Invalid slice");
167 }
168 }
169 }
170 const ds = await this.getDataset(path);
171 if (!ds) return undefined;
172 let urlToUse: string = this.url;
173 if (product(ds.shape) > 100) {
174 urlToUse = this.url;
175 }
177 const { slice, allowBigInt, canceler } = o;
178 const dummyCanceler = { onCancel: [] };
179 let resp;
180 try {
181 resp = await postRemoteH5WorkerRequest(
182 {
183 type: "getDatasetData",
184 url: urlToUse,
185 path,
186 slice,
187 chunkSize: this.o.chunkSize || defaultChunkSize,
188 },
189 canceler || dummyCanceler,
190 );
191 } catch {
192 return undefined;
193 }
194 const { data } = resp;
195 let x = data;
196 if (!allowBigInt) {
197 // check if x is a BigInt64Array
198 if (x && x.constructor && x.constructor.name === "BigInt64Array") {
199 // convert to Int32Array
200 const y = new Int32Array(x.length);
201 for (let i = 0; i < x.length; i++) {
202 y[i] = Number(x[i]);
203 }
204 x = y;
205 }
206 // check if x is a BigUint64Array
207 if (x && x.constructor && x.constructor.name === "BigUint64Array") {
208 // convert to Uint32Array
209 const y = new Uint32Array(x.length);
210 for (let i = 0; i < x.length; i++) {
211 y[i] = Number(x[i]);
212 }
213 x = y;
214 }
215 }
216 globalRemoteH5FileStats.getDatasetDataCount++;
217 return x;
218 }
219 getUrls() {
220 return [this.url];
221 }
222 get sourceUrls(): string[] | undefined {
223 return this.#sourceUrls;
224 }
225 set sourceUrls(v: string[] | undefined) {
226 this.#sourceUrls = v;
227 }
228}
230export class MergedRemoteH5File {
231 #files: RemoteH5FileX[];
232 #sourceUrls: string[] | undefined = undefined;
233 constructor(files: RemoteH5FileX[]) {
234 this.#files = files;
235 }
236 get dataIsRemote() {
237 return this.#files.some((f) => {
238 if (f instanceof RemoteH5File) {
239 return f.dataIsRemote;
240 } else if (f instanceof MergedRemoteH5File) {
241 // this case shouldn't happen - unfortunately we can't call f.dataIsRemote here because typescript doesn't allow it
242 return false;
243 } else {
244 throw Error("Unexpected");
245 }
246 });
247 }
248 async getGroup(path: string): Promise<RemoteH5Group | undefined> {
249 const allGroups: RemoteH5Group[] = [];
250 for (const f of this.#files) {
251 const gg = await f.getGroup(path);
252 if (gg) allGroups.push(gg);
253 }
254 console.log(`Got ${allGroups.length} groups`, path);
255 if (allGroups.length === 0) return undefined;
256 const ret = mergeGroups(allGroups);
257 return ret;
258 }
259 async getDataset(path: string): Promise<RemoteH5Dataset | undefined> {
260 for (const f of this.#files) {
261 const dd = await f.getDataset(path);
262 if (dd) {
263 // just return the first one
264 return dd;
265 }
266 }
267 return undefined;
268 }
269 async getDatasetData(
270 path: string,
271 o: {
272 slice?: [number, number][];
273 allowBigInt?: boolean;
274 canceler?: Canceler;
275 },
276 ): Promise<DatasetDataType | undefined> {
277 let canceled = false;
278 o.canceler?.onCancel.push(() => {
279 canceled = true;
280 });
281 for (const f of this.#files) {
282 const dd = await f.getDatasetData(path, o);
283 if (dd) {
284 // just return the first one
285 return dd;
286 }
287 if (canceled) return undefined;
288 }
289 return undefined;
290 }
291 getFiles() {
292 return this.#files;
293 }
294 getUrls(): string[] {
295 return this.#files.flatMap((f) => f.getUrls());
296 }
297 get sourceUrls(): string[] | undefined {
298 return this.#sourceUrls;
299 }
300 set sourceUrls(v: string[] | undefined) {
301 this.#sourceUrls = v;
302 }
303}
305const mergeGroups = (groups: RemoteH5Group[]): RemoteH5Group => {
306 if (groups.length === 0) throw Error("Unexpected groups.length == 0");
307 const ret: RemoteH5Group = {
308 path: groups[0].path,
309 subgroups: [],
310 datasets: [],
311 attrs: {},
312 };
313 const allSubgroupNames: string[] = [];
314 const allDatasetNames: string[] = [];
315 for (const g of groups) {
316 for (const sg of g.subgroups) {
317 if (!allSubgroupNames.includes(sg.name)) {
318 allSubgroupNames.push(sg.name);
319 }
320 }
321 for (const ds of g.datasets) {
322 if (!allDatasetNames.includes(ds.name)) {
323 allDatasetNames.push(ds.name);
324 }
325 }
326 }
327 for (const sgName of allSubgroupNames) {
328 const subgroups: RemoteH5Subgroup[] = [];
329 for (const g of groups) {
330 const sg = g.subgroups.find((s) => s.name === sgName);
331 if (sg) subgroups.push(sg);
332 }
333 ret.subgroups.push(mergeSubgroups(subgroups));
334 }
335 for (const dsName of allDatasetNames) {
336 const datasets: RemoteH5Subdataset[] = [];
337 for (const g of groups) {
338 const ds = g.datasets.find((d) => d.name === dsName);
339 if (ds) datasets.push(ds);
340 }
341 // for the datasets we just use the first one
342 if (datasets.length > 0) {
343 ret.datasets.push(datasets[0]);
344 }
345 }
346 for (const g of groups) {
347 for (const key in g.attrs) {
348 if (!(key in ret.attrs)) {
349 // the first takes precedence
350 ret.attrs[key] = g.attrs[key];
351 }
352 }
353 }
354 return ret;
355};
357const mergeSubgroups = (subgroups: RemoteH5Subgroup[]): RemoteH5Subgroup => {
358 if (subgroups.length === 0) throw Error("Unexpected subgroups.length == 0");
359 const ret: RemoteH5Subgroup = {
360 name: subgroups[0].name,
361 path: subgroups[0].path,
362 attrs: {},
363 };
364 for (const g of subgroups) {
365 for (const key in g.attrs) {
366 if (!(key in ret.attrs)) {
367 // the first takes precedence
368 ret.attrs[key] = g.attrs[key];
369 }
370 }
371 }
372 return ret;
373};
375const globalRemoteH5Files: { [url: string]: RemoteH5File } = {};
376export const getRemoteH5File = async (url: string) => {
377 const kk = url;
378 if (!globalRemoteH5Files[kk]) {
379 globalRemoteH5Files[kk] = new RemoteH5File(url, {});
380 }
381 return globalRemoteH5Files[kk];
382};
384const globalMergedRemoteH5Files: { [kk: string]: MergedRemoteH5File } = {};
385export const getMergedRemoteH5File = async (
386 urls: string[],
387 storageType: ("h5" | "zarr" | "lindi")[],
388) => {
389 if (urls.length === 0) throw Error(`Length of urls must be > 0`);
390 if (storageType.length !== urls.length)
391 throw Error(`Length of storageType must be equal to length of urls`);
392 if (urls.length === 1) {
393 if (storageType[0] === "lindi") {
394 return await getRemoteH5FileLindi(urls[0]);
395 } else {
396 return await getRemoteH5File(urls[0]);
397 }
398 }
399 const kk = urls.join("|");
400 if (!globalMergedRemoteH5Files[kk]) {
401 const files = await Promise.all(
402 urls.map((url, i) => {
403 if (storageType[i] === "lindi") {
404 return getRemoteH5FileLindi(url);
405 } else {
406 return getRemoteH5File(url);
407 }
408 }),
409 );
410 globalMergedRemoteH5Files[kk] = new MergedRemoteH5File(files);
411 }
412 return globalMergedRemoteH5Files[kk];
413};
415const product = (x: number[]) => {
416 let p = 1;
417 for (let i = 0; i < x.length; i++) p *= x[i];
418 return p;
419};