1// @ts-nocheck
2import { ZMetaDataZArray } from "./RemoteH5FileLindi";
3import zarrDecodeChunkArray from "./zarrDecodeChunkArray";
5/* eslint-disable @typescript-eslint/no-explicit-any */
6export type ReferenceFileSystemObject = {
7 version?: any;
8 refs: { [key: string]: string | [string, number, number] };
9 templates?: { [key: string]: string };
10};
12export const isReferenceFileSystemObject = (
13 x: any,
14): x is ReferenceFileSystemObject => {
15 if (!x) return false;
16 if (typeof x !== "object") return false;
17 if (!x.refs) return false;
18 return true;
19};
21export interface RemoteTarInterface {
22 url: string;
23 getByteRangeForFile: (
24 fileName: string,
25 ) => Promise<{ startByte: number; endByte: number }>;
26}
28export class ReferenceFileSystemClient {
29 #fileContentCache: {
30 [key: string]: { content: any | undefined; found: boolean };
31 } = {};
32 #inProgressReads: { [key: string]: boolean } = {};
33 constructor(
34 private obj: ReferenceFileSystemObject,
35 private remoteTar: RemoteTarInterface | undefined,
36 ) {}
37 async readJson(path: string): Promise<{ [key: string]: any } | undefined> {
38 const buf = await this.readBinary(path, { decodeArray: false });
39 if (!buf) return undefined;
40 const text = new TextDecoder().decode(buf);
41 // replace NaN by "NaN" so that JSON.parse doesn't choke on it
42 // text = text.replace(/NaN/g, '"___NaN___"'); // This is not ideal. See: https://stackoverflow.com/a/15228712
43 // BUT we want to make sure we don't replace NaN within quoted strings
44 // Here's an example where this matters: https://neurosift.app/?p=/nwb&dandisetId=000409&dandisetVersion=draft&url=https://api.dandiarchive.org/api/assets/54b277ce-2da7-4730-b86b-cfc8dbf9c6fd/download/
45 // raw/intervals/contrast_left
46 let newText: string;
47 if (text.includes("NaN")) {
48 newText = "";
49 let inString = false;
50 let isEscaped = false;
51 for (let i = 0; i < text.length; i++) {
52 const c = text[i];
53 if (c === '"' && !isEscaped) inString = !inString;
54 if (!inString && c === "N" && text.slice(i, i + 3) === "NaN") {
55 newText += '"___NaN___"';
56 i += 2;
57 } else {
58 newText += c;
59 }
60 isEscaped = c === "\\" && !isEscaped;
61 }
62 } else {
63 newText = text;
64 }
65 try {
66 return JSON.parse(newText, (_key, value) => {
67 if (value === "___NaN___") return NaN;
68 return value;
69 });
70 } catch (e) {
71 console.warn(text);
72 throw Error("Failed to parse JSON for " + path + ": " + e);
73 }
74 }
75 async readBinary(
76 path: string,
77 o: {
78 decodeArray?: boolean;
79 startByte?: number;
80 endByte?: number;
81 disableCache?: boolean;
82 },
83 ): Promise<any | undefined> {
84 if (o.startByte !== undefined) {
85 if (o.decodeArray)
86 throw Error("Cannot decode array and read a slice at the same time");
87 if (o.endByte === undefined)
88 throw Error("If you specify startByte, you must also specify endByte");
89 } else if (o.endByte !== undefined) {
90 throw Error("If you specify endByte, you must also specify startByte");
91 }
92 if (
93 o.endByte !== undefined &&
94 o.startByte !== undefined &&
95 o.endByte < o.startByte
96 ) {
97 throw Error(
98 `endByte must be greater than or equal to startByte: ${o.startByte} ${o.endByte} for ${path}`,
99 );
100 }
101 if (
102 o.endByte !== undefined &&
103 o.startByte !== undefined &&
104 o.endByte === o.startByte
105 ) {
106 return new ArrayBuffer(0);
107 }
108 const kk =
109 path +
110 "|" +
111 (o.decodeArray ? "decode" : "") +
112 "|" +
113 o.startByte +
114 "|" +
115 o.endByte;
116 while (this.#inProgressReads[kk]) {
117 await new Promise((resolve) => setTimeout(resolve, 100));
118 }
119 this.#inProgressReads[kk] = true;
120 try {
121 if (path.startsWith("/")) path = path.slice(1);
122 if (this.#fileContentCache[kk]) {
123 if (this.#fileContentCache[kk].found) {
124 return this.#fileContentCache[kk].content;
125 }
126 return undefined;
127 }
128 const ref = this.obj.refs[path];
129 if (!ref) return undefined;
130 let buf: ArrayBuffer | undefined;
131 if (typeof ref === "string") {
132 if (ref.startsWith("base64:")) {
133 buf = _base64ToArrayBuffer(ref.slice("base64:".length));
134 } else {
135 // just a string
136 buf = new TextEncoder().encode(ref).buffer;
137 }
138 if (o.startByte !== undefined) {
139 buf = buf.slice(o.startByte, o.endByte);
140 }
141 } else if (typeof ref === "object" && Array.isArray(ref)) {
142 if (ref.length !== 3) throw Error(`Invalid ref for ${path}`);
143 let refUrl = this._applyTemplates(ref[0]);
144 let start = ref[1];
145 let numBytes = ref[2];
146 if (refUrl.startsWith("./") && this.remoteTar) {
147 const { startByte } = await this._getByteRangeForFileInRemoteTar(
148 refUrl.slice("./".length),
149 );
150 refUrl = this.remoteTar.url;
151 start = startByte + start;
152 }
153 if (o.startByte !== undefined) {
154 start += o.startByte;
155 numBytes = o.endByte! - o.startByte;
156 }
157 let url0 = refUrl;
158 if (o.disableCache) {
159 url0 += `?cacheBust=${Date.now()}`;
160 }
161 const r = await fetch(url0, {
162 headers: {
163 Range: `bytes=${start}-${start + numBytes - 1}`,
164 },
165 });
166 if (!r.ok) throw Error("Failed to fetch " + refUrl);
167 buf = await r.arrayBuffer();
168 } else if (typeof ref === "object") {
169 buf = new TextEncoder().encode(JSON.stringify(ref)).buffer;
170 } else {
171 throw Error("Invalid ref for " + path);
172 }
173 if (o.decodeArray) {
174 const parentPath = path.split("/").slice(0, -1).join("/");
175 const zarray = (await this.readJson(parentPath + "/.zarray")) as
176 | ZMetaDataZArray
177 | undefined;
178 if (!zarray) throw Error("Failed to read .zarray for " + path);
179 try {
180 buf = await zarrDecodeChunkArray(
181 buf,
182 zarray.dtype,
183 zarray.compressor,
184 zarray.filters,
185 zarray.chunks,
186 );
187 } catch (e) {
188 throw Error(`Failed to decode chunk array for ${path}: ${e}`);
189 }
190 }
191 if (buf) {
192 this.#fileContentCache[kk] = { content: buf, found: true };
193 } else {
194 this.#fileContentCache[kk] = { content: undefined, found: false };
195 }
196 return buf;
197 } catch (e) {
198 this.#fileContentCache[kk] = { content: undefined, found: false }; // important to do this so we don't keep trying to read the same file
199 throw e;
200 } finally {
201 this.#inProgressReads[kk] = false;
202 }
203 }
204 get _refs() {
205 return this.obj.refs;
206 }
207 _applyTemplates(s: string): string {
208 if (s.includes("{{") && s.includes("}}") && this.obj.templates) {
209 for (const [k, v] of Object.entries(this.obj.templates)) {
210 s = s.replace("{{" + k + "}}", v);
211 }
212 return s;
213 } else {
214 return s;
215 }
216 }
217 private async _getByteRangeForFileInRemoteTar(fileName: string) {
218 if (!this.remoteTar) {
219 throw Error("Unexpected");
220 }
221 const { startByte, endByte } =
222 await this.remoteTar.getByteRangeForFile(fileName);
223 return { startByte, endByte };
224 }
225}
227function _base64ToArrayBuffer(base64: string) {
228 const binary_string = window.atob(base64);
229 const bytes = new Uint8Array(binary_string.length);
230 for (let i = 0; i < binary_string.length; i++) {
231 bytes[i] = binary_string.charCodeAt(i);
232 }
233 return bytes;
234}
236export default ReferenceFileSystemClient;