2 | Uint8Array
3 | Uint16Array
4 | Uint32Array
5 | Int16Array
6 | Int32Array;
8interface ChunkCache {
9 [key: number]: SupportedTypedArray;
10}
12type DType = "uint8" | "uint16" | "uint32" | "int16" | "int32";
14const TypedArrayConstructors = {
15 uint8: Uint8Array,
16 uint16: Uint16Array,
17 uint32: Uint32Array,
18 int16: Int16Array,
19 int32: Int32Array,
20} as const;
22export class TimeseriesDataClient {
23 private shape: number = 0;
24 private dtype: DType | null = null;
25 private chunkSize: number;
26 private cache: ChunkCache = {};
88280f2improve loading speedJeremy Magland 27 private inProgressFetches: { [key: number]: Promise<SupportedTypedArray> } =
28 {};
30 private datasetDataUrl: string;
32 constructor(
33 datasetJsonUrl: string,
34 datasetDataUrl: string,
35 chunkSize: number = 100000,
36 ) {
37 this.datasetJsonUrl = datasetJsonUrl;
38 this.datasetDataUrl = datasetDataUrl;
39 this.chunkSize = chunkSize;
40 }
42 static async create(
43 datasetJsonUrl: string,
44 datasetDataUrl: string,
45 chunkSize: number = 1000,
46 ): Promise<TimeseriesDataClient> {
47 const client = new TimeseriesDataClient(
48 datasetJsonUrl,
49 datasetDataUrl,
50 chunkSize,
51 );
52 await client.initialize();
53 return client;
54 }
56 private async initialize() {
57 const infoUrl = this.datasetJsonUrl;
58 const response = await fetch(infoUrl);
59 if (!response.ok) {
60 throw new Error(`Failed to fetch dataset info: ${response.statusText}`);
61 }
62 const info = await response.json();
63 this.shape = info.shape[0];
65 if (!this.isValidDType(info.dtype)) {
66 throw new Error(`Unsupported data type: ${info.dtype}`);
67 }
68 this.dtype = info.dtype;
69 }
71 private isValidDType(dtype: string): dtype is DType {
72 return dtype in TypedArrayConstructors;
73 }
75 private getChunkIndices(start: number, end: number): number[] {
76 const startChunk = Math.floor(start / this.chunkSize);
77 const endChunk = Math.floor(end / this.chunkSize);
78 const chunks: number[] = [];
79 for (let i = startChunk; i <= endChunk; i++) {
80 chunks.push(i);
81 }
82 return chunks;
83 }
85 private async fetchChunk(chunkIndex: number): Promise<SupportedTypedArray> {
88 return this.cache[chunkIndex];
89 }
88280f2improve loading speedJeremy Magland 91 // If this chunk is already being fetched, wait for it to complete
92 const inProgressFetch = this.inProgressFetches[chunkIndex];
93 if (inProgressFetch !== undefined) {
94 return inProgressFetch;
98 const fetchPromise = (async () => {
99 if (!this.dtype) {
100 throw new Error("Data type not initialized");
101 }
103 const start = chunkIndex * this.chunkSize;
104 const end = Math.min(start + this.chunkSize, this.shape);
105 const url = this.datasetDataUrl;
106 const itemSize = TypedArrayConstructors[this.dtype].BYTES_PER_ELEMENT;
107 const byteStart = start * itemSize;
108 const byteEnd = end * itemSize;
110 try {
111 const response = await fetch(url, {
112 headers: {
113 Range: `bytes=${byteStart}-${byteEnd - 1}`,
114 },
115 });
116 if (!response.ok) {
117 throw new Error(`Failed to fetch chunk: ${response.statusText}`);
118 }
120 const buffer = await response.arrayBuffer();
121 const ArrayConstructor = TypedArrayConstructors[this.dtype];
122 const data = new ArrayConstructor(buffer);
123 this.cache[chunkIndex] = data;
124 return data;
125 } finally {
126 // Clean up the in-progress fetch regardless of success/failure
127 delete this.inProgressFetches[chunkIndex];
128 }
129 })();
131 // Store the promise for other requests to wait on
132 this.inProgressFetches[chunkIndex] = fetchPromise;
133 return fetchPromise;
136 async fetchRange(start: number, end: number): Promise<SupportedTypedArray> {
137 if (!this.dtype) {
138 throw new Error("Data type not initialized");
139 }
141 const chunkIndices = this.getChunkIndices(start, end);
142 const chunks = await Promise.all(
143 chunkIndices.map((idx) => this.fetchChunk(idx)),
144 );
146 // Calculate total length needed
147 const length = end - start;
148 const ArrayConstructor = TypedArrayConstructors[this.dtype];
149 const result = new ArrayConstructor(length);
151 // Copy data from chunks into result array
152 let resultOffset = 0;
153 for (let i = 0; i < chunks.length; i++) {
154 const chunk = chunks[i];
155 const chunkStart = chunkIndices[i] * this.chunkSize;
156 const copyStart = Math.max(0, start - chunkStart);
157 const copyEnd = Math.min(chunk.length, end - chunkStart);
158 const copyLength = copyEnd - copyStart;
159 result.set(chunk.subarray(copyStart, copyEnd), resultOffset);
160 resultOffset += copyLength;
161 }
163 return result;
164 }
166 getShape(): number {
167 return this.shape;
168 }
170 getDType(): DType | null {
171 return this.dtype;
172 }
173}