1export type SupportedTypedArray =
2 | Uint8Array
3 | Uint16Array
4 | Uint32Array
5 | Int16Array
6 | Int32Array;
8interface ChunkCache {
9 [key: number]: SupportedTypedArray;
10}
12type DType = "uint8" | "uint16" | "uint32" | "int16" | "int32";
14const TypedArrayConstructors = {
15 uint8: Uint8Array,
16 uint16: Uint16Array,
17 uint32: Uint32Array,
18 int16: Int16Array,
19 int32: Int32Array,
20} as const;
22export class TimeseriesDataClient {
23 private shape: number = 0;
24 private dtype: DType | null = null;
25 private chunkSize: number;
26 private cache: ChunkCache = {};
27 private datasetJsonUrl: string;
28 private datasetDataUrl: string;
30 constructor(
31 datasetJsonUrl: string,
32 datasetDataUrl: string,
33 chunkSize: number = 100000,
34 ) {
35 this.datasetJsonUrl = datasetJsonUrl;
36 this.datasetDataUrl = datasetDataUrl;
37 this.chunkSize = chunkSize;
38 }
40 static async create(
41 datasetJsonUrl: string,
42 datasetDataUrl: string,
43 chunkSize: number = 1000,
44 ): Promise<TimeseriesDataClient> {
45 const client = new TimeseriesDataClient(
46 datasetJsonUrl,
47 datasetDataUrl,
48 chunkSize,
49 );
50 await client.initialize();
51 return client;
52 }
54 private async initialize() {
55 const infoUrl = this.datasetJsonUrl;
56 const response = await fetch(infoUrl);
57 if (!response.ok) {
58 throw new Error(`Failed to fetch dataset info: ${response.statusText}`);
59 }
60 const info = await response.json();
61 this.shape = info.shape[0];
63 if (!this.isValidDType(info.dtype)) {
64 throw new Error(`Unsupported data type: ${info.dtype}`);
65 }
66 this.dtype = info.dtype;
67 }
69 private isValidDType(dtype: string): dtype is DType {
70 return dtype in TypedArrayConstructors;
71 }
73 private getChunkIndices(start: number, end: number): number[] {
74 const startChunk = Math.floor(start / this.chunkSize);
75 const endChunk = Math.floor(end / this.chunkSize);
76 const chunks: number[] = [];
77 for (let i = startChunk; i <= endChunk; i++) {
78 chunks.push(i);
79 }
80 return chunks;
81 }
83 private async fetchChunk(chunkIndex: number): Promise<SupportedTypedArray> {
84 if (this.cache[chunkIndex]) {
85 return this.cache[chunkIndex];
86 }
88 if (!this.dtype) {
89 throw new Error("Data type not initialized");
90 }
92 const start = chunkIndex * this.chunkSize;
93 const end = Math.min(start + this.chunkSize, this.shape);
94 const url = this.datasetDataUrl;
95 const itemSize = TypedArrayConstructors[this.dtype].BYTES_PER_ELEMENT;
96 const byteStart = start * itemSize;
97 const byteEnd = end * itemSize;
99 const response = await fetch(url, {
100 headers: {
101 Range: `bytes=${byteStart}-${byteEnd - 1}`,
102 },
103 });
104 if (!response.ok) {
105 throw new Error(`Failed to fetch chunk: ${response.statusText}`);
106 }
108 const buffer = await response.arrayBuffer();
109 const ArrayConstructor = TypedArrayConstructors[this.dtype];
110 const data = new ArrayConstructor(buffer);
111 this.cache[chunkIndex] = data;
113 return data;
114 }
116 async fetchRange(start: number, end: number): Promise<SupportedTypedArray> {
117 if (!this.dtype) {
118 throw new Error("Data type not initialized");
119 }
121 const chunkIndices = this.getChunkIndices(start, end);
122 const chunks = await Promise.all(
123 chunkIndices.map((idx) => this.fetchChunk(idx)),
124 );
126 // Calculate total length needed
127 const length = end - start;
128 const ArrayConstructor = TypedArrayConstructors[this.dtype];
129 const result = new ArrayConstructor(length);
131 // Copy data from chunks into result array
132 let resultOffset = 0;
133 for (let i = 0; i < chunks.length; i++) {
134 const chunk = chunks[i];
135 const chunkStart = chunkIndices[i] * this.chunkSize;
136 const copyStart = Math.max(0, start - chunkStart);
137 const copyEnd = Math.min(chunk.length, end - chunkStart);
138 const copyLength = copyEnd - copyStart;
139 result.set(chunk.subarray(copyStart, copyEnd), resultOffset);
140 resultOffset += copyLength;
141 }
143 return result;
144 }
146 getShape(): number {
147 return this.shape;
148 }
150 getDType(): DType | null {
151 return this.dtype;
152 }
153}