caef63amip integration: system store in IndexedDB, shared across projectsJeremy Magland 1import { createIndexedDBFileSystem, type WorkspaceFileSystem } from 'minwebide';
2import { unzipSync } from 'fflate';
3import type { VfsFile } from './protocol';
5// The mip "system directory": numbl's package-manager tooling plus any
6// packages that scripts install with `mip load --install <pkg>`. It mirrors
7// the numbl browser IDE's __system__ store, with one difference by design:
8// it lives in its own IndexedDB database, shared by all projects, and is
9// never shown in the explorer — the mip directory is not part of a project.
10//
11// Freshness follows the numbl browser IDE: an activity stamp in
12// localStorage; when the system dir has been inactive longer than the wipe
13// interval, it is cleared and mip core is re-fetched (the mip-numbl release
14// tag is mutable, so this is what picks up new mip versions). Installed
15// packages are wiped too — scripts use `mip load --install`, which simply
16// reinstalls on demand.
18const SYSTEM_DB_NAME = 'numbl-web-ide-system';
19const ACTIVITY_KEY = 'numbl-web-ide.systemLastActivity';
20const INACTIVITY_WIPE_MS = 30 * 60 * 1000;
22const MHL_URL = 'https://github.com/mip-org/mip-core/releases/download/mip-numbl/mip-numbl-any.mhl';
23const MIP_CORE_PREFIX = '/mip/packages/gh/mip-org/core/mip';
24/** Search path (inside the run VFS) of the mip core package. */
25export const MIP_SEARCH_PATH = `/system${MIP_CORE_PREFIX}/mip`;
27/** GitHub release assets have no CORS headers; rewrite through the proxy
28 * numbl.org uses, cache-busting because the release tag is mutable. */
29function proxiedUrl(url: string): string {
30 if (/^https:\/\/github\.com\/.+\/releases\/download\/.+/.test(url)) {
31 let proxied = url.replace('https://github.com/', 'https://mip-cors-proxy.figurl.workers.dev/gh/');
32 proxied += proxied.includes('?') ? '&' : '?';
33 proxied += 't=' + Date.now();
34 return proxied;
35 }
36 return url;
37}
39export function markSystemActivity(): void {
40 localStorage.setItem(ACTIVITY_KEY, String(Date.now()));
41}
43function isSystemStale(): boolean {
44 const raw = localStorage.getItem(ACTIVITY_KEY);
45 const last = raw ? Number(raw) : 0;
46 return !Number.isFinite(last) || Date.now() - last > INACTIVITY_WIPE_MS;
47}
49async function listFiles(fs: WorkspaceFileSystem, path = '/'): Promise<{ path: string }[]> {
50 const result: { path: string }[] = [];
51 const stat = await fs.fileService.resolve(fs.root.with({ path }));
52 for (const child of stat.children ?? []) {
53 if (child.isDirectory) {
54 result.push(...await listFiles(fs, child.resource.path));
55 } else {
56 result.push({ path: child.resource.path });
57 }
58 }
59 return result;
60}
62async function wipeSystem(fs: WorkspaceFileSystem): Promise<void> {
63 const stat = await fs.fileService.resolve(fs.root);
64 for (const child of stat.children ?? []) {
65 await fs.fileService.del(child.resource, { recursive: true });
66 }
67}
69async function fetchMipCore(fs: WorkspaceFileSystem): Promise<void> {
70 const response = await fetch(proxiedUrl(MHL_URL));
71 if (!response.ok) {
72 throw new Error(`mip core download failed: HTTP ${response.status}`);
73 }
74 const zip = new Uint8Array(await response.arrayBuffer());
75 const entries = unzipSync(zip);
76 for (const [entryPath, content] of Object.entries(entries)) {
77 if (entryPath.endsWith('/')) {
78 continue; // directory entry
79 }
80 await fs.writeFile(`${MIP_CORE_PREFIX}/${entryPath}`, content);
81 }
82}
84export interface MipSystem {
85 /** Resolves when mip core is installed and fresh; memoized. */
86 ensureCore(): Promise<void>;
87 /** True once ensureCore() has resolved (i.e. a run won't have to wait). */
88 isReady(): boolean;
89 /** All system files, as run-VFS entries under /system/. */
90 collectFiles(): Promise<VfsFile[]>;
91 /** Persist a run's /system/ VFS changes back to the store. */
92 applyChanges(changes: { created: VfsFile[]; modified: VfsFile[]; deleted: string[] }): Promise<void>;
93 dispose(): void;
94}
96export async function createMipSystem(): Promise<MipSystem> {
97 const fs = await createIndexedDBFileSystem({ dbName: SYSTEM_DB_NAME });
98 let ready = false;
99 let corePromise: Promise<void> | undefined;
101 const ensureCore = (): Promise<void> => {
102 if (!corePromise) {
103 corePromise = (async () => {
104 if (isSystemStale()) {
105 await wipeSystem(fs);
106 }
107 const marker = fs.root.with({ path: `${MIP_CORE_PREFIX}/mip/mip.m` });
108 if (!(await fs.fileService.exists(marker))) {
109 await fetchMipCore(fs);
110 }
111 markSystemActivity();
112 ready = true;
113 })();
114 // a failed install should retry on the next run, not cache the error
115 corePromise.catch(() => { corePromise = undefined; });
116 }
117 return corePromise;
118 };
120 return {
121 ensureCore,
122 isReady: () => ready,
123 async collectFiles(): Promise<VfsFile[]> {
124 const files: VfsFile[] = [];
125 for (const { path } of await listFiles(fs)) {
126 const content = (await fs.fileService.readFile(fs.root.with({ path }))).value.buffer;
127 files.push({ path: `/system${path}`, content });
128 }
129 return files;
130 },
131 async applyChanges(changes): Promise<void> {
132 for (const file of [...changes.created, ...changes.modified]) {
133 if (file.path.startsWith('/system/')) {
134 await fs.writeFile(file.path.slice('/system'.length), file.content);
135 }
136 }
137 for (const path of changes.deleted) {
138 if (path.startsWith('/system/')) {
139 await fs.deleteFile(path.slice('/system'.length));
140 }
141 }
142 markSystemActivity();
143 },
144 dispose(): void {
145 fs.dispose();
146 },
147 };
148}