mip integration: system store in IndexedDB, shared across projects
- mipSystem.ts: dedicated 'numbl-web-ide-system' IndexedDB database holding
mip core + installed packages; never shown in the explorer
- mip core fetched via the mip CORS proxy (cache-busted) and unzipped with
fflate on IDE open; 30-minute inactivity wipe with localStorage activity
stamp, numbl.org-style
- runs merge system files into the worker VFS under /system/ and put the
mip core dir on the search path; /system/ vfsChanges (package installs)
are persisted back to the store after each run
- sample scripts/mip_demo.m (inpoly) + smoke checks for install, figure,
persistence, and reuse after reload
10 changed files+274−13
README.mdmodified+17−0View file
@@ -19,8 +19,25 @@ each with its own file system, addressed as `#/project/<id>`. Create a
1919 - Runs use the current editor contents (saved or not), execute in a Web
2020 Worker, and can be stopped with **⏹**.
2121 - Files a script writes (`fopen`/`fprintf`) appear in the Explorer.
22+- Scripts can install packages from the [mip](https://mip.sh) registry:
23+ `mip load --install inpoly` (see `scripts/mip_demo.m` in the sample).
2224 - The project name in the status bar takes you back to the project list.
2325
26+## mip packages
27+
28+The mip package manager and installed packages live in a **system store** —
29+a dedicated IndexedDB database shared by all projects — and never appear in
30+a project's Explorer (unlike numbl.org, which shows them under a `system/`
31+prefix). The mip core tooling is fetched in the background when the IDE
32+opens; every run merges the system files into the interpreter's virtual file
33+system and puts the mip directory on the search path, so `mip load`,
34+`mip install`, `mip list`, etc. work as in the numbl CLI/IDE. Packages a
35+script installs are written back to the system store after the run, so
36+they're reused across runs, reloads, and projects. After **30 minutes of
37+inactivity** the store is wiped and mip core is re-fetched on next use
38+(keeping the mutable `mip-numbl` release fresh); installed packages
39+re-install on demand via `mip load --install`.
40+
2441 ## Development
2542
2643 minwebide is consumed as a sibling checkout (`file:../minwebide`):
package-lock.jsonmodified+1−0View file
@@ -8,6 +8,7 @@
88 "name": "numbl-web-ide",
99 "version": "0.1.0",
1010 "dependencies": {
11+ "fflate": "^0.8.3",
1112 "minwebide": "file:../minwebide",
1213 "numbl": "^0.4.8",
1314 "react": "^19.2.7",
package.jsonmodified+1−0View file
@@ -11,6 +11,7 @@
1111 "smoke": "node scripts/smoke.mjs"
1212 },
1313 "dependencies": {
14+ "fflate": "^0.8.3",
1415 "minwebide": "file:../minwebide",
1516 "numbl": "^0.4.8",
1617 "react": "^19.2.7",
scripts/smoke.mjsmodified+25−0View file
@@ -86,6 +86,31 @@ try {
8686 check('written file appears in explorer', await page.getByText('summary.txt', { exact: true }).count() === 1);
8787 await page.screenshot({ path: out + '/n-results.png' });
8888
89+ // mip: first run downloads mip core + the inpoly package
90+ await page.getByText('mip_demo.m', { exact: true }).click();
91+ await page.waitForTimeout(600);
92+ await page.locator('.mw-tab-action[title="numbl"]').click();
93+ check('mip demo output (install + run)', await waitForOutput('points inside: 201 / 2000', 90000));
94+ await page.waitForTimeout(1000);
95+ check('mip demo figure', await page.locator('.mw-auxbar .mw-panel-tab', { hasText: 'Figure 1' }).count() === 1);
96+ await page.screenshot({ path: out + '/n-mip.png' });
97+ check('system store persisted', await page.evaluate(async () =>
98+ (await indexedDB.databases()).some((db) => db.name === 'numbl-web-ide-system')));
99+
100+ // mip: after a full reload the persisted install is reused
101+ const projectUrl = page.url();
102+ await page.goto('about:blank');
103+ await page.goto(projectUrl, { waitUntil: 'networkidle' });
104+ await page.waitForTimeout(1500);
105+ await page.getByText('scripts', { exact: true }).click();
106+ await page.waitForTimeout(400);
107+ await page.getByText('mip_demo.m', { exact: true }).click();
108+ await page.waitForTimeout(600);
109+ const reuseStart = Date.now();
110+ await page.locator('.mw-tab-action[title="numbl"]').click();
111+ check('mip reuse after reload', await waitForOutput('points inside: 201 / 2000', 30000));
112+ console.log(` (reuse run took ${((Date.now() - reuseStart) / 1000).toFixed(1)}s)`);
113+
89114 // project lifecycle basics
90115 await page.getByTitle('Back to projects').click();
91116 await page.waitForTimeout(800);
src/ide.tsmodified+7−1View file
@@ -1,4 +1,5 @@
11 import { createWorkbench, type WorkbenchTheme } from 'minwebide';
2+import { createMipSystem } from './numbl/mipSystem';
23 import { createNumblRunner } from './numbl/runner';
34 import { openProjectFileSystem, touchProject, type ProjectInfo } from './projects';
45
@@ -8,6 +9,10 @@ export async function openIde(container: HTMLElement, project: ProjectInfo, them
89 document.title = `${project.name} — numbl web IDE`;
910
1011 const fs = await openProjectFileSystem(project.id);
12+ const mip = await createMipSystem();
13+ // fetch/refresh mip core in the background; runs await it (and say so
14+ // in the output channel if they actually have to wait)
15+ void mip.ensureCore().catch(() => { /* retried on the next run */ });
1116
1217 const workbench = createWorkbench(container, {
1318 fileSystem: fs,
@@ -15,7 +20,7 @@ export async function openIde(container: HTMLElement, project: ProjectInfo, them
1520 workspaceName: project.name,
1621 });
1722
18- const numbl = createNumblRunner(fs, workbench);
23+ const numbl = createNumblRunner(fs, workbench, mip);
1924 workbench.registerRunner(numbl.runner);
2025
2126 // the project indicator: click to go back to the project list
@@ -41,6 +46,7 @@ export async function openIde(container: HTMLElement, project: ProjectInfo, them
4146 numbl.dispose();
4247 workbench.dispose();
4348 fs.dispose();
49+ mip.dispose();
4450 },
4551 };
4652 }
src/numbl/mipSystem.tsadded+148−0View file
@@ -0,0 +1,148 @@
1+import { createIndexedDBFileSystem, type WorkspaceFileSystem } from 'minwebide';
2+import { unzipSync } from 'fflate';
3+import type { VfsFile } from './protocol';
4+
5+// The mip "system directory": numbl's package-manager tooling plus any
6+// packages that scripts install with `mip load --install <pkg>`. It mirrors
7+// the numbl browser IDE's __system__ store, with one difference by design:
8+// it lives in its own IndexedDB database, shared by all projects, and is
9+// never shown in the explorer — the mip directory is not part of a project.
10+//
11+// Freshness follows the numbl browser IDE: an activity stamp in
12+// localStorage; when the system dir has been inactive longer than the wipe
13+// interval, it is cleared and mip core is re-fetched (the mip-numbl release
14+// tag is mutable, so this is what picks up new mip versions). Installed
15+// packages are wiped too — scripts use `mip load --install`, which simply
16+// reinstalls on demand.
17+
18+const SYSTEM_DB_NAME = 'numbl-web-ide-system';
19+const ACTIVITY_KEY = 'numbl-web-ide.systemLastActivity';
20+const INACTIVITY_WIPE_MS = 30 * 60 * 1000;
21+
22+const MHL_URL = 'https://github.com/mip-org/mip-core/releases/download/mip-numbl/mip-numbl-any.mhl';
23+const MIP_CORE_PREFIX = '/mip/packages/gh/mip-org/core/mip';
24+/** Search path (inside the run VFS) of the mip core package. */
25+export const MIP_SEARCH_PATH = `/system${MIP_CORE_PREFIX}/mip`;
26+
27+/** GitHub release assets have no CORS headers; rewrite through the proxy
28+ * numbl.org uses, cache-busting because the release tag is mutable. */
29+function proxiedUrl(url: string): string {
30+ if (/^https:\/\/github\.com\/.+\/releases\/download\/.+/.test(url)) {
31+ let proxied = url.replace('https://github.com/', 'https://mip-cors-proxy.figurl.workers.dev/gh/');
32+ proxied += proxied.includes('?') ? '&' : '?';
33+ proxied += 't=' + Date.now();
34+ return proxied;
35+ }
36+ return url;
37+}
38+
39+export function markSystemActivity(): void {
40+ localStorage.setItem(ACTIVITY_KEY, String(Date.now()));
41+}
42+
43+function isSystemStale(): boolean {
44+ const raw = localStorage.getItem(ACTIVITY_KEY);
45+ const last = raw ? Number(raw) : 0;
46+ return !Number.isFinite(last) || Date.now() - last > INACTIVITY_WIPE_MS;
47+}
48+
49+async function listFiles(fs: WorkspaceFileSystem, path = '/'): Promise<{ path: string }[]> {
50+ const result: { path: string }[] = [];
51+ const stat = await fs.fileService.resolve(fs.root.with({ path }));
52+ for (const child of stat.children ?? []) {
53+ if (child.isDirectory) {
54+ result.push(...await listFiles(fs, child.resource.path));
55+ } else {
56+ result.push({ path: child.resource.path });
57+ }
58+ }
59+ return result;
60+}
61+
62+async function wipeSystem(fs: WorkspaceFileSystem): Promise<void> {
63+ const stat = await fs.fileService.resolve(fs.root);
64+ for (const child of stat.children ?? []) {
65+ await fs.fileService.del(child.resource, { recursive: true });
66+ }
67+}
68+
69+async function fetchMipCore(fs: WorkspaceFileSystem): Promise<void> {
70+ const response = await fetch(proxiedUrl(MHL_URL));
71+ if (!response.ok) {
72+ throw new Error(`mip core download failed: HTTP ${response.status}`);
73+ }
74+ const zip = new Uint8Array(await response.arrayBuffer());
75+ const entries = unzipSync(zip);
76+ for (const [entryPath, content] of Object.entries(entries)) {
77+ if (entryPath.endsWith('/')) {
78+ continue; // directory entry
79+ }
80+ await fs.writeFile(`${MIP_CORE_PREFIX}/${entryPath}`, content);
81+ }
82+}
83+
84+export interface MipSystem {
85+ /** Resolves when mip core is installed and fresh; memoized. */
86+ ensureCore(): Promise<void>;
87+ /** True once ensureCore() has resolved (i.e. a run won't have to wait). */
88+ isReady(): boolean;
89+ /** All system files, as run-VFS entries under /system/. */
90+ collectFiles(): Promise<VfsFile[]>;
91+ /** Persist a run's /system/ VFS changes back to the store. */
92+ applyChanges(changes: { created: VfsFile[]; modified: VfsFile[]; deleted: string[] }): Promise<void>;
93+ dispose(): void;
94+}
95+
96+export async function createMipSystem(): Promise<MipSystem> {
97+ const fs = await createIndexedDBFileSystem({ dbName: SYSTEM_DB_NAME });
98+ let ready = false;
99+ let corePromise: Promise<void> | undefined;
100+
101+ const ensureCore = (): Promise<void> => {
102+ if (!corePromise) {
103+ corePromise = (async () => {
104+ if (isSystemStale()) {
105+ await wipeSystem(fs);
106+ }
107+ const marker = fs.root.with({ path: `${MIP_CORE_PREFIX}/mip/mip.m` });
108+ if (!(await fs.fileService.exists(marker))) {
109+ await fetchMipCore(fs);
110+ }
111+ markSystemActivity();
112+ ready = true;
113+ })();
114+ // a failed install should retry on the next run, not cache the error
115+ corePromise.catch(() => { corePromise = undefined; });
116+ }
117+ return corePromise;
118+ };
119+
120+ return {
121+ ensureCore,
122+ isReady: () => ready,
123+ async collectFiles(): Promise<VfsFile[]> {
124+ const files: VfsFile[] = [];
125+ for (const { path } of await listFiles(fs)) {
126+ const content = (await fs.fileService.readFile(fs.root.with({ path }))).value.buffer;
127+ files.push({ path: `/system${path}`, content });
128+ }
129+ return files;
130+ },
131+ async applyChanges(changes): Promise<void> {
132+ for (const file of [...changes.created, ...changes.modified]) {
133+ if (file.path.startsWith('/system/')) {
134+ await fs.writeFile(file.path.slice('/system'.length), file.content);
135+ }
136+ }
137+ for (const path of changes.deleted) {
138+ if (path.startsWith('/system/')) {
139+ await fs.deleteFile(path.slice('/system'.length));
140+ }
141+ }
142+ markSystemActivity();
143+ },
144+ dispose(): void {
145+ fs.dispose();
146+ },
147+ };
148+}
src/numbl/numblWorker.tsmodified+1−0View file
@@ -51,6 +51,7 @@ self.onmessage = (event: MessageEvent<RunRequest>) => {
5151 },
5252 msg.workspaceFiles,
5353 mainAbsPath,
54+ msg.searchPaths,
5455 );
5556 // onDrawnow flushes (and clears) the instruction buffer mid-run, so
5657 // result.plotInstructions is only the tail since the last drawnow
src/numbl/protocol.tsmodified+7−4View file
@@ -5,9 +5,10 @@ import type { PlotInstruction, WorkspaceFile } from 'numbl';
55 // npm): one "run" request in, streamed output/drawnow messages out, ending
66 // with "done" or "error".
77
8-/** A raw file placed into the run's virtual file system (under /project). */
8+/** A raw file placed into the run's virtual file system. */
99 export interface VfsFile {
10- /** Project-relative path, e.g. 'scripts/demo.m'. */
10+ /** Project-relative path ('scripts/demo.m' → /project) or an absolute
11+ * VFS path such as '/system/mip/...'. */
1112 path: string;
1213 content: Uint8Array;
1314 }
@@ -18,10 +19,12 @@ export interface RunRequest {
1819 code: string;
1920 /** Project-relative path of the file being run. */
2021 mainFileName: string;
21- /** Every project file, as bytes, for the virtual file system. */
22+ /** Every project + system file, as bytes, for the virtual file system. */
2223 vfsFiles: VfsFile[];
23- /** Every other project .m file, as text, callable as functions. */
24+ /** Every other project + system .m file, as text, callable as functions. */
2425 workspaceFiles: WorkspaceFile[];
26+ /** Extra search paths (e.g. the mip core package directory). */
27+ searchPaths?: string[];
2528 }
2629
2730 export interface VfsChanges {
src/numbl/runner.tsmodified+35−8View file
@@ -1,6 +1,7 @@
11 import { monaco, type FileRunner, type RunContext, type Workbench, type WorkspaceFileSystem } from 'minwebide';
22 import type { WorkspaceFile } from 'numbl';
33 import { FigureManager } from './figures';
4+import { MIP_SEARCH_PATH, type MipSystem } from './mipSystem';
45 import type { RunRequest, VfsChanges, VfsFile, WorkerResponse } from './protocol';
56
67 // The .m file runner. Follows numbl.org's IDE semantics: the whole project is
@@ -32,7 +33,7 @@ async function collectProjectFiles(fs: WorkspaceFileSystem): Promise<VfsFile[]>
3233 return files;
3334 }
3435
35-export function createNumblRunner(fs: WorkspaceFileSystem, workbench: Workbench): { runner: FileRunner; dispose(): void } {
36+export function createNumblRunner(fs: WorkspaceFileSystem, workbench: Workbench, mip: MipSystem): { runner: FileRunner; dispose(): void } {
3637 const figures = new FigureManager(workbench);
3738 const decoder = new TextDecoder();
3839
@@ -50,8 +51,9 @@ export function createNumblRunner(fs: WorkspaceFileSystem, workbench: Workbench)
5051 if (!changes) {
5152 return;
5253 }
53- // scripts can write/delete files (save, csvwrite, delete, ...) — sync
54- // those changes from the run's VFS back into the project
54+ // scripts can write/delete files (fopen/fprintf, delete, ...) — sync
55+ // project changes back into the project, and /system/ changes (mip
56+ // package installs) into the shared system store
5557 for (const file of [...changes.created, ...changes.modified]) {
5658 if (file.path.startsWith('/project/')) {
5759 await fs.writeFile(file.path.slice('/project'.length), file.content);
@@ -62,17 +64,35 @@ export function createNumblRunner(fs: WorkspaceFileSystem, workbench: Workbench)
6264 await fs.deleteFile(path.slice('/project'.length));
6365 }
6466 }
67+ await mip.applyChanges(changes);
6568 };
6669
6770 const run = async (context: RunContext): Promise<void> => {
6871 context.output.clear();
6972 figures.beginRun();
7073
74+ // mip core is fetched in the background when the IDE opens; only
75+ // narrate when a run actually has to wait for it
76+ if (!mip.isReady()) {
77+ context.output.appendLine('[mip] installing package manager…');
78+ }
79+ await mip.ensureCore();
80+
7181 const mainFileName = context.uri.path.replace(/^\//, '');
72- const [code, allFiles] = await Promise.all([context.getText(), collectProjectFiles(fs)]);
73- const workspaceFiles: WorkspaceFile[] = allFiles
74- .filter(f => f.path !== mainFileName && f.path.endsWith('.m'))
75- .map(f => ({ name: f.path, source: decoder.decode(f.content) }));
82+ const [code, projectFiles, systemFiles] = await Promise.all([
83+ context.getText(),
84+ collectProjectFiles(fs),
85+ mip.collectFiles(),
86+ ]);
87+ const workspaceFiles: WorkspaceFile[] = [
88+ ...projectFiles
89+ .filter(f => f.path !== mainFileName && f.path.endsWith('.m'))
90+ .map(f => ({ name: f.path, source: decoder.decode(f.content) })),
91+ ...systemFiles
92+ .filter(f => f.path.endsWith('.m'))
93+ .map(f => ({ name: f.path, source: decoder.decode(f.content) })),
94+ ];
95+ const allFiles = [...projectFiles, ...systemFiles];
7696
7797 await new Promise<void>((resolve) => {
7898 const w = ensureWorker();
@@ -117,7 +137,14 @@ export function createNumblRunner(fs: WorkspaceFileSystem, workbench: Workbench)
117137 break;
118138 }
119139 };
120- const request: RunRequest = { type: 'run', code, mainFileName, vfsFiles: allFiles, workspaceFiles };
140+ const request: RunRequest = {
141+ type: 'run',
142+ code,
143+ mainFileName,
144+ vfsFiles: allFiles,
145+ workspaceFiles,
146+ searchPaths: [MIP_SEARCH_PATH],
147+ };
121148 w.postMessage(request);
122149 });
123150 };
src/sampleWorkspace.tsmodified+32−0View file
@@ -15,6 +15,10 @@ tab bar:
1515 - \`scripts/animation.m\` — figures update live during the run (\`drawnow\`).
1616 - \`scripts/write_results.m\` — writes a file; it appears in the Explorer
1717 when the run finishes.
18+- \`scripts/mip_demo.m\` — installs the \`inpoly\` package from the
19+ [mip](https://mip.sh) registry on first run (\`mip load --install\`) and
20+ uses it. Installed packages are cached in your browser, shared by all
21+ projects, and refreshed automatically after 30 minutes of inactivity.
1822
1923 Edits are saved with **Ctrl+S** and persist in your browser. Runs use the
2024 current editor contents, saved or not. Press **⏹** to stop a runaway run.
@@ -99,6 +103,33 @@ fclose(fid);
99103 disp('wrote results/summary.txt');
100104 `;
101105
106+const mipDemoM = `% mip package demo: installs the inpoly package (point-in-polygon test)
107+% from the mip registry (https://mip.sh) on first run, then reuses the
108+% cached install. See https://mip.sh for available packages.
109+mip load --install inpoly
110+
111+% a star-shaped polygon
112+k = (0:9)';
113+r = 1 + 0.6 * (-1).^k;
114+node = [r .* cos(pi/5 * k + pi/2), r .* sin(pi/5 * k + pi/2)];
115+
116+% classify random points
117+rng(42);
118+pts = 4 * rand(2000, 2) - 2;
119+inside = inpoly2(pts, node);
120+fprintf('points inside: %d / %d\\n', sum(inside), length(inside));
121+
122+figure;
123+plot(pts(inside, 1), pts(inside, 2), '.');
124+hold on;
125+plot(pts(~inside, 1), pts(~inside, 2), '.');
126+plot([node(:, 1); node(1, 1)], [node(:, 2); node(1, 2)]);
127+hold off;
128+axis equal;
129+title('inpoly2: points inside a star');
130+legend('inside', 'outside', 'polygon');
131+`;
132+
102133 export const sampleWorkspace: Record<string, string> = {
103134 '/README.md': readme,
104135 '/waves.m': wavesM,
@@ -106,5 +137,6 @@ export const sampleWorkspace: Record<string, string> = {
106137 '/scripts/surface_demo.m': surfaceDemoM,
107138 '/scripts/animation.m': animationM,
108139 '/scripts/write_results.m': writeResultsM,
140+ '/scripts/mip_demo.m': mipDemoM,
109141 '/lib/mexican_hat.m': mexicanHatM,
110142 };