From 422bf3798d862af20010e871983ddec2c54e012c Mon Sep 17 00:00:00 2001 From: Jesus Armando Anaya <1445792+JArmandoAnaya@users.noreply.github.com> Date: Thu, 17 Sep 2026 11:34:10 -0700 Subject: [PATCH 01/15] feat(inference): validate browser model registry admissions --- .../data/browserInference/admissionCatalog.ts | 74 ++++++ .../browserInference/fixtures/manifests-v1.ts | 32 +++ .../fixtures/registry-v1.json | 40 ++++ .../app/src/data/browserInference/manifest.ts | 2 + .../browserInference/registryClient.test.ts | 153 ++++++++++++ .../data/browserInference/registryClient.ts | 217 ++++++++++++++++++ 6 files changed, 518 insertions(+) create mode 100644 frontend/app/src/data/browserInference/admissionCatalog.ts create mode 100644 frontend/app/src/data/browserInference/fixtures/manifests-v1.ts create mode 100644 frontend/app/src/data/browserInference/fixtures/registry-v1.json create mode 100644 frontend/app/src/data/browserInference/registryClient.test.ts create mode 100644 frontend/app/src/data/browserInference/registryClient.ts diff --git a/frontend/app/src/data/browserInference/admissionCatalog.ts b/frontend/app/src/data/browserInference/admissionCatalog.ts new file mode 100644 index 00000000..f707c9c5 --- /dev/null +++ b/frontend/app/src/data/browserInference/admissionCatalog.ts @@ -0,0 +1,74 @@ +export interface ArtifactAdmission { + readonly role: "encoder" | "decoder"; + readonly path: string; + readonly bytes: number; + readonly sha256: string; + readonly contentType: "application/octet-stream"; +} + +export interface BrowserModelAdmission { + readonly id: string; + readonly label: string; + readonly revision: string; + readonly modelRef: string; + readonly manifestPath: string; + readonly adapter: "efficient-sam-ti"; + readonly license: string; + readonly source: { + readonly repository: string; + readonly revision: string; + }; + readonly runtime: { + readonly format: "onnx"; + readonly opset: number; + readonly onnxruntimeWeb: string; + }; + readonly capabilities: { + readonly pointSuggest: true; + readonly positivePoints: true; + readonly negativePoints: false; + readonly maxPoints: number; + }; + readonly artifacts: readonly [ArtifactAdmission, ArtifactAdmission]; +} + +export const EFFICIENT_SAM_TI_ADMISSION: BrowserModelAdmission = Object.freeze({ + id: "efficient-sam-ti", + label: "EfficientSAM-Ti", + revision: "b19782d049c0-843761ca46f4", + modelRef: "robomous/efficient-sam-ti@b19782d049c0-843761ca46f4", + manifestPath: "/models/efficient-sam-ti/b19782d049c0-843761ca46f4/manifest.json", + adapter: "efficient-sam-ti", + license: "Apache-2.0", + source: { + repository: "https://github.com/yformer/EfficientSAM", + revision: "d525f622e6f640acf5a0fc37c7ca1f243da5bde0", + }, + runtime: { format: "onnx", opset: 17, onnxruntimeWeb: "1.29.0" }, + capabilities: { + pointSuggest: true, + positivePoints: true, + negativePoints: false, + maxPoints: 6, + }, + artifacts: [ + { + role: "encoder", + path: "encoder.onnx", + bytes: 24_799_777, + sha256: "b19782d049c09a8f1cc36ccc6029264ca23c8ac35e6379fd9ef9f1bc6d81e7f2", + contentType: "application/octet-stream", + }, + { + role: "decoder", + path: "decoder.onnx", + bytes: 16_501_901, + sha256: "843761ca46f4aa00b09fdcf0c94271321f76eece092a744296c742d682a86172", + contentType: "application/octet-stream", + }, + ] as const, +} satisfies BrowserModelAdmission); + +export const ADMITTED_BROWSER_MODELS: readonly BrowserModelAdmission[] = Object.freeze([ + EFFICIENT_SAM_TI_ADMISSION, +]); diff --git a/frontend/app/src/data/browserInference/fixtures/manifests-v1.ts b/frontend/app/src/data/browserInference/fixtures/manifests-v1.ts new file mode 100644 index 00000000..ae54cbbd --- /dev/null +++ b/frontend/app/src/data/browserInference/fixtures/manifests-v1.ts @@ -0,0 +1,32 @@ +export const EFFICIENT_SAM_TI_MANIFEST_V1 = { + schema_version: 1, + id: "efficient-sam-ti", + name: "EfficientSAM-Ti", + revision: "b19782d049c0-843761ca46f4", + model_ref: "robomous/efficient-sam-ti@b19782d049c0-843761ca46f4", + source: { + repository: "https://github.com/yformer/EfficientSAM", + revision: "d525f622e6f640acf5a0fc37c7ca1f243da5bde0", + }, + runtime: { format: "onnx", opset: 17, onnxruntime_web: "1.29.0" }, + capabilities: { + point_suggest: true, + positive_points: true, + negative_points: false, + max_points: 6, + }, + artifacts: { + encoder: { + path: "encoder.onnx", + bytes: 24_799_777, + sha256: "b19782d049c09a8f1cc36ccc6029264ca23c8ac35e6379fd9ef9f1bc6d81e7f2", + content_type: "application/octet-stream", + }, + decoder: { + path: "decoder.onnx", + bytes: 16_501_901, + sha256: "843761ca46f4aa00b09fdcf0c94271321f76eece092a744296c742d682a86172", + content_type: "application/octet-stream", + }, + }, +} as const; diff --git a/frontend/app/src/data/browserInference/fixtures/registry-v1.json b/frontend/app/src/data/browserInference/fixtures/registry-v1.json new file mode 100644 index 00000000..5f9f550b --- /dev/null +++ b/frontend/app/src/data/browserInference/fixtures/registry-v1.json @@ -0,0 +1,40 @@ +{ + "schema_version": 1, + "models": [ + { + "id": "efficient-sam-ti", + "name": "EfficientSAM-Ti", + "revision": "b19782d049c0-843761ca46f4", + "model_ref": "robomous/efficient-sam-ti@b19782d049c0-843761ca46f4", + "manifest": "/models/efficient-sam-ti/b19782d049c0-843761ca46f4/manifest.json" + }, + { + "id": "mobile-sam", + "name": "MobileSAM", + "revision": "359e37f2b168-7983079ab060", + "model_ref": "robomous/mobile-sam@359e37f2b168-7983079ab060", + "manifest": "/models/mobile-sam/359e37f2b168-7983079ab060/manifest.json" + }, + { + "id": "efficientvit-sam-l0", + "name": "EfficientViT-SAM-L0", + "revision": "e48dd681ba4b-1d3ba86d781b", + "model_ref": "robomous/efficientvit-sam-l0@e48dd681ba4b-1d3ba86d781b", + "manifest": "/models/efficientvit-sam-l0/e48dd681ba4b-1d3ba86d781b/manifest.json" + }, + { + "id": "slimsam-77-uniform", + "name": "SlimSAM-77-uniform", + "revision": "7f2c646efd21-e6eb3c03cdbd", + "model_ref": "robomous/slimsam-77-uniform@7f2c646efd21-e6eb3c03cdbd", + "manifest": "/models/slimsam-77-uniform/7f2c646efd21-e6eb3c03cdbd/manifest.json" + }, + { + "id": "sam2.1-hiera-tiny", + "name": "SAM2.1-Hiera-Tiny", + "revision": "7f000e65546d-6dbe21e6e60e", + "model_ref": "robomous/sam2.1-hiera-tiny@7f000e65546d-6dbe21e6e60e", + "manifest": "/models/sam2.1-hiera-tiny/7f000e65546d-6dbe21e6e60e/manifest.json" + } + ] +} diff --git a/frontend/app/src/data/browserInference/manifest.ts b/frontend/app/src/data/browserInference/manifest.ts index 65d32d40..a2300584 100644 --- a/frontend/app/src/data/browserInference/manifest.ts +++ b/frontend/app/src/data/browserInference/manifest.ts @@ -8,6 +8,8 @@ export const MODEL_CDN_BASE_URL: string = ( (import.meta.env["VITE_MODEL_CDN_BASE_URL"] as string | undefined) ?? "https://models.robomous.ai" ).replace(/\/+$/, ""); +export const MODEL_REGISTRY_URL = `${MODEL_CDN_BASE_URL}/registry/v1.json`; + export const EFFICIENT_SAM_TI_REVISION = "b19782d049c0-843761ca46f4"; /** diff --git a/frontend/app/src/data/browserInference/registryClient.test.ts b/frontend/app/src/data/browserInference/registryClient.test.ts new file mode 100644 index 00000000..1aced764 --- /dev/null +++ b/frontend/app/src/data/browserInference/registryClient.test.ts @@ -0,0 +1,153 @@ +/** @vitest-environment node */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import registryV1 from "./fixtures/registry-v1.json"; +import { EFFICIENT_SAM_TI_MANIFEST_V1 } from "./fixtures/manifests-v1.js"; +import { fetchAdmittedBrowserModels, resolveModelPath } from "./registryClient.js"; + +const BASE = "https://models.example"; + +function clone(value: T): T { + return structuredClone(value); +} + +function registryWith(model: Record): unknown { + return { schema_version: 1, models: [model] }; +} + +function admittedRow(): Record { + return clone(registryV1.models[0]) as Record; +} + +function fetchFixture(registry: unknown = registryV1, manifest: unknown = EFFICIENT_SAM_TI_MANIFEST_V1) { + return vi.fn(async (input: string | URL | Request) => { + const url = String(input); + if (url.endsWith("/registry/v1.json")) return new Response(JSON.stringify(registry)); + if (url.endsWith("/manifest.json")) return new Response(JSON.stringify(manifest)); + throw new Error(`unexpected fetch ${url}`); + }); +} + +describe("fetchAdmittedBrowserModels", () => { + beforeEach(() => vi.restoreAllMocks()); + + it("parses the deployed five-model registry but offers only the admitted executable release", async () => { + const fetch = fetchFixture(); + + const result = await fetchAdmittedBrowserModels(BASE, { fetch }); + + expect(result).toHaveLength(1); + expect(result[0]).toMatchObject({ + id: "efficient-sam-ti", + revision: "b19782d049c0-843761ca46f4", + modelRef: "robomous/efficient-sam-ti@b19782d049c0-843761ca46f4", + license: "Apache-2.0", + source: { repository: "https://github.com/yformer/EfficientSAM" }, + }); + expect(fetch).toHaveBeenCalledTimes(2); + }); + + it.each([ + null, + {}, + { schema_version: 2, models: [] }, + { schema_version: 1, models: "five" }, + registryWith({ id: "broken" }), + ])("rejects a malformed registry (%j)", async (registry) => { + await expect(fetchAdmittedBrowserModels(BASE, { fetch: fetchFixture(registry) })).rejects.toThrow( + /registry schema/i, + ); + }); + + it("rejects duplicate IDs even when their revisions differ", async () => { + const first = admittedRow(); + const second = { ...admittedRow(), revision: "another-immutable-revision" }; + await expect( + fetchAdmittedBrowserModels(BASE, { + fetch: fetchFixture({ schema_version: 1, models: [first, second] }), + }), + ).rejects.toThrow(/duplicate model id/i); + }); + + it("rejects an exact duplicate identity", async () => { + const row = admittedRow(); + await expect( + fetchAdmittedBrowserModels(BASE, { + fetch: fetchFixture({ schema_version: 1, models: [row, clone(row)] }), + }), + ).rejects.toThrow(/duplicate model id/i); + }); + + it("does not fetch or offer a valid but unadmitted model", async () => { + const fetch = fetchFixture(registryWith(clone(registryV1.models[1]))); + await expect(fetchAdmittedBrowserModels(BASE, { fetch })).resolves.toEqual([]); + expect(fetch).toHaveBeenCalledTimes(1); + }); + + it("rejects a registry revision mismatch for an admitted ID", async () => { + await expect( + fetchAdmittedBrowserModels(BASE, { + fetch: fetchFixture(registryWith({ ...admittedRow(), revision: "mutable-latest" })), + }), + ).rejects.toThrow(/revision mismatch/i); + }); + + it.each([ + ["source", { source: { ...EFFICIENT_SAM_TI_MANIFEST_V1.source, repository: "https://example.invalid/fork" } }], + ["license", { license: "MIT" }], + ["artifact hash", { + artifacts: { + ...EFFICIENT_SAM_TI_MANIFEST_V1.artifacts, + encoder: { ...EFFICIENT_SAM_TI_MANIFEST_V1.artifacts.encoder, sha256: "0".repeat(64) }, + }, + }], + ])("rejects a manifest %s mismatch against admission", async (_name, override) => { + const manifest = { ...clone(EFFICIENT_SAM_TI_MANIFEST_V1), ...override }; + await expect( + fetchAdmittedBrowserModels(BASE, { fetch: fetchFixture(registryWith(admittedRow()), manifest) }), + ).rejects.toThrow(/admission mismatch/i); + }); + + it("rejects a manifest identity mismatch", async () => { + const manifest = { ...clone(EFFICIENT_SAM_TI_MANIFEST_V1), revision: "mutable-latest" }; + await expect( + fetchAdmittedBrowserModels(BASE, { fetch: fetchFixture(registryWith(admittedRow()), manifest) }), + ).rejects.toThrow(/admission mismatch/i); + }); + + it.each([ + "../escape/manifest.json", + "/../escape/manifest.json", + "https://evil.example/manifest.json", + "//evil.example/manifest.json", + "/models/%2e%2e/escape/manifest.json", + "/models/model/manifest.json?mutable=1", + ])("rejects an unsafe manifest path %s", async (manifestPath) => { + await expect( + fetchAdmittedBrowserModels(BASE, { + fetch: fetchFixture(registryWith({ ...admittedRow(), manifest: manifestPath })), + }), + ).rejects.toThrow(/model path/i); + }); + + it.each(["../encoder.onnx", "/encoder.onnx", "nested/encoder.onnx", "https://evil.example/e.onnx"])( + "rejects an unsafe artifact path %s", + async (artifactPath) => { + const manifest = clone(EFFICIENT_SAM_TI_MANIFEST_V1) as { + artifacts: { encoder: { path: string } }; + }; + manifest.artifacts.encoder.path = artifactPath; + await expect( + fetchAdmittedBrowserModels(BASE, { fetch: fetchFixture(registryWith(admittedRow()), manifest) }), + ).rejects.toThrow(/admission mismatch/i); + }, + ); +}); + +describe("resolveModelPath", () => { + it("resolves a root-relative public path below a configured base prefix", () => { + expect(resolveModelPath("https://models.example/base", "/base/models/a/manifest.json").href).toBe( + "https://models.example/base/models/a/manifest.json", + ); + }); +}); diff --git a/frontend/app/src/data/browserInference/registryClient.ts b/frontend/app/src/data/browserInference/registryClient.ts new file mode 100644 index 00000000..be72f231 --- /dev/null +++ b/frontend/app/src/data/browserInference/registryClient.ts @@ -0,0 +1,217 @@ +import { + ADMITTED_BROWSER_MODELS, + type ArtifactAdmission, + type BrowserModelAdmission, +} from "./admissionCatalog.js"; + +interface RegistryRow { + readonly id: string; + readonly name: string; + readonly revision: string; + readonly modelRef: string; + readonly manifest: string; +} + +export interface AdmittedRegistryModel { + readonly admission: BrowserModelAdmission; + readonly id: string; + readonly label: string; + readonly revision: string; + readonly modelRef: string; + readonly license: string; + readonly source: BrowserModelAdmission["source"]; + readonly manifestUrl: URL; + readonly artifactUrls: Readonly>; +} + +interface FetchOptions { + readonly signal?: AbortSignal; + readonly fetch?: typeof globalThis.fetch; +} + +function record(value: unknown, at: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`unexpected registry schema at ${at}`); + } + return value as Record; +} + +function text(value: unknown, at: string): string { + if (typeof value !== "string" || value.length === 0) { + throw new Error(`unexpected registry schema at ${at}`); + } + return value; +} + +function number(value: unknown, at: string): number { + if (typeof value !== "number" || !Number.isFinite(value)) { + throw new Error(`unexpected registry schema at ${at}`); + } + return value; +} + +function boolean(value: unknown, at: string): boolean { + if (typeof value !== "boolean") throw new Error(`unexpected registry schema at ${at}`); + return value; +} + +function parseRegistry(value: unknown): readonly RegistryRow[] { + const root = record(value, "root"); + if (root["schema_version"] !== 1 || !Array.isArray(root["models"])) { + throw new Error("unexpected registry schema: expected schema_version 1 and a models array"); + } + const seen = new Set(); + return root["models"].map((item, index) => { + const row = record(item, `models[${index}]`); + const id = text(row["id"], `models[${index}].id`); + if (seen.has(id)) throw new Error(`duplicate model id in registry: ${id}`); + seen.add(id); + return { + id, + name: text(row["name"], `models[${index}].name`), + revision: text(row["revision"], `models[${index}].revision`), + modelRef: text(row["model_ref"], `models[${index}].model_ref`), + manifest: text(row["manifest"], `models[${index}].manifest`), + }; + }); +} + +function normalizedBase(baseUrl: string): URL { + const base = new URL(baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`); + if (base.protocol !== "https:" && base.protocol !== "http:") { + throw new Error("model base URL must use HTTP or HTTPS"); + } + return base; +} + +export function resolveModelPath(baseUrl: string, path: string): URL { + const base = normalizedBase(baseUrl); + if ( + path.length === 0 || + path.includes("\\") || + path.includes("?") || + path.includes("#") || + path.startsWith("//") || + /^[a-z][a-z\d+.-]*:/i.test(path) || + /%2f|%5c/i.test(path) + ) { + throw new Error(`unsafe model path: ${path}`); + } + let decoded: string; + try { + decoded = decodeURIComponent(path); + } catch { + throw new Error(`unsafe model path: ${path}`); + } + if (decoded.split("/").some((segment) => segment === "." || segment === "..")) { + throw new Error(`unsafe model path: ${path}`); + } + const resolved = new URL(path, base); + if (resolved.origin !== base.origin || !resolved.pathname.startsWith(base.pathname)) { + throw new Error(`unsafe model path outside configured base: ${path}`); + } + return resolved; +} + +function assertEqual(actual: unknown, expected: unknown, field: string): void { + if (actual !== expected) { + throw new Error(`model admission mismatch at ${field}`); + } +} + +function validateArtifact( + manifest: Record, + admission: ArtifactAdmission, +): string { + const artifacts = record(manifest["artifacts"], "manifest.artifacts"); + const artifact = record(artifacts[admission.role], `manifest.artifacts.${admission.role}`); + const path = text(artifact["path"], `manifest.artifacts.${admission.role}.path`); + if (path.includes("/") || path.includes("\\") || path === "." || path === "..") { + throw new Error(`model admission mismatch at artifacts.${admission.role}.path`); + } + assertEqual(path, admission.path, `artifacts.${admission.role}.path`); + assertEqual(number(artifact["bytes"], `artifacts.${admission.role}.bytes`), admission.bytes, `artifacts.${admission.role}.bytes`); + assertEqual(text(artifact["sha256"], `artifacts.${admission.role}.sha256`), admission.sha256, `artifacts.${admission.role}.sha256`); + assertEqual(text(artifact["content_type"], `artifacts.${admission.role}.content_type`), admission.contentType, `artifacts.${admission.role}.content_type`); + return path; +} + +function validateManifest(value: unknown, admission: BrowserModelAdmission): Record { + const manifest = record(value, "manifest"); + assertEqual(number(manifest["schema_version"], "manifest.schema_version"), 1, "schema_version"); + assertEqual(text(manifest["id"], "manifest.id"), admission.id, "id"); + assertEqual(text(manifest["name"], "manifest.name"), admission.label, "name"); + assertEqual(text(manifest["revision"], "manifest.revision"), admission.revision, "revision"); + assertEqual(text(manifest["model_ref"], "manifest.model_ref"), admission.modelRef, "model_ref"); + if (manifest["license"] !== undefined) { + assertEqual(text(manifest["license"], "manifest.license"), admission.license, "license"); + } + + const source = record(manifest["source"], "manifest.source"); + assertEqual(text(source["repository"], "manifest.source.repository"), admission.source.repository, "source.repository"); + assertEqual(text(source["revision"], "manifest.source.revision"), admission.source.revision, "source.revision"); + + const runtime = record(manifest["runtime"], "manifest.runtime"); + assertEqual(text(runtime["format"], "manifest.runtime.format"), admission.runtime.format, "runtime.format"); + assertEqual(number(runtime["opset"], "manifest.runtime.opset"), admission.runtime.opset, "runtime.opset"); + assertEqual(text(runtime["onnxruntime_web"], "manifest.runtime.onnxruntime_web"), admission.runtime.onnxruntimeWeb, "runtime.onnxruntime_web"); + + const capabilities = record(manifest["capabilities"], "manifest.capabilities"); + assertEqual(boolean(capabilities["point_suggest"], "manifest.capabilities.point_suggest"), admission.capabilities.pointSuggest, "capabilities.point_suggest"); + assertEqual(boolean(capabilities["positive_points"], "manifest.capabilities.positive_points"), admission.capabilities.positivePoints, "capabilities.positive_points"); + assertEqual(boolean(capabilities["negative_points"], "manifest.capabilities.negative_points"), admission.capabilities.negativePoints, "capabilities.negative_points"); + assertEqual(number(capabilities["max_points"], "manifest.capabilities.max_points"), admission.capabilities.maxPoints, "capabilities.max_points"); + + return Object.fromEntries( + admission.artifacts.map((artifact) => [artifact.role, validateArtifact(manifest, artifact)]), + ) as Record; +} + +async function json(response: Response, what: string): Promise { + if (!response.ok) throw new Error(`${what} fetch failed: ${response.status} ${response.statusText}`); + return response.json() as Promise; +} + +export async function fetchAdmittedBrowserModels( + baseUrl: string, + options: FetchOptions = {}, +): Promise { + const fetcher = options.fetch ?? globalThis.fetch; + const registryUrl = resolveModelPath(baseUrl, "registry/v1.json"); + const rows = parseRegistry(await json(await fetcher(registryUrl, { signal: options.signal }), "registry")); + + for (const row of rows) resolveModelPath(baseUrl, row.manifest); + + const result: AdmittedRegistryModel[] = []; + for (const admission of ADMITTED_BROWSER_MODELS) { + const row = rows.find((candidate) => candidate.id === admission.id); + if (row === undefined) continue; + if (row.revision !== admission.revision) { + throw new Error(`registry revision mismatch for admitted model ${admission.id}`); + } + assertEqual(row.name, admission.label, "registry.name"); + assertEqual(row.modelRef, admission.modelRef, "registry.model_ref"); + assertEqual(row.manifest, admission.manifestPath, "registry.manifest"); + const manifestUrl = resolveModelPath(baseUrl, row.manifest); + const artifactPaths = validateManifest( + await json(await fetcher(manifestUrl, { signal: options.signal }), "manifest"), + admission, + ); + const manifestDirectory = new URL("./", manifestUrl); + const artifactUrls = Object.fromEntries( + admission.artifacts.map((artifact) => [artifact.role, new URL(artifactPaths[artifact.role], manifestDirectory)]), + ) as Record; + result.push({ + admission, + id: admission.id, + label: admission.label, + revision: admission.revision, + modelRef: admission.modelRef, + license: admission.license, + source: admission.source, + manifestUrl, + artifactUrls, + }); + } + return result; +} From a2b6d537274e7d125f2acadb611b88608f31e23c Mon Sep 17 00:00:00 2001 From: Jesus Armando Anaya <1445792+JArmandoAnaya@users.noreply.github.com> Date: Thu, 17 Sep 2026 11:36:07 -0700 Subject: [PATCH 02/15] feat(inference): persist verified browser model artifacts --- .../browserInference/acquireEfficientSam.ts | 24 ++- .../browserInference/artifactStore.test.ts | 189 ++++++++++++++++++ .../data/browserInference/artifactStore.ts | 121 +++++++++++ 3 files changed, 326 insertions(+), 8 deletions(-) create mode 100644 frontend/app/src/data/browserInference/artifactStore.test.ts create mode 100644 frontend/app/src/data/browserInference/artifactStore.ts diff --git a/frontend/app/src/data/browserInference/acquireEfficientSam.ts b/frontend/app/src/data/browserInference/acquireEfficientSam.ts index 1bf87b27..2964c814 100644 --- a/frontend/app/src/data/browserInference/acquireEfficientSam.ts +++ b/frontend/app/src/data/browserInference/acquireEfficientSam.ts @@ -16,13 +16,27 @@ function artifactUrl(path: string): string { // wider `Uint8Array`) — TypeScript 6's `lib.dom.d.ts` types // `crypto.subtle.digest`'s `BufferSource` parameter as requiring the concrete // `ArrayBuffer` variant. -async function sha256Hex(bytes: Uint8Array): Promise { +export async function sha256Hex(bytes: Uint8Array): Promise { const digest = await crypto.subtle.digest("SHA-256", bytes); return Array.from(new Uint8Array(digest)) .map((byte) => byte.toString(16).padStart(2, "0")) .join(""); } +export async function verifyArtifact( + bytes: Uint8Array, + expected: { readonly bytes: number; readonly sha256: string }, + subject = "artifact", +): Promise { + if (bytes.byteLength !== expected.bytes) { + throw new Error(`${subject} size mismatch: got ${bytes.byteLength} bytes, expected ${expected.bytes}`); + } + const digest = await sha256Hex(bytes); + if (digest !== expected.sha256) { + throw new Error(`${subject} SHA-256 mismatch: got ${digest}, expected ${expected.sha256}`); + } +} + export async function fetchVerified( url: string, expected: { readonly bytes: number; readonly sha256: string }, @@ -31,13 +45,7 @@ export async function fetchVerified( const response = await fetch(url, { signal }); if (!response.ok) throw new Error(`artifact fetch failed: ${response.status} ${response.statusText}`); const bytes = new Uint8Array(await response.arrayBuffer()); - if (bytes.byteLength !== expected.bytes) { - throw new Error(`artifact size mismatch for ${url}: got ${bytes.byteLength} bytes, expected ${expected.bytes}`); - } - const digest = await sha256Hex(bytes); - if (digest !== expected.sha256) { - throw new Error(`artifact SHA-256 mismatch for ${url}: got ${digest}, expected ${expected.sha256}`); - } + await verifyArtifact(bytes, expected, `artifact ${url}`); return bytes; } diff --git a/frontend/app/src/data/browserInference/artifactStore.test.ts b/frontend/app/src/data/browserInference/artifactStore.test.ts new file mode 100644 index 00000000..3082075a --- /dev/null +++ b/frontend/app/src/data/browserInference/artifactStore.test.ts @@ -0,0 +1,189 @@ +/** @vitest-environment node */ +import { describe, expect, it, vi } from "vitest"; + +import type { BrowserModelAdmission } from "./admissionCatalog.js"; +import { + CACHE_NAMESPACE, + cacheKeyFor, + createCacheArtifactStore, + type ArtifactCache, + type ArtifactCacheStorage, +} from "./artifactStore.js"; + +type Artifacts = { readonly encoder: Uint8Array; readonly decoder: Uint8Array }; + +function bytes(value: string): Uint8Array { + return new TextEncoder().encode(value); +} + +async function digest(value: Uint8Array): Promise { + const hash = await crypto.subtle.digest("SHA-256", value); + return Array.from(new Uint8Array(hash), (part) => part.toString(16).padStart(2, "0")).join(""); +} + +async function fixture(revision = "rev-a"): Promise<{ admission: BrowserModelAdmission; artifacts: Artifacts }> { + const artifacts = { encoder: bytes("encoder"), decoder: bytes("decoder") }; + return { + artifacts, + admission: { + id: "fixture-model", + label: "Fixture", + revision, + modelRef: `fixture-model@${revision}`, + manifestPath: `/models/fixture-model/${revision}/manifest.json`, + adapter: "efficient-sam-ti", + license: "Apache-2.0", + source: { repository: "https://example.test/source", revision: "source-rev" }, + runtime: { format: "onnx", opset: 17, onnxruntimeWeb: "1.29.0" }, + capabilities: { pointSuggest: true, positivePoints: true, negativePoints: false, maxPoints: 6 }, + artifacts: [ + { + role: "encoder", + path: "encoder.onnx", + bytes: artifacts.encoder.byteLength, + sha256: await digest(artifacts.encoder), + contentType: "application/octet-stream", + }, + { + role: "decoder", + path: "decoder.onnx", + bytes: artifacts.decoder.byteLength, + sha256: await digest(artifacts.decoder), + contentType: "application/octet-stream", + }, + ], + }, + }; +} + +class MemoryCache implements ArtifactCache { + readonly entries = new Map(); + readonly put = vi.fn(async (request: RequestInfo | URL, response: Response) => { + this.entries.set(String(request), response.clone()); + }); + readonly match = vi.fn(async (request: RequestInfo | URL) => this.entries.get(String(request))?.clone()); + readonly delete = vi.fn(async (request: RequestInfo | URL) => this.entries.delete(String(request))); +} + +function storage(cache: MemoryCache): ArtifactCacheStorage { + return { open: vi.fn(async () => cache) }; +} + +describe("createCacheArtifactStore", () => { + it("reports a cache miss as not installed", async () => { + const cache = new MemoryCache(); + const { admission } = await fixture(); + await expect(createCacheArtifactStore(storage(cache)).inspect(admission)).resolves.toBe(false); + }); + + it("verifies every artifact before the first persistent write", async () => { + const cache = new MemoryCache(); + const { admission, artifacts } = await fixture(); + const corrupt = { ...artifacts, decoder: bytes("DECODEX") }; + + await expect(createCacheArtifactStore(storage(cache)).writeVerified(admission, corrupt)).rejects.toThrow( + /sha-256 mismatch/i, + ); + expect(cache.put).not.toHaveBeenCalled(); + }); + + it("writes both verified artifacts and then reports the model installed", async () => { + const cache = new MemoryCache(); + const { admission, artifacts } = await fixture(); + const store = createCacheArtifactStore(storage(cache)); + + await store.writeVerified(admission, artifacts); + + expect(cache.put).toHaveBeenCalledTimes(2); + expect(await store.inspect(admission)).toBe(true); + }); + + it("rolls back every revision key when a cache put fails", async () => { + const cache = new MemoryCache(); + cache.put.mockImplementationOnce(async (request, response) => { + cache.entries.set(String(request), response.clone()); + }); + cache.put.mockRejectedValueOnce(new DOMException("quota", "QuotaExceededError")); + const { admission, artifacts } = await fixture(); + const store = createCacheArtifactStore(storage(cache)); + + await expect(store.writeVerified(admission, artifacts)).rejects.toThrow(/quota/i); + + expect(await store.inspect(admission)).toBe(false); + expect(cache.entries.size).toBe(0); + }); + + it("cleans up a partial cache instead of treating it as installed", async () => { + const cache = new MemoryCache(); + const { admission, artifacts } = await fixture(); + const encoder = admission.artifacts[0]; + cache.entries.set(cacheKeyFor(admission, encoder), new Response(artifacts.encoder)); + + await expect(createCacheArtifactStore(storage(cache)).inspect(admission)).resolves.toBe(false); + + expect(cache.entries.size).toBe(0); + }); + + it("re-verifies cached bytes before returning them", async () => { + const cache = new MemoryCache(); + const { admission, artifacts } = await fixture(); + const store = createCacheArtifactStore(storage(cache)); + await store.writeVerified(admission, artifacts); + + const loaded = await store.readVerified(admission); + + expect(loaded).toEqual(artifacts); + }); + + it("deletes a corrupt revision and performs no network request", async () => { + const cache = new MemoryCache(); + const { admission, artifacts } = await fixture(); + const store = createCacheArtifactStore(storage(cache)); + await store.writeVerified(admission, artifacts); + cache.entries.set(cacheKeyFor(admission, admission.artifacts[0]), new Response(bytes("ENCODER"))); + const fetchSpy = vi.spyOn(globalThis, "fetch"); + + await expect(store.readVerified(admission)).rejects.toThrow(/sha-256 mismatch/i); + + expect(cache.entries.size).toBe(0); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it("removes only the selected revision even when another revision has the same bytes", async () => { + const cache = new MemoryCache(); + const first = await fixture("rev-a"); + const second = await fixture("rev-b"); + const store = createCacheArtifactStore(storage(cache)); + await store.writeVerified(first.admission, first.artifacts); + await store.writeVerified(second.admission, second.artifacts); + + await store.remove(first.admission); + + expect(await store.inspect(first.admission)).toBe(false); + expect(await store.inspect(second.admission)).toBe(true); + }); + + it("uses a versioned namespace and a model/revision/SHA cache identity", async () => { + const cache = new MemoryCache(); + const cacheStorage = storage(cache); + const { admission } = await fixture(); + const key = cacheKeyFor(admission, admission.artifacts[0]); + + await createCacheArtifactStore(cacheStorage).inspect(admission); + + expect(CACHE_NAMESPACE).toBe("visionset-browser-models-v1"); + expect(cacheStorage.open).toHaveBeenCalledWith(CACHE_NAMESPACE); + expect(key).toContain(encodeURIComponent(admission.id)); + expect(key).toContain(encodeURIComponent(admission.revision)); + expect(key).toContain(admission.artifacts[0].sha256); + expect(key.startsWith("https://cache.visionset.invalid/")).toBe(true); + }); + + it("degrades an unavailable Cache Storage API without claiming installation", async () => { + const { admission, artifacts } = await fixture(); + const store = createCacheArtifactStore(undefined); + await expect(store.inspect(admission)).resolves.toBe(false); + await expect(store.readVerified(admission)).resolves.toBeNull(); + await expect(store.writeVerified(admission, artifacts)).rejects.toThrow(/unavailable/i); + }); +}); diff --git a/frontend/app/src/data/browserInference/artifactStore.ts b/frontend/app/src/data/browserInference/artifactStore.ts new file mode 100644 index 00000000..e69612ac --- /dev/null +++ b/frontend/app/src/data/browserInference/artifactStore.ts @@ -0,0 +1,121 @@ +import type { ArtifactAdmission, BrowserModelAdmission } from "./admissionCatalog.js"; +import { verifyArtifact } from "./acquireEfficientSam.js"; + +export const CACHE_NAMESPACE = "visionset-browser-models-v1"; +const CACHE_ORIGIN = "https://cache.visionset.invalid"; + +export interface ArtifactCache { + match(request: RequestInfo | URL): Promise; + put(request: RequestInfo | URL, response: Response): Promise; + delete(request: RequestInfo | URL): Promise; +} + +export interface ArtifactCacheStorage { + open(cacheName: string): Promise; +} + +export interface BrowserModelArtifacts { + readonly encoder: Uint8Array; + readonly decoder: Uint8Array; +} + +export interface BrowserArtifactStore { + inspect(model: BrowserModelAdmission): Promise; + readVerified(model: BrowserModelAdmission): Promise; + writeVerified(model: BrowserModelAdmission, artifacts: BrowserModelArtifacts): Promise; + remove(model: BrowserModelAdmission): Promise; +} + +export function cacheKeyFor(model: BrowserModelAdmission, artifact: ArtifactAdmission): string { + return ( + `${CACHE_ORIGIN}/__visionset_model_cache__/` + + `${encodeURIComponent(model.id)}/${encodeURIComponent(model.revision)}/sha256/${artifact.sha256}` + ); +} + +function bytesFor(artifacts: BrowserModelArtifacts, role: ArtifactAdmission["role"]): Uint8Array { + return artifacts[role]; +} + +export function createCacheArtifactStore( + cacheStorage: ArtifactCacheStorage | undefined = globalThis.caches, +): BrowserArtifactStore { + async function cache(): Promise { + if (cacheStorage === undefined) return null; + return cacheStorage.open(CACHE_NAMESPACE); + } + + async function remove(model: BrowserModelAdmission): Promise { + const opened = await cache(); + if (opened === null) return; + await Promise.all(model.artifacts.map((artifact) => opened.delete(cacheKeyFor(model, artifact)))); + } + + return { + async inspect(model) { + const opened = await cache(); + if (opened === null) return false; + const present = await Promise.all( + model.artifacts.map(async (artifact) => (await opened.match(cacheKeyFor(model, artifact))) !== undefined), + ); + if (present.every(Boolean)) return true; + if (present.some(Boolean)) await remove(model); + return false; + }, + + async readVerified(model) { + const opened = await cache(); + if (opened === null) return null; + const found = await Promise.all( + model.artifacts.map(async (artifact) => ({ + artifact, + response: await opened.match(cacheKeyFor(model, artifact)), + })), + ); + if (found.some(({ response }) => response === undefined)) { + if (found.some(({ response }) => response !== undefined)) await remove(model); + return null; + } + try { + const verified = await Promise.all( + found.map(async ({ artifact, response }) => { + const value = new Uint8Array(await response!.arrayBuffer()); + await verifyArtifact(value, artifact, `cached ${artifact.role}`); + return [artifact.role, value] as const; + }), + ); + return Object.fromEntries(verified) as unknown as BrowserModelArtifacts; + } catch (error) { + await remove(model); + throw error; + } + }, + + async writeVerified(model, artifacts) { + // This entire pass completes before the cache is opened or the first put is issued. + // It is the transaction boundary that prevents a valid encoder being persisted before + // a corrupt decoder has even been checked. + await Promise.all( + model.artifacts.map((artifact) => + verifyArtifact(bytesFor(artifacts, artifact.role), artifact, `downloaded ${artifact.role}`), + ), + ); + const opened = await cache(); + if (opened === null) throw new Error("browser model storage is unavailable"); + try { + for (const artifact of model.artifacts) { + const value = bytesFor(artifacts, artifact.role); + await opened.put( + cacheKeyFor(model, artifact), + new Response(value, { headers: { "content-type": artifact.contentType } }), + ); + } + } catch (error) { + await remove(model); + throw error; + } + }, + + remove, + }; +} From 72e504f283d377e518baa41667295a9d6b27f558 Mon Sep 17 00:00:00 2001 From: Jesus Armando Anaya <1445792+JArmandoAnaya@users.noreply.github.com> Date: Thu, 17 Sep 2026 11:43:09 -0700 Subject: [PATCH 03/15] feat(inference): add persistent browser model catalog --- .../BrowserInferenceRuntime.test.ts | 41 +++- .../BrowserInferenceRuntime.ts | 158 +++++------- .../BrowserModelCatalog.test.ts | 197 +++++++++++++++ .../browserInference/BrowserModelCatalog.ts | 230 ++++++++++++++++++ .../browserInference/acquireEfficientSam.ts | 7 +- .../data/browserInference/admissionCatalog.ts | 2 + .../browserInference/artifactStore.test.ts | 2 +- frontend/ui-core/src/index.ts | 3 + .../ui-core/src/inference/browserPort.test.ts | 26 ++ frontend/ui-core/src/inference/browserPort.ts | 44 +++- 10 files changed, 591 insertions(+), 119 deletions(-) create mode 100644 frontend/app/src/data/browserInference/BrowserModelCatalog.test.ts create mode 100644 frontend/app/src/data/browserInference/BrowserModelCatalog.ts diff --git a/frontend/app/src/data/browserInference/BrowserInferenceRuntime.test.ts b/frontend/app/src/data/browserInference/BrowserInferenceRuntime.test.ts index f9783c9c..d8f37d67 100644 --- a/frontend/app/src/data/browserInference/BrowserInferenceRuntime.test.ts +++ b/frontend/app/src/data/browserInference/BrowserInferenceRuntime.test.ts @@ -2,9 +2,11 @@ import { describe, expect, it, vi } from "vitest"; import type { SuggestionRequest } from "@visionset/ui-core"; import { createOssBrowserInferenceRuntime } from "./BrowserInferenceRuntime.js"; import { EFFICIENT_SAM_TI_REVISION } from "./manifest.js"; +import type { VisionSetBrowserInferenceRuntime } from "@visionset/ui-core"; +import type { BrowserModelArtifacts } from "./artifactStore.js"; function fakeDeps(overrides?: { - acquire?: () => Promise<{ encoder: Uint8Array; decoder: Uint8Array }>; + acquire?: () => Promise; ready?: () => Promise; supported?: () => boolean; }) { @@ -20,7 +22,23 @@ function fakeDeps(overrides?: { return { acquire, createRuntime, supported: overrides?.supported ?? ((): boolean => true), prepareImage, dispose }; } +async function acquisition(runtime: VisionSetBrowserInferenceRuntime) { + // Registry/cache initialization is asynchronous in Phase G. `listTargets()` waits for that + // initial pass, after which the synchronous Phase F compatibility view is populated. + await runtime.listTargets(); + return runtime.listAcquisitions?.()[0]!; +} + describe("createOssBrowserInferenceRuntime", () => { + it("exposes the additive model catalog while preserving the Phase F runtime members", () => { + const runtime = createOssBrowserInferenceRuntime(fakeDeps()); + expect(runtime.modelCatalog).toBeDefined(); + expect(runtime.listTargets).toBeTypeOf("function"); + expect(runtime.executorFor).toBeTypeOf("function"); + expect(runtime.listAcquisitions).toBeTypeOf("function"); + expect(runtime.setActiveAsset).toBeTypeOf("function"); + }); + it("lists no targets and one acquisition before acquiring", async () => { const deps = fakeDeps(); const runtime = createOssBrowserInferenceRuntime(deps); @@ -31,7 +49,7 @@ describe("createOssBrowserInferenceRuntime", () => { it("lists the target and no acquisitions after acquiring", async () => { const deps = fakeDeps(); const runtime = createOssBrowserInferenceRuntime(deps); - await runtime.listAcquisitions?.()[0]!.acquire(); + await (await acquisition(runtime)).acquire(); expect(await runtime.listTargets()).toHaveLength(1); expect(runtime.listAcquisitions?.()).toEqual([]); expect(deps.acquire).toHaveBeenCalledTimes(1); @@ -41,17 +59,18 @@ describe("createOssBrowserInferenceRuntime", () => { it("leaves the model unacquired (retryable) when acquire() rejects", async () => { const deps = fakeDeps({ acquire: () => Promise.reject(new Error("network down")) }); const runtime = createOssBrowserInferenceRuntime(deps); - await expect(runtime.listAcquisitions?.()[0]!.acquire()).rejects.toThrow("network down"); + await expect((await acquisition(runtime)).acquire()).rejects.toThrow("network down"); expect(await runtime.listTargets()).toEqual([]); expect(runtime.listAcquisitions?.()).toHaveLength(1); }); it("dedupes a second acquire() call while the first is still in flight", async () => { - let resolveAcquire!: (value: { encoder: Uint8Array; decoder: Uint8Array }) => void; + let resolveAcquire!: (value: BrowserModelArtifacts) => void; const deps = fakeDeps({ acquire: () => new Promise((resolve) => (resolveAcquire = resolve)) }); const runtime = createOssBrowserInferenceRuntime(deps); - const first = runtime.listAcquisitions?.()[0]!.acquire(); - const second = runtime.listAcquisitions?.()[0]!.acquire(); + const available = await acquisition(runtime); + const first = available.acquire(); + const second = available.acquire(); resolveAcquire({ encoder: new Uint8Array(1), decoder: new Uint8Array(1) }); await Promise.all([first, second]); expect(deps.acquire).toHaveBeenCalledTimes(1); @@ -60,9 +79,9 @@ describe("createOssBrowserInferenceRuntime", () => { it("carries the pinned revision in the acquired target's modelRef", async () => { const deps = fakeDeps(); const runtime = createOssBrowserInferenceRuntime(deps); - await runtime.listAcquisitions?.()[0]!.acquire(); + await (await acquisition(runtime)).acquire(); const targets = await runtime.listTargets(); - expect(targets[0]!.modelRef).toBe(`efficient-sam-ti@${EFFICIENT_SAM_TI_REVISION}`); + expect(targets[0]!.modelRef).toBe(`robomous/efficient-sam-ti@${EFFICIENT_SAM_TI_REVISION}`); }); it("throws from executorFor before any acquisition has succeeded", () => { @@ -88,7 +107,7 @@ describe("createOssBrowserInferenceRuntime", () => { const deps = fakeDeps({ ready: () => Promise.reject(new Error("graph load failed")) }); const runtime = createOssBrowserInferenceRuntime(deps); - await expect(runtime.listAcquisitions?.()[0]!.acquire()).rejects.toThrow("graph load failed"); + await expect((await acquisition(runtime)).acquire()).rejects.toThrow("graph load failed"); expect(await runtime.listTargets()).toEqual([]); expect(runtime.listAcquisitions?.()).toHaveLength(1); @@ -100,7 +119,7 @@ describe("createOssBrowserInferenceRuntime", () => { describe("one encode per asset survives composition", () => { it("hands out the same executor on every executorFor call", async () => { const runtime = createOssBrowserInferenceRuntime(fakeDeps()); - await runtime.listAcquisitions?.()[0]!.acquire(); + await (await acquisition(runtime)).acquire(); expect(runtime.executorFor("efficient-sam-ti")).toBe(runtime.executorFor("efficient-sam-ti")); }); @@ -113,7 +132,7 @@ describe("createOssBrowserInferenceRuntime", () => { // refinements", proved where the composition actually happens. const deps = fakeDeps(); const runtime = createOssBrowserInferenceRuntime(deps); - await runtime.listAcquisitions?.()[0]!.acquire(); + await (await acquisition(runtime)).acquire(); const rgb = new Uint8Array(4 * 4 * 3); runtime.setActiveAsset?.({ diff --git a/frontend/app/src/data/browserInference/BrowserInferenceRuntime.ts b/frontend/app/src/data/browserInference/BrowserInferenceRuntime.ts index e41bd669..b609201a 100644 --- a/frontend/app/src/data/browserInference/BrowserInferenceRuntime.ts +++ b/frontend/app/src/data/browserInference/BrowserInferenceRuntime.ts @@ -7,36 +7,23 @@ import type { VisionSetBrowserInferenceRuntime, } from "@visionset/ui-core"; +import { EFFICIENT_SAM_TI_ADMISSION, type BrowserModelAdmission } from "./admissionCatalog.js"; +import { createCacheArtifactStore, type BrowserArtifactStore, type BrowserModelArtifacts } from "./artifactStore.js"; import { acquireEfficientSam } from "./acquireEfficientSam.js"; -import { EFFICIENT_SAM_TI_EXPECTED, EFFICIENT_SAM_TI_REVISION } from "./manifest.js"; +import { createBrowserModelCatalog } from "./BrowserModelCatalog.js"; import { createBrowserSuggestionExecutor } from "./BrowserSuggestionExecutor.js"; - -const MODEL_ID = "efficient-sam-ti"; -const MODEL_REF = `efficient-sam-ti@${EFFICIENT_SAM_TI_REVISION}`; +import { MODEL_CDN_BASE_URL } from "./manifest.js"; +import { fetchAdmittedBrowserModels } from "./registryClient.js"; interface Deps { - readonly acquire: (signal?: AbortSignal) => Promise<{ encoder: Uint8Array; decoder: Uint8Array }>; - readonly createRuntime: (artifacts: { encoder: Uint8Array; decoder: Uint8Array }) => PromptableSegmentationRuntime; - /** - * Whether a runtime can exist in this environment at all — a `Worker` to host ORT and - * WebAssembly to run it. Injected rather than called directly so both answers are - * reachable from a Node test; the real one is the package's own capability check. - */ + readonly acquire: (signal?: AbortSignal) => Promise; + readonly createRuntime: (artifacts: BrowserModelArtifacts) => PromptableSegmentationRuntime; readonly supported: () => boolean; + readonly store?: BrowserArtifactStore; + readonly discover?: (admission: BrowserModelAdmission, signal?: AbortSignal) => Promise; } -/** - * Where this deployment serves ONNX Runtime's WebAssembly artifacts. - * - * Stated rather than left to the worker's own default, which resolves `./ort/` against - * the worker's module URL: correct for the package as installed, wrong once vite has - * hashed the worker into `assets/`. `vite.config.ts` puts the directory at `ort/`, - * and `BASE_URL` is the only thing that knows what `` is — `/app/` in a build, - * because the wheel mounts the bundle there, and `/` under the dev server. - * - * Absolute, against the document: the worker resolves a relative `wasmPaths` against - * *its* location, which is the one place the path must not be relative to. - */ +/** Where this deployment serves the ONNX Runtime Web assets packaged with the app. */ function ortAssetBaseUrl(): string { return new URL(`${import.meta.env.BASE_URL}ort/`, window.location.href).href; } @@ -45,95 +32,64 @@ const REAL_DEPS: Deps = { acquire: acquireEfficientSam, createRuntime: (artifacts) => createEfficientSamRuntime({ ...artifacts, assetBaseUrl: ortAssetBaseUrl() }), supported: browserSupports, + store: createCacheArtifactStore(), + discover: async (admission, signal) => { + const models = await fetchAdmittedBrowserModels(MODEL_CDN_BASE_URL, { signal }); + return models.some((model) => model.id === admission.id && model.revision === admission.revision); + }, }; +function unavailableStore(): BrowserArtifactStore { + return createCacheArtifactStore(undefined); +} + /** - * The executor is held *here*, beside the runtime it wraps, rather than built per call. - * - * `createBrowserSuggestionExecutor`'s one-embedding slot is the only encode cache in the - * system, and it lives in that closure — so a fresh executor per `executorFor` call throws - * the embedding away. `AnnotationPage` calls `executorFor` during render, and it re-renders - * on every click, which turned "one encode per asset, N decodes per N refinements" into a - * full ~25 MB encoder pass per refinement click. Memoizing at the composition root keeps the - * invariant true whatever `ui-core`'s render behaviour does, which a `useMemo` over there - * would not. + * Compose discovery and persistence in front of the existing Phase F executor. Artifact bytes + * have exactly one exit from the catalog: this adapter creates the same EfficientSAM runtime and + * the same retained executor that the one-session acquisition path used. */ -type State = - | { readonly kind: "unacquired" } - | { - readonly kind: "ready"; - readonly runtime: PromptableSegmentationRuntime; - readonly executor: SuggestionExecutor; - }; - export function createOssBrowserInferenceRuntime(deps: Deps = REAL_DEPS): VisionSetBrowserInferenceRuntime { - let state: State = { kind: "unacquired" }; - let inFlight: Promise | null = null; let activeAssetSource: BrowserSuggestionAssetSource | null = null; + const admissions = deps.supported() ? [EFFICIENT_SAM_TI_ADMISSION] : []; + const catalog = createBrowserModelCatalog({ + admissions, + store: deps.store ?? unavailableStore(), + discover: deps.discover ?? (async () => true), + download: (_admission, signal) => deps.acquire(signal), + activate: async (admission, artifacts) => { + const runtime = deps.createRuntime(artifacts); + const target: BrowserSuggestionTarget = { + id: admission.id, + label: admission.label, + modelRef: admission.modelRef, + }; + const executor: SuggestionExecutor = createBrowserSuggestionExecutor({ + modelRef: admission.modelRef, + runtime, + getActiveSource: () => activeAssetSource, + }); + return { runtime, executor, target }; + }, + }); return { - async listTargets(): Promise { - return state.kind === "ready" ? [{ id: MODEL_ID, label: "EfficientSAM-Ti", modelRef: MODEL_REF }] : []; + modelCatalog: catalog, + async listTargets() { + await catalog.initialized; + return catalog.listTargets(); }, listAcquisitions() { - if (state.kind === "ready") return []; - // The port's own convention, one level down: a control that cannot be honoured is not - // offered. A browser with no `Worker` or no WebAssembly can only fail this download. - if (!deps.supported()) return []; - return [ - { - id: MODEL_ID, - label: "EfficientSAM-Ti", - approxBytes: EFFICIENT_SAM_TI_EXPECTED.encoder.bytes + EFFICIENT_SAM_TI_EXPECTED.decoder.bytes, - acquire(options?: { readonly signal?: AbortSignal }): Promise { - if (state.kind === "ready") return Promise.resolve(); - if (inFlight !== null) return inFlight; - inFlight = (async () => { - try { - const artifacts = await deps.acquire(options?.signal); - const runtime = deps.createRuntime(artifacts); - // `createRuntime` returns before the worker has loaded either graph or - // settled on an execution provider; `ready()` is what waits for that. - // Claiming "ready" on the constructor alone lists a target that every - // later click refuses — and, since a listed target hides the download - // control, refuses with no way back short of a page reload. - try { - await runtime.ready(); - } catch (error) { - try { - runtime.dispose(); - } catch { - // A runtime that could not start may not stop cleanly either; the - // load failure is the one worth reporting, and the worker still has - // to be let go. - } - throw error; - } - state = { - kind: "ready", - runtime, - executor: createBrowserSuggestionExecutor({ - modelRef: MODEL_REF, - runtime, - getActiveSource: () => activeAssetSource, - }), - }; - } finally { - inFlight = null; - } - })(); - return inFlight; - }, - }, - ]; - }, - executorFor(targetId: string): SuggestionExecutor { - if (state.kind !== "ready" || targetId !== MODEL_ID) { - throw new Error(`no ready browser target "${targetId}"`); - } - return state.executor; + return catalog.snapshot() + .filter((entry) => entry.state === "available" || entry.state === "failed") + .map((entry) => ({ + id: entry.id, + label: entry.label, + approxBytes: entry.bytes, + acquire: (options?: { readonly signal?: AbortSignal }) => catalog.acquire(entry.id, options), + })); }, - setActiveAsset(source: BrowserSuggestionAssetSource | null): void { + executorFor: (targetId) => catalog.executorFor(targetId), + setActiveAsset(source) { activeAssetSource = source; }, }; diff --git a/frontend/app/src/data/browserInference/BrowserModelCatalog.test.ts b/frontend/app/src/data/browserInference/BrowserModelCatalog.test.ts new file mode 100644 index 00000000..6263cc83 --- /dev/null +++ b/frontend/app/src/data/browserInference/BrowserModelCatalog.test.ts @@ -0,0 +1,197 @@ +/** @vitest-environment node */ +import { describe, expect, it, vi } from "vitest"; +import type { PromptableSegmentationRuntime } from "@visionset/browser-inference"; +import type { BrowserSuggestionTarget, SuggestionExecutor } from "@visionset/ui-core"; + +import type { BrowserModelAdmission } from "./admissionCatalog.js"; +import type { BrowserArtifactStore, BrowserModelArtifacts } from "./artifactStore.js"; +import { createBrowserModelCatalog } from "./BrowserModelCatalog.js"; + +const ADMISSION: BrowserModelAdmission = { + id: "efficient-sam-ti", + label: "EfficientSAM-Ti", + revision: "fixture-revision", + modelRef: "robomous/efficient-sam-ti@fixture-revision", + manifestPath: "/models/efficient-sam-ti/fixture-revision/manifest.json", + adapter: "efficient-sam-ti", + license: "Apache-2.0", + source: { label: "EfficientSAM", repository: "https://example.test/source", revision: "source-rev" }, + runtime: { format: "onnx", opset: 17, onnxruntimeWeb: "1.29.0" }, + capabilities: { pointSuggest: true, positivePoints: true, negativePoints: false, maxPoints: 6 }, + artifacts: [ + { role: "encoder", path: "encoder.onnx", bytes: 3, sha256: "a".repeat(64), contentType: "application/octet-stream" }, + { role: "decoder", path: "decoder.onnx", bytes: 3, sha256: "b".repeat(64), contentType: "application/octet-stream" }, + ], +}; +const ARTIFACTS: BrowserModelArtifacts = { + encoder: new Uint8Array([1, 2, 3]), + decoder: new Uint8Array([4, 5, 6]), +}; + +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((done) => (resolve = done)); + return { promise, resolve }; +} + +function harness(overrides: { + installed?: boolean; + discover?: () => Promise; + read?: () => Promise; + write?: () => Promise; + download?: () => Promise; + ready?: () => Promise; +} = {}) { + let installed = overrides.installed ?? false; + const store: BrowserArtifactStore = { + inspect: vi.fn(async () => installed), + readVerified: vi.fn(overrides.read ?? (async () => (installed ? ARTIFACTS : null))), + writeVerified: vi.fn(overrides.write ?? (async () => { installed = true; })), + remove: vi.fn(async () => { installed = false; }), + }; + const dispose = vi.fn(); + const runtime = { + ready: vi.fn(overrides.ready ?? (async () => [])), + prepareImage: vi.fn(), + suggest: vi.fn(), + dispose, + } as unknown as PromptableSegmentationRuntime; + const executor = { suggest: vi.fn() } as unknown as SuggestionExecutor; + const target: BrowserSuggestionTarget = { + id: ADMISSION.id, + label: ADMISSION.label, + modelRef: ADMISSION.modelRef, + }; + const download = vi.fn(overrides.download ?? (async () => ARTIFACTS)); + const catalog = createBrowserModelCatalog({ + admissions: [ADMISSION], + store, + discover: overrides.discover ?? (async () => true), + download, + activate: vi.fn(async () => ({ runtime, executor, target })), + }); + return { catalog, store, download, runtime, executor, dispose }; +} + +async function settles(catalog: ReturnType): Promise { + await catalog.initialized; +} + +describe("createBrowserModelCatalog", () => { + it("publishes an admitted registry model as available without downloading it", async () => { + const { catalog, download } = harness(); + await settles(catalog); + expect(catalog.snapshot()).toEqual([ + expect.objectContaining({ id: ADMISSION.id, state: "available", storage: "none" }), + ]); + expect(download).not.toHaveBeenCalled(); + }); + + it("publishes a complete persistent cache as installed without activating it", async () => { + const { catalog, runtime } = harness({ installed: true, discover: async () => Promise.reject(new Error("offline")) }); + await settles(catalog); + expect(catalog.snapshot()[0]).toMatchObject({ state: "installed", storage: "persistent" }); + expect(runtime.ready).not.toHaveBeenCalled(); + }); + + it("deduplicates an explicit acquisition and exposes every lifecycle transition", async () => { + const pending = deferred(); + const { catalog, download } = harness({ download: () => pending.promise }); + const states: string[] = []; + catalog.subscribe(() => states.push(catalog.snapshot()[0]?.state ?? "hidden")); + await settles(catalog); + + const first = catalog.acquire(ADMISSION.id); + const second = catalog.acquire(ADMISSION.id); + expect(catalog.snapshot()[0]?.state).toBe("downloading"); + pending.resolve(ARTIFACTS); + await Promise.all([first, second]); + + expect(download).toHaveBeenCalledTimes(1); + expect(states).toEqual(expect.arrayContaining(["available", "downloading", "installed", "activating", "ready"])); + expect(catalog.listTargets()).toHaveLength(1); + }); + + it("keeps verified bytes ready for this session when persistent writing fails", async () => { + const { catalog } = harness({ write: async () => Promise.reject(new DOMException("quota", "QuotaExceededError")) }); + await settles(catalog); + + await catalog.acquire(ADMISSION.id); + + expect(catalog.snapshot()[0]).toMatchObject({ state: "ready", storage: "session" }); + expect(catalog.listTargets()).toHaveLength(1); + }); + + it("activates installed bytes from cache without any artifact download", async () => { + const { catalog, download, runtime, store } = harness({ installed: true }); + await settles(catalog); + + await catalog.activate(ADMISSION.id); + + expect(store.readVerified).toHaveBeenCalledTimes(1); + expect(download).not.toHaveBeenCalled(); + expect(runtime.ready).toHaveBeenCalledTimes(1); + expect(catalog.snapshot()[0]?.state).toBe("ready"); + }); + + it("fails closed on corrupt cached bytes and never turns activation into a download", async () => { + const { catalog, download } = harness({ + installed: true, + read: async () => Promise.reject(new Error("cached encoder SHA-256 mismatch")), + }); + await settles(catalog); + + await expect(catalog.activate(ADMISSION.id)).rejects.toThrow(/sha-256 mismatch/i); + + expect(download).not.toHaveBeenCalled(); + expect(catalog.snapshot()[0]).toMatchObject({ state: "failed", storage: "none" }); + expect(catalog.listTargets()).toEqual([]); + }); + + it("does not treat a missing decoder as installed or ready", async () => { + const { catalog, download } = harness({ installed: false, discover: async () => true }); + await settles(catalog); + await expect(catalog.activate(ADMISSION.id)).rejects.toThrow(/not installed/i); + expect(download).not.toHaveBeenCalled(); + expect(catalog.listTargets()).toEqual([]); + }); + + it("removes cached artifacts after invalidating and disposing the active runtime", async () => { + const { catalog, dispose, store } = harness({ installed: true }); + await settles(catalog); + await catalog.activate(ADMISSION.id); + const events: string[] = []; + dispose.mockImplementation(() => events.push("disposed")); + vi.mocked(store.remove).mockImplementation(async () => { events.push("removed"); }); + + await catalog.remove(ADMISSION.id); + + expect(events).toEqual(["disposed", "removed"]); + expect(catalog.listTargets()).toEqual([]); + expect(catalog.snapshot()[0]).toMatchObject({ state: "available", storage: "none" }); + }); + + it("keeps a cached admitted model usable when registry discovery fails", async () => { + const { catalog, download } = harness({ installed: true, discover: async () => Promise.reject(new Error("503")) }); + await settles(catalog); + await catalog.activate(ADMISSION.id); + expect(catalog.listTargets()).toHaveLength(1); + expect(download).not.toHaveBeenCalled(); + }); + + it("knows an admitted but uninstalled preference without fabricating a ready target", async () => { + const { catalog } = harness(); + expect(catalog.isKnown(ADMISSION.id)).toBe(true); + expect(catalog.isKnown("registry-only-model")).toBe(false); + expect(catalog.listTargets()).toEqual([]); + }); + + it("disposes a runtime whose ready step fails and leaves acquisition retryable", async () => { + const { catalog, dispose } = harness({ ready: async () => Promise.reject(new Error("graph load failed")) }); + await settles(catalog); + await expect(catalog.acquire(ADMISSION.id)).rejects.toThrow(/graph load failed/i); + expect(dispose).toHaveBeenCalledTimes(1); + expect(catalog.listTargets()).toEqual([]); + expect(catalog.snapshot()[0]?.state).toBe("failed"); + }); +}); diff --git a/frontend/app/src/data/browserInference/BrowserModelCatalog.ts b/frontend/app/src/data/browserInference/BrowserModelCatalog.ts new file mode 100644 index 00000000..44db9d1c --- /dev/null +++ b/frontend/app/src/data/browserInference/BrowserModelCatalog.ts @@ -0,0 +1,230 @@ +import type { PromptableSegmentationRuntime } from "@visionset/browser-inference"; +import type { + BrowserModelCatalog, + BrowserModelCatalogEntry, + BrowserSuggestionTarget, + SuggestionExecutor, +} from "@visionset/ui-core"; + +import type { BrowserModelAdmission } from "./admissionCatalog.js"; +import type { BrowserArtifactStore, BrowserModelArtifacts } from "./artifactStore.js"; + +interface ActiveModel { + readonly runtime: PromptableSegmentationRuntime; + readonly executor: SuggestionExecutor; + readonly target: BrowserSuggestionTarget; +} + +interface CatalogDeps { + readonly admissions: readonly BrowserModelAdmission[]; + readonly store: BrowserArtifactStore; + /** Validates that this exact admission still appears in the public registry. */ + readonly discover: (admission: BrowserModelAdmission, signal?: AbortSignal) => Promise; + /** Explicit network acquisition. No other dependency callback may fetch artifact bytes. */ + readonly download: ( + admission: BrowserModelAdmission, + signal?: AbortSignal, + ) => Promise; + readonly activate: ( + admission: BrowserModelAdmission, + artifacts: BrowserModelArtifacts, + ) => Promise; +} + +interface ModelRecord { + readonly admission: BrowserModelAdmission; + visible: boolean; + discovered: boolean; + state: BrowserModelCatalogEntry["state"]; + storage: BrowserModelCatalogEntry["storage"]; + error?: string; + sessionArtifacts?: BrowserModelArtifacts; +} + +export interface OssBrowserModelCatalog extends BrowserModelCatalog { + /** Initialization is app-internal; tests and composition may await it, UI subscribes instead. */ + readonly initialized: Promise; + listTargets(): readonly BrowserSuggestionTarget[]; + executorFor(targetId: string): SuggestionExecutor; +} + +function message(error: unknown): string { + return error instanceof Error ? error.message : "Browser model operation failed."; +} + +function entryOf(record: ModelRecord): BrowserModelCatalogEntry { + const { admission } = record; + return { + id: admission.id, + label: admission.label, + modelRef: admission.modelRef, + revision: admission.revision, + bytes: admission.artifacts.reduce((total, artifact) => total + artifact.bytes, 0), + license: admission.license, + source: { label: admission.source.label, href: admission.source.repository }, + state: record.state, + storage: record.storage, + ...(record.error === undefined ? {} : { error: record.error }), + }; +} + +export function createBrowserModelCatalog(deps: CatalogDeps): OssBrowserModelCatalog { + const records = new Map( + deps.admissions.map((admission) => [ + admission.id, + { + admission, + visible: false, + discovered: false, + state: "available", + storage: "none", + } satisfies ModelRecord, + ]), + ); + const listeners = new Set<() => void>(); + const operations = new Map>(); + let active: { readonly id: string; readonly value: ActiveModel } | null = null; + let snapshot: readonly BrowserModelCatalogEntry[] = []; + + function publish(): void { + snapshot = [...records.values()].filter((record) => record.visible).map(entryOf); + for (const listener of listeners) listener(); + } + + function update( + record: ModelRecord, + patch: Partial>, + ): void { + Object.assign(record, patch); + if (patch.error === undefined && "error" in patch) delete record.error; + publish(); + } + + function required(id: string): ModelRecord { + const record = records.get(id); + if (record === undefined) throw new Error(`unknown browser model "${id}"`); + return record; + } + + async function initializeRecord(record: ModelRecord): Promise { + const [cacheResult, registryResult] = await Promise.allSettled([ + deps.store.inspect(record.admission), + deps.discover(record.admission), + ]); + const installed = cacheResult.status === "fulfilled" && cacheResult.value; + const discovered = registryResult.status === "fulfilled" && registryResult.value; + record.discovered = discovered; + if (installed) { + update(record, { visible: true, state: "installed", storage: "persistent", error: undefined }); + } else if (discovered) { + update(record, { visible: true, state: "available", storage: "none", error: undefined }); + } + } + + async function startRuntime(record: ModelRecord, artifacts: BrowserModelArtifacts): Promise { + update(record, { state: "activating", error: undefined }); + let next: ActiveModel | null = null; + try { + next = await deps.activate(record.admission, artifacts); + await next.runtime.ready(); + if (active !== null && active.id !== record.admission.id) { + active.value.runtime.dispose(); + } + active = { id: record.admission.id, value: next }; + update(record, { visible: true, state: "ready", error: undefined }); + } catch (error) { + try { + next?.runtime.dispose(); + } catch { + // The start error is the actionable one; disposal is best-effort for a failed worker. + } + update(record, { visible: true, state: "failed", error: message(error) }); + throw error; + } + } + + function once(id: string, operation: () => Promise): Promise { + const current = operations.get(id); + if (current !== undefined) return current; + const promise = operation().finally(() => operations.delete(id)); + operations.set(id, promise); + return promise; + } + + const initialized = Promise.all([...records.values()].map(initializeRecord)).then(() => undefined); + + const catalog: OssBrowserModelCatalog = { + initialized, + snapshot: () => snapshot, + subscribe(listener) { + listeners.add(listener); + return () => listeners.delete(listener); + }, + isKnown: (id) => records.has(id), + acquire(id, options) { + return once(id, async () => { + const record = required(id); + if (record.state === "ready") return; + if (!record.discovered) { + record.discovered = await deps.discover(record.admission, options?.signal); + if (!record.discovered) throw new Error(`browser model "${id}" is not available from this registry`); + } + update(record, { visible: true, state: "downloading", storage: "none", error: undefined }); + try { + const artifacts = await deps.download(record.admission, options?.signal); + let storage: BrowserModelCatalogEntry["storage"] = "persistent"; + try { + await deps.store.writeVerified(record.admission, artifacts); + } catch { + storage = "session"; + record.sessionArtifacts = artifacts; + } + update(record, { state: "installed", storage, error: undefined }); + await startRuntime(record, artifacts); + } catch (error) { + if (record.state !== "failed") { + update(record, { visible: true, state: "failed", storage: "none", error: message(error) }); + } + throw error; + } + }); + }, + activate(id) { + return once(id, async () => { + const record = required(id); + if (record.state === "ready") return; + try { + const artifacts = record.sessionArtifacts ?? (await deps.store.readVerified(record.admission)); + if (artifacts === null) throw new Error(`${record.admission.label} is not installed in this browser`); + await startRuntime(record, artifacts); + } catch (error) { + record.sessionArtifacts = undefined; + if (active?.id === id) active = null; + update(record, { visible: true, state: "failed", storage: "none", error: message(error) }); + throw error; + } + }); + }, + remove(id) { + return once(id, async () => { + const record = required(id); + const previous = active?.id === id ? active.value : null; + if (previous !== null) active = null; + record.sessionArtifacts = undefined; + update(record, { visible: true, state: "available", storage: "none", error: undefined }); + if (previous !== null) previous.runtime.dispose(); + await deps.store.remove(record.admission); + }); + }, + listTargets() { + return active === null ? [] : [active.value.target]; + }, + executorFor(targetId) { + if (active === null || active.id !== targetId) { + throw new Error(`no ready browser target "${targetId}"`); + } + return active.value.executor; + }, + }; + return catalog; +} diff --git a/frontend/app/src/data/browserInference/acquireEfficientSam.ts b/frontend/app/src/data/browserInference/acquireEfficientSam.ts index 2964c814..59979fc8 100644 --- a/frontend/app/src/data/browserInference/acquireEfficientSam.ts +++ b/frontend/app/src/data/browserInference/acquireEfficientSam.ts @@ -41,7 +41,7 @@ export async function fetchVerified( url: string, expected: { readonly bytes: number; readonly sha256: string }, signal?: AbortSignal, -): Promise { +): Promise> { const response = await fetch(url, { signal }); if (!response.ok) throw new Error(`artifact fetch failed: ${response.status} ${response.statusText}`); const bytes = new Uint8Array(await response.arrayBuffer()); @@ -52,7 +52,10 @@ export async function fetchVerified( /** Fetches nothing until called, and hard-fails rather than constructing a runtime on any mismatch. */ export async function acquireEfficientSam( signal?: AbortSignal, -): Promise<{ readonly encoder: Uint8Array; readonly decoder: Uint8Array }> { +): Promise<{ + readonly encoder: Uint8Array; + readonly decoder: Uint8Array; +}> { const manifest = await fetchEfficientSamManifest(signal); const encoder = await fetchVerified( artifactUrl(manifest.artifacts.encoder.path), diff --git a/frontend/app/src/data/browserInference/admissionCatalog.ts b/frontend/app/src/data/browserInference/admissionCatalog.ts index f707c9c5..1d75eab1 100644 --- a/frontend/app/src/data/browserInference/admissionCatalog.ts +++ b/frontend/app/src/data/browserInference/admissionCatalog.ts @@ -15,6 +15,7 @@ export interface BrowserModelAdmission { readonly adapter: "efficient-sam-ti"; readonly license: string; readonly source: { + readonly label: string; readonly repository: string; readonly revision: string; }; @@ -41,6 +42,7 @@ export const EFFICIENT_SAM_TI_ADMISSION: BrowserModelAdmission = Object.freeze({ adapter: "efficient-sam-ti", license: "Apache-2.0", source: { + label: "EfficientSAM", repository: "https://github.com/yformer/EfficientSAM", revision: "d525f622e6f640acf5a0fc37c7ca1f243da5bde0", }, diff --git a/frontend/app/src/data/browserInference/artifactStore.test.ts b/frontend/app/src/data/browserInference/artifactStore.test.ts index 3082075a..88b4be14 100644 --- a/frontend/app/src/data/browserInference/artifactStore.test.ts +++ b/frontend/app/src/data/browserInference/artifactStore.test.ts @@ -33,7 +33,7 @@ async function fixture(revision = "rev-a"): Promise<{ admission: BrowserModelAdm manifestPath: `/models/fixture-model/${revision}/manifest.json`, adapter: "efficient-sam-ti", license: "Apache-2.0", - source: { repository: "https://example.test/source", revision: "source-rev" }, + source: { label: "Fixture upstream", repository: "https://example.test/source", revision: "source-rev" }, runtime: { format: "onnx", opset: 17, onnxruntimeWeb: "1.29.0" }, capabilities: { pointSuggest: true, positivePoints: true, negativePoints: false, maxPoints: 6 }, artifacts: [ diff --git a/frontend/ui-core/src/index.ts b/frontend/ui-core/src/index.ts index abc11b12..253bcf2c 100644 --- a/frontend/ui-core/src/index.ts +++ b/frontend/ui-core/src/index.ts @@ -124,6 +124,9 @@ export { } from "./inference/VisionSetBrowserInferenceProvider.js"; export type { ActiveSuggestionTarget, + BrowserModelCatalog, + BrowserModelCatalogEntry, + BrowserModelCatalogState, BrowserModelAcquisition, BrowserSuggestionAssetSource, BrowserSuggestionTarget, diff --git a/frontend/ui-core/src/inference/browserPort.test.ts b/frontend/ui-core/src/inference/browserPort.test.ts index f53095c1..e085372b 100644 --- a/frontend/ui-core/src/inference/browserPort.test.ts +++ b/frontend/ui-core/src/inference/browserPort.test.ts @@ -20,4 +20,30 @@ describe("VisionSetBrowserInferenceRuntime", () => { }; expect(runtime.listAcquisitions?.()).toHaveLength(1); }); + + it("accepts the additive reactive catalog without requiring it from existing hosts", () => { + const runtime: VisionSetBrowserInferenceRuntime = { + listTargets: async () => [], + executorFor: () => ({ suggest: async () => { throw new Error("unused"); } }), + modelCatalog: { + snapshot: () => [{ + id: "m", + label: "Model", + modelRef: "model@revision", + revision: "revision", + bytes: 10, + license: "Apache-2.0", + source: { label: "Upstream", href: "https://example.test/upstream" }, + state: "available", + storage: "none", + }], + subscribe: () => () => {}, + isKnown: (id) => id === "m", + acquire: async () => {}, + activate: async () => {}, + remove: async () => {}, + }, + }; + expect(runtime.modelCatalog?.isKnown("m")).toBe(true); + }); }); diff --git a/frontend/ui-core/src/inference/browserPort.ts b/frontend/ui-core/src/inference/browserPort.ts index b850fee3..f6692a16 100644 --- a/frontend/ui-core/src/inference/browserPort.ts +++ b/frontend/ui-core/src/inference/browserPort.ts @@ -15,10 +15,10 @@ * mechanism, because a host that answers these two questions is the whole of what the UI needs * and everything else would be this package guessing at an implementation it does not own. * - * It is an *execution* contract, not a catalog: it says what can answer now, and says nothing - * about what could be obtained. Finding and acquiring a model is a question about things that - * cannot answer yet, and it is answered elsewhere, later, by whatever seam the work that needs - * it earns. + * The execution members remain narrower than the optional catalog: `listTargets()` says what + * can answer now, while `modelCatalog` says what this host can acquire, activate, or remove. + * Keeping those answers separate prevents a downloaded-but-not-running model from becoming a + * target a click cannot actually use. */ import type { RgbPixels } from "@visionset/annotator"; import type { SuggestionExecutor } from "./suggestionExecutor.js"; @@ -67,6 +67,40 @@ export interface BrowserModelAcquisition { acquire(options?: { readonly signal?: AbortSignal }): Promise; } +export type BrowserModelCatalogState = + | "available" + | "downloading" + | "installed" + | "activating" + | "ready" + | "failed"; + +export interface BrowserModelCatalogEntry { + readonly id: string; + readonly label: string; + readonly modelRef: string; + readonly revision: string; + readonly bytes: number; + readonly license: string; + readonly source: { readonly label: string; readonly href: string }; + readonly state: BrowserModelCatalogState; + /** `session` means verified bytes are usable now but were not saved persistently. */ + readonly storage: "none" | "persistent" | "session"; + readonly error?: string; +} + +/** A host-owned catalog. Remote metadata is data; implementations never execute values from it. */ +export interface BrowserModelCatalog { + /** Stable by identity until a subscribed change is published; suitable for useSyncExternalStore. */ + snapshot(): readonly BrowserModelCatalogEntry[]; + subscribe(listener: () => void): () => void; + /** Whether this build admits an ID, including while registry discovery is still pending. */ + isKnown(id: string): boolean; + acquire(id: string, options?: { readonly signal?: AbortSignal }): Promise; + activate(id: string, options?: { readonly signal?: AbortSignal }): Promise; + remove(id: string): Promise; +} + /** * Which kind of thing answers a suggestion now: a workspace connection, or a browser * target. Never persisted as an `InferenceConnection` — a browser target answers @@ -88,6 +122,8 @@ export interface VisionSetBrowserInferenceRuntime { listTargets(): Promise; /** How to ask one of them. The same contract the server path answers through. */ executorFor(targetId: string): SuggestionExecutor; + /** Reactive discovery/acquisition state. Additive: Phase F hosts may omit it. */ + readonly modelCatalog?: BrowserModelCatalog; /** Models not yet ready, each with its own explicit `acquire()`. Absent hosts offer none. */ listAcquisitions?(): readonly BrowserModelAcquisition[]; /** The displayed asset, or `null` between assets. Feeds the executor's race-safety checks. */ From cb74f8db31ce059a914b613c45535a756e510e60 Mon Sep 17 00:00:00 2001 From: Jesus Armando Anaya <1445792+JArmandoAnaya@users.noreply.github.com> Date: Thu, 17 Sep 2026 11:48:40 -0700 Subject: [PATCH 04/15] feat(inference): manage browser models in suggest panel --- .../ui-core/src/annotator/AnnotationPage.tsx | 60 +++++-- .../ui-core/src/annotator/SuggestPanel.tsx | 146 +++++++++++++++++- .../src/annotator/suggestPanel.test.tsx | 104 ++++++++++++- .../src/inference/browserRuntime.test.tsx | 6 + 4 files changed, 302 insertions(+), 14 deletions(-) diff --git a/frontend/ui-core/src/annotator/AnnotationPage.tsx b/frontend/ui-core/src/annotator/AnnotationPage.tsx index d21c2930..779a1d40 100644 --- a/frontend/ui-core/src/annotator/AnnotationPage.tsx +++ b/frontend/ui-core/src/annotator/AnnotationPage.tsx @@ -137,6 +137,7 @@ import { useMemo, useRef, useState, + useSyncExternalStore, type JSX, type ReactNode, } from "react"; @@ -241,13 +242,19 @@ export function staleStoredBrowserTarget( storedTarget: StoredSuggestTarget, browserTargets: readonly BrowserSuggestionTarget[] | undefined, explicitlyChosen: boolean, + knownByCatalog = false, ): boolean { if (browserTargets === undefined) return false; if (storedTarget.kind !== "browser") return false; if (explicitlyChosen) return false; + if (knownByCatalog) return false; return !browserTargets.some((row) => row.id === storedTarget.targetId); } +const EMPTY_BROWSER_MODELS = Object.freeze([]); +const emptyBrowserModels = () => EMPTY_BROWSER_MODELS; +const subscribeToNothing = () => () => {}; + /** * Where "a trackpad has been seen on this browser" is remembered. * @@ -970,6 +977,12 @@ function Workspace({ const [adjusting, setAdjusting] = useState(false); const browserRuntime = useBrowserInferenceRuntime(); + const browserCatalog = browserRuntime?.modelCatalog; + const browserModels = useSyncExternalStore( + browserCatalog?.subscribe ?? subscribeToNothing, + browserCatalog?.snapshot ?? emptyBrowserModels, + browserCatalog?.snapshot ?? emptyBrowserModels, + ); useEffect(() => { return () => browserRuntime?.setActiveAsset?.(null); @@ -1023,17 +1036,13 @@ function Workspace({ return () => { cancelled = true; }; - }, [browserRuntime, browserTargetsRefreshKey]); + }, [browserRuntime, browserTargetsRefreshKey, browserModels]); const [storedTarget, setStoredTarget] = useState(() => readStoredSuggestTarget(projectId)); - // Whether *this session* picked a browser target through `chooseTarget`, as opposed to one - // merely read back from a persisted preference. Acquired model bytes are never persisted - // across page loads (a later phase's work), so "the stored browser target isn't in - // `listTargets()`'s answer" is the ordinary shape of every reload for someone who picked - // "This device" last time — not a rare failure. That case must fall back to Server - // silently. A target this session explicitly chose and which later drops out is a - // different thing (surfaced via `blocker`/`refusal`, never auto-switched), and this ref is - // what tells the two apart. + // Whether this session picked a browser target through `chooseTarget`, as opposed to one + // merely read back from a preference. The catalog distinguishes a known uninstalled model + // from a genuinely unknown stale ID; this ref still prevents an explicit in-session failure + // from silently switching to Server. const explicitlyChosenBrowser = useRef(false); // A stale/unavailable *stored* preference falls back to Server silently, once the browser @@ -1043,15 +1052,37 @@ function Workspace({ // `suggest.target.` key, so a later reload re-checks the same id once that // model has actually been re-acquired. useEffect(() => { - if (staleStoredBrowserTarget(storedTarget, browserTargets, explicitlyChosenBrowser.current)) { + const knownByCatalog = + storedTarget.kind === "browser" && browserCatalog?.isKnown(storedTarget.targetId) === true; + if ( + staleStoredBrowserTarget( + storedTarget, + browserTargets, + explicitlyChosenBrowser.current, + knownByCatalog, + ) + ) { setStoredTarget({ kind: "server" }); } - }, [browserTargets, storedTarget]); + }, [browserCatalog, browserTargets, browserModels, storedTarget]); const activeTarget: ActiveSuggestionTarget = browserRuntime !== null && storedTarget.kind === "browser" ? { kind: "browser", targetId: storedTarget.targetId } : { kind: "server", connectionId: connection?.id ?? "" }; + const activeBrowserTargetId = activeTarget.kind === "browser" ? activeTarget.targetId : null; + const suggestArmed = session !== null; + + useEffect(() => { + if (!suggestArmed || activeBrowserTargetId === null || browserCatalog === undefined) return; + const selected = browserModels.find((entry) => entry.id === activeBrowserTargetId); + if (selected?.state !== "installed") return; + // This reads verified local bytes only. `acquire()` remains the sole operation allowed to + // fetch a missing artifact, and the catalog publishes any activation failure for the panel. + void browserCatalog + .activate(activeBrowserTargetId) + .catch(() => setBrowserTargetsRefreshKey((key) => key + 1)); + }, [activeBrowserTargetId, browserCatalog, browserModels, suggestArmed]); const blocker = computeSuggestBlocker(activeTarget, serverBlocker, browserTargets); // `executorFor` throws for a target that isn't actually ready yet — which is exactly the @@ -2900,6 +2931,13 @@ function Workspace({ blocker={blocker} browserTargets={browserRuntime === null ? undefined : browserTargets} browserAcquisitions={browserRuntime?.listAcquisitions?.()} + {...(browserCatalog === undefined + ? {} + : { + browserModels, + onAcquireBrowserModel: (id: string) => browserCatalog.acquire(id), + onRemoveBrowserModel: (id: string) => browserCatalog.remove(id), + })} activeTarget={activeTarget} onChooseTarget={chooseTarget} onAcquired={() => setBrowserTargetsRefreshKey((key) => key + 1)} diff --git a/frontend/ui-core/src/annotator/SuggestPanel.tsx b/frontend/ui-core/src/annotator/SuggestPanel.tsx index a6a3bb92..cbdc6662 100644 --- a/frontend/ui-core/src/annotator/SuggestPanel.tsx +++ b/frontend/ui-core/src/annotator/SuggestPanel.tsx @@ -84,6 +84,7 @@ import type { Connection, SuggestBlocker } from "../data/inferenceQueries"; import type { ActiveSuggestionTarget, BrowserModelAcquisition, + BrowserModelCatalogEntry, BrowserSuggestionTarget, } from "../inference/browserPort.js"; @@ -134,6 +135,10 @@ export interface SuggestPanelProps { readonly browserTargets?: readonly BrowserSuggestionTarget[]; /** Models not yet acquired. `undefined` when no browser runtime is wired at all. */ readonly browserAcquisitions?: readonly BrowserModelAcquisition[]; + /** Reactive install/activation state. Absent keeps the Phase F acquisition UI. */ + readonly browserModels?: readonly BrowserModelCatalogEntry[]; + readonly onAcquireBrowserModel?: (id: string) => Promise; + readonly onRemoveBrowserModel?: (id: string) => Promise; readonly activeTarget?: ActiveSuggestionTarget; readonly onChooseTarget?: (target: ActiveSuggestionTarget) => void; /** Called once an `acquire()` this panel started resolves, so the host can re-read `listTargets()`. */ @@ -222,6 +227,9 @@ export function SuggestPanel({ onConfigure, browserTargets, browserAcquisitions, + browserModels, + onAcquireBrowserModel, + onRemoveBrowserModel, activeTarget, onChooseTarget, onAcquired, @@ -237,7 +245,8 @@ export function SuggestPanel({ // apart — which is exactly the shape of the bug fixed alongside this line: // the guard used to fire on a wired runtime's own "not-ready" and blank out // the chooser it should have deferred to instead. - const runtimeWired = browserTargets !== undefined || browserAcquisitions !== undefined; + const runtimeWired = + browserTargets !== undefined || browserAcquisitions !== undefined || browserModels !== undefined; /* Parked outranks even the blocker. A connection this tool will not use @@ -491,6 +500,9 @@ export function SuggestPanel({ onDiscard={onDiscard} browserTargets={browserTargets ?? []} browserAcquisitions={browserAcquisitions ?? []} + {...(browserModels === undefined ? {} : { browserModels })} + {...(onAcquireBrowserModel === undefined ? {} : { onAcquireBrowserModel })} + {...(onRemoveBrowserModel === undefined ? {} : { onRemoveBrowserModel })} activeTarget={activeTarget} {...(onChooseTarget === undefined ? {} : { onChooseTarget })} {...(onAcquired === undefined ? {} : { onAcquired })} @@ -583,6 +595,9 @@ function TargetChooser({ onDiscard, browserTargets, browserAcquisitions, + browserModels, + onAcquireBrowserModel, + onRemoveBrowserModel, activeTarget, onChooseTarget, onAcquired, @@ -596,6 +611,9 @@ function TargetChooser({ readonly onDiscard: () => void; readonly browserTargets: readonly BrowserSuggestionTarget[]; readonly browserAcquisitions: readonly BrowserModelAcquisition[]; + readonly browserModels?: readonly BrowserModelCatalogEntry[]; + readonly onAcquireBrowserModel?: (id: string) => Promise; + readonly onRemoveBrowserModel?: (id: string) => Promise; readonly activeTarget: ActiveSuggestionTarget | undefined; readonly onChooseTarget?: (target: ActiveSuggestionTarget) => void; readonly onAcquired?: () => void; @@ -612,7 +630,7 @@ function TargetChooser({ onChooseTarget?.({ kind: "server", connectionId: connectionId ?? "" }); return; } - const targetId = browserTargets[0]?.id ?? browserAcquisitions[0]?.id; + const targetId = browserTargets[0]?.id ?? browserModels?.[0]?.id ?? browserAcquisitions[0]?.id; if (targetId !== undefined) { onChooseTarget?.({ kind: "browser", targetId }); } @@ -656,6 +674,9 @@ function TargetChooser({ @@ -729,15 +750,32 @@ function BlockedMessage({ function DeviceTab({ targets, acquisitions, + models, + onAcquireModel, + onRemoveModel, onAcquired, }: { readonly targets: readonly BrowserSuggestionTarget[]; readonly acquisitions: readonly BrowserModelAcquisition[]; + readonly models?: readonly BrowserModelCatalogEntry[]; + readonly onAcquireModel?: (id: string) => Promise; + readonly onRemoveModel?: (id: string) => Promise; readonly onAcquired?: () => void; }): JSX.Element | null { const [acquiring, setAcquiring] = useState(false); const [failed, setFailed] = useState(false); + const model = models?.[0]; + if (model !== undefined) { + return ( + + ); + } + const target = targets[0]; if (target !== undefined) { return ( @@ -783,6 +821,110 @@ function DeviceTab({ ); } +function catalogFailure(error: string | undefined): string | null { + if (error === undefined) return null; + if (/sha-256|size mismatch|verification/i.test(error)) { + return "This model failed verification. Download it again to use it on this device."; + } + return "That browser model operation failed. Try again."; +} + +function CatalogModel({ + model, + onAcquire, + onRemove, +}: { + readonly model: BrowserModelCatalogEntry; + readonly onAcquire?: (id: string) => Promise; + readonly onRemove?: (id: string) => Promise; +}): JSX.Element { + const [localError, setLocalError] = useState(null); + const [removing, setRemoving] = useState(false); + const busy = model.state === "downloading" || model.state === "activating"; + const canRemove = model.storage !== "none" || model.state === "installed" || model.state === "ready"; + const failure = localError ?? catalogFailure(model.error); + const stateLabel = + model.state === "downloading" + ? "Downloading…" + : model.state === "installed" + ? "Installed" + : model.state === "activating" + ? "Loading…" + : model.state === "ready" + ? "Ready" + : null; + + return ( +
+
+ {model.label} + {stateLabel !== null && ( + {stateLabel} + )} +
+

+ ~{Math.round(model.bytes / 1_000_000)} MB · {model.license} ·{" "} + + {model.source.label} + +

+ {(model.state === "available" || model.state === "failed") && onAcquire !== undefined && ( + + )} + {busy && ( +

+

+ )} + {model.state === "ready" && model.storage === "session" && ( +

+ Ready for this session, but it was not saved in this browser. +

+ )} + {canRemove && onRemove !== undefined && ( + + )} + {failure !== null &&

{failure}

} +
+ ); +} + /** The take-back, where a state has something to take back and nothing to accept. */ function Discard({ onDiscard }: { readonly onDiscard: () => void }): JSX.Element { return ( diff --git a/frontend/ui-core/src/annotator/suggestPanel.test.tsx b/frontend/ui-core/src/annotator/suggestPanel.test.tsx index 00dd7621..cbf9a1ad 100644 --- a/frontend/ui-core/src/annotator/suggestPanel.test.tsx +++ b/frontend/ui-core/src/annotator/suggestPanel.test.tsx @@ -20,7 +20,11 @@ import type { Suggestion, SuggestionState } from "@visionset/annotator"; import { SuggestPanel } from "./SuggestPanel"; import type { Answer } from "@visionset/annotator"; import { usableConnection, type Connection } from "../data/inferenceQueries"; -import type { BrowserModelAcquisition, BrowserSuggestionTarget } from "../inference/browserPort.js"; +import type { + BrowserModelAcquisition, + BrowserModelCatalogEntry, + BrowserSuggestionTarget, +} from "../inference/browserPort.js"; const A_BOX = { type: "bbox", x: 10, y: 20, width: 30, height: 40 } as const; @@ -658,6 +662,104 @@ describe("this device, once a browser runtime is wired", () => { }; } + function catalogEntry(overrides: Partial = {}): BrowserModelCatalogEntry { + return { + id: "efficient-sam-ti", + label: "EfficientSAM-Ti", + modelRef: "robomous/efficient-sam-ti@revision", + revision: "revision", + bytes: 41_301_678, + license: "Apache-2.0", + source: { label: "EfficientSAM", href: "https://github.com/yformer/EfficientSAM" }, + state: "available", + storage: "none", + ...overrides, + }; + } + + it("shows admitted model identity and explicitly acquires an available model", async () => { + const acquire = vi.fn().mockResolvedValue(undefined); + const user = userEvent.setup(); + render(mount({ + browserTargets: [], + browserModels: [catalogEntry()], + activeTarget: { kind: "browser", targetId: "efficient-sam-ti" }, + onChooseTarget: vi.fn(), + onAcquireBrowserModel: acquire, + })); + + const section = screen.getByTestId("suggest-device-section"); + expect(section.textContent).toContain("EfficientSAM-Ti"); + expect(section.textContent).toContain("41 MB"); + expect(section.textContent).toContain("Apache-2.0"); + expect(screen.getByRole("link", { name: "EfficientSAM" }).getAttribute("href")).toBe( + "https://github.com/yformer/EfficientSAM", + ); + await user.click(screen.getByTestId("suggest-device-acquire-efficient-sam-ti")); + expect(acquire).toHaveBeenCalledWith("efficient-sam-ti"); + }); + + it.each([ + ["downloading", "Downloading…"], + ["installed", "Installed"], + ["activating", "Loading…"], + ["ready", "Ready"], + ] as const)("renders the catalog %s state as %s", (state, label) => { + render(mount({ + browserTargets: state === "ready" ? [READY] : [], + browserModels: [catalogEntry({ state, storage: state === "downloading" ? "none" : "persistent" })], + activeTarget: { kind: "browser", targetId: READY.id }, + onChooseTarget: vi.fn(), + onAcquireBrowserModel: vi.fn(), + onRemoveBrowserModel: vi.fn(), + })); + expect(screen.getByTestId("suggest-device-section").textContent).toContain(label); + }); + + it("removes an installed model through an explicit packaged control", async () => { + const remove = vi.fn().mockResolvedValue(undefined); + const user = userEvent.setup(); + render(mount({ + browserTargets: [], + browserModels: [catalogEntry({ state: "installed", storage: "persistent" })], + activeTarget: { kind: "browser", targetId: READY.id }, + onChooseTarget: vi.fn(), + onRemoveBrowserModel: remove, + })); + await user.click(screen.getByTestId("suggest-device-remove-efficient-sam-ti")); + expect(remove).toHaveBeenCalledWith("efficient-sam-ti"); + }); + + it("reports session-only readiness without claiming the model is installed", () => { + render(mount({ + browserTargets: [READY], + browserModels: [catalogEntry({ state: "ready", storage: "session" })], + activeTarget: { kind: "browser", targetId: READY.id }, + onChooseTarget: vi.fn(), + onRemoveBrowserModel: vi.fn(), + })); + expect(screen.getByTestId("suggest-device-session-only").textContent).toMatch( + /ready for this session.*not saved/i, + ); + expect(screen.getByTestId("suggest-device-section").textContent).not.toContain("Installed"); + }); + + it("turns a cached integrity failure into useful prose and another explicit Download", () => { + render(mount({ + browserTargets: [], + browserModels: [catalogEntry({ + state: "failed", + storage: "none", + error: "cached encoder SHA-256 mismatch", + })], + activeTarget: { kind: "browser", targetId: READY.id }, + onChooseTarget: vi.fn(), + onAcquireBrowserModel: vi.fn(), + })); + expect(screen.getByRole("alert").textContent).toMatch(/failed verification.*download/i); + expect(screen.getByTestId("suggest-device-acquire-efficient-sam-ti")).toBeTruthy(); + }); + it("renders no device section, and no tab chooser, when no runtime is wired at all", () => { render(mount()); diff --git a/frontend/ui-core/src/inference/browserRuntime.test.tsx b/frontend/ui-core/src/inference/browserRuntime.test.tsx index a439a789..bfea0d07 100644 --- a/frontend/ui-core/src/inference/browserRuntime.test.tsx +++ b/frontend/ui-core/src/inference/browserRuntime.test.tsx @@ -445,6 +445,12 @@ describe("staleStoredBrowserTarget", () => { // fails must keep surfacing through `blocker`/`refusal`, not silently revert. expect(staleStoredBrowserTarget({ kind: "browser", targetId: "gone" }, [], true)).toBe(false); }); + + it("keeps a known admitted preference when the model is not installed yet", () => { + expect(staleStoredBrowserTarget({ kind: "browser", targetId: "efficient-sam-ti" }, [], false, true)).toBe( + false, + ); + }); }); describe("BrowserSuggestionAssetSource", () => { From 4ce489099c8173136ed9c9f7814c7e23da51f8ba Mon Sep 17 00:00:00 2001 From: Jesus Armando Anaya <1445792+JArmandoAnaya@users.noreply.github.com> Date: Thu, 17 Sep 2026 11:58:46 -0700 Subject: [PATCH 05/15] test(inference): prove persistent browser model lifecycle --- frontend/app/e2e/browserSuggestion.spec.ts | 137 +++++++++++++++++- .../BrowserModelCatalog.test.ts | 17 +++ .../browserInference/BrowserModelCatalog.ts | 13 +- .../ui-core/src/annotator/AnnotationPage.tsx | 37 +++-- .../src/inference/browserRuntime.test.tsx | 35 ++++- 5 files changed, 219 insertions(+), 20 deletions(-) diff --git a/frontend/app/e2e/browserSuggestion.spec.ts b/frontend/app/e2e/browserSuggestion.spec.ts index 26fec77c..6c6726c1 100644 --- a/frontend/app/e2e/browserSuggestion.spec.ts +++ b/frontend/app/e2e/browserSuggestion.spec.ts @@ -64,20 +64,45 @@ const DECODER_BYTES = HAS_ARTIFACTS ? readFileSync(DECODER_PATH) : Buffer.alloc( * transform and would throw under this suite's plain Node/tsx loader. */ const REAL_ENCODER_BYTE_LENGTH = 24_799_777; +const REVISION = "b19782d049c0-843761ca46f4"; +const MODEL_REF = `robomous/efficient-sam-ti@${REVISION}`; +const REGISTRY = { + schema_version: 1, + models: [ + { id: "efficient-sam-ti", name: "EfficientSAM-Ti", revision: REVISION, model_ref: MODEL_REF, manifest: `/models/efficient-sam-ti/${REVISION}/manifest.json` }, + { id: "mobile-sam", name: "MobileSAM", revision: "359e37f2b168-7983079ab060", model_ref: "robomous/mobile-sam@359e37f2b168-7983079ab060", manifest: "/models/mobile-sam/359e37f2b168-7983079ab060/manifest.json" }, + { id: "efficientvit-sam-l0", name: "EfficientViT-SAM-L0", revision: "e48dd681ba4b-1d3ba86d781b", model_ref: "robomous/efficientvit-sam-l0@e48dd681ba4b-1d3ba86d781b", manifest: "/models/efficientvit-sam-l0/e48dd681ba4b-1d3ba86d781b/manifest.json" }, + { id: "slimsam-77-uniform", name: "SlimSAM-77-uniform", revision: "7f2c646efd21-e6eb3c03cdbd", model_ref: "robomous/slimsam-77-uniform@7f2c646efd21-e6eb3c03cdbd", manifest: "/models/slimsam-77-uniform/7f2c646efd21-e6eb3c03cdbd/manifest.json" }, + { id: "sam2.1-hiera-tiny", name: "SAM2.1-hiera-tiny", revision: "7f000e65546d-6dbe21e6e60e", model_ref: "robomous/sam2.1-hiera-tiny@7f000e65546d-6dbe21e6e60e", manifest: "/models/sam2.1-hiera-tiny/7f000e65546d-6dbe21e6e60e/manifest.json" }, + ], +}; +const MANIFEST = { + schema_version: 1, + id: "efficient-sam-ti", + name: "EfficientSAM-Ti", + revision: REVISION, + model_ref: MODEL_REF, + source: { repository: "https://github.com/yformer/EfficientSAM", revision: "d525f622e6f640acf5a0fc37c7ca1f243da5bde0" }, + runtime: { format: "onnx", opset: 17, onnxruntime_web: "1.29.0" }, + capabilities: { point_suggest: true, positive_points: true, negative_points: false, max_points: 6 }, + artifacts: { + encoder: { path: "encoder.onnx", bytes: 24_799_777, sha256: "b19782d049c09a8f1cc36ccc6029264ca23c8ac35e6379fd9ef9f1bc6d81e7f2", content_type: "application/octet-stream" }, + decoder: { path: "decoder.onnx", bytes: 16_501_901, sha256: "843761ca46f4aa00b09fdcf0c94271321f76eece092a744296c742d682a86172", content_type: "application/octet-stream" }, + }, +}; /** Routes the CDN manifest + artifacts to the local fixture — no network call ever leaves the page. */ async function mockCdn(page: Page, encoderBytes = ENCODER_BYTES, decoderBytes = DECODER_BYTES): Promise { // Registered first, so it is matched *last* (Playwright tries the most-recently-added - // handler first): anything at this host the three specific routes below don't + // handler first): anything at this host the specific routes below don't // recognise is hard-aborted rather than silently reaching the real CDN — including if // `VITE_MODEL_CDN_BASE_URL` or the manifest layout ever drifts out from under this stub. await page.route("**/models.robomous.ai/**", (route) => route.abort()); - // The real, live manifest shape (fixed in manifest.ts/acquireEfficientSam.ts after a - // production incident): artifacts nest under `artifacts`, and each `path` is a bare - // filename resolved relative to the manifest's own revision directory — never a - // leading-slash, top-level path. + await page.route("**/models.robomous.ai/registry/v1.json", (route) => route.fulfill({ json: REGISTRY })); + // The complete deployed v1 shape: registry admission validates every field before + // acquisition, while the pinned build record remains the integrity anchor. await page.route("**/models.robomous.ai/models/efficient-sam-ti/**/manifest.json", (route) => - route.fulfill({ json: { artifacts: { encoder: { path: "encoder.onnx" }, decoder: { path: "decoder.onnx" } } } }), + route.fulfill({ json: MANIFEST }), ); await page.route("**/models.robomous.ai/models/efficient-sam-ti/**/encoder.onnx", (route) => route.fulfill({ body: encoderBytes, contentType: "application/octet-stream" }), @@ -185,7 +210,7 @@ function suggestCallsOf(sent: Request[]): Request[] { function countModelRequestsFromNow(page: Page): () => number { let count = 0; page.on("request", (request) => { - if (request.url().includes("models.robomous.ai")) count += 1; + if (/models\.robomous\.ai\/.*\/(encoder|decoder)\.onnx$/.test(request.url())) count += 1; }); return () => count; } @@ -211,7 +236,15 @@ async function armSuggestTool(page: Page): Promise { async function acquireAndSelectBrowserTarget(page: Page): Promise { await page.getByTestId("suggest-target-browser").click(); await page.getByTestId("suggest-device-acquire-efficient-sam-ti").click(); - await expect(page.getByTestId("suggest-device-section").getByText(/ready/i)).toBeVisible({ timeout: 60_000 }); + await expect(page.getByTestId("suggest-device-section").getByText("Ready", { exact: true })).toBeVisible({ + timeout: 60_000, + }); +} + +async function makeBrowserSuggestion(page: Page): Promise { + const picture = (await page.getByTestId("annotator-canvas").boundingBox())!; + await page.mouse.click(picture.x + picture.width / 2, picture.y + picture.height / 2); + await expect(page.getByTestId("suggestion-shape")).toBeVisible({ timeout: 30_000 }); } test.describe("browser suggestion", () => { @@ -239,6 +272,94 @@ test.describe("browser suggestion", () => { expect(suggestCallsOf(sent)).toHaveLength(0); }); + test("an installed model survives reload and suggests again without an artifact GET", async ({ page }) => { + const sent: Request[] = []; + const artifactRequests = countModelRequestsFromNow(page); + await openJobWithBrowserRuntime(page, sent, true); + await armSuggestTool(page); + await acquireAndSelectBrowserTarget(page); + await makeBrowserSuggestion(page); + expect(artifactRequests()).toBe(2); + + await page.reload(); + await expect(page.getByTestId("annotation-page")).toBeVisible(); + await armSuggestTool(page); + await expect(page.getByTestId("suggest-device-section").getByText(/ready/i)).toBeVisible({ timeout: 60_000 }); + expect(artifactRequests()).toBe(2); + await makeBrowserSuggestion(page); + expect(suggestCallsOf(sent)).toHaveLength(0); + }); + + test("an installed model remains usable when registry and artifact routes fail", async ({ page }) => { + const sent: Request[] = []; + await openJobWithBrowserRuntime(page, sent, true); + await armSuggestTool(page); + await acquireAndSelectBrowserTarget(page); + await makeBrowserSuggestion(page); + + await page.route("**/models.robomous.ai/registry/v1.json", (route) => + route.fulfill({ status: 503, body: "offline fixture" }), + ); + await page.route("**/models.robomous.ai/**/*.onnx", (route) => + route.fulfill({ status: 503, body: "offline fixture" }), + ); + const artifactRequests = countModelRequestsFromNow(page); + await page.reload(); + await expect(page.getByTestId("annotation-page")).toBeVisible(); + await armSuggestTool(page); + await expect(page.getByTestId("suggest-device-section").getByText(/ready/i)).toBeVisible({ timeout: 60_000 }); + expect(artifactRequests()).toBe(0); + await makeBrowserSuggestion(page); + expect(suggestCallsOf(sent)).toHaveLength(0); + }); + + test("Remove from this browser clears artifacts, disposes readiness, and survives reload", async ({ page }) => { + const sent: Request[] = []; + await openJobWithBrowserRuntime(page, sent, true); + await armSuggestTool(page); + await acquireAndSelectBrowserTarget(page); + + await page.getByTestId("suggest-device-remove-efficient-sam-ti").click(); + await expect(page.getByTestId("suggest-device-acquire-efficient-sam-ti")).toBeVisible(); + expect(await page.evaluate(async () => (await caches.open("visionset-browser-models-v1")).keys().then((keys) => keys.length))).toBe(0); + + const artifactRequests = countModelRequestsFromNow(page); + await page.reload(); + await expect(page.getByTestId("annotation-page")).toBeVisible(); + await armSuggestTool(page); + await expect(page.getByTestId("suggest-device-acquire-efficient-sam-ti")).toBeVisible(); + expect(artifactRequests()).toBe(0); + }); + + test("a persistent storage refusal is reported as session-only readiness", async ({ page }) => { + await page.addInitScript(() => { + const nativeOpen = caches.open.bind(caches); + caches.open = async (name: string): Promise => { + const cache = await nativeOpen(name); + return { + add: cache.add.bind(cache), + addAll: cache.addAll.bind(cache), + match: cache.match.bind(cache), + matchAll: cache.matchAll.bind(cache), + delete: cache.delete.bind(cache), + keys: cache.keys.bind(cache), + put: async () => { throw new DOMException("fixture quota", "QuotaExceededError"); }, + }; + }; + }); + const sent: Request[] = []; + await openJobWithBrowserRuntime(page, sent, true); + await armSuggestTool(page); + await acquireAndSelectBrowserTarget(page); + await expect(page.getByTestId("suggest-device-session-only")).toBeVisible(); + await makeBrowserSuggestion(page); + + await page.reload(); + await expect(page.getByTestId("annotation-page")).toBeVisible(); + await armSuggestTool(page); + await expect(page.getByTestId("suggest-device-acquire-efficient-sam-ti")).toBeVisible(); + }); + test("a ready browser target is never blocked by a server-connection blocker", async ({ page }) => { const sent: Request[] = []; await openJobWithBrowserRuntime(page, sent, false); // no server connections diff --git a/frontend/app/src/data/browserInference/BrowserModelCatalog.test.ts b/frontend/app/src/data/browserInference/BrowserModelCatalog.test.ts index 6263cc83..4e84c271 100644 --- a/frontend/app/src/data/browserInference/BrowserModelCatalog.test.ts +++ b/frontend/app/src/data/browserInference/BrowserModelCatalog.test.ts @@ -171,6 +171,23 @@ describe("createBrowserModelCatalog", () => { expect(catalog.snapshot()[0]).toMatchObject({ state: "available", storage: "none" }); }); + it("does not claim cached artifacts were removed when persistent deletion fails", async () => { + const { catalog, dispose, store } = harness({ installed: true }); + await settles(catalog); + await catalog.activate(ADMISSION.id); + vi.mocked(store.remove).mockRejectedValue(new Error("storage delete failed")); + + await expect(catalog.remove(ADMISSION.id)).rejects.toThrow(/storage delete failed/i); + + expect(dispose).toHaveBeenCalledTimes(1); + expect(catalog.listTargets()).toEqual([]); + expect(catalog.snapshot()[0]).toMatchObject({ + state: "failed", + storage: "persistent", + error: "storage delete failed", + }); + }); + it("keeps a cached admitted model usable when registry discovery fails", async () => { const { catalog, download } = harness({ installed: true, discover: async () => Promise.reject(new Error("503")) }); await settles(catalog); diff --git a/frontend/app/src/data/browserInference/BrowserModelCatalog.ts b/frontend/app/src/data/browserInference/BrowserModelCatalog.ts index 44db9d1c..9eb6faf5 100644 --- a/frontend/app/src/data/browserInference/BrowserModelCatalog.ts +++ b/frontend/app/src/data/browserInference/BrowserModelCatalog.ts @@ -209,11 +209,22 @@ export function createBrowserModelCatalog(deps: CatalogDeps): OssBrowserModelCat return once(id, async () => { const record = required(id); const previous = active?.id === id ? active.value : null; + const previousStorage = record.storage; if (previous !== null) active = null; record.sessionArtifacts = undefined; update(record, { visible: true, state: "available", storage: "none", error: undefined }); if (previous !== null) previous.runtime.dispose(); - await deps.store.remove(record.admission); + try { + await deps.store.remove(record.admission); + } catch (error) { + update(record, { + visible: true, + state: "failed", + storage: previousStorage === "persistent" ? "persistent" : "none", + error: message(error), + }); + throw error; + } }); }, listTargets() { diff --git a/frontend/ui-core/src/annotator/AnnotationPage.tsx b/frontend/ui-core/src/annotator/AnnotationPage.tsx index 779a1d40..8425a3ed 100644 --- a/frontend/ui-core/src/annotator/AnnotationPage.tsx +++ b/frontend/ui-core/src/annotator/AnnotationPage.tsx @@ -191,7 +191,12 @@ import type { SuggestionOut } from "../data/inferenceQueries"; import { useServerSuggestionExecutor } from "../inference/suggestionExecutor"; import type { SuggestionExecutor } from "../inference/suggestionExecutor"; import { useBrowserInferenceRuntime } from "../inference/VisionSetBrowserInferenceProvider.js"; -import type { ActiveSuggestionTarget, BrowserSuggestionAssetSource, BrowserSuggestionTarget } from "../inference/browserPort.js"; +import type { + ActiveSuggestionTarget, + BrowserModelCatalogEntry, + BrowserSuggestionAssetSource, + BrowserSuggestionTarget, +} from "../inference/browserPort.js"; import { computeSuggestBlocker } from "../inference/targetBlocker.js"; import { readPref, writePref } from "../data/prefs"; @@ -231,12 +236,10 @@ function writeStoredSuggestTarget(projectId: string, target: ActiveSuggestionTar * Whether a stored browser-target preference is stale enough to fall back to Server * silently. * - * Acquired model bytes are never persisted across page loads, so "the stored browser - * target isn't in `listTargets()`'s answer" is the ordinary shape of every reload for - * someone who previously picked a browser target — not a rare failure, and it must not - * surface as a `blocker`. `explicitlyChosen` is what keeps this from also catching a - * target this *session* picked and which later drops out: that one is pinned, and stays - * pinned to `not-ready`/`refusal` rather than silently reverting. + * A catalog-known target may be installed, activating, or awaiting an explicit download; + * none of those make its preference stale. Only an ID unknown to the build falls back. + * `explicitlyChosen` separately keeps a target picked in this session pinned if it later + * drops out, so failure stays visible instead of silently switching to Server. */ export function staleStoredBrowserTarget( storedTarget: StoredSuggestTarget, @@ -251,6 +254,20 @@ export function staleStoredBrowserTarget( return !browserTargets.some((row) => row.id === storedTarget.targetId); } +/** + * Reconciles the asynchronously resolved Phase F target list with the catalog's synchronous + * lifecycle snapshot. Removal and corruption invalidate an executor before the next + * `listTargets()` promise settles, so a target is answerable only while both views say ready. + */ +export function readyBrowserTargets( + browserTargets: readonly BrowserSuggestionTarget[] | undefined, + browserModels: readonly BrowserModelCatalogEntry[], +): readonly BrowserSuggestionTarget[] | undefined { + if (browserTargets === undefined) return browserTargets; + const readyIds = new Set(browserModels.filter((entry) => entry.state === "ready").map((entry) => entry.id)); + return browserTargets.filter((target) => readyIds.has(target.id)); +} + const EMPTY_BROWSER_MODELS = Object.freeze([]); const emptyBrowserModels = () => EMPTY_BROWSER_MODELS; const subscribeToNothing = () => () => {}; @@ -1037,6 +1054,8 @@ function Workspace({ cancelled = true; }; }, [browserRuntime, browserTargetsRefreshKey, browserModels]); + const answerableBrowserTargets = + browserCatalog === undefined ? browserTargets : readyBrowserTargets(browserTargets, browserModels); const [storedTarget, setStoredTarget] = useState(() => readStoredSuggestTarget(projectId)); // Whether this session picked a browser target through `chooseTarget`, as opposed to one @@ -1084,7 +1103,7 @@ function Workspace({ .catch(() => setBrowserTargetsRefreshKey((key) => key + 1)); }, [activeBrowserTargetId, browserCatalog, browserModels, suggestArmed]); - const blocker = computeSuggestBlocker(activeTarget, serverBlocker, browserTargets); + const blocker = computeSuggestBlocker(activeTarget, serverBlocker, answerableBrowserTargets); // `executorFor` throws for a target that isn't actually ready yet — which is exactly the // state selecting "This device" starts in, before a download ever completes — so this must // check readiness itself rather than trust `browserRuntime !== null` alone. Derived from @@ -2929,7 +2948,7 @@ function Workspace({ // matching while parked — the one reading that has to name it. heldClass={activeClass} blocker={blocker} - browserTargets={browserRuntime === null ? undefined : browserTargets} + browserTargets={browserRuntime === null ? undefined : answerableBrowserTargets} browserAcquisitions={browserRuntime?.listAcquisitions?.()} {...(browserCatalog === undefined ? {} diff --git a/frontend/ui-core/src/inference/browserRuntime.test.tsx b/frontend/ui-core/src/inference/browserRuntime.test.tsx index bfea0d07..02acaf0e 100644 --- a/frontend/ui-core/src/inference/browserRuntime.test.tsx +++ b/frontend/ui-core/src/inference/browserRuntime.test.tsx @@ -8,9 +8,17 @@ import { useBrowserInferenceRuntime, VisionSetBrowserInferenceProvider, } from "./VisionSetBrowserInferenceProvider"; -import type { BrowserSuggestionAssetSource, VisionSetBrowserInferenceRuntime } from "./browserPort"; +import type { + BrowserModelCatalogEntry, + BrowserSuggestionAssetSource, + VisionSetBrowserInferenceRuntime, +} from "./browserPort"; import { clearPrefs, writePref } from "../data/prefs"; -import { AnnotationPage, staleStoredBrowserTarget } from "../annotator/AnnotationPage"; +import { + AnnotationPage, + readyBrowserTargets, + staleStoredBrowserTarget, +} from "../annotator/AnnotationPage"; import { TooltipProvider } from "@robomous/ui-core"; import { renderWithData } from "../testing/dataHarness"; import { stubResizeObserver } from "../testing/resizeObserver.js"; @@ -424,6 +432,29 @@ describe("executor selection never calls executorFor on an unready browser targe }); }); +describe("readyBrowserTargets", () => { + const target = { id: "efficient-sam-ti", label: "EfficientSAM-Ti", modelRef: "model@revision" }; + const model: BrowserModelCatalogEntry = { + id: target.id, + label: target.label, + modelRef: target.modelRef, + revision: "revision", + bytes: 41_301_678, + license: "Apache-2.0", + source: { label: "EfficientSAM", href: "https://example.test/upstream" }, + state: "ready", + storage: "persistent", + }; + + it("removes a stale ready target synchronously when catalog removal publishes", () => { + expect(readyBrowserTargets([target], [{ ...model, state: "available", storage: "none" }])).toEqual([]); + }); + + it("keeps a target while its catalog entry is ready", () => { + expect(readyBrowserTargets([target], [model])).toEqual([target]); + }); +}); + describe("staleStoredBrowserTarget", () => { const listed = [{ id: "t1", label: "T1", modelRef: "m@rev" }]; From 7e37786068ae9e1dcecd105ba5a9a2ad79917a3a Mon Sep 17 00:00:00 2001 From: Jesus Armando Anaya <1445792+JArmandoAnaya@users.noreply.github.com> Date: Thu, 17 Sep 2026 12:09:55 -0700 Subject: [PATCH 06/15] docs(inference): define persistent browser model acquisition --- docs/content/architecture/frontend/app.md | 31 ++++++++ .../frontend/browser-inference.md | 9 +-- docs/content/ui.md | 19 +++-- frontend/app/e2e/browserSuggestion.spec.ts | 71 ++++++++++++++++++- frontend/app/src/data/OssSession.tsx | 4 +- .../BrowserInferenceRuntime.test.ts | 14 +++- .../BrowserInferenceRuntime.ts | 13 ++++ tests/packaging/test_wheel.py | 14 +++- 8 files changed, 161 insertions(+), 14 deletions(-) diff --git a/docs/content/architecture/frontend/app.md b/docs/content/architecture/frontend/app.md index 1b0348be..cbb0f419 100644 --- a/docs/content/architecture/frontend/app.md +++ b/docs/content/architecture/frontend/app.md @@ -40,6 +40,37 @@ managed host replaces only the `FrameSink`'s destination, the same way it replac `ossClient.ts` today, which is what proves the boundary actually sits where [ui-core.md](ui-core.md) says it does. +The browser-model acquisition system is composed here for the same reason. The public +registry and immutable manifests are discovery data, **not a trust anchor**. The app validates +their schema and intersects them with a build-owned admission catalog that pins the model ID, +revision, graph contract, source and license metadata, artifact roles, byte counts and SHA-256 +digests. Registry entries the build cannot execute are not offered. Registry metadata cannot +name code, dynamic imports, preprocessing functions, or paths outside the configured public +model base. + +An explicit Download action fetches admitted artifacts. Every complete download is checked for +its pinned size and digest before any byte is written to the versioned Cache Storage namespace; +unverified or partial models are never installed. Storage keys include the model revision and +artifact digest, so mirrors of identical immutable content share an identity without confusing +different revisions. Activation reads the whole admitted set back and repeats both checks. A +missing or corrupt entry is removed and leaves the model unavailable; activation never turns +that failure into a network download. + +The catalog and the execution port deliberately answer different questions. Catalog states +describe known, downloading, installed, activating, ready, and failed models. `listTargets()` +continues to list only models that can answer now. Startup inspects storage but creates no ONNX +Runtime worker or session. A cached model activates lazily when the armed Suggest surface needs +the selected target; runtime sessions and image embeddings remain memory-only. Removal first +invalidates and disposes that runtime, then deletes only the admitted revision's model +artifacts. + +Cache Storage is a browser-managed persistence layer, not a permanence guarantee. If a verified +download cannot be written, the app may run it for that session while stating that it was not +saved. An admitted cached model can be verified and activated when registry and artifact routes +are unavailable, while Server inference remains independent of the catalog and public model +source. No service worker, Python-side browser cache, or automatic revision update participates +in this flow. + ## What a route does ```mermaid diff --git a/docs/content/architecture/frontend/browser-inference.md b/docs/content/architecture/frontend/browser-inference.md index a8fe9d14..f9c67b5e 100644 --- a/docs/content/architecture/frontend/browser-inference.md +++ b/docs/content/architecture/frontend/browser-inference.md @@ -17,10 +17,11 @@ that is supposed to be *injected with* runtimes - the dependency arrow pointing through the seam it exists to serve. A host that holds both adapts one to the other, which is what [host composition](../decisions/browser-inference-is-host-injected.md) is for. -Nothing in this repository composes that adapter yet, and nothing imports this package. -**A package existing is not the product offering a browser target.** There is no "This device" -control and no user-visible change - a caller who wants EfficientSAM-Ti running still has to -supply the weights and wire the adapter itself. +The OSS app composes that adapter at its host boundary. **A package existing is still not the +product offering a browser target:** the app supplies admitted, verified graph bytes and adapts +the resulting runtime to `ui-core`; this package neither discovers models nor decides which +ones VisionSet trusts. Another host can omit browser inference or provide a different +implementation of the same optional port. ## Core and adapter are two entrypoints, for the same reason as media diff --git a/docs/content/ui.md b/docs/content/ui.md index b25b44c4..bb64e4cb 100644 --- a/docs/content/ui.md +++ b/docs/content/ui.md @@ -627,11 +627,20 @@ and without a connection being configured at all - useful where the workspace ha model that can answer a click, or where the server is slow to reach. It has to be downloaded first, and the panel says so: about 41 MB, on an explicit -press, never on its own. **The download does not survive a page reload.** It is held -for the session only, so reopening the editor tomorrow - or reloading today - means -downloading it again before "This device" can answer. Until then the tab shows the -Download button rather than inviting a click that would do nothing, and the choice -falls back to Server. +press, never on its own. Before that press the panel names the model's upstream source, +license and download size. Verified weights are stored in this browser and normally +survive a reload; the browser can still evict its storage, and a removed or evicted +model can be downloaded again. If storage is unavailable, a verified download may be +ready for the current session without being described as installed. + +Opening the app, arming Suggest, or choosing **This device** never downloads missing +weights. A stored choice for a known model remains selected and shows the Download +action when that model is not installed. An installed model is loaded only when the +suggestion surface needs it, and loading from browser storage does not fetch the model +again. **Remove from this browser** releases the running model and removes its stored +weights. The model source receives artifact requests during an explicit download, but +the image and points being annotated remain local during browser inference. Self-hosted +deployments can replace the public model source with `VITE_MODEL_CDN_BASE_URL`. **Alt-click needs Server.** The model running here takes only points that are *on* the object; a point marking something that is not part of it is refused rather than diff --git a/frontend/app/e2e/browserSuggestion.spec.ts b/frontend/app/e2e/browserSuggestion.spec.ts index 6c6726c1..af131064 100644 --- a/frontend/app/e2e/browserSuggestion.spec.ts +++ b/frontend/app/e2e/browserSuggestion.spec.ts @@ -186,8 +186,9 @@ async function openJobWithBrowserRuntime( page: Page, sent: Request[], suggestible: boolean, + modelSource: "fixture" | "live" = "fixture", ): Promise { - await mockCdn(page); + if (modelSource === "fixture") await mockCdn(page); await serveApi(page, sent, undefined, undefined, undefined, undefined, suggestible); await mockAssetImage(page); await page.goto(`/jobs/${JOB}`); @@ -290,6 +291,74 @@ test.describe("browser suggestion", () => { expect(suggestCallsOf(sent)).toHaveLength(0); }); + test("live CDN smoke: admitted artifacts persist and reactivate without a second download", async ({ page }) => { + test.skip(process.env.VISIONSET_LIVE_MODEL_SMOKE !== "1", "manual smoke against models.robomous.ai"); + test.setTimeout(120_000); + const sent: Request[] = []; + const responses: { url: string; bytes: number; milliseconds: number }[] = []; + page.on("response", async (response) => { + if (!response.url().startsWith("https://models.robomous.ai/")) return; + await response.finished(); + const timing = response.request().timing(); + const declaredBytes = Number(response.headers()["content-length"] ?? 0); + const bytes = declaredBytes > 0 ? declaredBytes : (await response.body()).byteLength; + responses.push({ url: response.url(), bytes, milliseconds: timing.responseEnd }); + }); + // Warm Chromium's connection to the public origin with the same registry document the + // application will validate. This keeps a one-off DNS/TLS stall from consuming the whole + // UI timeout while still exercising the app's own fetch, parser, and admission match below. + await page.goto("https://models.robomous.ai/registry/v1.json"); + await expect(page.locator("body")).toContainText('"schema_version": 1'); + await openJobWithBrowserRuntime(page, sent, true, "live"); + await armSuggestTool(page); + const coldStarted = Date.now(); + await acquireAndSelectBrowserTarget(page); + const coldMilliseconds = Date.now() - coldStarted; + await makeBrowserSuggestion(page); + + const cacheMeasurements = await page.evaluate(async () => { + const cache = await caches.open("visionset-browser-models-v1"); + const keys = await cache.keys(); + let bytes = 0; + let readMilliseconds = 0; + let shaMilliseconds = 0; + const lookupStarted = performance.now(); + await Promise.all(keys.map((key) => cache.match(key))); + const lookupMilliseconds = performance.now() - lookupStarted; + for (const key of keys) { + const response = await cache.match(key); + if (response === undefined) continue; + const readStarted = performance.now(); + const body = await response.arrayBuffer(); + readMilliseconds += performance.now() - readStarted; + bytes += body.byteLength; + const shaStarted = performance.now(); + await crypto.subtle.digest("SHA-256", body); + shaMilliseconds += performance.now() - shaStarted; + } + return { entries: keys.length, bytes, lookupMilliseconds, readMilliseconds, shaMilliseconds }; + }); + + const artifactRequests = countModelRequestsFromNow(page); + const reloadStarted = Date.now(); + await page.reload(); + await expect(page.getByTestId("annotation-page")).toBeVisible(); + await armSuggestTool(page); + await expect(page.getByTestId("suggest-device-section").getByText("Ready", { exact: true })).toBeVisible({ + timeout: 60_000, + }); + const reloadActivationMilliseconds = Date.now() - reloadStarted; + expect(artifactRequests()).toBe(0); + await makeBrowserSuggestion(page); + + console.info("VISIONSET_LIVE_MODEL_SMOKE", JSON.stringify({ + responses, + coldMilliseconds, + reloadActivationMilliseconds, + ...cacheMeasurements, + })); + }); + test("an installed model remains usable when registry and artifact routes fail", async ({ page }) => { const sent: Request[] = []; await openJobWithBrowserRuntime(page, sent, true); diff --git a/frontend/app/src/data/OssSession.tsx b/frontend/app/src/data/OssSession.tsx index 60c834d2..3c8c0d22 100644 --- a/frontend/app/src/data/OssSession.tsx +++ b/frontend/app/src/data/OssSession.tsx @@ -37,7 +37,7 @@ import type { QueryClient } from "@tanstack/react-query"; import { VisionSetBrowserInferenceProvider, VisionSetDataProvider, VisionSetMediaProvider } from "@visionset/ui-core"; import { MediabunnyVideoMaterializer } from "@visionset/media/mediabunny"; -import { createOssBrowserInferenceRuntime } from "./browserInference/BrowserInferenceRuntime"; +import { getSharedOssBrowserInferenceRuntime } from "./browserInference/BrowserInferenceRuntime"; import { createOssDataClient, requestSession } from "./ossClient"; import { createLocalApiFrameSink } from "./frameSink"; import { clearToken, readToken, writeToken } from "./token"; @@ -164,7 +164,7 @@ export function OssSessionProvider({ * and SHA-256 verification. Built once per session — its acquisition state (unacquired * vs. ready) must survive across asset navigation, not reset on every render. */ - const browserInferenceRuntime = useMemo(() => createOssBrowserInferenceRuntime(), []); + const browserInferenceRuntime = useMemo(() => getSharedOssBrowserInferenceRuntime(), []); const signIn = useCallback((next: string) => { writeToken(next); diff --git a/frontend/app/src/data/browserInference/BrowserInferenceRuntime.test.ts b/frontend/app/src/data/browserInference/BrowserInferenceRuntime.test.ts index d8f37d67..8feac0fe 100644 --- a/frontend/app/src/data/browserInference/BrowserInferenceRuntime.test.ts +++ b/frontend/app/src/data/browserInference/BrowserInferenceRuntime.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it, vi } from "vitest"; import type { SuggestionRequest } from "@visionset/ui-core"; -import { createOssBrowserInferenceRuntime } from "./BrowserInferenceRuntime.js"; +import { + createOssBrowserInferenceRuntime, + getSharedOssBrowserInferenceRuntime, +} from "./BrowserInferenceRuntime.js"; import { EFFICIENT_SAM_TI_REVISION } from "./manifest.js"; import type { VisionSetBrowserInferenceRuntime } from "@visionset/ui-core"; import type { BrowserModelArtifacts } from "./artifactStore.js"; @@ -30,6 +33,15 @@ async function acquisition(runtime: VisionSetBrowserInferenceRuntime) { } describe("createOssBrowserInferenceRuntime", () => { + it("shares the host runtime across repeated composition calls", () => { + const runtime = createOssBrowserInferenceRuntime(fakeDeps()); + const factory = vi.fn(() => runtime); + + expect(getSharedOssBrowserInferenceRuntime(factory)).toBe(runtime); + expect(getSharedOssBrowserInferenceRuntime(factory)).toBe(runtime); + expect(factory).toHaveBeenCalledTimes(1); + }); + it("exposes the additive model catalog while preserving the Phase F runtime members", () => { const runtime = createOssBrowserInferenceRuntime(fakeDeps()); expect(runtime.modelCatalog).toBeDefined(); diff --git a/frontend/app/src/data/browserInference/BrowserInferenceRuntime.ts b/frontend/app/src/data/browserInference/BrowserInferenceRuntime.ts index b609201a..e3256c49 100644 --- a/frontend/app/src/data/browserInference/BrowserInferenceRuntime.ts +++ b/frontend/app/src/data/browserInference/BrowserInferenceRuntime.ts @@ -94,3 +94,16 @@ export function createOssBrowserInferenceRuntime(deps: Deps = REAL_DEPS): Vision }, }; } + +let sharedRuntime: VisionSetBrowserInferenceRuntime | undefined; + +/** + * The browser model catalog belongs to the page, not to an authenticated server-data scope. + * Sharing it also keeps React development strict mounts from starting duplicate discovery. + */ +export function getSharedOssBrowserInferenceRuntime( + factory: () => VisionSetBrowserInferenceRuntime = createOssBrowserInferenceRuntime, +): VisionSetBrowserInferenceRuntime { + sharedRuntime ??= factory(); + return sharedRuntime; +} diff --git a/tests/packaging/test_wheel.py b/tests/packaging/test_wheel.py index d7694e9e..634fef0d 100644 --- a/tests/packaging/test_wheel.py +++ b/tests/packaging/test_wheel.py @@ -100,7 +100,19 @@ #: Media suffixes. `_static/` legitimately holds none today — the app ships as #: HTML, CSS, JavaScript and one WebAssembly runtime — so any of these is #: something nobody meant to ship. -FORBIDDEN_SUFFIXES = (".mp4", ".mov", ".avi", ".jpg", ".jpeg", ".tiff", ".bmp") +FORBIDDEN_SUFFIXES = ( + ".mp4", + ".mov", + ".avi", + ".jpg", + ".jpeg", + ".tiff", + ".bmp", + # Browser model weights belong to the explicit CDN/cache acquisition flow. + # ONNX Runtime's own `.wasm` payload remains the deliberately packaged exception. + ".onnx", + ".pt", +) #: How long the freshly installed server gets to bind a socket. #: From f23c6bf32bffe6be8abec2536c9ab5943f4b0d06 Mon Sep 17 00:00:00 2001 From: Jesus Armando Anaya <1445792+JArmandoAnaya@users.noreply.github.com> Date: Thu, 17 Sep 2026 12:12:51 -0700 Subject: [PATCH 07/15] fix(inference): keep catalog storage state truthful --- .../BrowserModelCatalog.test.ts | 13 +++++++++++++ .../browserInference/BrowserModelCatalog.ts | 11 ++++++++--- .../app/src/data/browserInference/manifest.ts | 16 ++++++++++------ .../ui-core/src/annotator/SuggestPanel.tsx | 19 ++++++++++++++++--- .../src/annotator/suggestPanel.test.tsx | 14 ++++++++++++++ 5 files changed, 61 insertions(+), 12 deletions(-) diff --git a/frontend/app/src/data/browserInference/BrowserModelCatalog.test.ts b/frontend/app/src/data/browserInference/BrowserModelCatalog.test.ts index 4e84c271..f511b004 100644 --- a/frontend/app/src/data/browserInference/BrowserModelCatalog.test.ts +++ b/frontend/app/src/data/browserInference/BrowserModelCatalog.test.ts @@ -148,6 +148,19 @@ describe("createBrowserModelCatalog", () => { expect(catalog.listTargets()).toEqual([]); }); + it("keeps persistent storage truthful when runtime startup fails after verified cache read", async () => { + const { catalog, download } = harness({ + installed: true, + ready: async () => Promise.reject(new Error("runtime startup failed")), + }); + await settles(catalog); + + await expect(catalog.activate(ADMISSION.id)).rejects.toThrow(/runtime startup failed/i); + + expect(download).not.toHaveBeenCalled(); + expect(catalog.snapshot()[0]).toMatchObject({ state: "failed", storage: "persistent" }); + }); + it("does not treat a missing decoder as installed or ready", async () => { const { catalog, download } = harness({ installed: false, discover: async () => true }); await settles(catalog); diff --git a/frontend/app/src/data/browserInference/BrowserModelCatalog.ts b/frontend/app/src/data/browserInference/BrowserModelCatalog.ts index 9eb6faf5..da2afb36 100644 --- a/frontend/app/src/data/browserInference/BrowserModelCatalog.ts +++ b/frontend/app/src/data/browserInference/BrowserModelCatalog.ts @@ -193,16 +193,21 @@ export function createBrowserModelCatalog(deps: CatalogDeps): OssBrowserModelCat return once(id, async () => { const record = required(id); if (record.state === "ready") return; + let artifacts: BrowserModelArtifacts; try { - const artifacts = record.sessionArtifacts ?? (await deps.store.readVerified(record.admission)); - if (artifacts === null) throw new Error(`${record.admission.label} is not installed in this browser`); - await startRuntime(record, artifacts); + const stored = record.sessionArtifacts ?? (await deps.store.readVerified(record.admission)); + if (stored === null) throw new Error(`${record.admission.label} is not installed in this browser`); + artifacts = stored; } catch (error) { record.sessionArtifacts = undefined; if (active?.id === id) active = null; update(record, { visible: true, state: "failed", storage: "none", error: message(error) }); throw error; } + // `startRuntime` owns its failure state. At this point bytes have already passed the + // cache integrity check, so a worker/session failure must not pretend persistent + // storage disappeared or was corrupt. + await startRuntime(record, artifacts); }); }, remove(id) { diff --git a/frontend/app/src/data/browserInference/manifest.ts b/frontend/app/src/data/browserInference/manifest.ts index a2300584..a3834cb1 100644 --- a/frontend/app/src/data/browserInference/manifest.ts +++ b/frontend/app/src/data/browserInference/manifest.ts @@ -4,13 +4,15 @@ * Self-hosted deployments override VITE_MODEL_CDN_BASE_URL to point at their own mirror * of this same manifest layout. */ +import { EFFICIENT_SAM_TI_ADMISSION } from "./admissionCatalog.js"; + export const MODEL_CDN_BASE_URL: string = ( (import.meta.env["VITE_MODEL_CDN_BASE_URL"] as string | undefined) ?? "https://models.robomous.ai" ).replace(/\/+$/, ""); export const MODEL_REGISTRY_URL = `${MODEL_CDN_BASE_URL}/registry/v1.json`; -export const EFFICIENT_SAM_TI_REVISION = "b19782d049c0-843761ca46f4"; +export const EFFICIENT_SAM_TI_REVISION = EFFICIENT_SAM_TI_ADMISSION.revision; /** * Shared by the manifest URL and every artifact URL, so the CDN's directory layout @@ -21,11 +23,13 @@ export const EFFICIENT_SAM_TI_BASE_URL = `${MODEL_CDN_BASE_URL}/models/efficient export const EFFICIENT_SAM_TI_MANIFEST_URL = `${EFFICIENT_SAM_TI_BASE_URL}/manifest.json`; -/** Verified 2026-09-16 against the live models.robomous.ai release — see the design doc §5. */ -export const EFFICIENT_SAM_TI_EXPECTED = { - encoder: { sha256: "b19782d049c09a8f1cc36ccc6029264ca23c8ac35e6379fd9ef9f1bc6d81e7f2", bytes: 24_799_777 }, - decoder: { sha256: "843761ca46f4aa00b09fdcf0c94271321f76eece092a744296c742d682a86172", bytes: 16_501_901 }, -} as const; +/** Phase F compatibility view; the build admission record is the single trust anchor. */ +export const EFFICIENT_SAM_TI_EXPECTED = Object.fromEntries( + EFFICIENT_SAM_TI_ADMISSION.artifacts.map((artifact) => [ + artifact.role, + { sha256: artifact.sha256, bytes: artifact.bytes }, + ]), +) as Record<"encoder" | "decoder", { readonly sha256: string; readonly bytes: number }>; /** * Mirrors the real, already-deployed manifest shape (nested under `artifacts`, diff --git a/frontend/ui-core/src/annotator/SuggestPanel.tsx b/frontend/ui-core/src/annotator/SuggestPanel.tsx index cbdc6662..1d8be971 100644 --- a/frontend/ui-core/src/annotator/SuggestPanel.tsx +++ b/frontend/ui-core/src/annotator/SuggestPanel.tsx @@ -443,6 +443,14 @@ export function SuggestPanel({ // thing that is actually available: the Download control the tab below already draws. const browserTabUnacquired = runtimeWired && activeTarget?.kind === "browser" && blocker === "not-ready"; + const activeBrowserModel = + activeTarget?.kind === "browser" + ? browserModels?.find((model) => model.id === activeTarget.targetId) + : undefined; + const browserModelBusy = + activeBrowserModel?.state === "downloading" || + activeBrowserModel?.state === "installed" || + activeBrowserModel?.state === "activating"; return ( }> @@ -450,11 +458,16 @@ export function SuggestPanel({ (browserTabUnacquired ? ( <>

- Download the model first + {activeBrowserModel?.state === "downloading" + ? "Downloading the model…" + : browserModelBusy + ? "Loading the model…" + : "Download the model first"}

- “This device” has nothing to answer with yet, so a click does nothing. Download - it below, or switch back to Server. + {browserModelBusy + ? "“This device” is getting the selected model ready. A click will work once loading finishes." + : "“This device” has nothing to answer with yet, so a click does nothing. Download it below, or switch back to Server."}

) : ( diff --git a/frontend/ui-core/src/annotator/suggestPanel.test.tsx b/frontend/ui-core/src/annotator/suggestPanel.test.tsx index cbf9a1ad..658f5e9d 100644 --- a/frontend/ui-core/src/annotator/suggestPanel.test.tsx +++ b/frontend/ui-core/src/annotator/suggestPanel.test.tsx @@ -716,6 +716,20 @@ describe("this device, once a browser runtime is wired", () => { expect(screen.getByTestId("suggest-device-section").textContent).toContain(label); }); + it("describes a cached model as loading instead of asking for another download", () => { + render(mount({ + blocker: "not-ready", + browserTargets: [], + browserModels: [catalogEntry({ state: "installed", storage: "persistent" })], + activeTarget: { kind: "browser", targetId: READY.id }, + onChooseTarget: vi.fn(), + onRemoveBrowserModel: vi.fn(), + })); + + expect(screen.getByTestId("suggest-idle-unacquired").textContent).toMatch(/loading/i); + expect(screen.getByTestId("suggest-panel").textContent).not.toContain("Download the model first"); + }); + it("removes an installed model through an explicit packaged control", async () => { const remove = vi.fn().mockResolvedValue(undefined); const user = userEvent.setup(); From 8d70f297ca01138166e24a7df558746a2c254043 Mon Sep 17 00:00:00 2001 From: Jesus Armando Anaya <1445792+JArmandoAnaya@users.noreply.github.com> Date: Thu, 17 Sep 2026 12:14:52 -0700 Subject: [PATCH 08/15] test(inference): keep runtime fixture type safe --- .../src/data/browserInference/BrowserInferenceRuntime.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/app/src/data/browserInference/BrowserInferenceRuntime.test.ts b/frontend/app/src/data/browserInference/BrowserInferenceRuntime.test.ts index 8feac0fe..3f910d2a 100644 --- a/frontend/app/src/data/browserInference/BrowserInferenceRuntime.test.ts +++ b/frontend/app/src/data/browserInference/BrowserInferenceRuntime.test.ts @@ -29,7 +29,9 @@ async function acquisition(runtime: VisionSetBrowserInferenceRuntime) { // Registry/cache initialization is asynchronous in Phase G. `listTargets()` waits for that // initial pass, after which the synchronous Phase F compatibility view is populated. await runtime.listTargets(); - return runtime.listAcquisitions?.()[0]!; + const available = runtime.listAcquisitions?.()[0]; + if (available === undefined) throw new Error("expected an available browser model fixture"); + return available; } describe("createOssBrowserInferenceRuntime", () => { From 88e228f2a9322dad387b3c118fef3bda174b6e59 Mon Sep 17 00:00:00 2001 From: Jesus Armando Anaya <1445792+JArmandoAnaya@users.noreply.github.com> Date: Thu, 17 Sep 2026 12:21:49 -0700 Subject: [PATCH 09/15] test(inference): require complete artifact removal --- frontend/app/src/data/browserInference/artifactStore.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frontend/app/src/data/browserInference/artifactStore.test.ts b/frontend/app/src/data/browserInference/artifactStore.test.ts index 88b4be14..6f5b8a59 100644 --- a/frontend/app/src/data/browserInference/artifactStore.test.ts +++ b/frontend/app/src/data/browserInference/artifactStore.test.ts @@ -159,6 +159,9 @@ describe("createCacheArtifactStore", () => { await store.remove(first.admission); + for (const artifact of first.admission.artifacts) { + expect(cache.entries.has(cacheKeyFor(first.admission, artifact))).toBe(false); + } expect(await store.inspect(first.admission)).toBe(false); expect(await store.inspect(second.admission)).toBe(true); }); From 5eeec0520fd857acdf8569ebe24d5c4f91fe5c68 Mon Sep 17 00:00:00 2001 From: Jesus Armando Anaya <1445792+JArmandoAnaya@users.noreply.github.com> Date: Thu, 17 Sep 2026 12:30:05 -0700 Subject: [PATCH 10/15] test(inference): isolate real-model functional timing --- frontend/app/e2e/browserSuggestion.spec.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frontend/app/e2e/browserSuggestion.spec.ts b/frontend/app/e2e/browserSuggestion.spec.ts index af131064..976ddd76 100644 --- a/frontend/app/e2e/browserSuggestion.spec.ts +++ b/frontend/app/e2e/browserSuggestion.spec.ts @@ -249,6 +249,10 @@ async function makeBrowserSuggestion(page: Page): Promise { } test.describe("browser suggestion", () => { + // Real ONNX work competes with the rest of the fully-parallel app suite on local machines. + // This is a functional ceiling, not a performance assertion; measured timings are reported + // by the opt-in live smoke instead of turning shared-runner wall clock into a gate. + test.setTimeout(60_000); test.skip(!HAS_ARTIFACTS, MISSING_MESSAGE); test("Server target: a click issues exactly one /inference/suggest HTTP request", async ({ page }) => { From 74b817912cd3b5a87b547471e2ea718113d7786f Mon Sep 17 00:00:00 2001 From: Jesus Armando Anaya <1445792+JArmandoAnaya@users.noreply.github.com> Date: Thu, 17 Sep 2026 12:42:48 -0700 Subject: [PATCH 11/15] fix(inference): serialize browser model lifecycle operations --- .../BrowserModelCatalog.test.ts | 31 +++++++++++++++++ .../browserInference/BrowserModelCatalog.ts | 34 +++++++++++++++---- .../acquireEfficientSam.test.ts | 26 +++++++------- .../browserInference/acquireEfficientSam.ts | 22 +++++++----- .../app/src/data/browserInference/manifest.ts | 5 +++ 5 files changed, 90 insertions(+), 28 deletions(-) diff --git a/frontend/app/src/data/browserInference/BrowserModelCatalog.test.ts b/frontend/app/src/data/browserInference/BrowserModelCatalog.test.ts index f511b004..e942e1fd 100644 --- a/frontend/app/src/data/browserInference/BrowserModelCatalog.test.ts +++ b/frontend/app/src/data/browserInference/BrowserModelCatalog.test.ts @@ -184,6 +184,25 @@ describe("createBrowserModelCatalog", () => { expect(catalog.snapshot()[0]).toMatchObject({ state: "available", storage: "none" }); }); + it("queues removal behind activation instead of mistaking activation for removal", async () => { + const pendingReady = deferred(); + const { catalog, dispose, store } = harness({ installed: true, ready: () => pendingReady.promise }); + await settles(catalog); + + const activating = catalog.activate(ADMISSION.id); + await vi.waitFor(() => expect(catalog.snapshot()[0]?.state).toBe("activating")); + const removing = catalog.remove(ADMISSION.id); + expect(store.remove).not.toHaveBeenCalled(); + + pendingReady.resolve([]); + await Promise.all([activating, removing]); + + expect(dispose).toHaveBeenCalledTimes(1); + expect(store.remove).toHaveBeenCalledTimes(1); + expect(catalog.listTargets()).toEqual([]); + expect(catalog.snapshot()[0]).toMatchObject({ state: "available", storage: "none" }); + }); + it("does not claim cached artifacts were removed when persistent deletion fails", async () => { const { catalog, dispose, store } = harness({ installed: true }); await settles(catalog); @@ -209,6 +228,18 @@ describe("createBrowserModelCatalog", () => { expect(download).not.toHaveBeenCalled(); }); + it("keeps an admitted uninstalled model visible and retryable when registry discovery fails", async () => { + const { catalog } = harness({ discover: async () => Promise.reject(new Error("registry unavailable")) }); + await settles(catalog); + + expect(catalog.snapshot()[0]).toMatchObject({ + state: "failed", + storage: "none", + error: "registry unavailable", + }); + expect(catalog.isKnown(ADMISSION.id)).toBe(true); + }); + it("knows an admitted but uninstalled preference without fabricating a ready target", async () => { const { catalog } = harness(); expect(catalog.isKnown(ADMISSION.id)).toBe(true); diff --git a/frontend/app/src/data/browserInference/BrowserModelCatalog.ts b/frontend/app/src/data/browserInference/BrowserModelCatalog.ts index da2afb36..ebff08a2 100644 --- a/frontend/app/src/data/browserInference/BrowserModelCatalog.ts +++ b/frontend/app/src/data/browserInference/BrowserModelCatalog.ts @@ -82,7 +82,10 @@ export function createBrowserModelCatalog(deps: CatalogDeps): OssBrowserModelCat ]), ); const listeners = new Set<() => void>(); - const operations = new Map>(); + const operations = new Map< + string, + { readonly kind: "acquire" | "activate" | "remove"; readonly promise: Promise } + >(); let active: { readonly id: string; readonly value: ActiveModel } | null = null; let snapshot: readonly BrowserModelCatalogEntry[] = []; @@ -118,6 +121,13 @@ export function createBrowserModelCatalog(deps: CatalogDeps): OssBrowserModelCat update(record, { visible: true, state: "installed", storage: "persistent", error: undefined }); } else if (discovered) { update(record, { visible: true, state: "available", storage: "none", error: undefined }); + } else if (registryResult.status === "rejected") { + update(record, { + visible: true, + state: "failed", + storage: "none", + error: message(registryResult.reason), + }); } } @@ -143,11 +153,21 @@ export function createBrowserModelCatalog(deps: CatalogDeps): OssBrowserModelCat } } - function once(id: string, operation: () => Promise): Promise { + function once( + id: string, + kind: "acquire" | "activate" | "remove", + operation: () => Promise, + ): Promise { const current = operations.get(id); - if (current !== undefined) return current; + if (current !== undefined) { + if (current.kind === kind) return current.promise; + return current.promise.then( + () => once(id, kind, operation), + () => once(id, kind, operation), + ); + } const promise = operation().finally(() => operations.delete(id)); - operations.set(id, promise); + operations.set(id, { kind, promise }); return promise; } @@ -162,7 +182,7 @@ export function createBrowserModelCatalog(deps: CatalogDeps): OssBrowserModelCat }, isKnown: (id) => records.has(id), acquire(id, options) { - return once(id, async () => { + return once(id, "acquire", async () => { const record = required(id); if (record.state === "ready") return; if (!record.discovered) { @@ -190,7 +210,7 @@ export function createBrowserModelCatalog(deps: CatalogDeps): OssBrowserModelCat }); }, activate(id) { - return once(id, async () => { + return once(id, "activate", async () => { const record = required(id); if (record.state === "ready") return; let artifacts: BrowserModelArtifacts; @@ -211,7 +231,7 @@ export function createBrowserModelCatalog(deps: CatalogDeps): OssBrowserModelCat }); }, remove(id) { - return once(id, async () => { + return once(id, "remove", async () => { const record = required(id); const previous = active?.id === id ? active.value : null; const previousStorage = record.storage; diff --git a/frontend/app/src/data/browserInference/acquireEfficientSam.test.ts b/frontend/app/src/data/browserInference/acquireEfficientSam.test.ts index d0fecc57..aea18f2a 100644 --- a/frontend/app/src/data/browserInference/acquireEfficientSam.test.ts +++ b/frontend/app/src/data/browserInference/acquireEfficientSam.test.ts @@ -150,23 +150,25 @@ describe("acquireEfficientSam", () => { expect(result.decoder).toEqual(decoderBytes); }); - it("fails closed on a manifest artifact path shaped like a traversal, rather than building whatever URL it names", async () => { - vi.stubGlobal( - "fetch", - vi.fn(async (url: string) => { + it.each(["../secrets.onnx", "..", "%2e%2e", "\\..\\private"])( + "fails closed on manifest artifact path %s before any artifact request", + async (path) => { + const fetchMock = vi.fn(async (url: string) => { if (url.endsWith("manifest.json")) { return new Response( - JSON.stringify({ artifacts: { encoder: { path: "../secrets.onnx" }, decoder: { path: "decoder.onnx" } } }), + JSON.stringify({ artifacts: { encoder: { path }, decoder: { path: "decoder.onnx" } } }), ); } throw new Error(`unexpected url ${url}`); - }), - ); - vi.resetModules(); - const { acquireEfficientSam } = await import("./acquireEfficientSam.js"); - - await expect(acquireEfficientSam()).rejects.toThrow(/unexpected manifest artifact path/i); - }); + }); + vi.stubGlobal("fetch", fetchMock); + vi.resetModules(); + const { acquireEfficientSam } = await import("./acquireEfficientSam.js"); + + await expect(acquireEfficientSam()).rejects.toThrow(/unexpected manifest artifact path/i); + expect(fetchMock).toHaveBeenCalledTimes(1); + }, + ); }); describe("fetchEfficientSamManifest", () => { diff --git a/frontend/app/src/data/browserInference/acquireEfficientSam.ts b/frontend/app/src/data/browserInference/acquireEfficientSam.ts index 59979fc8..6396711d 100644 --- a/frontend/app/src/data/browserInference/acquireEfficientSam.ts +++ b/frontend/app/src/data/browserInference/acquireEfficientSam.ts @@ -1,14 +1,18 @@ -import { EFFICIENT_SAM_TI_BASE_URL, EFFICIENT_SAM_TI_EXPECTED, fetchEfficientSamManifest } from "./manifest.js"; +import { + EFFICIENT_SAM_TI_ARTIFACT_PATHS, + EFFICIENT_SAM_TI_BASE_URL, + EFFICIENT_SAM_TI_EXPECTED, + fetchEfficientSamManifest, +} from "./manifest.js"; // The manifest's `artifacts.*.path` is a bare filename, resolved relative to the // manifest's own directory (`EFFICIENT_SAM_TI_BASE_URL`, the same prefix // `EFFICIENT_SAM_TI_MANIFEST_URL` is built from) — not an absolute path safe to -// append directly to `MODEL_CDN_BASE_URL`. A `path` containing a slash is rejected -// outright: it should always be a bare filename, and a mutated manifest asking to -// climb out of its own directory fails closed here rather than silently building -// whatever URL it names. -function artifactUrl(path: string): string { - if (path.includes("/")) throw new Error(`unexpected manifest artifact path: ${path}`); +// append directly to `MODEL_CDN_BASE_URL`. The mutable value must equal the +// build-admitted filename, so normalized or encoded traversal syntax fails before +// an artifact request rather than silently building whatever URL it names. +function artifactUrl(path: string, expectedPath: string): string { + if (path !== expectedPath) throw new Error(`unexpected manifest artifact path: ${path}`); return `${EFFICIENT_SAM_TI_BASE_URL}/${path}`; } @@ -58,12 +62,12 @@ export async function acquireEfficientSam( }> { const manifest = await fetchEfficientSamManifest(signal); const encoder = await fetchVerified( - artifactUrl(manifest.artifacts.encoder.path), + artifactUrl(manifest.artifacts.encoder.path, EFFICIENT_SAM_TI_ARTIFACT_PATHS.encoder), EFFICIENT_SAM_TI_EXPECTED.encoder, signal, ); const decoder = await fetchVerified( - artifactUrl(manifest.artifacts.decoder.path), + artifactUrl(manifest.artifacts.decoder.path, EFFICIENT_SAM_TI_ARTIFACT_PATHS.decoder), EFFICIENT_SAM_TI_EXPECTED.decoder, signal, ); diff --git a/frontend/app/src/data/browserInference/manifest.ts b/frontend/app/src/data/browserInference/manifest.ts index a3834cb1..2ea274f2 100644 --- a/frontend/app/src/data/browserInference/manifest.ts +++ b/frontend/app/src/data/browserInference/manifest.ts @@ -31,6 +31,11 @@ export const EFFICIENT_SAM_TI_EXPECTED = Object.fromEntries( ]), ) as Record<"encoder" | "decoder", { readonly sha256: string; readonly bytes: number }>; +/** Admitted filenames used to bind the actual artifact request to the build's trust record. */ +export const EFFICIENT_SAM_TI_ARTIFACT_PATHS = Object.fromEntries( + EFFICIENT_SAM_TI_ADMISSION.artifacts.map((artifact) => [artifact.role, artifact.path]), +) as Record<"encoder" | "decoder", string>; + /** * Mirrors the real, already-deployed manifest shape (nested under `artifacts`, * with each `path` a bare filename relative to the manifest's own directory) — not From 749642f848d49a0f799949f4d76a689bec48a9be Mon Sep 17 00:00:00 2001 From: Jesus Armando Anaya <1445792+JArmandoAnaya@users.noreply.github.com> Date: Thu, 17 Sep 2026 12:48:02 -0700 Subject: [PATCH 12/15] fix(inference): expose unavailable admitted models --- .../browserInference/BrowserModelCatalog.test.ts | 12 ++++++++++++ .../src/data/browserInference/BrowserModelCatalog.ts | 7 +++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/frontend/app/src/data/browserInference/BrowserModelCatalog.test.ts b/frontend/app/src/data/browserInference/BrowserModelCatalog.test.ts index e942e1fd..a5002e45 100644 --- a/frontend/app/src/data/browserInference/BrowserModelCatalog.test.ts +++ b/frontend/app/src/data/browserInference/BrowserModelCatalog.test.ts @@ -240,6 +240,18 @@ describe("createBrowserModelCatalog", () => { expect(catalog.isKnown(ADMISSION.id)).toBe(true); }); + it("does not strand an admitted preference when a valid registry omits its release", async () => { + const { catalog } = harness({ discover: async () => false }); + await settles(catalog); + + expect(catalog.snapshot()[0]).toMatchObject({ + state: "failed", + storage: "none", + error: expect.stringMatching(/not available.*registry/i), + }); + expect(catalog.isKnown(ADMISSION.id)).toBe(true); + }); + it("knows an admitted but uninstalled preference without fabricating a ready target", async () => { const { catalog } = harness(); expect(catalog.isKnown(ADMISSION.id)).toBe(true); diff --git a/frontend/app/src/data/browserInference/BrowserModelCatalog.ts b/frontend/app/src/data/browserInference/BrowserModelCatalog.ts index ebff08a2..d5c9ffff 100644 --- a/frontend/app/src/data/browserInference/BrowserModelCatalog.ts +++ b/frontend/app/src/data/browserInference/BrowserModelCatalog.ts @@ -121,12 +121,15 @@ export function createBrowserModelCatalog(deps: CatalogDeps): OssBrowserModelCat update(record, { visible: true, state: "installed", storage: "persistent", error: undefined }); } else if (discovered) { update(record, { visible: true, state: "available", storage: "none", error: undefined }); - } else if (registryResult.status === "rejected") { + } else { update(record, { visible: true, state: "failed", storage: "none", - error: message(registryResult.reason), + error: + registryResult.status === "rejected" + ? message(registryResult.reason) + : `${record.admission.label} is not available from the configured registry`, }); } } From b5ed3fa8b797bf3c7cfb823810e0e8cf73aff0cc Mon Sep 17 00:00:00 2001 From: Jesus Armando Anaya <1445792+JArmandoAnaya@users.noreply.github.com> Date: Thu, 17 Sep 2026 12:57:38 -0700 Subject: [PATCH 13/15] test(inference): await live registry admission --- frontend/app/e2e/browserSuggestion.spec.ts | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/frontend/app/e2e/browserSuggestion.spec.ts b/frontend/app/e2e/browserSuggestion.spec.ts index 976ddd76..ddb7c4d1 100644 --- a/frontend/app/e2e/browserSuggestion.spec.ts +++ b/frontend/app/e2e/browserSuggestion.spec.ts @@ -300,21 +300,32 @@ test.describe("browser suggestion", () => { test.setTimeout(120_000); const sent: Request[] = []; const responses: { url: string; bytes: number; milliseconds: number }[] = []; + page.on("request", (request) => { + if (request.url().startsWith("https://models.robomous.ai/")) { + console.info("VISIONSET_LIVE_MODEL_REQUEST", request.url()); + } + }); + page.on("requestfailed", (request) => { + if (request.url().startsWith("https://models.robomous.ai/")) { + console.info("VISIONSET_LIVE_MODEL_REQUEST_FAILED", request.url(), request.failure()?.errorText); + } + }); page.on("response", async (response) => { if (!response.url().startsWith("https://models.robomous.ai/")) return; + console.info("VISIONSET_LIVE_MODEL_RESPONSE", response.status(), response.url()); await response.finished(); + console.info("VISIONSET_LIVE_MODEL_RESPONSE_FINISHED", response.url()); const timing = response.request().timing(); const declaredBytes = Number(response.headers()["content-length"] ?? 0); const bytes = declaredBytes > 0 ? declaredBytes : (await response.body()).byteLength; responses.push({ url: response.url(), bytes, milliseconds: timing.responseEnd }); }); - // Warm Chromium's connection to the public origin with the same registry document the - // application will validate. This keeps a one-off DNS/TLS stall from consuming the whole - // UI timeout while still exercising the app's own fetch, parser, and admission match below. - await page.goto("https://models.robomous.ai/registry/v1.json"); - await expect(page.locator("body")).toContainText('"schema_version": 1'); await openJobWithBrowserRuntime(page, sent, true, "live"); await armSuggestTool(page); + // Registry discovery is deliberately asynchronous and does not block the editor. Wait for + // its immutable manifest validation before selecting the controlled This device tab; a + // machine-speed click before the catalog has any target is intentionally a no-op. + await expect.poll(() => responses.some(({ url }) => url.endsWith("/manifest.json"))).toBe(true); const coldStarted = Date.now(); await acquireAndSelectBrowserTarget(page); const coldMilliseconds = Date.now() - coldStarted; From 98c6e58a0589b34b899c4edf834d0c52b8b0d4f6 Mon Sep 17 00:00:00 2001 From: Jesus Armando Anaya <1445792+JArmandoAnaya@users.noreply.github.com> Date: Thu, 17 Sep 2026 13:41:19 -0700 Subject: [PATCH 14/15] fix(inference): harden browser model lifecycle --- .../BrowserInferenceRuntime.test.ts | 34 ++++- .../BrowserModelCatalog.test.ts | 92 +++++++++++- .../browserInference/BrowserModelCatalog.ts | 114 ++++++++++---- .../acquireEfficientSam.test.ts | 142 ++++++++++++------ .../browserInference/artifactStore.test.ts | 46 ++++++ .../data/browserInference/artifactStore.ts | 50 +++++- .../app/src/data/browserInference/manifest.ts | 7 +- .../browserInference/registryClient.test.ts | 50 +++++- .../data/browserInference/registryClient.ts | 78 +++++++--- .../ui-core/src/annotator/SuggestPanel.tsx | 10 ++ .../src/annotator/suggestPanel.test.tsx | 30 ++++ frontend/ui-core/src/inference/browserPort.ts | 10 +- 12 files changed, 551 insertions(+), 112 deletions(-) diff --git a/frontend/app/src/data/browserInference/BrowserInferenceRuntime.test.ts b/frontend/app/src/data/browserInference/BrowserInferenceRuntime.test.ts index 3f910d2a..9e3f74cf 100644 --- a/frontend/app/src/data/browserInference/BrowserInferenceRuntime.test.ts +++ b/frontend/app/src/data/browserInference/BrowserInferenceRuntime.test.ts @@ -6,12 +6,14 @@ import { } from "./BrowserInferenceRuntime.js"; import { EFFICIENT_SAM_TI_REVISION } from "./manifest.js"; import type { VisionSetBrowserInferenceRuntime } from "@visionset/ui-core"; -import type { BrowserModelArtifacts } from "./artifactStore.js"; +import type { BrowserArtifactStore, BrowserModelArtifacts } from "./artifactStore.js"; function fakeDeps(overrides?: { acquire?: () => Promise; ready?: () => Promise; supported?: () => boolean; + store?: BrowserArtifactStore; + discover?: () => Promise; }) { const prepareImage = vi.fn(async () => ({ width: 4, height: 4 })); const dispose = vi.fn(); @@ -22,7 +24,15 @@ function fakeDeps(overrides?: { dispose, })); const acquire = vi.fn(overrides?.acquire ?? (async () => ({ encoder: new Uint8Array(1), decoder: new Uint8Array(1) }))); - return { acquire, createRuntime, supported: overrides?.supported ?? ((): boolean => true), prepareImage, dispose }; + return { + acquire, + createRuntime, + supported: overrides?.supported ?? ((): boolean => true), + ...(overrides?.store === undefined ? {} : { store: overrides.store }), + ...(overrides?.discover === undefined ? {} : { discover: overrides.discover }), + prepareImage, + dispose, + }; } async function acquisition(runtime: VisionSetBrowserInferenceRuntime) { @@ -60,6 +70,26 @@ describe("createOssBrowserInferenceRuntime", () => { expect(runtime.listAcquisitions?.()).toHaveLength(1); }); + it("does not wait for a hanging registry before exposing and activating an installed model", async () => { + const artifacts = { encoder: new Uint8Array(1), decoder: new Uint8Array(1) }; + const store: BrowserArtifactStore = { + inspect: vi.fn(async () => true), + readVerified: vi.fn(async () => artifacts), + writeVerified: vi.fn(async () => undefined), + remove: vi.fn(async () => undefined), + }; + const runtime = createOssBrowserInferenceRuntime(fakeDeps({ + store, + discover: () => new Promise(() => undefined), + })); + + // This awaits cache inspection only. A mutable registry must not become a prerequisite for + // an already-admitted local model, including after an offline reload. + await expect(runtime.listTargets()).resolves.toEqual([]); + await runtime.modelCatalog!.activate("efficient-sam-ti"); + await expect(runtime.listTargets()).resolves.toHaveLength(1); + }); + it("lists the target and no acquisitions after acquiring", async () => { const deps = fakeDeps(); const runtime = createOssBrowserInferenceRuntime(deps); diff --git a/frontend/app/src/data/browserInference/BrowserModelCatalog.test.ts b/frontend/app/src/data/browserInference/BrowserModelCatalog.test.ts index a5002e45..92f919ae 100644 --- a/frontend/app/src/data/browserInference/BrowserModelCatalog.test.ts +++ b/frontend/app/src/data/browserInference/BrowserModelCatalog.test.ts @@ -4,7 +4,12 @@ import type { PromptableSegmentationRuntime } from "@visionset/browser-inference import type { BrowserSuggestionTarget, SuggestionExecutor } from "@visionset/ui-core"; import type { BrowserModelAdmission } from "./admissionCatalog.js"; -import type { BrowserArtifactStore, BrowserModelArtifacts } from "./artifactStore.js"; +import { + BrowserArtifactRollbackError, + BrowserArtifactStorageIndeterminateError, + type BrowserArtifactStore, + type BrowserModelArtifacts, +} from "./artifactStore.js"; import { createBrowserModelCatalog } from "./BrowserModelCatalog.js"; const ADMISSION: BrowserModelAdmission = { @@ -36,6 +41,7 @@ function deferred() { function harness(overrides: { installed?: boolean; + inspect?: () => Promise; discover?: () => Promise; read?: () => Promise; write?: () => Promise; @@ -44,7 +50,7 @@ function harness(overrides: { } = {}) { let installed = overrides.installed ?? false; const store: BrowserArtifactStore = { - inspect: vi.fn(async () => installed), + inspect: vi.fn(overrides.inspect ?? (async () => installed)), readVerified: vi.fn(overrides.read ?? (async () => (installed ? ARTIFACTS : null))), writeVerified: vi.fn(overrides.write ?? (async () => { installed = true; })), remove: vi.fn(async () => { installed = false; }), @@ -63,14 +69,15 @@ function harness(overrides: { modelRef: ADMISSION.modelRef, }; const download = vi.fn(overrides.download ?? (async () => ARTIFACTS)); + const activate = vi.fn(async () => ({ runtime, executor, target })); const catalog = createBrowserModelCatalog({ admissions: [ADMISSION], store, discover: overrides.discover ?? (async () => true), download, - activate: vi.fn(async () => ({ runtime, executor, target })), + activate, }); - return { catalog, store, download, runtime, executor, dispose }; + return { catalog, store, download, runtime, executor, dispose, activate }; } async function settles(catalog: ReturnType): Promise { @@ -94,6 +101,36 @@ describe("createBrowserModelCatalog", () => { expect(runtime.ready).not.toHaveBeenCalled(); }); + it("publishes cached installation and permits activation while registry discovery never settles", async () => { + const never = new Promise(() => undefined); + const { catalog, download, runtime } = harness({ installed: true, discover: async () => never }); + + await settles(catalog); + expect(catalog.snapshot()[0]).toMatchObject({ state: "installed", storage: "persistent" }); + + await catalog.activate(ADMISSION.id); + expect(catalog.listTargets()).toHaveLength(1); + expect(runtime.ready).toHaveBeenCalledTimes(1); + expect(download).not.toHaveBeenCalled(); + }); + + it("serializes activation requested before cache initialization and creates one runtime", async () => { + const inspection = deferred(); + const { catalog, activate, dispose, runtime } = harness({ installed: true, inspect: () => inspection.promise }); + + const first = catalog.activate(ADMISSION.id); + const second = catalog.activate(ADMISSION.id); + expect(activate).not.toHaveBeenCalled(); + inspection.resolve(true); + await Promise.all([first, second]); + + expect(activate).toHaveBeenCalledTimes(1); + expect(runtime.ready).toHaveBeenCalledTimes(1); + expect(catalog.snapshot()[0]).toMatchObject({ state: "ready", storage: "persistent" }); + await catalog.remove(ADMISSION.id); + expect(dispose).toHaveBeenCalledTimes(1); + }); + it("deduplicates an explicit acquisition and exposes every lifecycle transition", async () => { const pending = deferred(); const { catalog, download } = harness({ download: () => pending.promise }); @@ -122,6 +159,38 @@ describe("createBrowserModelCatalog", () => { expect(catalog.listTargets()).toHaveLength(1); }); + it("keeps removal available when a failed cache write could not be rolled back", async () => { + const rollbackFailure = new BrowserArtifactRollbackError( + new DOMException("quota", "QuotaExceededError"), + new Error("cache delete failed"), + ); + const { catalog, store } = harness({ write: async () => Promise.reject(rollbackFailure) }); + await settles(catalog); + + await catalog.acquire(ADMISSION.id); + + expect(catalog.snapshot()[0]).toMatchObject({ state: "ready", storage: "unknown" }); + await catalog.remove(ADMISSION.id); + expect(store.remove).toHaveBeenCalledTimes(1); + expect(catalog.snapshot()[0]).toMatchObject({ state: "available", storage: "none" }); + }); + + it("keeps removal available when Cache Storage could not open", async () => { + const openFailure = new BrowserArtifactStorageIndeterminateError( + "Browser model cache could not be opened (cache namespace unavailable).", + new Error("cache namespace unavailable"), + ); + const { catalog, store } = harness({ write: async () => Promise.reject(openFailure) }); + await settles(catalog); + + await catalog.acquire(ADMISSION.id); + + expect(catalog.snapshot()[0]).toMatchObject({ state: "ready", storage: "unknown" }); + await catalog.remove(ADMISSION.id); + expect(store.remove).toHaveBeenCalledTimes(1); + expect(catalog.snapshot()[0]).toMatchObject({ state: "available", storage: "none" }); + }); + it("activates installed bytes from cache without any artifact download", async () => { const { catalog, download, runtime, store } = harness({ installed: true }); await settles(catalog); @@ -144,7 +213,7 @@ describe("createBrowserModelCatalog", () => { await expect(catalog.activate(ADMISSION.id)).rejects.toThrow(/sha-256 mismatch/i); expect(download).not.toHaveBeenCalled(); - expect(catalog.snapshot()[0]).toMatchObject({ state: "failed", storage: "none" }); + expect(catalog.snapshot()[0]).toMatchObject({ state: "failed", storage: "unknown" }); expect(catalog.listTargets()).toEqual([]); }); @@ -215,14 +284,25 @@ describe("createBrowserModelCatalog", () => { expect(catalog.listTargets()).toEqual([]); expect(catalog.snapshot()[0]).toMatchObject({ state: "failed", - storage: "persistent", + storage: "unknown", error: "storage delete failed", }); }); + it("keeps removal available when inspection cannot determine whether artifacts remain", async () => { + const { catalog, store } = harness({ inspect: async () => Promise.reject(new Error("cache match failed")) }); + await settles(catalog); + + expect(catalog.snapshot()[0]).toMatchObject({ state: "failed", storage: "unknown" }); + await catalog.remove(ADMISSION.id); + expect(store.remove).toHaveBeenCalledTimes(1); + expect(catalog.snapshot()[0]).toMatchObject({ state: "available", storage: "none" }); + }); + it("keeps a cached admitted model usable when registry discovery fails", async () => { const { catalog, download } = harness({ installed: true, discover: async () => Promise.reject(new Error("503")) }); await settles(catalog); + await vi.waitFor(() => expect(catalog.snapshot()[0]).toMatchObject({ warning: "503" })); await catalog.activate(ADMISSION.id); expect(catalog.listTargets()).toHaveLength(1); expect(download).not.toHaveBeenCalled(); diff --git a/frontend/app/src/data/browserInference/BrowserModelCatalog.ts b/frontend/app/src/data/browserInference/BrowserModelCatalog.ts index d5c9ffff..504f7bdd 100644 --- a/frontend/app/src/data/browserInference/BrowserModelCatalog.ts +++ b/frontend/app/src/data/browserInference/BrowserModelCatalog.ts @@ -7,7 +7,11 @@ import type { } from "@visionset/ui-core"; import type { BrowserModelAdmission } from "./admissionCatalog.js"; -import type { BrowserArtifactStore, BrowserModelArtifacts } from "./artifactStore.js"; +import { + BrowserArtifactStorageIndeterminateError, + type BrowserArtifactStore, + type BrowserModelArtifacts, +} from "./artifactStore.js"; interface ActiveModel { readonly runtime: PromptableSegmentationRuntime; @@ -38,6 +42,7 @@ interface ModelRecord { state: BrowserModelCatalogEntry["state"]; storage: BrowserModelCatalogEntry["storage"]; error?: string; + registryFailure?: string; sessionArtifacts?: BrowserModelArtifacts; } @@ -65,6 +70,7 @@ function entryOf(record: ModelRecord): BrowserModelCatalogEntry { state: record.state, storage: record.storage, ...(record.error === undefined ? {} : { error: record.error }), + ...(record.registryFailure === undefined ? {} : { warning: record.registryFailure }), }; } @@ -84,10 +90,14 @@ export function createBrowserModelCatalog(deps: CatalogDeps): OssBrowserModelCat const listeners = new Set<() => void>(); const operations = new Map< string, - { readonly kind: "acquire" | "activate" | "remove"; readonly promise: Promise } + { readonly kind: "initialize" | "acquire" | "activate" | "remove"; readonly promise: Promise } >(); let active: { readonly id: string; readonly value: ActiveModel } | null = null; let snapshot: readonly BrowserModelCatalogEntry[] = []; + // One lifecycle lane makes the active runtime a real singleton, not merely a best-effort + // per-model convention. It also means an initial cache inspection cannot publish over a + // caller that has already started activation. + let lifecycleTail: Promise | null = null; function publish(): void { snapshot = [...records.values()].filter((record) => record.visible).map(entryOf); @@ -110,27 +120,47 @@ export function createBrowserModelCatalog(deps: CatalogDeps): OssBrowserModelCat } async function initializeRecord(record: ModelRecord): Promise { - const [cacheResult, registryResult] = await Promise.allSettled([ - deps.store.inspect(record.admission), - deps.discover(record.admission), - ]); - const installed = cacheResult.status === "fulfilled" && cacheResult.value; - const discovered = registryResult.status === "fulfilled" && registryResult.value; - record.discovered = discovered; - if (installed) { - update(record, { visible: true, state: "installed", storage: "persistent", error: undefined }); - } else if (discovered) { - update(record, { visible: true, state: "available", storage: "none", error: undefined }); - } else { - update(record, { - visible: true, + try { + const installed = await deps.store.inspect(record.admission); + if (installed) { + update(record, { visible: true, state: "installed", storage: "persistent", error: undefined }); + } else if (record.registryFailure !== undefined) { + update(record, { visible: true, state: "failed", storage: "none", error: record.registryFailure }); + } else { + update(record, { visible: true, state: "available", storage: "none", error: undefined }); + } + } catch (error) { + // An inspection may fail while opening storage, matching an entry, or cleaning a partial + // model. In each case we cannot honestly say that no persistent bytes remain. + update(record, { visible: true, state: "failed", storage: "unknown", error: message(error) }); + } + } + + async function discoverRecord(record: ModelRecord): Promise { + try { + const discovered = await deps.discover(record.admission); + record.discovered = discovered; + record.registryFailure = discovered + ? undefined + : `${record.admission.label} is not available from the configured registry`; + // Discovery is advisory for a locally verified admission. Never let a late registry + // answer replace installed, activating, or ready local state. + if (discovered || record.state !== "available" || record.storage !== "none") { + publish(); + return; + } + update(record, record.registryFailure === undefined ? { error: undefined } : { state: "failed", - storage: "none", - error: - registryResult.status === "rejected" - ? message(registryResult.reason) - : `${record.admission.label} is not available from the configured registry`, + error: record.registryFailure, }); + } catch (error) { + record.discovered = false; + record.registryFailure = message(error); + if (record.state === "available" && record.storage === "none") { + update(record, { state: "failed", error: record.registryFailure }); + } else { + publish(); + } } } @@ -158,7 +188,7 @@ export function createBrowserModelCatalog(deps: CatalogDeps): OssBrowserModelCat function once( id: string, - kind: "acquire" | "activate" | "remove", + kind: "initialize" | "acquire" | "activate" | "remove", operation: () => Promise, ): Promise { const current = operations.get(id); @@ -169,12 +199,30 @@ export function createBrowserModelCatalog(deps: CatalogDeps): OssBrowserModelCat () => once(id, kind, operation), ); } - const promise = operation().finally(() => operations.delete(id)); + const promise = lifecycleTail === null ? operation() : lifecycleTail.then(operation); + const settled = promise.catch(() => undefined); + lifecycleTail = settled; operations.set(id, { kind, promise }); + void promise.then( + () => { + if (operations.get(id)?.promise === promise) operations.delete(id); + if (lifecycleTail === settled) lifecycleTail = null; + }, + () => { + if (operations.get(id)?.promise === promise) operations.delete(id); + if (lifecycleTail === settled) lifecycleTail = null; + }, + ); return promise; } - const initialized = Promise.all([...records.values()].map(initializeRecord)).then(() => undefined); + // Cache inspection must settle the public initialization boundary. Registry discovery is + // deliberately background metadata: an offline/hanging registry cannot strand an admitted + // cached model or delay server-independent browser activation. + const initialized = Promise.all( + [...records.values()].map((record) => once(record.admission.id, "initialize", () => initializeRecord(record))), + ).then(() => undefined); + for (const record of records.values()) void discoverRecord(record); const catalog: OssBrowserModelCatalog = { initialized, @@ -191,6 +239,7 @@ export function createBrowserModelCatalog(deps: CatalogDeps): OssBrowserModelCat if (!record.discovered) { record.discovered = await deps.discover(record.admission, options?.signal); if (!record.discovered) throw new Error(`browser model "${id}" is not available from this registry`); + record.registryFailure = undefined; } update(record, { visible: true, state: "downloading", storage: "none", error: undefined }); try { @@ -198,8 +247,11 @@ export function createBrowserModelCatalog(deps: CatalogDeps): OssBrowserModelCat let storage: BrowserModelCatalogEntry["storage"] = "persistent"; try { await deps.store.writeVerified(record.admission, artifacts); - } catch { - storage = "session"; + } catch (error) { + // Verified bytes remain usable in memory even when persistence is refused. The + // artifact store rolls back partial writes before rejecting. If rollback itself + // failed, keep removal available because any subset of the revision may remain. + storage = error instanceof BrowserArtifactStorageIndeterminateError ? "unknown" : "session"; record.sessionArtifacts = artifacts; } update(record, { state: "installed", storage, error: undefined }); @@ -224,7 +276,9 @@ export function createBrowserModelCatalog(deps: CatalogDeps): OssBrowserModelCat } catch (error) { record.sessionArtifacts = undefined; if (active?.id === id) active = null; - update(record, { visible: true, state: "failed", storage: "none", error: message(error) }); + // `readVerified` may itself fail while removing a corrupt or partial entry. Preserve + // uncertainty so the UI still offers explicit removal instead of hiding remnants. + update(record, { visible: true, state: "failed", storage: "unknown", error: message(error) }); throw error; } // `startRuntime` owns its failure state. At this point bytes have already passed the @@ -237,18 +291,18 @@ export function createBrowserModelCatalog(deps: CatalogDeps): OssBrowserModelCat return once(id, "remove", async () => { const record = required(id); const previous = active?.id === id ? active.value : null; - const previousStorage = record.storage; if (previous !== null) active = null; record.sessionArtifacts = undefined; - update(record, { visible: true, state: "available", storage: "none", error: undefined }); if (previous !== null) previous.runtime.dispose(); try { await deps.store.remove(record.admission); + update(record, { visible: true, state: "available", storage: "none", error: undefined }); } catch (error) { update(record, { visible: true, state: "failed", - storage: previousStorage === "persistent" ? "persistent" : "none", + // Cache deletion is multi-artifact. A failure can leave any subset behind. + storage: "unknown", error: message(error), }); throw error; diff --git a/frontend/app/src/data/browserInference/acquireEfficientSam.test.ts b/frontend/app/src/data/browserInference/acquireEfficientSam.test.ts index aea18f2a..6b965b36 100644 --- a/frontend/app/src/data/browserInference/acquireEfficientSam.test.ts +++ b/frontend/app/src/data/browserInference/acquireEfficientSam.test.ts @@ -10,8 +10,11 @@ */ import { beforeEach, describe, expect, it, vi } from "vitest"; +import { EFFICIENT_SAM_TI_MANIFEST_V1 } from "./fixtures/manifests-v1.js"; +import registryV1 from "./fixtures/registry-v1.json"; import { fetchVerified } from "./acquireEfficientSam.js"; import { fetchEfficientSamManifest } from "./manifest.js"; +import { fetchAdmittedBrowserModels } from "./registryClient.js"; // `Uint8Array`, not the bare `Uint8Array` — see the same note in // acquireEfficientSam.ts: TypeScript 6's `lib.dom.d.ts` requires the concrete @@ -25,8 +28,33 @@ async function sha256Of(bytes: Uint8Array): Promise { return Array.from(new Uint8Array(digest)).map((b) => b.toString(16).padStart(2, "0")).join(""); } +interface ManifestArtifactOverride { + readonly path?: string; + readonly bytes?: number; + readonly sha256?: string; + readonly content_type?: string; +} + +function manifestWithArtifactOverrides( + encoder: ManifestArtifactOverride = {}, + decoder: ManifestArtifactOverride = {}, +) { + return { + ...EFFICIENT_SAM_TI_MANIFEST_V1, + artifacts: { + encoder: { ...EFFICIENT_SAM_TI_MANIFEST_V1.artifacts.encoder, ...encoder }, + decoder: { ...EFFICIENT_SAM_TI_MANIFEST_V1.artifacts.decoder, ...decoder }, + }, + }; +} + describe("fetchVerified", () => { - beforeEach(() => vi.restoreAllMocks()); + beforeEach(() => { + vi.restoreAllMocks(); + vi.resetModules(); + vi.doUnmock("./admissionCatalog.js"); + vi.doUnmock("./manifest.js"); + }); it("returns the bytes when size and SHA-256 both match", async () => { const bytes = bytesOf("hello world"); @@ -55,17 +83,50 @@ describe("fetchVerified", () => { }); describe("acquireEfficientSam", () => { + beforeEach(() => { + vi.restoreAllMocks(); + vi.resetModules(); + vi.doUnmock("./admissionCatalog.js"); + vi.doUnmock("./manifest.js"); + }); + + it("rejects a mutated acquisition-time manifest even after catalog discovery admitted an earlier copy", async () => { + let manifestRequests = 0; + const mutatedManifest = { + ...structuredClone(EFFICIENT_SAM_TI_MANIFEST_V1), + runtime: { ...EFFICIENT_SAM_TI_MANIFEST_V1.runtime, opset: 18 }, + }; + const fetchMock = vi.fn(async (url: string | URL | Request) => { + const href = String(url); + if (href.endsWith("registry/v1.json")) return new Response(JSON.stringify(registryV1)); + if (href.endsWith("manifest.json")) { + manifestRequests += 1; + return new Response(JSON.stringify(manifestRequests === 1 ? EFFICIENT_SAM_TI_MANIFEST_V1 : mutatedManifest)); + } + throw new Error(`artifact must not be fetched after a manifest mismatch: ${href}`); + }); + vi.stubGlobal("fetch", fetchMock); + + await expect(fetchAdmittedBrowserModels("https://models.robomous.ai")).resolves.toHaveLength(1); + await expect((await import("./acquireEfficientSam.js")).acquireEfficientSam()).rejects.toThrow( + /admission mismatch at runtime\.opset/i, + ); + expect(fetchMock).toHaveBeenCalledTimes(3); + }); + it("fetches the manifest, then each artifact — never before the manifest resolves", async () => { const encoderBytes = bytesOf("encoder-fixture"); const decoderBytes = bytesOf("decoder-fixture"); + const manifest = manifestWithArtifactOverrides( + { bytes: encoderBytes.byteLength, sha256: await sha256Of(encoderBytes) }, + { bytes: decoderBytes.byteLength, sha256: await sha256Of(decoderBytes) }, + ); const fetchMock = vi.fn(async (url: string) => { if (url.endsWith("manifest.json")) { // The real, already-deployed manifest shape: artifacts nested under // `artifacts`, each `path` a bare filename relative to the manifest's own // directory — not `{ encoder: { path: "/encoder.onnx" } }` at the top level. - return new Response( - JSON.stringify({ artifacts: { encoder: { path: "encoder.onnx" }, decoder: { path: "decoder.onnx" } } }), - ); + return new Response(JSON.stringify(manifest)); } if (url.endsWith("encoder.onnx")) return new Response(encoderBytes); if (url.endsWith("decoder.onnx")) return new Response(decoderBytes); @@ -79,15 +140,24 @@ describe("acquireEfficientSam", () => { // the cache first, forcing the dynamic `import()` below to re-evaluate both // modules fresh, this time picking up the mock. vi.resetModules(); - vi.doMock("./manifest.js", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - EFFICIENT_SAM_TI_EXPECTED: { - encoder: { sha256: await sha256Of(encoderBytes), bytes: encoderBytes.byteLength }, - decoder: { sha256: await sha256Of(decoderBytes), bytes: decoderBytes.byteLength }, - }, + vi.doMock("./admissionCatalog.js", async (importOriginal) => { + const actual = await importOriginal(); + const admitted = { + ...actual.EFFICIENT_SAM_TI_ADMISSION, + artifacts: [ + { + ...actual.EFFICIENT_SAM_TI_ADMISSION.artifacts[0], + bytes: encoderBytes.byteLength, + sha256: await sha256Of(encoderBytes), + }, + { + ...actual.EFFICIENT_SAM_TI_ADMISSION.artifacts[1], + bytes: decoderBytes.byteLength, + sha256: await sha256Of(decoderBytes), + }, + ] as const, }; + return { ...actual, EFFICIENT_SAM_TI_ADMISSION: admitted, ADMITTED_BROWSER_MODELS: [admitted] }; }); const { acquireEfficientSam } = await import("./acquireEfficientSam.js"); @@ -108,46 +178,21 @@ describe("acquireEfficientSam", () => { ); }); - it("never trusts the manifest's own bytes/sha256 — a manifest that lies about both still verifies against the pinned constants", async () => { - const encoderBytes = bytesOf("encoder-fixture"); - const decoderBytes = bytesOf("decoder-fixture"); + it("rejects a manifest that lies about an admitted artifact hash before fetching artifact bytes", async () => { const fetchMock = vi.fn(async (url: string) => { if (url.endsWith("manifest.json")) { return new Response( - JSON.stringify({ - artifacts: { - // Deliberately wrong `bytes`/`sha256` alongside the real `path` — a - // manifest that lies about its own artifacts' hashes. If acquisition - // ever read these instead of `EFFICIENT_SAM_TI_EXPECTED`, this fixture - // would either reject the correct fixture bytes or accept forged ones. - encoder: { path: "encoder.onnx", bytes: 1, sha256: "0".repeat(64) }, - decoder: { path: "decoder.onnx", bytes: 1, sha256: "0".repeat(64) }, - }, - }), + JSON.stringify(manifestWithArtifactOverrides({ sha256: "0".repeat(64) })), ); } - if (url.endsWith("encoder.onnx")) return new Response(encoderBytes); - if (url.endsWith("decoder.onnx")) return new Response(decoderBytes); - throw new Error(`unexpected url ${url}`); + throw new Error(`artifact must not be fetched: ${url}`); }); vi.stubGlobal("fetch", fetchMock); vi.resetModules(); - vi.doMock("./manifest.js", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - EFFICIENT_SAM_TI_EXPECTED: { - encoder: { sha256: await sha256Of(encoderBytes), bytes: encoderBytes.byteLength }, - decoder: { sha256: await sha256Of(decoderBytes), bytes: decoderBytes.byteLength }, - }, - }; - }); const { acquireEfficientSam } = await import("./acquireEfficientSam.js"); - const result = await acquireEfficientSam(); - - expect(result.encoder).toEqual(encoderBytes); - expect(result.decoder).toEqual(decoderBytes); + await expect(acquireEfficientSam()).rejects.toThrow(/admission mismatch at artifacts\.encoder\.sha256/i); + expect(fetchMock).toHaveBeenCalledTimes(1); }); it.each(["../secrets.onnx", "..", "%2e%2e", "\\..\\private"])( @@ -156,7 +201,7 @@ describe("acquireEfficientSam", () => { const fetchMock = vi.fn(async (url: string) => { if (url.endsWith("manifest.json")) { return new Response( - JSON.stringify({ artifacts: { encoder: { path }, decoder: { path: "decoder.onnx" } } }), + JSON.stringify(manifestWithArtifactOverrides({ path })), ); } throw new Error(`unexpected url ${url}`); @@ -165,14 +210,19 @@ describe("acquireEfficientSam", () => { vi.resetModules(); const { acquireEfficientSam } = await import("./acquireEfficientSam.js"); - await expect(acquireEfficientSam()).rejects.toThrow(/unexpected manifest artifact path/i); + await expect(acquireEfficientSam()).rejects.toThrow(/admission mismatch at artifacts\.encoder\.path/i); expect(fetchMock).toHaveBeenCalledTimes(1); }, ); }); describe("fetchEfficientSamManifest", () => { - beforeEach(() => vi.restoreAllMocks()); + beforeEach(() => { + vi.restoreAllMocks(); + vi.resetModules(); + vi.doUnmock("./admissionCatalog.js"); + vi.doUnmock("./manifest.js"); + }); it("throws a diagnosable error, not a bare TypeError, when the CDN's manifest schema has moved", async () => { vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response(JSON.stringify({ artifacts: { encoder: {} } })))); @@ -184,7 +234,7 @@ describe("fetchEfficientSamManifest", () => { "fetch", vi.fn().mockResolvedValue( new Response( - JSON.stringify({ artifacts: { encoder: { path: "encoder.onnx" }, decoder: { path: "decoder.onnx" } } }), + JSON.stringify(EFFICIENT_SAM_TI_MANIFEST_V1), ), ), ); diff --git a/frontend/app/src/data/browserInference/artifactStore.test.ts b/frontend/app/src/data/browserInference/artifactStore.test.ts index 6f5b8a59..34139a9d 100644 --- a/frontend/app/src/data/browserInference/artifactStore.test.ts +++ b/frontend/app/src/data/browserInference/artifactStore.test.ts @@ -4,6 +4,8 @@ import { describe, expect, it, vi } from "vitest"; import type { BrowserModelAdmission } from "./admissionCatalog.js"; import { CACHE_NAMESPACE, + BrowserArtifactRollbackError, + BrowserArtifactStorageIndeterminateError, cacheKeyFor, createCacheArtifactStore, type ArtifactCache, @@ -113,6 +115,50 @@ describe("createCacheArtifactStore", () => { expect(cache.entries.size).toBe(0); }); + it("distinguishes a failed write whose rollback also fails and preserves the write cause", async () => { + const cache = new MemoryCache(); + const quota = new DOMException("quota", "QuotaExceededError"); + const cleanup = new Error("cache delete failed"); + cache.put.mockImplementationOnce(async (request, response) => { + cache.entries.set(String(request), response.clone()); + }); + cache.put.mockRejectedValueOnce(quota); + cache.delete.mockRejectedValue(cleanup); + const { admission, artifacts } = await fixture(); + const store = createCacheArtifactStore(storage(cache)); + + let thrown: unknown; + try { + await store.writeVerified(admission, artifacts); + } catch (error) { + thrown = error; + } + + expect(thrown).toBeInstanceOf(BrowserArtifactRollbackError); + expect(thrown).toMatchObject({ cause: quota, cleanupCause: cleanup }); + expect((thrown as Error).message).toMatch(/quota.*cleanup failed/i); + // The failed cleanup leaves the written encoder potentially resident; callers must offer + // explicit removal rather than claiming a clean session-only fallback. + expect(cache.entries.size).toBe(1); + }); + + it("marks a rejected Cache Storage open as indeterminate while retaining the original cause", async () => { + const openFailure = new Error("cache namespace unavailable"); + const cacheStorage: ArtifactCacheStorage = { open: vi.fn(async () => Promise.reject(openFailure)) }; + const { admission, artifacts } = await fixture(); + + let thrown: unknown; + try { + await createCacheArtifactStore(cacheStorage).writeVerified(admission, artifacts); + } catch (error) { + thrown = error; + } + + expect(thrown).toBeInstanceOf(BrowserArtifactStorageIndeterminateError); + expect(thrown).toMatchObject({ cause: openFailure }); + expect((thrown as Error).message).toMatch(/could not be opened.*namespace unavailable/i); + }); + it("cleans up a partial cache instead of treating it as installed", async () => { const cache = new MemoryCache(); const { admission, artifacts } = await fixture(); diff --git a/frontend/app/src/data/browserInference/artifactStore.ts b/frontend/app/src/data/browserInference/artifactStore.ts index e69612ac..8728fddc 100644 --- a/frontend/app/src/data/browserInference/artifactStore.ts +++ b/frontend/app/src/data/browserInference/artifactStore.ts @@ -26,6 +26,40 @@ export interface BrowserArtifactStore { remove(model: BrowserModelAdmission): Promise; } +/** + * Cache Storage could not establish a trustworthy persistent state. Callers must retain a + * removal affordance: pre-existing entries may still be resident even when this operation could + * not inspect or clean them. + */ +export class BrowserArtifactStorageIndeterminateError extends Error { + constructor(message: string, cause: unknown) { + super(message, { cause }); + this.name = "BrowserArtifactStorageIndeterminateError"; + } +} + +/** + * A failed cache write normally leaves no model bytes behind because the store rolls its keys + * back. This error is deliberately distinct: the write failed *and* that rollback failed, so a + * caller must not describe the model as merely session-only or hide its removal affordance. + */ +export class BrowserArtifactRollbackError extends BrowserArtifactStorageIndeterminateError { + readonly cleanupCause: unknown; + + constructor(writeCause: unknown, cleanupCause: unknown) { + super( + `Browser model cache write failed (${messageFor(writeCause)}) and cleanup failed (${messageFor(cleanupCause)}).`, + writeCause, + ); + this.name = "BrowserArtifactRollbackError"; + this.cleanupCause = cleanupCause; + } +} + +function messageFor(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + export function cacheKeyFor(model: BrowserModelAdmission, artifact: ArtifactAdmission): string { return ( `${CACHE_ORIGIN}/__visionset_model_cache__/` + @@ -100,7 +134,15 @@ export function createCacheArtifactStore( verifyArtifact(bytesFor(artifacts, artifact.role), artifact, `downloaded ${artifact.role}`), ), ); - const opened = await cache(); + let opened: ArtifactCache | null; + try { + opened = await cache(); + } catch (error) { + throw new BrowserArtifactStorageIndeterminateError( + `Browser model cache could not be opened (${messageFor(error)}).`, + error, + ); + } if (opened === null) throw new Error("browser model storage is unavailable"); try { for (const artifact of model.artifacts) { @@ -111,7 +153,11 @@ export function createCacheArtifactStore( ); } } catch (error) { - await remove(model); + try { + await remove(model); + } catch (cleanupError) { + throw new BrowserArtifactRollbackError(error, cleanupError); + } throw error; } }, diff --git a/frontend/app/src/data/browserInference/manifest.ts b/frontend/app/src/data/browserInference/manifest.ts index 2ea274f2..05296afe 100644 --- a/frontend/app/src/data/browserInference/manifest.ts +++ b/frontend/app/src/data/browserInference/manifest.ts @@ -5,6 +5,7 @@ * of this same manifest layout. */ import { EFFICIENT_SAM_TI_ADMISSION } from "./admissionCatalog.js"; +import { validateManifestAgainstAdmission } from "./registryClient.js"; export const MODEL_CDN_BASE_URL: string = ( (import.meta.env["VITE_MODEL_CDN_BASE_URL"] as string | undefined) ?? "https://models.robomous.ai" @@ -39,8 +40,9 @@ export const EFFICIENT_SAM_TI_ARTIFACT_PATHS = Object.fromEntries( /** * Mirrors the real, already-deployed manifest shape (nested under `artifacts`, * with each `path` a bare filename relative to the manifest's own directory) — not - * an assumed flat shape. Only `path` is read from this; `bytes`/`sha256` are never - * trusted from the manifest itself (see `EFFICIENT_SAM_TI_EXPECTED`). + * an assumed flat shape. Before an acquisition may use it, every supported field + * is validated against the build admission record. The admission record remains + * the integrity anchor for the artifact verification that follows. */ export interface EfficientSamManifest { readonly artifacts: { @@ -66,5 +68,6 @@ export async function fetchEfficientSamManifest(signal?: AbortSignal): Promise { expect(fetch).toHaveBeenCalledTimes(2); }); + it("accepts an equivalent relative admitted manifest path and resolves it below a mirror base prefix", async () => { + const row = admittedRow(); + row.manifest = "models/efficient-sam-ti/b19782d049c0-843761ca46f4/manifest.json"; + const fetch = fetchFixture(registryWith(row)); + + const result = await fetchAdmittedBrowserModels("https://models.example/mirror", { fetch }); + + expect(result).toHaveLength(1); + expect(result[0]!.manifestUrl.href).toBe( + "https://models.example/mirror/models/efficient-sam-ti/b19782d049c0-843761ca46f4/manifest.json", + ); + expect(fetch.mock.calls.map(([input]) => String(input))).toEqual([ + "https://models.example/mirror/registry/v1.json", + "https://models.example/mirror/models/efficient-sam-ti/b19782d049c0-843761ca46f4/manifest.json", + ]); + }); + + it("validates and preserves a registry license when the deployed registry supplies one", async () => { + const row = { ...admittedRow(), license: "Apache-2.0" }; + + const [model] = await fetchAdmittedBrowserModels(BASE, { + fetch: fetchFixture(registryWith(row)), + }); + + expect(model?.license).toBe("Apache-2.0"); + expect(model?.registryLicense).toBe("Apache-2.0"); + }); + + it("rejects a registry license mismatch against the admission", async () => { + await expect( + fetchAdmittedBrowserModels(BASE, { + fetch: fetchFixture(registryWith({ ...admittedRow(), license: "MIT" })), + }), + ).rejects.toThrow(/admission mismatch at registry\.license/i); + }); + it.each([ null, {}, @@ -59,6 +95,14 @@ describe("fetchAdmittedBrowserModels", () => { ); }); + it("rejects a malformed optional registry license", async () => { + await expect( + fetchAdmittedBrowserModels(BASE, { + fetch: fetchFixture(registryWith({ ...admittedRow(), license: 42 })), + }), + ).rejects.toThrow(/registry schema/i); + }); + it("rejects duplicate IDs even when their revisions differ", async () => { const first = admittedRow(); const second = { ...admittedRow(), revision: "another-immutable-revision" }; @@ -122,6 +166,8 @@ describe("fetchAdmittedBrowserModels", () => { "//evil.example/manifest.json", "/models/%2e%2e/escape/manifest.json", "/models/model/manifest.json?mutable=1", + "/models/model/manifest.json%3Fmutable%3D1", + "https%3A%2F%2Fevil.example/manifest.json", ])("rejects an unsafe manifest path %s", async (manifestPath) => { await expect( fetchAdmittedBrowserModels(BASE, { @@ -145,8 +191,8 @@ describe("fetchAdmittedBrowserModels", () => { }); describe("resolveModelPath", () => { - it("resolves a root-relative public path below a configured base prefix", () => { - expect(resolveModelPath("https://models.example/base", "/base/models/a/manifest.json").href).toBe( + it("resolves a root-relative logical path below a configured base prefix", () => { + expect(resolveModelPath("https://models.example/base", "/models/a/manifest.json").href).toBe( "https://models.example/base/models/a/manifest.json", ); }); diff --git a/frontend/app/src/data/browserInference/registryClient.ts b/frontend/app/src/data/browserInference/registryClient.ts index be72f231..de30e604 100644 --- a/frontend/app/src/data/browserInference/registryClient.ts +++ b/frontend/app/src/data/browserInference/registryClient.ts @@ -10,6 +10,8 @@ interface RegistryRow { readonly revision: string; readonly modelRef: string; readonly manifest: string; + /** Optional in the measured v1 registry schema; preserve it when supplied. */ + readonly license?: string; } export interface AdmittedRegistryModel { @@ -19,6 +21,8 @@ export interface AdmittedRegistryModel { readonly revision: string; readonly modelRef: string; readonly license: string; + /** The registry's matching license declaration, when its schema supplied one. */ + readonly registryLicense?: string; readonly source: BrowserModelAdmission["source"]; readonly manifestUrl: URL; readonly artifactUrls: Readonly>; @@ -43,6 +47,11 @@ function text(value: unknown, at: string): string { return value; } +function optionalText(value: unknown, at: string): string | undefined { + if (value === undefined) return undefined; + return text(value, at); +} + function number(value: unknown, at: string): number { if (typeof value !== "number" || !Number.isFinite(value)) { throw new Error(`unexpected registry schema at ${at}`); @@ -72,6 +81,7 @@ function parseRegistry(value: unknown): readonly RegistryRow[] { revision: text(row["revision"], `models[${index}].revision`), modelRef: text(row["model_ref"], `models[${index}].model_ref`), manifest: text(row["manifest"], `models[${index}].manifest`), + license: optionalText(row["license"], `models[${index}].license`), }; }); } @@ -84,17 +94,14 @@ function normalizedBase(baseUrl: string): URL { return base; } -export function resolveModelPath(baseUrl: string, path: string): URL { - const base = normalizedBase(baseUrl); - if ( - path.length === 0 || - path.includes("\\") || - path.includes("?") || - path.includes("#") || - path.startsWith("//") || - /^[a-z][a-z\d+.-]*:/i.test(path) || - /%2f|%5c/i.test(path) - ) { +/** + * Registry paths identify objects inside a configured model source, rather than + * origin-root URLs. Canonicalize the accepted spelling so `/models/x` and + * `models/x` compare as the same admitted logical path, then resolve below the + * base's (possibly non-root) path prefix. + */ +export function normalizeModelPath(path: string): string { + if (path.length === 0 || /%2f|%5c/i.test(path)) { throw new Error(`unsafe model path: ${path}`); } let decoded: string; @@ -103,14 +110,27 @@ export function resolveModelPath(baseUrl: string, path: string): URL { } catch { throw new Error(`unsafe model path: ${path}`); } - if (decoded.split("/").some((segment) => segment === "." || segment === "..")) { + if ( + decoded.includes("\\") || + decoded.includes("?") || + decoded.includes("#") || + decoded.startsWith("//") || + /^[a-z][a-z\d+.-]*:/i.test(decoded) + ) { throw new Error(`unsafe model path: ${path}`); } - const resolved = new URL(path, base); - if (resolved.origin !== base.origin || !resolved.pathname.startsWith(base.pathname)) { - throw new Error(`unsafe model path outside configured base: ${path}`); + if (decoded.split("/").some((segment) => segment === "." || segment === "..")) { + throw new Error(`unsafe model path: ${path}`); } - return resolved; + const relative = decoded.replace(/^\/+/, ""); + if (relative.length === 0) throw new Error(`unsafe model path: ${path}`); + return `/${relative}`; +} + +export function resolveModelPath(baseUrl: string, path: string): URL { + const base = normalizedBase(baseUrl); + const logicalPath = normalizeModelPath(path); + return new URL(logicalPath.slice(1), base); } function assertEqual(actual: unknown, expected: unknown, field: string): void { @@ -136,7 +156,15 @@ function validateArtifact( return path; } -function validateManifest(value: unknown, admission: BrowserModelAdmission): Record { +/** + * Validates remote metadata against this build's admission record. The manifest + * describes a release but never becomes its trust anchor: artifact size and hash + * must still exactly equal the build-pinned admission values. + */ +export function validateManifestAgainstAdmission( + value: unknown, + admission: BrowserModelAdmission, +): Record { const manifest = record(value, "manifest"); assertEqual(number(manifest["schema_version"], "manifest.schema_version"), 1, "schema_version"); assertEqual(text(manifest["id"], "manifest.id"), admission.id, "id"); @@ -162,6 +190,14 @@ function validateManifest(value: unknown, admission: BrowserModelAdmission): Rec assertEqual(boolean(capabilities["negative_points"], "manifest.capabilities.negative_points"), admission.capabilities.negativePoints, "capabilities.negative_points"); assertEqual(number(capabilities["max_points"], "manifest.capabilities.max_points"), admission.capabilities.maxPoints, "capabilities.max_points"); + const artifacts = record(manifest["artifacts"], "manifest.artifacts"); + const expectedRoles = new Set(admission.artifacts.map((artifact) => artifact.role)); + for (const role of Object.keys(artifacts)) { + if (!expectedRoles.has(role as ArtifactAdmission["role"])) { + throw new Error(`model admission mismatch at artifacts.${role}`); + } + } + return Object.fromEntries( admission.artifacts.map((artifact) => [artifact.role, validateArtifact(manifest, artifact)]), ) as Record; @@ -191,9 +227,10 @@ export async function fetchAdmittedBrowserModels( } assertEqual(row.name, admission.label, "registry.name"); assertEqual(row.modelRef, admission.modelRef, "registry.model_ref"); - assertEqual(row.manifest, admission.manifestPath, "registry.manifest"); + assertEqual(normalizeModelPath(row.manifest), normalizeModelPath(admission.manifestPath), "registry.manifest"); + if (row.license !== undefined) assertEqual(row.license, admission.license, "registry.license"); const manifestUrl = resolveModelPath(baseUrl, row.manifest); - const artifactPaths = validateManifest( + const artifactPaths = validateManifestAgainstAdmission( await json(await fetcher(manifestUrl, { signal: options.signal }), "manifest"), admission, ); @@ -207,7 +244,8 @@ export async function fetchAdmittedBrowserModels( label: admission.label, revision: admission.revision, modelRef: admission.modelRef, - license: admission.license, + license: row.license ?? admission.license, + registryLicense: row.license, source: admission.source, manifestUrl, artifactUrls, diff --git a/frontend/ui-core/src/annotator/SuggestPanel.tsx b/frontend/ui-core/src/annotator/SuggestPanel.tsx index 1d8be971..77c1c4ee 100644 --- a/frontend/ui-core/src/annotator/SuggestPanel.tsx +++ b/frontend/ui-core/src/annotator/SuggestPanel.tsx @@ -911,6 +911,16 @@ function CatalogModel({ Ready for this session, but it was not saved in this browser.

)} + {model.storage === "unknown" && ( +

+ Browser storage could not be checked. Remove this model to clear any saved files. +

+ )} + {model.warning !== undefined && model.warning !== model.error && ( +

+ The model registry could not be checked. A saved model can still run on this device. +

+ )} {canRemove && onRemove !== undefined && (