From 66221f52fc9cf7160799d364ad8a3a770dc45f58 Mon Sep 17 00:00:00 2001
From: Jesus Armando Anaya <1445792+JArmandoAnaya@users.noreply.github.com>
Date: Wed, 16 Sep 2026 16:24:50 -0700
Subject: [PATCH 1/6] feat(ui-core,annotator): retain the displayed asset as a
guarded, lazy browser pixel source
AssetImage retains the exact fetched Blob behind its object URL (one
content request, one Blob, one URL; abort+revoke on replacement or
unmount) while still exposing only src to the existing render-prop
consumers.
AnnotatorCanvas exposes an optional onImageReady callback carrying a
generation-aware decoded-image lease over the real rendered
annotator-image element. The lease's readRgb performs lazy,
descriptor-frame RGBA-to-RGB conversion and refuses once its
generation or src is stale, so a retained source can never read a
replacement asset through a reused DOM node.
No annotator/browser-inference dependency, model change, or exposed
inference target is introduced; the headless annotator-core boundary
is untouched.
---
docs/content/architecture/frontend/ui-core.md | 16 +++++
.../src/adapters/react/AnnotatorCanvas.tsx | 28 ++++++++
.../adapters/react/decodedAssetImage.test.ts | 34 ++++++++++
.../src/adapters/react/decodedAssetImage.ts | 55 ++++++++++++++++
.../annotator/src/adapters/react/index.ts | 1 +
frontend/ui-core/src/annotator/AssetImage.tsx | 18 +++--
.../ui-core/src/annotator/assetImage.test.tsx | 66 ++++++++++++++++++-
.../src/annotator/decodedImageLease.test.tsx | 55 ++++++++++++++++
8 files changed, 264 insertions(+), 9 deletions(-)
create mode 100644 frontend/annotator/src/adapters/react/decodedAssetImage.test.ts
create mode 100644 frontend/annotator/src/adapters/react/decodedAssetImage.ts
create mode 100644 frontend/ui-core/src/annotator/decodedImageLease.test.tsx
diff --git a/docs/content/architecture/frontend/ui-core.md b/docs/content/architecture/frontend/ui-core.md
index 714491c2..da2751be 100644
--- a/docs/content/architecture/frontend/ui-core.md
+++ b/docs/content/architecture/frontend/ui-core.md
@@ -89,6 +89,22 @@ how it is grown instead.
> something this browser can do. They are not two spellings of one idea, and neither is
> derivable from the other.
+## The displayed asset is the browser pixel source
+
+`AssetImage` fetches an asset once through the credentialed data client, retains that exact Blob
+only while the asset is mounted, and owns the object URL it gives the existing annotator image.
+The URL is revoked and an unfinished request is aborted when the asset is replaced or unmounted.
+There is no JavaScript Blob cache: revisiting an asset relies on the server's immutable HTTP cache.
+
+The React annotator adapter can report a generation-scoped lease over the same visible decoded
+`![]()
`. A future host that needs pixels reads them lazily from that lease, in the asset
+descriptor's coordinate frame, through a canvas RGBA-to-RGB copy. It does not fetch or decode a
+second copy of the asset. A lease refuses after its image source is replaced, which prevents a
+retained reference for one asset from reading pixels of the next asset through a reused DOM node.
+
+This is resource plumbing only. It neither selects a browser model nor exposes an inference
+target; composing pixels with a browser runtime remains a later host decision.
+
## Asking for a suggestion is not sending one
`inference/suggestionExecutor.ts` is the seam the runtime above would plug into, and it exists
diff --git a/frontend/annotator/src/adapters/react/AnnotatorCanvas.tsx b/frontend/annotator/src/adapters/react/AnnotatorCanvas.tsx
index e8cd32a7..94a7ec2b 100644
--- a/frontend/annotator/src/adapters/react/AnnotatorCanvas.tsx
+++ b/frontend/annotator/src/adapters/react/AnnotatorCanvas.tsx
@@ -207,6 +207,7 @@ import {
} from "../viewport";
import type { Viewport } from "../viewport";
import { AnnotationLayer } from "./AnnotationLayer";
+import { createDecodedAssetImage, type DecodedAssetImage } from "./decodedAssetImage";
import { useAnnotatorSnapshot } from "./hooks";
import { digitFromCode, isComposing, isTextEntry } from "./keyboard";
import { classColor, editedId, paintAnnotation, paintSuggestions } from "./paint";
@@ -257,6 +258,14 @@ export interface AnnotatorCanvasProps {
* individually plausible and uniformly wrong.
*/
readonly imageSrc: string;
+ /**
+ * The existing rendered image once its exact source has decoded.
+ *
+ * The source is generation-scoped: after `imageSrc` changes, a previously
+ * delivered source refuses pixel reads rather than reading the replacement
+ * through React's reused image node.
+ */
+ readonly onImageReady?: (source: DecodedAssetImage) => void;
/** The class a drawing gesture will carry. `null` is select mode. */
readonly activeClass: string | null;
/**
@@ -488,6 +497,7 @@ const EMPTY_SUGGESTIONS: readonly PaintedSuggestion[] = [];
export function AnnotatorCanvas({
store,
imageSrc,
+ onImageReady,
activeClass,
activeTool = null,
onActivateClass,
@@ -514,6 +524,12 @@ export function AnnotatorCanvas({
const rootRef = useRef(null);
const paneRef = useRef(null);
+ const imageSrcNow = useRef(imageSrc);
+ const imageGeneration = useRef(0);
+ if (imageSrcNow.current !== imageSrc) {
+ imageSrcNow.current = imageSrc;
+ imageGeneration.current += 1;
+ }
// The fallback, built once — see the prop's docstring for why `useState`.
const [ownClipboard] = useState(createClipboard);
@@ -1465,6 +1481,18 @@ export function AnnotatorCanvas({
>
{
+ const image = event.currentTarget;
+ if (imageSrcNow.current !== imageSrc || image.getAttribute("src") !== imageSrc) return;
+ onImageReady?.(
+ createDecodedAssetImage(
+ image,
+ imageSrc,
+ imageGeneration.current,
+ () => imageGeneration.current,
+ ),
+ );
+ }}
alt=""
aria-hidden="true"
// Named so the pixelated-at-depth rule is asserted against the
diff --git a/frontend/annotator/src/adapters/react/decodedAssetImage.test.ts b/frontend/annotator/src/adapters/react/decodedAssetImage.test.ts
new file mode 100644
index 00000000..3235348e
--- /dev/null
+++ b/frontend/annotator/src/adapters/react/decodedAssetImage.test.ts
@@ -0,0 +1,34 @@
+import { describe, expect, it, vi } from "vitest";
+
+import { createDecodedAssetImage, rgbPixelsFromDecodedImage } from "./decodedAssetImage";
+
+describe("decoded asset image", () => {
+ it("refuses a retained A source once its image generation is replaced by B", () => {
+ let current = 1;
+ const image = {} as HTMLImageElement;
+ const read = vi.fn(() => ({ width: 1, height: 1, rgb: new Uint8Array([1, 2, 3]) }));
+ const source = createDecodedAssetImage(image, "blob:a", 1, () => current, read);
+
+ current = 2;
+
+ expect(() => source.readRgb(1, 1)).toThrow("image source is no longer current");
+ expect(read).not.toHaveBeenCalled();
+ });
+
+ it("converts descriptor-frame RGBA pixels into exact row-major RGB", () => {
+ const drawImage = vi.fn();
+ const context = {
+ drawImage,
+ getImageData: () => ({ data: new Uint8ClampedArray([1, 2, 3, 4, 5, 6, 7, 8]) }),
+ };
+ vi.stubGlobal("document", {
+ createElement: () => ({ width: 0, height: 0, getContext: () => context }),
+ });
+
+ const pixels = rgbPixelsFromDecodedImage({} as HTMLImageElement, 2, 1);
+
+ expect(pixels).toEqual({ width: 2, height: 1, rgb: new Uint8Array([1, 2, 3, 5, 6, 7]) });
+ expect(drawImage).toHaveBeenCalledWith(expect.anything(), 0, 0, 2, 1);
+ vi.unstubAllGlobals();
+ });
+});
diff --git a/frontend/annotator/src/adapters/react/decodedAssetImage.ts b/frontend/annotator/src/adapters/react/decodedAssetImage.ts
new file mode 100644
index 00000000..d888625a
--- /dev/null
+++ b/frontend/annotator/src/adapters/react/decodedAssetImage.ts
@@ -0,0 +1,55 @@
+/** A browser-only, generation-scoped lease over the image the adapter already renders. */
+
+export interface RgbPixels {
+ readonly width: number;
+ readonly height: number;
+ readonly rgb: Uint8Array;
+}
+
+export interface DecodedAssetImage {
+ readonly image: HTMLImageElement;
+ readonly src: string;
+ readRgb(width: number, height: number): RgbPixels;
+}
+
+type RgbReader = (image: HTMLImageElement, width: number, height: number) => RgbPixels;
+
+export function createDecodedAssetImage(
+ image: HTMLImageElement,
+ src: string,
+ generation: number,
+ currentGeneration: () => number,
+ read: RgbReader = rgbPixelsFromDecodedImage,
+): DecodedAssetImage {
+ return {
+ image,
+ src,
+ readRgb(width, height) {
+ if (currentGeneration() !== generation || image.getAttribute("src") !== src) {
+ throw new Error("image source is no longer current");
+ }
+ return read(image, width, height);
+ },
+ };
+}
+
+export function rgbPixelsFromDecodedImage(
+ image: HTMLImageElement,
+ width: number,
+ height: number,
+): RgbPixels {
+ const canvas = document.createElement("canvas");
+ canvas.width = width;
+ canvas.height = height;
+ const context = canvas.getContext("2d");
+ if (context === null) throw new Error("2D canvas context is unavailable");
+ context.drawImage(image, 0, 0, width, height);
+ const rgba = context.getImageData(0, 0, width, height).data;
+ const rgb = new Uint8Array(width * height * 3);
+ for (let source = 0, target = 0; source < rgba.length; source += 4, target += 3) {
+ rgb[target] = rgba[source];
+ rgb[target + 1] = rgba[source + 1];
+ rgb[target + 2] = rgba[source + 2];
+ }
+ return { width, height, rgb };
+}
diff --git a/frontend/annotator/src/adapters/react/index.ts b/frontend/annotator/src/adapters/react/index.ts
index 9265b40d..035b28e0 100644
--- a/frontend/annotator/src/adapters/react/index.ts
+++ b/frontend/annotator/src/adapters/react/index.ts
@@ -12,6 +12,7 @@ export {
type AnnotatorCanvasProps,
type AnnotatorView,
} from "./AnnotatorCanvas";
+export type { DecodedAssetImage, RgbPixels } from "./decodedAssetImage";
export {
useAnnotatorSnapshot,
useAnnotatorStore,
diff --git a/frontend/ui-core/src/annotator/AssetImage.tsx b/frontend/ui-core/src/annotator/AssetImage.tsx
index 4e58784c..3f84252d 100644
--- a/frontend/ui-core/src/annotator/AssetImage.tsx
+++ b/frontend/ui-core/src/annotator/AssetImage.tsx
@@ -33,9 +33,14 @@ export interface AssetImageProps {
readonly children: (src: string) => ReactNode;
}
+interface AssetImageResource {
+ readonly src: string;
+ readonly blob: Blob;
+}
+
export function AssetImage({ projectId, assetId, children }: AssetImageProps): JSX.Element {
const client = useApiClient();
- const [src, setSrc] = useState(null);
+ const [resource, setResource] = useState(null);
const [failed, setFailed] = useState(false);
useEffect(() => {
@@ -46,7 +51,7 @@ export function AssetImage({ projectId, assetId, children }: AssetImageProps): J
// running to completion (#572).
const controller = new AbortController();
let objectUrl: string | null = null;
- setSrc(null);
+ setResource(null);
setFailed(false);
void (async () => {
@@ -63,8 +68,9 @@ export function AssetImage({ projectId, assetId, children }: AssetImageProps): J
setFailed(true);
return;
}
- objectUrl = URL.createObjectURL(result.data as unknown as Blob);
- setSrc(objectUrl);
+ const blob = result.data as unknown as Blob;
+ objectUrl = URL.createObjectURL(blob);
+ setResource({ src: objectUrl, blob });
} catch {
// The abort lands here by design. Anything else is a network that
// died, which the error state answers better than an eternal skeleton.
@@ -92,9 +98,9 @@ export function AssetImage({ projectId, assetId, children }: AssetImageProps): J
);
}
- if (src === null) {
+ if (resource === null) {
return ;
}
- return <>{children(src)};
+ return <>{children(resource.src)};
}
diff --git a/frontend/ui-core/src/annotator/assetImage.test.tsx b/frontend/ui-core/src/annotator/assetImage.test.tsx
index 25a5597b..8889276b 100644
--- a/frontend/ui-core/src/annotator/assetImage.test.tsx
+++ b/frontend/ui-core/src/annotator/assetImage.test.tsx
@@ -3,11 +3,12 @@
* honest about a network that is down (#572).
*/
-import { screen, waitFor } from "@testing-library/react";
+import { render, screen, waitFor } from "@testing-library/react";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
-import type { ReactNode } from "react";
+import type { JSX, ReactNode } from "react";
-import { renderWithData } from "../testing/dataHarness";
+import { VisionSetDataProvider } from "../data/VisionSetDataProvider";
+import { harnessClient, harnessQueryClient, renderWithData } from "../testing/dataHarness";
import { AssetImage } from "./AssetImage";
const PROJECT = "11111111-1111-4111-8111-111111111111";
@@ -56,6 +57,14 @@ function image(): ReactNode {
);
}
+function asset(assetId: string): ReactNode {
+ return (
+
+ {(src) => 
}
+
+ );
+}
+
describe("the asset's pixels", () => {
it("fetches them with the credential and hands an object URL to the child", async () => {
on("GET", /\/content$/, { status: 200, body: null });
@@ -65,6 +74,35 @@ describe("the asset's pixels", () => {
expect(frame.getAttribute("src") ?? "").toMatch(/^blob:/);
});
+ it("hands createObjectURL a Blob carrying the exact fetched bytes, not merely a Blob", async () => {
+ // A trivial `expect.any(Blob)`-shaped check would pass for any Blob at all
+ // (jsdom's blob has no enumerable own properties, so `toHaveBeenCalledWith`
+ // would even accept one with unrelated content) — the content check below is
+ // what actually distinguishes "the fetched resource" from "a Blob".
+ const bytes = "the-exact-fetched-bytes";
+ vi.stubGlobal("fetch", async (request: Request) => {
+ sent.push(request);
+ // A string body, not a `Blob` one: this jsdom/undici pairing mangles a
+ // `Response` constructed directly from a `Blob` (its `.blob()` comes back
+ // stringified to "[object Blob]"), a test-environment quirk unrelated to
+ // the code under test. A string body round-trips through `.blob()`
+ // correctly and is what a real network response looks like anyway.
+ return new Response(bytes, { status: 200, headers: { "content-type": "image/png" } });
+ });
+ const createObjectURL = vi.spyOn(URL, "createObjectURL");
+
+ renderWithData(image());
+ await screen.findByTestId("the-frame");
+
+ expect(sent).toHaveLength(1);
+ expect(createObjectURL).toHaveBeenCalledTimes(1);
+ const received = createObjectURL.mock.calls[0]?.[0];
+ expect(received).toBeInstanceOf(Blob);
+ await expect((received as Blob).text()).resolves.toBe(bytes);
+
+ createObjectURL.mockRestore();
+ });
+
it("aborts the transfer when it unmounts mid-flight (#572)", async () => {
// Walking a job with the arrow keys unmounts each frame's image; before
// the abort, every skipped frame's full-size download ran to completion.
@@ -76,6 +114,28 @@ describe("the asset's pixels", () => {
expect(sent[0].signal.aborted).toBe(true);
});
+ it("revokes the replaced asset URL and aborts its completed request", async () => {
+ on("GET", /\/content$/, { status: 200, body: null });
+ const revoke = vi.spyOn(URL, "revokeObjectURL");
+ const client = harnessClient();
+ const queries = harnessQueryClient();
+ const scope = Symbol("asset-image-test");
+ const wrap = (assetId: string): JSX.Element => (
+ queries}>
+ {asset(assetId)}
+
+ );
+ const view = render(wrap("asset-a"));
+ const first = await screen.findByTestId("the-frame");
+ const firstSrc = first.getAttribute("src");
+
+ view.rerender(wrap("asset-b"));
+ await waitFor(() => expect(sent).toHaveLength(2));
+
+ expect(sent[0]?.signal.aborted).toBe(true);
+ expect(revoke).toHaveBeenCalledWith(firstSrc);
+ });
+
it("shows the failure state when the fetch itself throws", async () => {
// A rejected fetch (network down) used to be an unhandled rejection and an
// eternal loading skeleton; the abort turned rejection into an ordinary
diff --git a/frontend/ui-core/src/annotator/decodedImageLease.test.tsx b/frontend/ui-core/src/annotator/decodedImageLease.test.tsx
new file mode 100644
index 00000000..d8e9ec1f
--- /dev/null
+++ b/frontend/ui-core/src/annotator/decodedImageLease.test.tsx
@@ -0,0 +1,55 @@
+import { AnnotatorCanvas, AnnotatorStore, documentFromWire } from "@visionset/annotator";
+import { fireEvent, render, screen } from "@testing-library/react";
+import { describe, expect, it, vi } from "vitest";
+
+function store(): AnnotatorStore {
+ return new AnnotatorStore(
+ documentFromWire({
+ asset: { id: "asset", width: 2, height: 1 },
+ schema: { project_id: "project", version: 1, classes: [] },
+ annotations: [],
+ }),
+ );
+}
+
+function canvas(imageSrc: string, ready: (source: { image: HTMLImageElement; readRgb(w: number, h: number): unknown }) => void) {
+ return ;
+}
+
+describe("decoded image lease", () => {
+ it("hands the exact rendered image to the ready callback", () => {
+ const ready = vi.fn();
+ render(canvas("blob:a", ready));
+ const image = screen.getByTestId("annotator-image") as HTMLImageElement;
+ fireEvent.load(image);
+ expect(ready.mock.calls[0][0].image).toBe(image);
+ });
+
+ it("does not let retained A read through the DOM node reused for B", () => {
+ const ready = vi.fn();
+ const view = render(canvas("blob:a", ready));
+ const image = screen.getByTestId("annotator-image") as HTMLImageElement;
+ fireEvent.load(image);
+ const sourceA = ready.mock.calls[0][0];
+ view.rerender(canvas("blob:b", ready));
+ expect(screen.getByTestId("annotator-image")).toBe(image);
+ expect(() => sourceA.readRgb(2, 1)).toThrow("image source is no longer current");
+ });
+
+ it("never touches canvas pixel extraction during ordinary rendering", () => {
+ // `rgbPixelsFromDecodedImage` unconditionally calls `getContext("2d")` on a
+ // canvas it creates, so a spy that observes zero calls to that prototype
+ // method is proof the lazy helper never ran — ordinary mounting and an
+ // image `load` only ever hand back the lease, they never read it.
+ const getContext = vi.spyOn(HTMLCanvasElement.prototype, "getContext");
+ const ready = vi.fn();
+ render(canvas("blob:a", ready));
+ const image = screen.getByTestId("annotator-image") as HTMLImageElement;
+ fireEvent.load(image);
+
+ expect(ready).toHaveBeenCalledTimes(1);
+ expect(getContext).not.toHaveBeenCalled();
+
+ getContext.mockRestore();
+ });
+});
From 445b8dedea22cf8215979cb716f4271f03d0cb3b Mon Sep 17 00:00:00 2001
From: Jesus Armando Anaya <1445792+JArmandoAnaya@users.noreply.github.com>
Date: Wed, 16 Sep 2026 16:42:24 -0700
Subject: [PATCH 2/6] docs(ui-core): define displayed assets as browser pixel
sources
`docs/content/architecture/frontend/ui-core.md`'s durable rule now states
laziness, no-second-Image, and no-exposed-inference-capability explicitly, and
draws the line on what is proved versus claimed: no universal browser/Pillow
byte parity, only what a real browser measurably does.
`e2e/assetPixels.spec.ts` and its test-only harness
(`app/src/demo/AssetPixelsFixture.tsx`, routed at `/asset-pixels-fixture`,
ungated like `/demo` and `/styleguide`) are the real-browser proof: a tiny RGBA
image encoded to a PNG at runtime, no binary fixture, showing one content
request, the same decoded `![]()
` reused as the pixel source, and the exact
descriptor-frame RGB a real 2D context produced.
`decodedAssetImage.test.ts`'s existing pixel-math test is strengthened to
assert the exact image reference and exact `getImageData` args, closing the
one gap jsdom's canvas leaves for a unit test to catch on its own.
---
docs/content/architecture/frontend/ui-core.md | 23 ++-
.../adapters/react/decodedAssetImage.test.ts | 19 +-
frontend/app/e2e/assetPixels.spec.ts | 22 +++
frontend/app/src/demo/AssetPixelsFixture.tsx | 168 ++++++++++++++++++
frontend/app/src/routes.tsx | 7 +
5 files changed, 228 insertions(+), 11 deletions(-)
create mode 100644 frontend/app/e2e/assetPixels.spec.ts
create mode 100644 frontend/app/src/demo/AssetPixelsFixture.tsx
diff --git a/docs/content/architecture/frontend/ui-core.md b/docs/content/architecture/frontend/ui-core.md
index da2751be..53842643 100644
--- a/docs/content/architecture/frontend/ui-core.md
+++ b/docs/content/architecture/frontend/ui-core.md
@@ -97,13 +97,26 @@ The URL is revoked and an unfinished request is aborted when the asset is replac
There is no JavaScript Blob cache: revisiting an asset relies on the server's immutable HTTP cache.
The React annotator adapter can report a generation-scoped lease over the same visible decoded
-`![]()
`. A future host that needs pixels reads them lazily from that lease, in the asset
-descriptor's coordinate frame, through a canvas RGBA-to-RGB copy. It does not fetch or decode a
-second copy of the asset. A lease refuses after its image source is replaced, which prevents a
-retained reference for one asset from reading pixels of the next asset through a reused DOM node.
+`![]()
` — `AnnotatorCanvas`'s `onImageReady`, fired from that image's own `onLoad`. A future host
+that needs pixels reads them lazily from that lease: nothing draws to a canvas or calls
+`getImageData` until a caller asks for `readRgb(width, height)`, in the asset descriptor's
+coordinate frame, and ordinary mounting and display never asks. It does not fetch or decode a
+second copy of the asset, and it does not construct a second `Image` to read from — the element
+handed back is the exact node the person is looking at. A lease refuses after its image source is
+replaced, which prevents a retained reference for one asset from reading pixels of the next asset
+through a reused DOM node.
This is resource plumbing only. It neither selects a browser model nor exposes an inference
-target; composing pixels with a browser runtime remains a later host decision.
+target — no capability in this phase lets a host ask for local inference at all; composing pixels
+with a browser runtime remains a later host decision.
+
+**What is proved, and what is not claimed.** `e2e/assetPixels.spec.ts` drives a real Chromium
+against a tiny runtime-generated image and confirms, in that browser, on that image: one content
+request, the same decoded `![]()
` handed back as the pixel source, and the exact descriptor-frame
+RGB a real 2D context produced. That is evidence about this seam's wiring, not a claim that a
+browser's canvas decode agrees with the server's own (Pillow) decode byte-for-byte in general —
+this phase changes no server decoding, and takes no position on cross-decoder parity beyond what
+is measured here.
## Asking for a suggestion is not sending one
diff --git a/frontend/annotator/src/adapters/react/decodedAssetImage.test.ts b/frontend/annotator/src/adapters/react/decodedAssetImage.test.ts
index 3235348e..91387bcb 100644
--- a/frontend/annotator/src/adapters/react/decodedAssetImage.test.ts
+++ b/frontend/annotator/src/adapters/react/decodedAssetImage.test.ts
@@ -16,19 +16,26 @@ describe("decoded asset image", () => {
});
it("converts descriptor-frame RGBA pixels into exact row-major RGB", () => {
+ // A distinct object, unlike `{}` — asserted below by identity, not merely by
+ // shape. `rgbPixelsFromDecodedImage` must draw the exact rendered image it
+ // was handed, never a second `Image` it constructs to read from.
+ const image = { tag: "the rendered image" } as unknown as HTMLImageElement;
const drawImage = vi.fn();
- const context = {
- drawImage,
- getImageData: () => ({ data: new Uint8ClampedArray([1, 2, 3, 4, 5, 6, 7, 8]) }),
- };
+ const getImageData = vi.fn(() => ({ data: new Uint8ClampedArray([1, 2, 3, 4, 5, 6, 7, 8]) }));
+ const context = { drawImage, getImageData };
vi.stubGlobal("document", {
createElement: () => ({ width: 0, height: 0, getContext: () => context }),
});
- const pixels = rgbPixelsFromDecodedImage({} as HTMLImageElement, 2, 1);
+ const pixels = rgbPixelsFromDecodedImage(image, 2, 1);
expect(pixels).toEqual({ width: 2, height: 1, rgb: new Uint8Array([1, 2, 3, 5, 6, 7]) });
- expect(drawImage).toHaveBeenCalledWith(expect.anything(), 0, 0, 2, 1);
+ // The exact requested frame, not the image's own natural size (`image` above
+ // carries no `naturalWidth`/`naturalHeight` at all, so a mutation reading
+ // those instead would call both of these with `undefined`) and not a
+ // transposed pair either.
+ expect(drawImage).toHaveBeenCalledWith(image, 0, 0, 2, 1);
+ expect(getImageData).toHaveBeenCalledWith(0, 0, 2, 1);
vi.unstubAllGlobals();
});
});
diff --git a/frontend/app/e2e/assetPixels.spec.ts b/frontend/app/e2e/assetPixels.spec.ts
new file mode 100644
index 00000000..ceaf8eee
--- /dev/null
+++ b/frontend/app/e2e/assetPixels.spec.ts
@@ -0,0 +1,22 @@
+/**
+ * The displayed asset is the browser pixel source — proved in a real browser.
+ *
+ * `docs/content/architecture/frontend/ui-core.md` states the rule and the unit
+ * tests beside `AssetImage` and `decodedAssetImage.ts` hold most of it, but jsdom's
+ * canvas does not decode a real image: nothing in that suite can show that a real
+ * `![]()
`, drawn through a real 2D context, yields the exact bytes the descriptor
+ * frame promises. `/asset-pixels-fixture` (`src/demo/AssetPixelsFixture.tsx`) is a
+ * test-only host that composes the real `AssetImage` and `AnnotatorCanvas` against
+ * a tiny RGBA image encoded to a PNG at runtime, and this is the one scenario that
+ * reads it.
+ */
+
+import { expect, test } from "@playwright/test";
+
+test("reuses the visible decoded image for exact descriptor-frame RGB", async ({ page }) => {
+ await page.goto("/asset-pixels-fixture");
+ await expect(page.getByTestId("annotator-image")).toBeVisible();
+ await expect(page.getByTestId("pixel-source-same-image")).toHaveText("true");
+ await expect(page.getByTestId("pixel-rgb")).toHaveText("1,2,3,5,6,7,9,10,11");
+ await expect.poll(() => page.evaluate(() => window.__assetContentRequests)).toBe(1);
+});
diff --git a/frontend/app/src/demo/AssetPixelsFixture.tsx b/frontend/app/src/demo/AssetPixelsFixture.tsx
new file mode 100644
index 00000000..3ba6ab76
--- /dev/null
+++ b/frontend/app/src/demo/AssetPixelsFixture.tsx
@@ -0,0 +1,168 @@
+/**
+ * A test-only harness proving the displayed asset is the browser pixel source.
+ *
+ * `docs/content/architecture/frontend/ui-core.md`'s rule — the visible `![]()
` is
+ * reused as the browser pixel source, with no second fetch and no independent
+ * decode — has one half no unit test can reach: jsdom's canvas does not decode a
+ * real image, so `getImageData` there is either absent or a stub a test supplies
+ * itself. This page composes the real `AssetImage` and `AnnotatorCanvas` exactly
+ * as `AnnotationPage` does, against a tiny RGBA image **encoded to a PNG blob at
+ * runtime** (no binary fixture file, so the exact bytes are never stale or
+ * regenerated by hand) — so `e2e/assetPixels.spec.ts` can prove the seam in a
+ * real browser: one content request, the same decoded `![]()
` handed back as the
+ * pixel source, and the exact descriptor-frame RGB a real canvas produced.
+ *
+ * No token, no server, not linked from anywhere in the product — the same
+ * standing as `/demo` and `/styleguide`.
+ */
+
+import { AnnotatorCanvas, AnnotatorStore, documentFromWire } from "@visionset/annotator";
+import { AssetImage, VisionSetDataProvider } from "@visionset/ui-core";
+import type { DataResult, VisionSetDataClient } from "@visionset/ui-core";
+import { useEffect, useMemo, useState, type JSX } from "react";
+
+declare global {
+ interface Window {
+ /** How many times the stub client has delivered content. Reset on load. */
+ __assetContentRequests?: number;
+ }
+}
+
+const PROJECT_ID = "fixture-project";
+const ASSET_ID = "fixture-asset";
+const WIDTH = 3;
+const HEIGHT = 1;
+
+/**
+ * Three RGBA pixels, alpha fully opaque.
+ *
+ * A low alpha would make the PNG round-trip lossy — canvas premultiplies alpha
+ * on encode, and an opaque pixel is the one value that survives unpremultiplying
+ * exactly. `1,2,3` / `5,6,7` / `9,10,11` is what `pixel-rgb` below must read back
+ * once the fourth (alpha) byte of each RGBA quadruple is dropped.
+ */
+const RGBA = new Uint8ClampedArray([1, 2, 3, 255, 5, 6, 7, 255, 9, 10, 11, 255]);
+
+function fixtureStore(): AnnotatorStore {
+ return new AnnotatorStore(
+ documentFromWire({
+ asset: { id: ASSET_ID, width: WIDTH, height: HEIGHT },
+ schema: { project_id: PROJECT_ID, version: 1, classes: [] },
+ annotations: [],
+ }),
+ );
+}
+
+/** A real PNG, built from the exact bytes above — never a checked-in fixture. */
+async function encodeFixturePng(): Promise {
+ const canvas = document.createElement("canvas");
+ canvas.width = WIDTH;
+ canvas.height = HEIGHT;
+ const context = canvas.getContext("2d");
+ if (context === null) throw new Error("2D canvas context is unavailable");
+ context.putImageData(new ImageData(RGBA, WIDTH, HEIGHT), 0, 0);
+ return await new Promise((resolve, reject) => {
+ canvas.toBlob((blob) => {
+ if (blob === null) reject(new Error("canvas produced no blob"));
+ else resolve(blob);
+ }, "image/png");
+ });
+}
+
+/**
+ * A minimal `VisionSetDataClient`: it answers the one route `AssetImage` calls,
+ * counts a delivery only once it actually settles, and rejects a request whose
+ * signal is already or becomes aborted — the same contract clause a real host
+ * honours (`data/port.ts`'s "a request cancelled through its signal rejects").
+ *
+ * Delivery is deferred one macrotask so a synchronous `StrictMode` remount's
+ * cleanup — the abort of the first mount's controller — has already landed on
+ * `signal` before this settles, the ordering a real network transfer has for
+ * free. Without it, a mutation that dropped the abort could still read as one
+ * request here purely by winning a race, which would prove nothing.
+ */
+function fixtureClient(png: Blob, onDelivered: () => void): VisionSetDataClient {
+ const GET = (path: string, init?: { readonly signal?: AbortSignal }): Promise => {
+ if (!path.endsWith("/content")) return Promise.resolve({ ok: false, failure: "unreachable" });
+ const signal = init?.signal;
+ return new Promise((resolve, reject) => {
+ if (signal?.aborted === true) {
+ reject(new DOMException("The operation was aborted.", "AbortError"));
+ return;
+ }
+ const onAbort = (): void => {
+ window.clearTimeout(timer);
+ reject(new DOMException("The operation was aborted.", "AbortError"));
+ };
+ const timer = window.setTimeout(() => {
+ signal?.removeEventListener("abort", onAbort);
+ if (signal?.aborted === true) {
+ reject(new DOMException("The operation was aborted.", "AbortError"));
+ return;
+ }
+ onDelivered();
+ resolve({ ok: true, data: png });
+ }, 0);
+ signal?.addEventListener("abort", onAbort, { once: true });
+ });
+ };
+ const refuse = (): Promise => Promise.resolve({ ok: false, failure: "unreachable" });
+ return {
+ GET,
+ POST: refuse,
+ PUT: refuse,
+ PATCH: refuse,
+ DELETE: refuse,
+ } as unknown as VisionSetDataClient;
+}
+
+export function AssetPixelsFixture(): JSX.Element {
+ const [png, setPng] = useState(null);
+ const [sameImage, setSameImage] = useState("pending");
+ const [rgb, setRgb] = useState("pending");
+ const store = useMemo(fixtureStore, []);
+
+ useEffect(() => {
+ window.__assetContentRequests = 0;
+ let cancelled = false;
+ void encodeFixturePng().then((blob) => {
+ if (!cancelled) setPng(blob);
+ });
+ return () => {
+ cancelled = true;
+ };
+ }, []);
+
+ const client = useMemo(() => {
+ if (png === null) return null;
+ return fixtureClient(png, () => {
+ window.__assetContentRequests = (window.__assetContentRequests ?? 0) + 1;
+ });
+ }, [png]);
+
+ if (client === null) return ;
+
+ return (
+
+
+
+ {(src) => (
+ {}}
+ onImageReady={(source) => {
+ const visible = window.document.querySelector('[data-testid="annotator-image"]');
+ setSameImage(source.image === visible ? "true" : "false");
+ setRgb(Array.from(source.readRgb(WIDTH, HEIGHT).rgb).join(","));
+ }}
+ />
+ )}
+
+
![]()
still carried
the old src, so readRgb's guard could pass and hand back the previous
asset's pixels. Bump the generation ref in an unmount-only effect
cleanup so every outstanding lease from that instance genuinely refuses
afterward, through the same guard readRgb already checks.
Covering tests in decodedImageLease.test.tsx: an unmount() + readRgb
refusal test for the invalidation path, and a compile-time check (typed
via the real @visionset/annotator import, not a relative path) proving
DecodedAssetImage/RgbPixels actually surface publicly. Verified by
temporarily reverting the export and confirming ui-core's typecheck
fails with TS2459 before restoring it.
Updates the architecture doc's lease-refusal sentence to cover both
the imageSrc-replacement and the unmount path.
---
docs/content/architecture/frontend/ui-core.md | 17 +++++++-
.../src/adapters/react/AnnotatorCanvas.tsx | 13 +++++++
frontend/annotator/src/index.ts | 2 +
.../src/annotator/decodedImageLease.test.tsx | 39 ++++++++++++++++++-
4 files changed, 67 insertions(+), 4 deletions(-)
diff --git a/docs/content/architecture/frontend/ui-core.md b/docs/content/architecture/frontend/ui-core.md
index 53842643..023b1945 100644
--- a/docs/content/architecture/frontend/ui-core.md
+++ b/docs/content/architecture/frontend/ui-core.md
@@ -103,8 +103,10 @@ that needs pixels reads them lazily from that lease: nothing draws to a canvas o
coordinate frame, and ordinary mounting and display never asks. It does not fetch or decode a
second copy of the asset, and it does not construct a second `Image` to read from — the element
handed back is the exact node the person is looking at. A lease refuses after its image source is
-replaced, which prevents a retained reference for one asset from reading pixels of the next asset
-through a reused DOM node.
+replaced *or* after its owning `AnnotatorCanvas` instance unmounts, which prevents a retained
+reference for one asset from reading pixels of the next asset either through a reused DOM node or
+through the detached one a host's real asset-switch (unmount the old canvas, mount a fresh one)
+leaves behind.
This is resource plumbing only. It neither selects a browser model nor exposes an inference
target — no capability in this phase lets a host ask for local inference at all; composing pixels
@@ -118,6 +120,17 @@ browser's canvas decode agrees with the server's own (Pillow) decode byte-for-by
this phase changes no server decoding, and takes no position on cross-decoder parity beyond what
is measured here.
+**Measured browser-canvas-vs-Pillow divergence.** A separate, informal comparison against the
+server's `convert("RGB")` decode found three real categories of disagreement: alpha-premultiplication
+rounding loss on partially-transparent pixels, EXIF-orientation auto-rotation (a browser applies an
+image's EXIF orientation when decoding to canvas; the server's direct-bytes decode path does not),
+and AdobeRGB ICC-profile color management (a browser color-manages a tagged profile toward sRGB on
+decode; the server's path does not). None of these are exact figures worth repeating here — they
+are a known limitation, not a benchmark — and no consumer of this lease has yet needed to reconcile
+them, since nothing in this phase reads pixels for inference. A future phase that feeds this lease's
+`readRgb` output to a model must account for these divergences before treating browser-decoded
+pixels as equivalent to the server's own decode of the same asset.
+
## Asking for a suggestion is not sending one
`inference/suggestionExecutor.ts` is the seam the runtime above would plug into, and it exists
diff --git a/frontend/annotator/src/adapters/react/AnnotatorCanvas.tsx b/frontend/annotator/src/adapters/react/AnnotatorCanvas.tsx
index 94a7ec2b..53c029ca 100644
--- a/frontend/annotator/src/adapters/react/AnnotatorCanvas.tsx
+++ b/frontend/annotator/src/adapters/react/AnnotatorCanvas.tsx
@@ -531,6 +531,19 @@ export function AnnotatorCanvas({
imageGeneration.current += 1;
}
+ // A host switches assets by unmounting this component and mounting a fresh
+ // one for the next asset, not by changing `imageSrc` on a live instance —
+ // so the ordinary generation bump above never runs. Without this, a lease
+ // handed out before teardown still finds its captured generation equal to
+ // `imageGeneration.current` and its detached `![]()
` still `complete` with
+ // its old `src` attribute intact, and `readRgb` would hand back the wrong
+ // asset's pixels from a component nothing renders any more.
+ useEffect(() => {
+ return () => {
+ imageGeneration.current += 1;
+ };
+ }, []);
+
// The fallback, built once — see the prop's docstring for why `useState`.
const [ownClipboard] = useState(createClipboard);
const clipboard = hostClipboard ?? ownClipboard;
diff --git a/frontend/annotator/src/index.ts b/frontend/annotator/src/index.ts
index a0fe7e86..fa6e5c24 100644
--- a/frontend/annotator/src/index.ts
+++ b/frontend/annotator/src/index.ts
@@ -344,12 +344,14 @@ export {
type AnnotatorCanvasProps,
type AnnotatorView,
type CompositionProbe,
+ type DecodedAssetImage,
type PaintedAnnotation,
type PaintedSuggestion,
type PendingIndicator,
type PendingIndicatorState,
type PendingPhase,
type PendingPolygon,
+ type RgbPixels,
type TextEntryProbe,
type TransientLayerProps,
} from "./adapters/react";
diff --git a/frontend/ui-core/src/annotator/decodedImageLease.test.tsx b/frontend/ui-core/src/annotator/decodedImageLease.test.tsx
index d8e9ec1f..07363dd3 100644
--- a/frontend/ui-core/src/annotator/decodedImageLease.test.tsx
+++ b/frontend/ui-core/src/annotator/decodedImageLease.test.tsx
@@ -1,4 +1,10 @@
-import { AnnotatorCanvas, AnnotatorStore, documentFromWire } from "@visionset/annotator";
+import {
+ AnnotatorCanvas,
+ AnnotatorStore,
+ documentFromWire,
+ type DecodedAssetImage,
+ type RgbPixels,
+} from "@visionset/annotator";
import { fireEvent, render, screen } from "@testing-library/react";
import { describe, expect, it, vi } from "vitest";
@@ -12,7 +18,11 @@ function store(): AnnotatorStore {
);
}
-function canvas(imageSrc: string, ready: (source: { image: HTMLImageElement; readRgb(w: number, h: number): unknown }) => void) {
+// `DecodedAssetImage` resolving here, from the package's real entry point
+// rather than a path into its internals, is the regression guard: the type
+// is otherwise easy to re-export from the React adapter alone and miss the
+// package barrel that actually reaches a host.
+function canvas(imageSrc: string, ready: (source: DecodedAssetImage) => void) {
return ;
}
@@ -36,6 +46,31 @@ describe("decoded image lease", () => {
expect(() => sourceA.readRgb(2, 1)).toThrow("image source is no longer current");
});
+ it("refuses a retained lease once its owning component has unmounted", () => {
+ // The real host swaps assets by unmounting the old `AnnotatorCanvas`
+ // instance and mounting a fresh one, not by changing `imageSrc` on a live
+ // instance — so this is the path the generation-bump-on-`imageSrc`-change
+ // above does not cover. A lease taken out before teardown must refuse
+ // afterward even though nothing ever changed its `src` attribute.
+ const ready = vi.fn();
+ const view = render(canvas("blob:a", ready));
+ const image = screen.getByTestId("annotator-image") as HTMLImageElement;
+ fireEvent.load(image);
+ const source = ready.mock.calls[0][0];
+
+ view.unmount();
+
+ expect(() => source.readRgb(2, 1)).toThrow("image source is no longer current");
+ });
+
+ it("surfaces RgbPixels from the package entry point, shaped as readRgb returns it", () => {
+ // A compile-time proof standing in beside the runtime ones above: if
+ // `RgbPixels` stopped being re-exported from `@visionset/annotator`, this
+ // file would fail to typecheck rather than merely fail at runtime.
+ const pixels: RgbPixels = { width: 1, height: 1, rgb: new Uint8Array([1, 2, 3]) };
+ expect(pixels.rgb.length).toBe(pixels.width * pixels.height * 3);
+ });
+
it("never touches canvas pixel extraction during ordinary rendering", () => {
// `rgbPixelsFromDecodedImage` unconditionally calls `getContext("2d")` on a
// canvas it creates, so a spy that observes zero calls to that prototype
From 2017627f021d4261e5bdbd21c9e21852be30193c Mon Sep 17 00:00:00 2001
From: Jesus Armando Anaya <1445792+JArmandoAnaya@users.noreply.github.com>
Date: Wed, 16 Sep 2026 17:59:17 -0700
Subject: [PATCH 5/6] refactor(app): fold the asset-pixels Playwright fixture
into /demo
The Chromium proof for the browser pixel-reuse seam had its own standalone
route, asset-pixels-fixture, which is a test-only production surface with no
product link. Move it behind the existing /demo route's scene query
parameter, the same pattern the annotator benchmark already uses, so no
dedicated route exists purely to host a Playwright harness.
---
frontend/app/e2e/assetPixels.spec.ts | 4 ++--
frontend/app/src/routes.tsx | 19 ++++++++++---------
2 files changed, 12 insertions(+), 11 deletions(-)
diff --git a/frontend/app/e2e/assetPixels.spec.ts b/frontend/app/e2e/assetPixels.spec.ts
index 2ad07f5d..a219e588 100644
--- a/frontend/app/e2e/assetPixels.spec.ts
+++ b/frontend/app/e2e/assetPixels.spec.ts
@@ -5,7 +5,7 @@
* tests beside `AssetImage` and `decodedAssetImage.ts` hold most of it, but jsdom's
* canvas does not decode a real image: nothing in that suite can show that a real
* `![]()
`, drawn through a real 2D context, yields the exact bytes the descriptor
- * frame promises. `/asset-pixels-fixture` (`src/demo/AssetPixelsFixture.tsx`) is a
+ * frame promises. `/demo?scene=asset-pixels` (`src/demo/AssetPixelsFixture.tsx`) is a
* test-only host that composes the real `AssetImage` and `AnnotatorCanvas` against
* a tiny RGBA image encoded to a PNG at runtime, and this is the one scenario that
* reads it.
@@ -14,7 +14,7 @@
import { expect, test } from "@playwright/test";
test("reuses the visible decoded image for exact descriptor-frame RGB", async ({ page }) => {
- await page.goto("/asset-pixels-fixture");
+ await page.goto("/demo?scene=asset-pixels");
await expect(page.getByTestId("annotator-image")).toBeVisible();
await expect(page.getByTestId("pixel-source-same-image")).toHaveText("true");
await expect(page.getByTestId("pixel-rgb")).toHaveText("1,2,3,5,6,7,9,10,11");
diff --git a/frontend/app/src/routes.tsx b/frontend/app/src/routes.tsx
index 51d4fd05..d394e8f9 100644
--- a/frontend/app/src/routes.tsx
+++ b/frontend/app/src/routes.tsx
@@ -30,7 +30,9 @@
* The benchmark keeps its query parameter (`/demo?scene=bench`) rather than
* gaining a route: it is an instrument, its recorded numbers were taken against
* that exact page, and moving it would change what it measures for no reason
- * anybody asked for.
+ * anybody asked for. `e2e/assetPixels.spec.ts`'s fixture rides the same route
+ * behind `/demo?scene=asset-pixels`, for the same reason: test-only, not linked
+ * from the product, so it earns no route of its own either.
*
* ## Every route has a screen
*
@@ -136,20 +138,19 @@ export function AppRoutes(): JSX.Element {
{/* No token, no server. Also what the browser suite drives. */}
} />
} />
- {/*
- A test-only harness for `e2e/assetPixels.spec.ts` — proving the displayed
- asset is the browser pixel source in a real browser. Not linked from
- anywhere else, the same standing as the two routes above.
- */}
- } />
);
}
-/** The showcase, and the benchmark behind its query parameter. */
+/**
+ * The showcase, the benchmark and the asset-pixels test fixture, each behind
+ * its own `scene` query value.
+ */
function Showcase(): JSX.Element {
const [query] = useSearchParams();
- const bench = query.get("scene") === "bench";
+ const scene = query.get("scene");
+ if (scene === "asset-pixels") return ;
+ const bench = scene === "bench";
return (
{bench ? : }
From 9d17a0308426730283b741ac614c3878d33a7c0c Mon Sep 17 00:00:00 2001
From: Jesus Armando Anaya <1445792+JArmandoAnaya@users.noreply.github.com>
Date: Wed, 16 Sep 2026 18:12:47 -0700
Subject: [PATCH 6/6] docs(annotator): document the live-instance stale-load
residual on onImageReady
The final review for this seam found that React always invokes the most
recently committed onLoad handler, never the one attached when a particular
load event was queued, so the existing generation/src guard cannot by itself
refuse a delayed load delivered after imageSrc changes twice on a live,
non-remounted instance. No host exercises that path today: every real asset
switch unmounts this component, which the separate unmount effect already
covers, and browsers do not dispatch load/error for an abandoned image
request in the first place. Record the residual in the two places a future
consumer would read them, rather than carry an unverifiable code change for
a race nothing here can exercise or test.
---
.../src/adapters/react/AnnotatorCanvas.tsx | 25 ++++++++++++++++---
1 file changed, 22 insertions(+), 3 deletions(-)
diff --git a/frontend/annotator/src/adapters/react/AnnotatorCanvas.tsx b/frontend/annotator/src/adapters/react/AnnotatorCanvas.tsx
index 53c029ca..24913823 100644
--- a/frontend/annotator/src/adapters/react/AnnotatorCanvas.tsx
+++ b/frontend/annotator/src/adapters/react/AnnotatorCanvas.tsx
@@ -261,9 +261,15 @@ export interface AnnotatorCanvasProps {
/**
* The existing rendered image once its exact source has decoded.
*
- * The source is generation-scoped: after `imageSrc` changes, a previously
- * delivered source refuses pixel reads rather than reading the replacement
- * through React's reused image node.
+ * The source is generation-scoped: after `imageSrc` changes on a live
+ * instance, or after this component unmounts, a previously delivered source
+ * refuses pixel reads rather than reading the replacement through React's
+ * reused image node or a detached one.
+ *
+ * Every host today switches assets by unmounting this component rather than
+ * changing `imageSrc` in place — see the comment beside the `onLoad` handler
+ * for the one race that leaves open on a live-instance switch this contract
+ * has never had to close.
*/
readonly onImageReady?: (source: DecodedAssetImage) => void;
/** The class a drawing gesture will carry. `null` is select mode. */
@@ -1495,6 +1501,19 @@ export function AnnotatorCanvas({
{
+ // React always calls the handler from the most recently committed
+ // render, never the one attached when this particular `load` was
+ // queued — so both checks below compare the latest `imageSrc`
+ // against itself on a live instance whose source changed twice in
+ // a row before the first `load` fired, and cannot by themselves
+ // refuse a stale event delivered after such a change. No caller
+ // does this today: every host switches assets by unmounting this
+ // component (the effect below covers that), and a delayed load
+ // for an abandoned request is a case browsers do not dispatch —
+ // they fire `load`/`error` only for an image element's current
+ // request. A future host that mutates `imageSrc` on a live
+ // instance without remounting should re-examine this before
+ // relying on it.
const image = event.currentTarget;
if (imageSrcNow.current !== imageSrc || image.getAttribute("src") !== imageSrc) return;
onImageReady?.(