refactor(inference): reuse the displayed asset for browser pixel preparation

docs/content/architecture/frontend/ui-core.md

@@ -89,6 +89,48 @@ how it is grown instead.
 > something this browser can do. They are not two spellings of one idea, and neither is
 > derivable from the other.
 
+## The displayed asset is the browser pixel source
+
+`AssetImage` fetches an asset once through the credentialed data client, retains that exact Blob
+only while the asset is mounted, and owns the object URL it gives the existing annotator image.
+The URL is revoked and an unfinished request is aborted when the asset is replaced or unmounted.
+There is no JavaScript Blob cache: revisiting an asset relies on the server's immutable HTTP cache.
+
+The React annotator adapter can report a generation-scoped lease over the same visible decoded
+`<img>` — `AnnotatorCanvas`'s `onImageReady`, fired from that image's own `onLoad`. A future host
+that needs pixels reads them lazily from that lease: nothing draws to a canvas or calls
+`getImageData` until a caller asks for `readRgb(width, height)`, in the asset descriptor's
+coordinate frame, and ordinary mounting and display never asks. It does not fetch or decode a
+second copy of the asset, and it does not construct a second `Image` to read from — the element
+handed back is the exact node the person is looking at. A lease refuses after its image source is
+replaced *or* after its owning `AnnotatorCanvas` instance unmounts, which prevents a retained
+reference for one asset from reading pixels of the next asset either through a reused DOM node or
+through the detached one a host's real asset-switch (unmount the old canvas, mount a fresh one)
+leaves behind.
+
+This is resource plumbing only. It neither selects a browser model nor exposes an inference
+target — no capability in this phase lets a host ask for local inference at all; composing pixels
+with a browser runtime remains a later host decision.
+
+**What is proved, and what is not claimed.** `e2e/assetPixels.spec.ts` drives a real Chromium
+against a tiny runtime-generated image and confirms, in that browser, on that image: one content
+request, the same decoded `<img>` handed back as the pixel source, and the exact descriptor-frame
+RGB a real 2D context produced. That is evidence about this seam's wiring, not a claim that a
+browser's canvas decode agrees with the server's own (Pillow) decode byte-for-byte in general —
+this phase changes no server decoding, and takes no position on cross-decoder parity beyond what
+is measured here.
+
+**Measured browser-canvas-vs-Pillow divergence.** A separate, informal comparison against the
+server's `convert("RGB")` decode found three real categories of disagreement: alpha-premultiplication
+rounding loss on partially-transparent pixels, EXIF-orientation auto-rotation (a browser applies an
+image's EXIF orientation when decoding to canvas; the server's direct-bytes decode path does not),
+and AdobeRGB ICC-profile color management (a browser color-manages a tagged profile toward sRGB on
+decode; the server's path does not). None of these are exact figures worth repeating here — they
+are a known limitation, not a benchmark — and no consumer of this lease has yet needed to reconcile
+them, since nothing in this phase reads pixels for inference. A future phase that feeds this lease's
+`readRgb` output to a model must account for these divergences before treating browser-decoded
+pixels as equivalent to the server's own decode of the same asset.
+
 ## Asking for a suggestion is not sending one
 
 `inference/suggestionExecutor.ts` is the seam the runtime above would plug into, and it exists

frontend/annotator/src/adapters/react/AnnotatorCanvas.tsx

@@ -207,6 +207,7 @@ import {
 } from "../viewport";
 import type { Viewport } from "../viewport";
 import { AnnotationLayer } from "./AnnotationLayer";
+import { createDecodedAssetImage, type DecodedAssetImage } from "./decodedAssetImage";
 import { useAnnotatorSnapshot } from "./hooks";
 import { digitFromCode, isComposing, isTextEntry } from "./keyboard";
 import { classColor, editedId, paintAnnotation, paintSuggestions } from "./paint";
@@ -257,6 +258,20 @@ export interface AnnotatorCanvasProps {
    * individually plausible and uniformly wrong.
    */
   readonly imageSrc: string;
+  /**
+   * The existing rendered image once its exact source has decoded.
+   *
+   * The source is generation-scoped: after `imageSrc` changes on a live
+   * instance, or after this component unmounts, a previously delivered source
+   * refuses pixel reads rather than reading the replacement through React's
+   * reused image node or a detached one.
+   *
+   * Every host today switches assets by unmounting this component rather than
+   * changing `imageSrc` in place — see the comment beside the `onLoad` handler
+   * for the one race that leaves open on a live-instance switch this contract
+   * has never had to close.
+   */
+  readonly onImageReady?: (source: DecodedAssetImage) => void;
   /** The class a drawing gesture will carry. `null` is select mode. */
   readonly activeClass: string | null;
   /**
@@ -488,6 +503,7 @@ const EMPTY_SUGGESTIONS: readonly PaintedSuggestion[] = [];
 export function AnnotatorCanvas({
   store,
   imageSrc,
+  onImageReady,
   activeClass,
   activeTool = null,
   onActivateClass,
@@ -514,6 +530,25 @@ export function AnnotatorCanvas({
 
   const rootRef = useRef<HTMLDivElement | null>(null);
   const paneRef = useRef<HTMLDivElement | null>(null);
+  const imageSrcNow = useRef(imageSrc);
+  const imageGeneration = useRef(0);
+  if (imageSrcNow.current !== imageSrc) {
+    imageSrcNow.current = imageSrc;
+    imageGeneration.current += 1;
+  }
+
+  // A host switches assets by unmounting this component and mounting a fresh
+  // one for the next asset, not by changing `imageSrc` on a live instance —
+  // so the ordinary generation bump above never runs. Without this, a lease
+  // handed out before teardown still finds its captured generation equal to
+  // `imageGeneration.current` and its detached `<img>` still `complete` with
+  // its old `src` attribute intact, and `readRgb` would hand back the wrong
+  // asset's pixels from a component nothing renders any more.
+  useEffect(() => {
+    return () => {
+      imageGeneration.current += 1;
+    };
+  }, []);
 
   // The fallback, built once — see the prop's docstring for why `useState`.
   const [ownClipboard] = useState(createClipboard);
@@ -1465,6 +1500,31 @@ export function AnnotatorCanvas({
         >
           <img
             src={imageSrc}
+            onLoad={(event) => {
+              // React always calls the handler from the most recently committed
+              // render, never the one attached when this particular `load` was
+              // queued — so both checks below compare the latest `imageSrc`
+              // against itself on a live instance whose source changed twice in
+              // a row before the first `load` fired, and cannot by themselves
+              // refuse a stale event delivered after such a change. No caller
+              // does this today: every host switches assets by unmounting this
+              // component (the effect below covers that), and a delayed load
+              // for an abandoned request is a case browsers do not dispatch —
+              // they fire `load`/`error` only for an image element's current
+              // request. A future host that mutates `imageSrc` on a live
+              // instance without remounting should re-examine this before
+              // relying on it.
+              const image = event.currentTarget;
+              if (imageSrcNow.current !== imageSrc || image.getAttribute("src") !== imageSrc) return;
+              onImageReady?.(
+                createDecodedAssetImage(
+                  image,
+                  imageSrc,
+                  imageGeneration.current,
+                  () => imageGeneration.current,
+                ),
+              );
+            }}
             alt=""
             aria-hidden="true"
             // Named so the pixelated-at-depth rule is asserted against the

frontend/annotator/src/adapters/react/decodedAssetImage.test.ts

@@ -0,0 +1,41 @@
+import { describe, expect, it, vi } from "vitest";
+
+import { createDecodedAssetImage, rgbPixelsFromDecodedImage } from "./decodedAssetImage";
+
+describe("decoded asset image", () => {
+  it("refuses a retained A source once its image generation is replaced by B", () => {
+    let current = 1;
+    const image = {} as HTMLImageElement;
+    const read = vi.fn(() => ({ width: 1, height: 1, rgb: new Uint8Array([1, 2, 3]) }));
+    const source = createDecodedAssetImage(image, "blob:a", 1, () => current, read);
+
+    current = 2;
+
+    expect(() => source.readRgb(1, 1)).toThrow("image source is no longer current");
+    expect(read).not.toHaveBeenCalled();
+  });
+
+  it("converts descriptor-frame RGBA pixels into exact row-major RGB", () => {
+    // A distinct object, unlike `{}` — asserted below by identity, not merely by
+    // shape. `rgbPixelsFromDecodedImage` must draw the exact rendered image it
+    // was handed, never a second `Image` it constructs to read from.
+    const image = { tag: "the rendered image" } as unknown as HTMLImageElement;
+    const drawImage = vi.fn();
+    const getImageData = vi.fn(() => ({ data: new Uint8ClampedArray([1, 2, 3, 4, 5, 6, 7, 8]) }));
+    const context = { drawImage, getImageData };
+    vi.stubGlobal("document", {
+      createElement: () => ({ width: 0, height: 0, getContext: () => context }),
+    });
+
+    const pixels = rgbPixelsFromDecodedImage(image, 2, 1);
+
+    expect(pixels).toEqual({ width: 2, height: 1, rgb: new Uint8Array([1, 2, 3, 5, 6, 7]) });
+    // The exact requested frame, not the image's own natural size (`image` above
+    // carries no `naturalWidth`/`naturalHeight` at all, so a mutation reading
+    // those instead would call both of these with `undefined`) and not a
+    // transposed pair either.
+    expect(drawImage).toHaveBeenCalledWith(image, 0, 0, 2, 1);
+    expect(getImageData).toHaveBeenCalledWith(0, 0, 2, 1);
+    vi.unstubAllGlobals();
+  });
+});

frontend/annotator/src/adapters/react/decodedAssetImage.ts

@@ -0,0 +1,55 @@
+/** A browser-only, generation-scoped lease over the image the adapter already renders. */
+
+export interface RgbPixels {
+  readonly width: number;
+  readonly height: number;
+  readonly rgb: Uint8Array;
+}
+
+export interface DecodedAssetImage {
+  readonly image: HTMLImageElement;
+  readonly src: string;
+  readRgb(width: number, height: number): RgbPixels;
+}
+
+type RgbReader = (image: HTMLImageElement, width: number, height: number) => RgbPixels;
+
+export function createDecodedAssetImage(
+  image: HTMLImageElement,
+  src: string,
+  generation: number,
+  currentGeneration: () => number,
+  read: RgbReader = rgbPixelsFromDecodedImage,
+): DecodedAssetImage {
+  return {
+    image,
+    src,
+    readRgb(width, height) {
+      if (currentGeneration() !== generation || image.getAttribute("src") !== src) {
+        throw new Error("image source is no longer current");
+      }
+      return read(image, width, height);
+    },
+  };
+}
+
+export function rgbPixelsFromDecodedImage(
+  image: HTMLImageElement,
+  width: number,
+  height: number,
+): RgbPixels {
+  const canvas = document.createElement("canvas");
+  canvas.width = width;
+  canvas.height = height;
+  const context = canvas.getContext("2d");
+  if (context === null) throw new Error("2D canvas context is unavailable");
+  context.drawImage(image, 0, 0, width, height);
+  const rgba = context.getImageData(0, 0, width, height).data;
+  const rgb = new Uint8Array(width * height * 3);
+  for (let source = 0, target = 0; source < rgba.length; source += 4, target += 3) {
+    rgb[target] = rgba[source];
+    rgb[target + 1] = rgba[source + 1];
+    rgb[target + 2] = rgba[source + 2];
+  }
+  return { width, height, rgb };
+}

frontend/annotator/src/adapters/react/index.ts

@@ -12,6 +12,7 @@ export {
   type AnnotatorCanvasProps,
   type AnnotatorView,
 } from "./AnnotatorCanvas";
+export type { DecodedAssetImage, RgbPixels } from "./decodedAssetImage";
 export {
   useAnnotatorSnapshot,
   useAnnotatorStore,

frontend/annotator/src/index.ts

@@ -344,12 +344,14 @@ export {
   type AnnotatorCanvasProps,
   type AnnotatorView,
   type CompositionProbe,
+  type DecodedAssetImage,
   type PaintedAnnotation,
   type PaintedSuggestion,
   type PendingIndicator,
   type PendingIndicatorState,
   type PendingPhase,
   type PendingPolygon,
+  type RgbPixels,
   type TextEntryProbe,
   type TransientLayerProps,
 } from "./adapters/react";

frontend/app/e2e/assetPixels.spec.ts

@@ -0,0 +1,28 @@
+/**
+ * The displayed asset is the browser pixel source — proved in a real browser.
+ *
+ * `docs/content/architecture/frontend/ui-core.md` states the rule and the unit
+ * tests beside `AssetImage` and `decodedAssetImage.ts` hold most of it, but jsdom's
+ * canvas does not decode a real image: nothing in that suite can show that a real
+ * `<img>`, drawn through a real 2D context, yields the exact bytes the descriptor
+ * frame promises. `/demo?scene=asset-pixels` (`src/demo/AssetPixelsFixture.tsx`) is a
+ * test-only host that composes the real `AssetImage` and `AnnotatorCanvas` against
+ * a tiny RGBA image encoded to a PNG at runtime, and this is the one scenario that
+ * reads it.
+ */
+
+import { expect, test } from "@playwright/test";
+
+test("reuses the visible decoded image for exact descriptor-frame RGB", async ({ page }) => {
+  await page.goto("/demo?scene=asset-pixels");
+  await expect(page.getByTestId("annotator-image")).toBeVisible();
+  await expect(page.getByTestId("pixel-source-same-image")).toHaveText("true");
+  await expect(page.getByTestId("pixel-rgb")).toHaveText("1,2,3,5,6,7,9,10,11");
+  await expect
+    .poll(() =>
+      page.evaluate(
+        () => (window as unknown as { __assetContentRequests?: number }).__assetContentRequests,
+      ),
+    )
+    .toBe(1);
+});