diff --git a/docs/content/architecture/frontend/ui-core.md b/docs/content/architecture/frontend/ui-core.md index 714491c2..023b1945 100644 --- a/docs/content/architecture/frontend/ui-core.md +++ b/docs/content/architecture/frontend/ui-core.md @@ -89,6 +89,48 @@ how it is grown instead. > something this browser can do. They are not two spellings of one idea, and neither is > derivable from the other. +## The displayed asset is the browser pixel source + +`AssetImage` fetches an asset once through the credentialed data client, retains that exact Blob +only while the asset is mounted, and owns the object URL it gives the existing annotator image. +The URL is revoked and an unfinished request is aborted when the asset is replaced or unmounted. +There is no JavaScript Blob cache: revisiting an asset relies on the server's immutable HTTP cache. + +The React annotator adapter can report a generation-scoped lease over the same visible decoded +`` — `AnnotatorCanvas`'s `onImageReady`, fired from that image's own `onLoad`. A future host +that needs pixels reads them lazily from that lease: nothing draws to a canvas or calls +`getImageData` until a caller asks for `readRgb(width, height)`, in the asset descriptor's +coordinate frame, and ordinary mounting and display never asks. It does not fetch or decode a +second copy of the asset, and it does not construct a second `Image` to read from — the element +handed back is the exact node the person is looking at. A lease refuses after its image source is +replaced *or* after its owning `AnnotatorCanvas` instance unmounts, which prevents a retained +reference for one asset from reading pixels of the next asset either through a reused DOM node or +through the detached one a host's real asset-switch (unmount the old canvas, mount a fresh one) +leaves behind. + +This is resource plumbing only. It neither selects a browser model nor exposes an inference +target — no capability in this phase lets a host ask for local inference at all; composing pixels +with a browser runtime remains a later host decision. + +**What is proved, and what is not claimed.** `e2e/assetPixels.spec.ts` drives a real Chromium +against a tiny runtime-generated image and confirms, in that browser, on that image: one content +request, the same decoded `` handed back as the pixel source, and the exact descriptor-frame +RGB a real 2D context produced. That is evidence about this seam's wiring, not a claim that a +browser's canvas decode agrees with the server's own (Pillow) decode byte-for-byte in general — +this phase changes no server decoding, and takes no position on cross-decoder parity beyond what +is measured here. + +**Measured browser-canvas-vs-Pillow divergence.** A separate, informal comparison against the +server's `convert("RGB")` decode found three real categories of disagreement: alpha-premultiplication +rounding loss on partially-transparent pixels, EXIF-orientation auto-rotation (a browser applies an +image's EXIF orientation when decoding to canvas; the server's direct-bytes decode path does not), +and AdobeRGB ICC-profile color management (a browser color-manages a tagged profile toward sRGB on +decode; the server's path does not). None of these are exact figures worth repeating here — they +are a known limitation, not a benchmark — and no consumer of this lease has yet needed to reconcile +them, since nothing in this phase reads pixels for inference. A future phase that feeds this lease's +`readRgb` output to a model must account for these divergences before treating browser-decoded +pixels as equivalent to the server's own decode of the same asset. + ## Asking for a suggestion is not sending one `inference/suggestionExecutor.ts` is the seam the runtime above would plug into, and it exists diff --git a/frontend/annotator/src/adapters/react/AnnotatorCanvas.tsx b/frontend/annotator/src/adapters/react/AnnotatorCanvas.tsx index e8cd32a7..24913823 100644 --- a/frontend/annotator/src/adapters/react/AnnotatorCanvas.tsx +++ b/frontend/annotator/src/adapters/react/AnnotatorCanvas.tsx @@ -207,6 +207,7 @@ import { } from "../viewport"; import type { Viewport } from "../viewport"; import { AnnotationLayer } from "./AnnotationLayer"; +import { createDecodedAssetImage, type DecodedAssetImage } from "./decodedAssetImage"; import { useAnnotatorSnapshot } from "./hooks"; import { digitFromCode, isComposing, isTextEntry } from "./keyboard"; import { classColor, editedId, paintAnnotation, paintSuggestions } from "./paint"; @@ -257,6 +258,20 @@ export interface AnnotatorCanvasProps { * individually plausible and uniformly wrong. */ readonly imageSrc: string; + /** + * The existing rendered image once its exact source has decoded. + * + * The source is generation-scoped: after `imageSrc` changes on a live + * instance, or after this component unmounts, a previously delivered source + * refuses pixel reads rather than reading the replacement through React's + * reused image node or a detached one. + * + * Every host today switches assets by unmounting this component rather than + * changing `imageSrc` in place — see the comment beside the `onLoad` handler + * for the one race that leaves open on a live-instance switch this contract + * has never had to close. + */ + readonly onImageReady?: (source: DecodedAssetImage) => void; /** The class a drawing gesture will carry. `null` is select mode. */ readonly activeClass: string | null; /** @@ -488,6 +503,7 @@ const EMPTY_SUGGESTIONS: readonly PaintedSuggestion[] = []; export function AnnotatorCanvas({ store, imageSrc, + onImageReady, activeClass, activeTool = null, onActivateClass, @@ -514,6 +530,25 @@ export function AnnotatorCanvas({ const rootRef = useRef(null); const paneRef = useRef(null); + const imageSrcNow = useRef(imageSrc); + const imageGeneration = useRef(0); + if (imageSrcNow.current !== imageSrc) { + imageSrcNow.current = imageSrc; + imageGeneration.current += 1; + } + + // A host switches assets by unmounting this component and mounting a fresh + // one for the next asset, not by changing `imageSrc` on a live instance — + // so the ordinary generation bump above never runs. Without this, a lease + // handed out before teardown still finds its captured generation equal to + // `imageGeneration.current` and its detached `` still `complete` with + // its old `src` attribute intact, and `readRgb` would hand back the wrong + // asset's pixels from a component nothing renders any more. + useEffect(() => { + return () => { + imageGeneration.current += 1; + }; + }, []); // The fallback, built once — see the prop's docstring for why `useState`. const [ownClipboard] = useState(createClipboard); @@ -1465,6 +1500,31 @@ export function AnnotatorCanvas({ > { + // React always calls the handler from the most recently committed + // render, never the one attached when this particular `load` was + // queued — so both checks below compare the latest `imageSrc` + // against itself on a live instance whose source changed twice in + // a row before the first `load` fired, and cannot by themselves + // refuse a stale event delivered after such a change. No caller + // does this today: every host switches assets by unmounting this + // component (the effect below covers that), and a delayed load + // for an abandoned request is a case browsers do not dispatch — + // they fire `load`/`error` only for an image element's current + // request. A future host that mutates `imageSrc` on a live + // instance without remounting should re-examine this before + // relying on it. + const image = event.currentTarget; + if (imageSrcNow.current !== imageSrc || image.getAttribute("src") !== imageSrc) return; + onImageReady?.( + createDecodedAssetImage( + image, + imageSrc, + imageGeneration.current, + () => imageGeneration.current, + ), + ); + }} alt="" aria-hidden="true" // Named so the pixelated-at-depth rule is asserted against the diff --git a/frontend/annotator/src/adapters/react/decodedAssetImage.test.ts b/frontend/annotator/src/adapters/react/decodedAssetImage.test.ts new file mode 100644 index 00000000..91387bcb --- /dev/null +++ b/frontend/annotator/src/adapters/react/decodedAssetImage.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it, vi } from "vitest"; + +import { createDecodedAssetImage, rgbPixelsFromDecodedImage } from "./decodedAssetImage"; + +describe("decoded asset image", () => { + it("refuses a retained A source once its image generation is replaced by B", () => { + let current = 1; + const image = {} as HTMLImageElement; + const read = vi.fn(() => ({ width: 1, height: 1, rgb: new Uint8Array([1, 2, 3]) })); + const source = createDecodedAssetImage(image, "blob:a", 1, () => current, read); + + current = 2; + + expect(() => source.readRgb(1, 1)).toThrow("image source is no longer current"); + expect(read).not.toHaveBeenCalled(); + }); + + it("converts descriptor-frame RGBA pixels into exact row-major RGB", () => { + // A distinct object, unlike `{}` — asserted below by identity, not merely by + // shape. `rgbPixelsFromDecodedImage` must draw the exact rendered image it + // was handed, never a second `Image` it constructs to read from. + const image = { tag: "the rendered image" } as unknown as HTMLImageElement; + const drawImage = vi.fn(); + const getImageData = vi.fn(() => ({ data: new Uint8ClampedArray([1, 2, 3, 4, 5, 6, 7, 8]) })); + const context = { drawImage, getImageData }; + vi.stubGlobal("document", { + createElement: () => ({ width: 0, height: 0, getContext: () => context }), + }); + + const pixels = rgbPixelsFromDecodedImage(image, 2, 1); + + expect(pixels).toEqual({ width: 2, height: 1, rgb: new Uint8Array([1, 2, 3, 5, 6, 7]) }); + // The exact requested frame, not the image's own natural size (`image` above + // carries no `naturalWidth`/`naturalHeight` at all, so a mutation reading + // those instead would call both of these with `undefined`) and not a + // transposed pair either. + expect(drawImage).toHaveBeenCalledWith(image, 0, 0, 2, 1); + expect(getImageData).toHaveBeenCalledWith(0, 0, 2, 1); + vi.unstubAllGlobals(); + }); +}); diff --git a/frontend/annotator/src/adapters/react/decodedAssetImage.ts b/frontend/annotator/src/adapters/react/decodedAssetImage.ts new file mode 100644 index 00000000..d888625a --- /dev/null +++ b/frontend/annotator/src/adapters/react/decodedAssetImage.ts @@ -0,0 +1,55 @@ +/** A browser-only, generation-scoped lease over the image the adapter already renders. */ + +export interface RgbPixels { + readonly width: number; + readonly height: number; + readonly rgb: Uint8Array; +} + +export interface DecodedAssetImage { + readonly image: HTMLImageElement; + readonly src: string; + readRgb(width: number, height: number): RgbPixels; +} + +type RgbReader = (image: HTMLImageElement, width: number, height: number) => RgbPixels; + +export function createDecodedAssetImage( + image: HTMLImageElement, + src: string, + generation: number, + currentGeneration: () => number, + read: RgbReader = rgbPixelsFromDecodedImage, +): DecodedAssetImage { + return { + image, + src, + readRgb(width, height) { + if (currentGeneration() !== generation || image.getAttribute("src") !== src) { + throw new Error("image source is no longer current"); + } + return read(image, width, height); + }, + }; +} + +export function rgbPixelsFromDecodedImage( + image: HTMLImageElement, + width: number, + height: number, +): RgbPixels { + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + const context = canvas.getContext("2d"); + if (context === null) throw new Error("2D canvas context is unavailable"); + context.drawImage(image, 0, 0, width, height); + const rgba = context.getImageData(0, 0, width, height).data; + const rgb = new Uint8Array(width * height * 3); + for (let source = 0, target = 0; source < rgba.length; source += 4, target += 3) { + rgb[target] = rgba[source]; + rgb[target + 1] = rgba[source + 1]; + rgb[target + 2] = rgba[source + 2]; + } + return { width, height, rgb }; +} diff --git a/frontend/annotator/src/adapters/react/index.ts b/frontend/annotator/src/adapters/react/index.ts index 9265b40d..035b28e0 100644 --- a/frontend/annotator/src/adapters/react/index.ts +++ b/frontend/annotator/src/adapters/react/index.ts @@ -12,6 +12,7 @@ export { type AnnotatorCanvasProps, type AnnotatorView, } from "./AnnotatorCanvas"; +export type { DecodedAssetImage, RgbPixels } from "./decodedAssetImage"; export { useAnnotatorSnapshot, useAnnotatorStore, diff --git a/frontend/annotator/src/index.ts b/frontend/annotator/src/index.ts index a0fe7e86..fa6e5c24 100644 --- a/frontend/annotator/src/index.ts +++ b/frontend/annotator/src/index.ts @@ -344,12 +344,14 @@ export { type AnnotatorCanvasProps, type AnnotatorView, type CompositionProbe, + type DecodedAssetImage, type PaintedAnnotation, type PaintedSuggestion, type PendingIndicator, type PendingIndicatorState, type PendingPhase, type PendingPolygon, + type RgbPixels, type TextEntryProbe, type TransientLayerProps, } from "./adapters/react"; diff --git a/frontend/app/e2e/assetPixels.spec.ts b/frontend/app/e2e/assetPixels.spec.ts new file mode 100644 index 00000000..a219e588 --- /dev/null +++ b/frontend/app/e2e/assetPixels.spec.ts @@ -0,0 +1,28 @@ +/** + * The displayed asset is the browser pixel source — proved in a real browser. + * + * `docs/content/architecture/frontend/ui-core.md` states the rule and the unit + * tests beside `AssetImage` and `decodedAssetImage.ts` hold most of it, but jsdom's + * canvas does not decode a real image: nothing in that suite can show that a real + * ``, drawn through a real 2D context, yields the exact bytes the descriptor + * frame promises. `/demo?scene=asset-pixels` (`src/demo/AssetPixelsFixture.tsx`) is a + * test-only host that composes the real `AssetImage` and `AnnotatorCanvas` against + * a tiny RGBA image encoded to a PNG at runtime, and this is the one scenario that + * reads it. + */ + +import { expect, test } from "@playwright/test"; + +test("reuses the visible decoded image for exact descriptor-frame RGB", async ({ page }) => { + await page.goto("/demo?scene=asset-pixels"); + await expect(page.getByTestId("annotator-image")).toBeVisible(); + await expect(page.getByTestId("pixel-source-same-image")).toHaveText("true"); + await expect(page.getByTestId("pixel-rgb")).toHaveText("1,2,3,5,6,7,9,10,11"); + await expect + .poll(() => + page.evaluate( + () => (window as unknown as { __assetContentRequests?: number }).__assetContentRequests, + ), + ) + .toBe(1); +}); diff --git a/frontend/app/src/demo/AssetPixelsFixture.tsx b/frontend/app/src/demo/AssetPixelsFixture.tsx new file mode 100644 index 00000000..3ba6ab76 --- /dev/null +++ b/frontend/app/src/demo/AssetPixelsFixture.tsx @@ -0,0 +1,168 @@ +/** + * A test-only harness proving the displayed asset is the browser pixel source. + * + * `docs/content/architecture/frontend/ui-core.md`'s rule — the visible `` is + * reused as the browser pixel source, with no second fetch and no independent + * decode — has one half no unit test can reach: jsdom's canvas does not decode a + * real image, so `getImageData` there is either absent or a stub a test supplies + * itself. This page composes the real `AssetImage` and `AnnotatorCanvas` exactly + * as `AnnotationPage` does, against a tiny RGBA image **encoded to a PNG blob at + * runtime** (no binary fixture file, so the exact bytes are never stale or + * regenerated by hand) — so `e2e/assetPixels.spec.ts` can prove the seam in a + * real browser: one content request, the same decoded `` handed back as the + * pixel source, and the exact descriptor-frame RGB a real canvas produced. + * + * No token, no server, not linked from anywhere in the product — the same + * standing as `/demo` and `/styleguide`. + */ + +import { AnnotatorCanvas, AnnotatorStore, documentFromWire } from "@visionset/annotator"; +import { AssetImage, VisionSetDataProvider } from "@visionset/ui-core"; +import type { DataResult, VisionSetDataClient } from "@visionset/ui-core"; +import { useEffect, useMemo, useState, type JSX } from "react"; + +declare global { + interface Window { + /** How many times the stub client has delivered content. Reset on load. */ + __assetContentRequests?: number; + } +} + +const PROJECT_ID = "fixture-project"; +const ASSET_ID = "fixture-asset"; +const WIDTH = 3; +const HEIGHT = 1; + +/** + * Three RGBA pixels, alpha fully opaque. + * + * A low alpha would make the PNG round-trip lossy — canvas premultiplies alpha + * on encode, and an opaque pixel is the one value that survives unpremultiplying + * exactly. `1,2,3` / `5,6,7` / `9,10,11` is what `pixel-rgb` below must read back + * once the fourth (alpha) byte of each RGBA quadruple is dropped. + */ +const RGBA = new Uint8ClampedArray([1, 2, 3, 255, 5, 6, 7, 255, 9, 10, 11, 255]); + +function fixtureStore(): AnnotatorStore { + return new AnnotatorStore( + documentFromWire({ + asset: { id: ASSET_ID, width: WIDTH, height: HEIGHT }, + schema: { project_id: PROJECT_ID, version: 1, classes: [] }, + annotations: [], + }), + ); +} + +/** A real PNG, built from the exact bytes above — never a checked-in fixture. */ +async function encodeFixturePng(): Promise { + const canvas = document.createElement("canvas"); + canvas.width = WIDTH; + canvas.height = HEIGHT; + const context = canvas.getContext("2d"); + if (context === null) throw new Error("2D canvas context is unavailable"); + context.putImageData(new ImageData(RGBA, WIDTH, HEIGHT), 0, 0); + return await new Promise((resolve, reject) => { + canvas.toBlob((blob) => { + if (blob === null) reject(new Error("canvas produced no blob")); + else resolve(blob); + }, "image/png"); + }); +} + +/** + * A minimal `VisionSetDataClient`: it answers the one route `AssetImage` calls, + * counts a delivery only once it actually settles, and rejects a request whose + * signal is already or becomes aborted — the same contract clause a real host + * honours (`data/port.ts`'s "a request cancelled through its signal rejects"). + * + * Delivery is deferred one macrotask so a synchronous `StrictMode` remount's + * cleanup — the abort of the first mount's controller — has already landed on + * `signal` before this settles, the ordering a real network transfer has for + * free. Without it, a mutation that dropped the abort could still read as one + * request here purely by winning a race, which would prove nothing. + */ +function fixtureClient(png: Blob, onDelivered: () => void): VisionSetDataClient { + const GET = (path: string, init?: { readonly signal?: AbortSignal }): Promise => { + if (!path.endsWith("/content")) return Promise.resolve({ ok: false, failure: "unreachable" }); + const signal = init?.signal; + return new Promise((resolve, reject) => { + if (signal?.aborted === true) { + reject(new DOMException("The operation was aborted.", "AbortError")); + return; + } + const onAbort = (): void => { + window.clearTimeout(timer); + reject(new DOMException("The operation was aborted.", "AbortError")); + }; + const timer = window.setTimeout(() => { + signal?.removeEventListener("abort", onAbort); + if (signal?.aborted === true) { + reject(new DOMException("The operation was aborted.", "AbortError")); + return; + } + onDelivered(); + resolve({ ok: true, data: png }); + }, 0); + signal?.addEventListener("abort", onAbort, { once: true }); + }); + }; + const refuse = (): Promise => Promise.resolve({ ok: false, failure: "unreachable" }); + return { + GET, + POST: refuse, + PUT: refuse, + PATCH: refuse, + DELETE: refuse, + } as unknown as VisionSetDataClient; +} + +export function AssetPixelsFixture(): JSX.Element { + const [png, setPng] = useState(null); + const [sameImage, setSameImage] = useState("pending"); + const [rgb, setRgb] = useState("pending"); + const store = useMemo(fixtureStore, []); + + useEffect(() => { + window.__assetContentRequests = 0; + let cancelled = false; + void encodeFixturePng().then((blob) => { + if (!cancelled) setPng(blob); + }); + return () => { + cancelled = true; + }; + }, []); + + const client = useMemo(() => { + if (png === null) return null; + return fixtureClient(png, () => { + window.__assetContentRequests = (window.__assetContentRequests ?? 0) + 1; + }); + }, [png]); + + if (client === null) return
; + + return ( + +
+ + {(src) => ( + {}} + onImageReady={(source) => { + const visible = window.document.querySelector('[data-testid="annotator-image"]'); + setSameImage(source.image === visible ? "true" : "false"); + setRgb(Array.from(source.readRgb(WIDTH, HEIGHT).rgb).join(",")); + }} + /> + )} + +
+ {sameImage} + {rgb} + + ); +} diff --git a/frontend/app/src/routes.tsx b/frontend/app/src/routes.tsx index 2e9475e1..d394e8f9 100644 --- a/frontend/app/src/routes.tsx +++ b/frontend/app/src/routes.tsx @@ -30,7 +30,9 @@ * The benchmark keeps its query parameter (`/demo?scene=bench`) rather than * gaining a route: it is an instrument, its recorded numbers were taken against * that exact page, and moving it would change what it measures for no reason - * anybody asked for. + * anybody asked for. `e2e/assetPixels.spec.ts`'s fixture rides the same route + * behind `/demo?scene=asset-pixels`, for the same reason: test-only, not linked + * from the product, so it earns no route of its own either. * * ## Every route has a screen * @@ -59,6 +61,7 @@ import { Navigate, Route, Routes, useNavigate, useParams, useSearchParams } from import type { JSX } from "react"; import { AnnotatorDemo } from "./demo/AnnotatorDemo"; +import { AssetPixelsFixture } from "./demo/AssetPixelsFixture"; import { BenchmarkHost } from "./demo/BenchmarkHost"; import { ShowcaseFrame } from "./demo/ShowcaseFrame"; import { AppShell, FullBleedPane, PaddedPane, ProjectPane } from "./shell/AppShell"; @@ -139,10 +142,15 @@ export function AppRoutes(): JSX.Element { ); } -/** The showcase, and the benchmark behind its query parameter. */ +/** + * The showcase, the benchmark and the asset-pixels test fixture, each behind + * its own `scene` query value. + */ function Showcase(): JSX.Element { const [query] = useSearchParams(); - const bench = query.get("scene") === "bench"; + const scene = query.get("scene"); + if (scene === "asset-pixels") return ; + const bench = scene === "bench"; return ( {bench ? : } diff --git a/frontend/ui-core/src/annotator/AssetImage.tsx b/frontend/ui-core/src/annotator/AssetImage.tsx index 4e58784c..3f84252d 100644 --- a/frontend/ui-core/src/annotator/AssetImage.tsx +++ b/frontend/ui-core/src/annotator/AssetImage.tsx @@ -33,9 +33,14 @@ export interface AssetImageProps { readonly children: (src: string) => ReactNode; } +interface AssetImageResource { + readonly src: string; + readonly blob: Blob; +} + export function AssetImage({ projectId, assetId, children }: AssetImageProps): JSX.Element { const client = useApiClient(); - const [src, setSrc] = useState(null); + const [resource, setResource] = useState(null); const [failed, setFailed] = useState(false); useEffect(() => { @@ -46,7 +51,7 @@ export function AssetImage({ projectId, assetId, children }: AssetImageProps): J // running to completion (#572). const controller = new AbortController(); let objectUrl: string | null = null; - setSrc(null); + setResource(null); setFailed(false); void (async () => { @@ -63,8 +68,9 @@ export function AssetImage({ projectId, assetId, children }: AssetImageProps): J setFailed(true); return; } - objectUrl = URL.createObjectURL(result.data as unknown as Blob); - setSrc(objectUrl); + const blob = result.data as unknown as Blob; + objectUrl = URL.createObjectURL(blob); + setResource({ src: objectUrl, blob }); } catch { // The abort lands here by design. Anything else is a network that // died, which the error state answers better than an eternal skeleton. @@ -92,9 +98,9 @@ export function AssetImage({ projectId, assetId, children }: AssetImageProps): J ); } - if (src === null) { + if (resource === null) { return
; } - return <>{children(src)}; + return <>{children(resource.src)}; } diff --git a/frontend/ui-core/src/annotator/assetImage.test.tsx b/frontend/ui-core/src/annotator/assetImage.test.tsx index 25a5597b..8889276b 100644 --- a/frontend/ui-core/src/annotator/assetImage.test.tsx +++ b/frontend/ui-core/src/annotator/assetImage.test.tsx @@ -3,11 +3,12 @@ * honest about a network that is down (#572). */ -import { screen, waitFor } from "@testing-library/react"; +import { render, screen, waitFor } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { ReactNode } from "react"; +import type { JSX, ReactNode } from "react"; -import { renderWithData } from "../testing/dataHarness"; +import { VisionSetDataProvider } from "../data/VisionSetDataProvider"; +import { harnessClient, harnessQueryClient, renderWithData } from "../testing/dataHarness"; import { AssetImage } from "./AssetImage"; const PROJECT = "11111111-1111-4111-8111-111111111111"; @@ -56,6 +57,14 @@ function image(): ReactNode { ); } +function asset(assetId: string): ReactNode { + return ( + + {(src) => frame} + + ); +} + describe("the asset's pixels", () => { it("fetches them with the credential and hands an object URL to the child", async () => { on("GET", /\/content$/, { status: 200, body: null }); @@ -65,6 +74,35 @@ describe("the asset's pixels", () => { expect(frame.getAttribute("src") ?? "").toMatch(/^blob:/); }); + it("hands createObjectURL a Blob carrying the exact fetched bytes, not merely a Blob", async () => { + // A trivial `expect.any(Blob)`-shaped check would pass for any Blob at all + // (jsdom's blob has no enumerable own properties, so `toHaveBeenCalledWith` + // would even accept one with unrelated content) — the content check below is + // what actually distinguishes "the fetched resource" from "a Blob". + const bytes = "the-exact-fetched-bytes"; + vi.stubGlobal("fetch", async (request: Request) => { + sent.push(request); + // A string body, not a `Blob` one: this jsdom/undici pairing mangles a + // `Response` constructed directly from a `Blob` (its `.blob()` comes back + // stringified to "[object Blob]"), a test-environment quirk unrelated to + // the code under test. A string body round-trips through `.blob()` + // correctly and is what a real network response looks like anyway. + return new Response(bytes, { status: 200, headers: { "content-type": "image/png" } }); + }); + const createObjectURL = vi.spyOn(URL, "createObjectURL"); + + renderWithData(image()); + await screen.findByTestId("the-frame"); + + expect(sent).toHaveLength(1); + expect(createObjectURL).toHaveBeenCalledTimes(1); + const received = createObjectURL.mock.calls[0]?.[0]; + expect(received).toBeInstanceOf(Blob); + await expect((received as Blob).text()).resolves.toBe(bytes); + + createObjectURL.mockRestore(); + }); + it("aborts the transfer when it unmounts mid-flight (#572)", async () => { // Walking a job with the arrow keys unmounts each frame's image; before // the abort, every skipped frame's full-size download ran to completion. @@ -76,6 +114,28 @@ describe("the asset's pixels", () => { expect(sent[0].signal.aborted).toBe(true); }); + it("revokes the replaced asset URL and aborts its completed request", async () => { + on("GET", /\/content$/, { status: 200, body: null }); + const revoke = vi.spyOn(URL, "revokeObjectURL"); + const client = harnessClient(); + const queries = harnessQueryClient(); + const scope = Symbol("asset-image-test"); + const wrap = (assetId: string): JSX.Element => ( + queries}> + {asset(assetId)} + + ); + const view = render(wrap("asset-a")); + const first = await screen.findByTestId("the-frame"); + const firstSrc = first.getAttribute("src"); + + view.rerender(wrap("asset-b")); + await waitFor(() => expect(sent).toHaveLength(2)); + + expect(sent[0]?.signal.aborted).toBe(true); + expect(revoke).toHaveBeenCalledWith(firstSrc); + }); + it("shows the failure state when the fetch itself throws", async () => { // A rejected fetch (network down) used to be an unhandled rejection and an // eternal loading skeleton; the abort turned rejection into an ordinary diff --git a/frontend/ui-core/src/annotator/decodedImageLease.test.tsx b/frontend/ui-core/src/annotator/decodedImageLease.test.tsx new file mode 100644 index 00000000..07363dd3 --- /dev/null +++ b/frontend/ui-core/src/annotator/decodedImageLease.test.tsx @@ -0,0 +1,90 @@ +import { + AnnotatorCanvas, + AnnotatorStore, + documentFromWire, + type DecodedAssetImage, + type RgbPixels, +} from "@visionset/annotator"; +import { fireEvent, render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; + +function store(): AnnotatorStore { + return new AnnotatorStore( + documentFromWire({ + asset: { id: "asset", width: 2, height: 1 }, + schema: { project_id: "project", version: 1, classes: [] }, + annotations: [], + }), + ); +} + +// `DecodedAssetImage` resolving here, from the package's real entry point +// rather than a path into its internals, is the regression guard: the type +// is otherwise easy to re-export from the React adapter alone and miss the +// package barrel that actually reaches a host. +function canvas(imageSrc: string, ready: (source: DecodedAssetImage) => void) { + return ; +} + +describe("decoded image lease", () => { + it("hands the exact rendered image to the ready callback", () => { + const ready = vi.fn(); + render(canvas("blob:a", ready)); + const image = screen.getByTestId("annotator-image") as HTMLImageElement; + fireEvent.load(image); + expect(ready.mock.calls[0][0].image).toBe(image); + }); + + it("does not let retained A read through the DOM node reused for B", () => { + const ready = vi.fn(); + const view = render(canvas("blob:a", ready)); + const image = screen.getByTestId("annotator-image") as HTMLImageElement; + fireEvent.load(image); + const sourceA = ready.mock.calls[0][0]; + view.rerender(canvas("blob:b", ready)); + expect(screen.getByTestId("annotator-image")).toBe(image); + expect(() => sourceA.readRgb(2, 1)).toThrow("image source is no longer current"); + }); + + it("refuses a retained lease once its owning component has unmounted", () => { + // The real host swaps assets by unmounting the old `AnnotatorCanvas` + // instance and mounting a fresh one, not by changing `imageSrc` on a live + // instance — so this is the path the generation-bump-on-`imageSrc`-change + // above does not cover. A lease taken out before teardown must refuse + // afterward even though nothing ever changed its `src` attribute. + const ready = vi.fn(); + const view = render(canvas("blob:a", ready)); + const image = screen.getByTestId("annotator-image") as HTMLImageElement; + fireEvent.load(image); + const source = ready.mock.calls[0][0]; + + view.unmount(); + + expect(() => source.readRgb(2, 1)).toThrow("image source is no longer current"); + }); + + it("surfaces RgbPixels from the package entry point, shaped as readRgb returns it", () => { + // A compile-time proof standing in beside the runtime ones above: if + // `RgbPixels` stopped being re-exported from `@visionset/annotator`, this + // file would fail to typecheck rather than merely fail at runtime. + const pixels: RgbPixels = { width: 1, height: 1, rgb: new Uint8Array([1, 2, 3]) }; + expect(pixels.rgb.length).toBe(pixels.width * pixels.height * 3); + }); + + it("never touches canvas pixel extraction during ordinary rendering", () => { + // `rgbPixelsFromDecodedImage` unconditionally calls `getContext("2d")` on a + // canvas it creates, so a spy that observes zero calls to that prototype + // method is proof the lazy helper never ran — ordinary mounting and an + // image `load` only ever hand back the lease, they never read it. + const getContext = vi.spyOn(HTMLCanvasElement.prototype, "getContext"); + const ready = vi.fn(); + render(canvas("blob:a", ready)); + const image = screen.getByTestId("annotator-image") as HTMLImageElement; + fireEvent.load(image); + + expect(ready).toHaveBeenCalledTimes(1); + expect(getContext).not.toHaveBeenCalled(); + + getContext.mockRestore(); + }); +});