Fix color image thumbnails

src/core/thumbnailers/vtk-image.ts

@@ -2,40 +2,73 @@ import vtkImageData from '@kitware/vtk.js/Common/DataModel/ImageData';
 import type { TypedArray } from '@kitware/vtk.js/types';
 import { ThumbnailSlice } from '.';
 
-function scalarImageToImageData(
-  values: TypedArray,
-  width: number,
-  height: number,
-  scaleMin: number,
-  scaleMax: number
-) {
+type ThumbnailPixels = {
+  values: TypedArray;
+  width: number;
+  height: number;
+  components: number;
+  pixelIndex: (x: number, y: number) => number;
+  scaleMin: number;
+  scaleMax: number;
+};
+
+function colorChannelScale(values: TypedArray) {
+  if (values instanceof Uint8Array || values instanceof Uint8ClampedArray) {
+    return 1;
+  }
+  if (values instanceof Uint16Array) return 255 / 65535;
+  return 255;
+}
+
+function imageSliceToImageData({
+  values,
+  width,
+  height,
+  components,
+  pixelIndex,
+  scaleMin,
+  scaleMax,
+}: ThumbnailPixels) {
   const im = new ImageData(width, height);
-  const arr32 = new Uint32Array(im.data.buffer);
-  // scale to 1 unsigned byte
   const factor = 255 / (scaleMax - scaleMin);
-  for (let i = 0; i < values.length; i += 1) {
-    const byte = Math.floor((values[i] - scaleMin) * factor);
-    // ABGR order
-
-    arr32[i] = (255 << 24) | (byte << 16) | (byte << 8) | byte;
+  const byteFactor = colorChannelScale(values);
+  const toByte = (value: number) => {
+    if (components >= 3) return value * byteFactor;
+    if (scaleMax === scaleMin) return 0;
+    return (value - scaleMin) * factor;
+  };
+  for (let y = 0; y < height; y += 1) {
+    for (let x = 0; x < width; x += 1) {
+      const source = pixelIndex(x, y) * components;
+      const target = (y * width + x) * 4;
+      const grayscale = components < 3;
+      im.data[target] = toByte(values[source]);
+      im.data[target + 1] = grayscale
+        ? im.data[target]
+        : toByte(values[source + 1]);
+      im.data[target + 2] = grayscale
+        ? im.data[target]
+        : toByte(values[source + 2]);
+      im.data[target + 3] =
+        components === 2 || components === 4
+          ? values[source + components - 1] * byteFactor
+          : 255;
+    }
   }
 
   return im;
 }
 
-/**
- * Generates a thumbnail given an image data.
- *
- * Assumption: image is comprised of single-component scalars
- */
+/** Generates a thumbnail from one image plane. */
 function generateThumbnail(
   imageData: vtkImageData,
   axis: 0 | 1 | 2 = 2,
   whichSlice = ThumbnailSlice.Middle
 ) {
   const scalars = imageData.getPointData().getScalars();
   const data = scalars.getData() as TypedArray;
-  const dataRange = scalars.getRange();
+  const components = scalars.getNumberOfComponents();
+  const [scaleMin, scaleMax] = scalars.getRange(0);
   const dims = imageData.getDimensions();
 
   // ThumbnailSlice.First
@@ -46,52 +79,30 @@ function generateThumbnail(
     slice = dims[axis] - 1;
   }
 
-  let sliceData: TypedArray;
   let width: number;
   let height: number;
+  let pixelIndex: (x: number, y: number) => number;
 
   if (axis === 0) {
-    // work-around for typing data.constructor.
-    // data is not necessarily of type Uint8Array.
-    sliceData = new (<Uint8ArrayConstructor>data.constructor)(
-      dims[1] * dims[2]
-    );
     [, width, height] = dims;
-    for (let k = 0; k < dims[2]; k++) {
-      for (let j = 0; j < dims[1]; j++) {
-        const index = slice + j * dims[0] + k * dims[0] * dims[1];
-        const offset = k * dims[1] + j;
-        sliceData[offset] = data[index];
-      }
-    }
+    pixelIndex = (x, y) => slice + x * dims[0] + y * dims[0] * dims[1];
   } else if (axis === 1) {
-    sliceData = new (<Uint8ArrayConstructor>data.constructor)(
-      dims[0] * dims[2]
-    );
     [width, , height] = dims;
-    for (let k = 0; k < dims[2]; k++) {
-      for (let i = 0; i < dims[0]; i++) {
-        const index = i + slice * dims[0] + k * dims[0] * dims[1];
-        const offset = k * dims[0] + i;
-        sliceData[offset] = data[index];
-      }
-    }
-  } else if (axis === 2) {
+    pixelIndex = (x, y) => x + slice * dims[0] + y * dims[0] * dims[1];
+  } else {
     [width, height] = dims;
-    const skip = dims[0] * dims[1];
-    const sliceOffset = slice * skip;
-    sliceData = Array.isArray(data)
-      ? data.slice(sliceOffset, sliceOffset + skip)
-      : data.subarray(sliceOffset, sliceOffset + skip);
+    pixelIndex = (x, y) => x + y * dims[0] + slice * dims[0] * dims[1];
   }
 
-  return scalarImageToImageData(
-    sliceData!,
-    width!,
-    height!,
-    dataRange[0],
-    dataRange[1]
-  );
+  return imageSliceToImageData({
+    values: data,
+    width,
+    height,
+    components,
+    pixelIndex,
+    scaleMin,
+    scaleMax,
+  });
 }
 
 export function createVTKImageThumbnailer() {

tests/specs/raster-thumbnail.e2e.ts

@@ -0,0 +1,122 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { TEMP_DIR } from '../../wdio.shared.conf';
+import { volViewPage } from '../pageobjects/volview.page';
+import { createRasterFixture } from './rasterThumbnailFixtures';
+import { writeManifestToFile } from './utils';
+
+type RGB = [number, number, number];
+
+async function openImages(names: string[], manifestName: string) {
+  for (const name of names) {
+    fs.writeFileSync(path.join(TEMP_DIR, name), createRasterFixture(name));
+  }
+  await writeManifestToFile(
+    { resources: names.map((name) => ({ url: `/tmp/${name}`, name })) },
+    manifestName
+  );
+  await volViewPage.open(`?urls=[tmp/${manifestName}]`);
+}
+
+async function thumbnailPixels(name: string, points: [number, number][]) {
+  await browser.waitUntil(
+    () =>
+      browser.execute((imageName) => {
+        const card = Array.from(
+          document.querySelectorAll('.image-list-card')
+        ).find((element) => element.textContent?.includes(imageName));
+        const image = card?.querySelector('img');
+        return (
+          image instanceof HTMLImageElement &&
+          image.complete &&
+          image.naturalWidth > 0
+        );
+      }, name),
+    { timeoutMsg: `Expected a rendered thumbnail for ${name}` }
+  );
+
+  return browser.execute(
+    (imageName, locations) => {
+      const card = Array.from(
+        document.querySelectorAll('.image-list-card')
+      ).find((element) => element.textContent?.includes(imageName));
+      const image = card?.querySelector('img') as HTMLImageElement;
+      const canvas = document.createElement('canvas');
+      canvas.width = image.naturalWidth;
+      canvas.height = image.naturalHeight;
+      const context = canvas.getContext('2d')!;
+      context.drawImage(image, 0, 0);
+      return locations.map(
+        ([x, y]) =>
+          Array.from(context.getImageData(x, y, 1, 1).data.slice(0, 3)) as RGB
+      );
+    },
+    name,
+    points
+  );
+}
+
+describe('raster image thumbnails', () => {
+  it('shows RGB PNG and JPEG colors and a grayscale gradient', async () => {
+    const names = [
+      'rgb-thumbnail.png',
+      'rgb-thumbnail.jpg',
+      'gray-thumbnail.png',
+    ];
+    await openImages(names, 'raster-thumbnail-manifest.json');
+    const expected: RGB[] = [
+      [128, 64, 64],
+      [64, 128, 64],
+      [192, 32, 64],
+      [32, 192, 64],
+    ];
+    for (const name of names.slice(0, 2)) {
+      const colors = (await thumbnailPixels(name, [
+        [25, 25],
+        [75, 25],
+        [25, 75],
+        [75, 75],
+      ])) as RGB[];
+      for (let pixel = 0; pixel < expected.length; pixel += 1) {
+        for (let channel = 0; channel < 3; channel += 1) {
+          expect(
+            Math.abs(colors[pixel][channel] - expected[pixel][channel])
+          ).toBeLessThan(16);
+        }
+      }
+    }
+
+    const [dark, light] = await thumbnailPixels(names[2], [
+      [15, 50],
+      [85, 50],
+    ]);
+    expect(dark[0]).toBeLessThan(80);
+    expect(light[0]).toBeGreaterThan(170);
+  });
+
+  it('composites RGBA pixels into the displayed thumbnail', async () => {
+    const name = 'rgba-thumbnail.png';
+    await openImages([name], 'rgba-thumbnail-manifest.json');
+    const [opaque, partial, transparent, yellow] = (await thumbnailPixels(
+      name,
+      [
+        [25, 25],
+        [75, 25],
+        [25, 75],
+        [75, 75],
+      ]
+    )) as RGB[];
+
+    expect(opaque[0]).toBeGreaterThan(200);
+    expect(opaque[1]).toBeLessThan(50);
+    expect(opaque[2]).toBeLessThan(50);
+    expect(partial[0]).toBeLessThan(30);
+    expect(partial[1]).toBeGreaterThan(100);
+    expect(partial[1]).toBeLessThan(160);
+    expect(partial[2]).toBeLessThan(30);
+    expect(transparent.every((channel) => channel < 30)).toBe(true);
+    expect(yellow[0]).toBeGreaterThan(200);
+    expect(yellow[1]).toBeGreaterThan(200);
+    expect(yellow[2]).toBeLessThan(50);
+  });
+});

tests/specs/rasterThumbnailFixtures.ts

@@ -0,0 +1,74 @@
+import * as zlib from 'zlib';
+
+const SIZE = 128;
+const RGB_JPEG_BASE64 =
+  '/9j//gARTGF2YzU4LjEzNC4xMDAA/9sAQwAIBAQEBAQFBQUFBQUGBgYGBgYGBgYGBgYGBwcHCAgIBwcHBgYHBwgICAgJCQkICAgICQkKCgoMDAsLDg4OEREU/8QAUQABAQEAAAAAAAAAAAAAAAAAAAUGAQEAAwEBAAAAAAAAAAAAAAAABggEBQcQAQAAAAAAAAAAAAAAAAAAAAARAQAAAAAAAAAAAAAAAAAAAAD/wAARCACAAIADARIAAhIAAxIA/9oADAMBAAIRAxEAPwDJDIJAAAAAAAAAAAAAAAKYyiCAAAAAAAAAAAAAACYNQnYAAAAAAAAAAAAAApjKIIAAAAAAAAAAAAAAJg1CdgAAAAAAAAAAAAACmMoggAAAAAAAAAAAAAAmDUJ2AAAAAAAAAAAAAAKYyiCAAAAAAAAAAAAAACYNQ9oAAAAAAAAAAAAAAGvHFFbAAAAAAAAAAAAAABkB2hZMAAAAAAAAAAAAAAa8cUVsAAAAAAAAAAAAAAGQHaFkwAAAAAAAAAAAAABrxxRWwAAAAAAAAAAAAAAZAdoWTAAAAAAAAAAAAAAGvHFFbAAAAAAAAAAAAAAB/9k=';
+
+function crc32(bytes: Buffer) {
+  let crc = 0xffffffff;
+  for (const byte of bytes) {
+    crc ^= byte;
+    for (let bit = 0; bit < 8; bit += 1) {
+      crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1));
+    }
+  }
+  return (crc ^ 0xffffffff) >>> 0;
+}
+
+function pngChunk(type: string, data: Buffer) {
+  const payload = Buffer.concat([Buffer.from(type), data]);
+  const length = Buffer.alloc(4);
+  length.writeUInt32BE(data.length);
+  const checksum = Buffer.alloc(4);
+  checksum.writeUInt32BE(crc32(payload));
+  return Buffer.concat([length, payload, checksum]);
+}
+
+function createPng(
+  channels: 1 | 3 | 4,
+  pixel: (x: number, y: number) => number[]
+) {
+  const rowLength = 1 + SIZE * channels;
+  const pixels = Buffer.alloc(rowLength * SIZE);
+  for (let y = 0; y < SIZE; y += 1) {
+    for (let x = 0; x < SIZE; x += 1) {
+      pixels.set(pixel(x, y), y * rowLength + 1 + x * channels);
+    }
+  }
+  const header = Buffer.alloc(13);
+  header.writeUInt32BE(SIZE, 0);
+  header.writeUInt32BE(SIZE, 4);
+  header[8] = 8;
+  const colorTypes = { 1: 0, 3: 2, 4: 6 } as const;
+  header[9] = colorTypes[channels];
+  return Buffer.concat([
+    Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]),
+    pngChunk('IHDR', header),
+    pngChunk('IDAT', zlib.deflateSync(pixels)),
+    pngChunk('IEND', Buffer.alloc(0)),
+  ]);
+}
+
+const rgbColors = (x: number, y: number) => {
+  if (x < SIZE / 2 && y < SIZE / 2) return [128, 64, 64];
+  if (x >= SIZE / 2 && y < SIZE / 2) return [64, 128, 64];
+  if (x < SIZE / 2) return [192, 32, 64];
+  return [32, 192, 64];
+};
+
+const colorsWithAlpha = (x: number, y: number) => {
+  if (x < SIZE / 2 && y < SIZE / 2) return [255, 0, 0, 255];
+  if (x >= SIZE / 2 && y < SIZE / 2) return [0, 255, 0, 128];
+  if (x < SIZE / 2) return [0, 0, 255, 0];
+  return [255, 255, 0, 255];
+};
+
+export function createRasterFixture(name: string) {
+  if (name === 'rgb-thumbnail.png') return createPng(3, rgbColors);
+  if (name === 'rgba-thumbnail.png') return createPng(4, colorsWithAlpha);
+  if (name === 'gray-thumbnail.png')
+    return createPng(1, (x) => [Math.floor((x * 255) / (SIZE - 1))]);
+  if (name === 'rgb-thumbnail.jpg')
+    return Buffer.from(RGB_JPEG_BASE64, 'base64');
+  throw new Error(`Unknown raster fixture: ${name}`);
+}