Server-side interactive segmentation and stereo for the web client

client/dive-common/components/TrackSettingsPanel.vue

@@ -58,7 +58,7 @@ export default defineComponent({
       help,
       modes,
       typeList,
-      stereoMatchMethods: stereoMatchMethodsFor(isDesktopRuntime()),
+      stereoMatchMethods: computed(() => stereoMatchMethodsFor(isDesktopRuntime())),
     };
   },
 });

client/dive-common/use/stereo/StereoServerMatcher.spec.ts

@@ -0,0 +1,37 @@
+import {
+  describe, expect, it, vi,
+} from 'vitest';
+import StereoServerMatcher, { parseFrameKey } from './StereoServerMatcher';
+
+const RANGE = { min: 0, max: 0 } as never;
+
+describe('StereoServerMatcher', () => {
+  it('reads the frame pair out of the transfer key', () => {
+    expect(parseFrameKey('left>right@12')).toEqual({ source: 'left', target: 'right', frame: 12 });
+    expect(() => parseFrameKey(undefined)).toThrow('which frame');
+  });
+
+  it('asks the server for the points of that frame and marks the misses', async () => {
+    const api = {
+      transferPoints: vi.fn(async () => ({
+        success: true, transferredPoints: [[10, 11], [NaN, NaN]] as [number, number][], validMatches: [true, false],
+      })),
+      setFrame: vi.fn(async () => ({})),
+    };
+    const matcher = new StereoServerMatcher(api, 'foundation');
+    const results = await matcher.warpPoints([[1, 2], [3, 4]], null, null, null, { range: RANGE, frameKey: 'left>right@3' });
+    expect(api.transferPoints).toHaveBeenCalledWith({
+      frame: 3, sourceCamera: 'left', points: [[1, 2], [3, 4]], method: 'foundation',
+    });
+    expect(results.map((r) => r.accepted)).toEqual([true, false]);
+    expect([results[0].x, results[0].y]).toEqual([10, 11]);
+    await matcher.prepare('right>left@7', null, null, null);
+    expect(api.setFrame).toHaveBeenCalledWith(7, 'foundation');
+  });
+
+  it('surfaces the server reason when the warp fails', async () => {
+    const api = { transferPoints: vi.fn(async () => ({ success: false, error: 'no calibration' })), setFrame: vi.fn() };
+    const matcher = new StereoServerMatcher(api, 'ncc');
+    await expect(matcher.warpPoints([[1, 2]], null, null, null, { range: RANGE, frameKey: 'a>b@1' })).rejects.toThrow('no calibration');
+  });
+});

client/dive-common/use/stereo/StereoServerMatcher.ts

@@ -0,0 +1,77 @@
+import type { StereoMatcher, StereoMatchMethod } from './stereoMatcher';
+import type { WarpOptions, WarpResult } from './StereoOnnxMatcher';
+
+/** Identity of the frame pair a warp belongs to, from the transfer's frameKey. */
+export interface FramePair {
+  source: string;
+  target: string;
+  frame: number;
+}
+
+/** The transfer keys frames as `source>target@frame`. */
+export function parseFrameKey(key: string | undefined): FramePair {
+  const match = key?.match(/^(.+?)>(.+?)@(\d+)$/);
+  if (!match) throw new Error('A server-side warp needs to know which frame it is for.');
+  return { source: match[1], target: match[2], frame: Number(match[3]) };
+}
+
+export interface ServerStereoApi {
+  transferPoints(request: {
+    frame: number; sourceCamera: string; points: [number, number][]; method: StereoMatchMethod;
+  }): Promise<{
+    success: boolean; error?: string; transferredPoints?: [number, number][]; validMatches?: boolean[];
+  }>;
+  setFrame(frame: number, method: StereoMatchMethod): Promise<unknown>;
+}
+
+/**
+ * A matcher whose correspondence search runs in VIAME's interactive service
+ * on the server. The frame pixels and rig the transfer hands over are unused:
+ * the server reads the same frames from its own copy of the dataset.
+ */
+export default class StereoServerMatcher implements StereoMatcher {
+  private api: ServerStereoApi;
+
+  private method: StereoMatchMethod;
+
+  constructor(api: ServerStereoApi, method: StereoMatchMethod) {
+    this.api = api;
+    this.method = method;
+  }
+
+  async warpPoints(
+    points: [number, number][],
+    _source: unknown,
+    _target: unknown,
+    _rig: unknown,
+    opts: WarpOptions,
+  ): Promise<WarpResult[]> {
+    if (!points.length) return [];
+    const pair = parseFrameKey(opts.frameKey);
+    const response = await this.api.transferPoints({
+      frame: pair.frame, sourceCamera: pair.source, points, method: this.method,
+    });
+    if (!response.success) {
+      throw new Error(response.error || 'The server could not map the points to the other camera.');
+    }
+    const mapped = response.transferredPoints ?? [];
+    return points.map((_, i) => {
+      const point = mapped[i];
+      const valid = response.validMatches?.[i] !== false
+        && !!point && Number.isFinite(point[0]) && Number.isFinite(point[1]);
+      return {
+        x: valid ? point[0] : NaN,
+        y: valid ? point[1] : NaN,
+        score: valid ? 1 : 0,
+        secondScore: 0,
+        accepted: valid,
+      };
+    });
+  }
+
+  async prepare(frameKey: string, _s: unknown, _t: unknown, _r: unknown, stillWanted?: () => boolean) {
+    const pair = parseFrameKey(frameKey);
+    if (stillWanted && !stillWanted()) return;
+    await this.api.setFrame(pair.frame, this.method);
+  }
+}

client/dive-common/use/stereo/stereoMatcher.ts

@@ -1,3 +1,4 @@
+import { ref } from 'vue';
 /**
  * The contract both correspondence methods satisfy, so the transfer composable
  * and its callers never branch on which one is selected.
@@ -45,8 +46,13 @@ export const STEREO_MATCH_METHODS: { value: StereoMatchMethod; text: string; des
   { value: 'ncc', text: 'Lower Quality, Faster' },
 ];
 
+/** Methods a server-side interactive service can run for the web client. */
+export const serverStereoMethods = ref<StereoMatchMethod[]>([]);
+
 export function stereoMatchMethodsFor(desktop: boolean) {
-  return STEREO_MATCH_METHODS.filter((m) => desktop || !m.desktopOnly);
+  return STEREO_MATCH_METHODS.filter(
+    (m) => desktop || !m.desktopOnly || serverStereoMethods.value.includes(m.value),
+  );
 }
 
 export function isStereoMatchMethod(value: unknown, desktop: boolean): value is StereoMatchMethod {

client/dive-common/use/stereo/useStereoOnnxTransfer.ts

@@ -603,6 +603,7 @@ export default function useStereoOnnxTransfer(config: StereoOnnxTransferConfig)
     warpAllFromCamera,
     warpPoint,
     measureAtFrame,
+    measureAndReport,
     precomputeFrame,
   };
 }

client/platform/web-girder/api/configuration.service.ts

@@ -65,6 +65,11 @@ export interface DiveConfiguration {
   trainingEnabled?: boolean;
   jobsDisabled?: boolean;
   jobsDisabledMessage?: string;
+  /** Server-side interactive segmentation and stereo (the `interactive` service). */
+  interactiveEnabled?: boolean;
+  interactiveMessage?: string;
+  interactiveStereoMethods?: string[];
+  interactiveTextQuery?: boolean;
 }
 
 export interface JobsDisabledConfig {

client/platform/web-girder/api/index.ts

@@ -10,6 +10,7 @@ export * from './configuration.service';
 export * from './dataset.service';
 export { default as loadFrameMetadata } from './frameMetadata.service';
 export * from './girder.service';
+export * from './interactive.service';
 export * from './multicamResolve';
 export * from './rpc.service';
 export * from './scoring.service';

client/platform/web-girder/api/interactive.service.ts

@@ -0,0 +1,174 @@
+/**
+ * Server-side interactive segmentation and stereo: the GPU host runs VIAME's
+ * interactive service and girder resolves dataset frames to its inputs.
+ */
+import type {
+  SegmentationPolygon, SegmentationPolygonKeypointsResponse, SegmentationPredictResponse,
+  SegmentationStereoSegmentResponse, TextQueryResponse,
+} from 'dive-common/apispec';
+import type { StereoMatchMethod } from 'dive-common/use/stereo/stereoMatcher';
+import girderRest from 'platform/web-girder/plugins/girder';
+
+export interface InteractiveStatus {
+  interactiveEnabled: boolean;
+  interactiveMessage: string;
+  interactiveStereoMethods?: StereoMatchMethod[];
+  interactiveTextQuery?: boolean;
+}
+
+interface FrameRequest {
+  datasetId: string;
+  /** Camera name of a multi-camera dataset; ignored for single-camera ones. */
+  camera?: string;
+  frame: number;
+}
+
+export interface InteractivePredictRequest extends FrameRequest {
+  points: [number, number][];
+  pointLabels: number[];
+  box?: [number, number, number, number];
+  line?: [number, number][];
+  multimaskOutput?: boolean;
+}
+
+export interface InteractiveStereoSegmentRequest {
+  datasetId: string;
+  frame: number;
+  sourceCamera: string;
+  points: [number, number][];
+  pointLabels: number[];
+  polygon?: [number, number][];
+  polygons?: SegmentationPolygon[];
+  method?: StereoMatchMethod;
+}
+
+export interface InteractiveTransferPointsRequest {
+  datasetId: string;
+  frame: number;
+  sourceCamera: string;
+  points: [number, number][];
+  strict?: boolean;
+  method?: StereoMatchMethod;
+}
+
+export interface InteractiveTransferPointsResponse {
+  success: boolean;
+  error?: string;
+  transferredPoints?: [number, number][];
+  validMatches?: boolean[];
+  disparityValues?: number[];
+}
+
+export interface InteractiveMeasurement {
+  length: number;
+  midpoint_x: number;
+  midpoint_y: number;
+  midpoint_z: number;
+  midpoint_range: number;
+  stereo_rms: number;
+}
+
+export interface InteractiveTransferLineResponse {
+  success: boolean;
+  error?: string;
+  transferredLine?: [number, number][];
+  length?: number;
+  measurement?: InteractiveMeasurement;
+}
+
+const BASE = 'dive_interactive';
+
+async function post<T>(path: string, body: Record<string, unknown>): Promise<T> {
+  return (await girderRest.post<T>(`${BASE}/${path}`, body)).data;
+}
+
+export async function getInteractiveStatus(): Promise<InteractiveStatus> {
+  return (await girderRest.get<InteractiveStatus>(`${BASE}/status`)).data;
+}
+
+export function interactiveSegmentationPredict(
+  request: InteractivePredictRequest,
+): Promise<SegmentationPredictResponse> {
+  return post<SegmentationPredictResponse>('segmentation/predict', { ...request });
+}
+
+export function interactiveSegmentationKeypoints(datasetId: string, polygon: [number, number][], polygons?: SegmentationPolygon[]): Promise<SegmentationPolygonKeypointsResponse> {
+  return post<SegmentationPolygonKeypointsResponse>('segmentation/keypoints', { datasetId, polygon, polygons });
+}
+
+export async function interactiveStereoSegment(
+  request: InteractiveStereoSegmentRequest,
+): Promise<SegmentationStereoSegmentResponse> {
+  const raw = await post<Record<string, unknown>>('segmentation/stereo_segment', { ...request });
+  return {
+    id: '',
+    success: !!raw.success,
+    error: raw.error as string | undefined,
+    polygon: raw.polygon as SegmentationStereoSegmentResponse['polygon'],
+    polygons: raw.polygons as SegmentationPolygon[] | undefined,
+    bounds: raw.bounds as SegmentationStereoSegmentResponse['bounds'],
+    score: raw.score as number | undefined,
+    seedPoints: raw.seed_points as [number, number][] | undefined,
+    seedLabels: raw.seed_labels as number[] | undefined,
+    generateLine: raw.generate_line as boolean | undefined,
+    lineSource: raw.line_source as SegmentationStereoSegmentResponse['lineSource'],
+    lineOther: raw.line_other as SegmentationStereoSegmentResponse['lineOther'],
+    measurement: raw.measurement as InteractiveMeasurement | undefined,
+  };
+}
+
+export function interactiveTextQuery(
+  request: FrameRequest & { text: string },
+): Promise<TextQueryResponse> {
+  return post<TextQueryResponse>('text_query', { ...request });
+}
+
+export function interactiveStereoSetFrame(datasetId: string, frame: number, method?: StereoMatchMethod): Promise<{ success: boolean; error?: string; disparity_ready?: boolean }> {
+  return post('stereo/set_frame', { datasetId, frame, method });
+}
+
+export async function interactiveStereoTransferPoints(
+  request: InteractiveTransferPointsRequest,
+): Promise<InteractiveTransferPointsResponse> {
+  const raw = await post<Record<string, unknown>>('stereo/transfer_points', { ...request });
+  return {
+    success: !!raw.success,
+    error: raw.error as string | undefined,
+    transferredPoints: raw.transferred_points as [number, number][] | undefined,
+    validMatches: raw.valid_matches as boolean[] | undefined,
+    disparityValues: raw.disparity_values as number[] | undefined,
+  };
+}
+
+export async function interactiveStereoTransferLine(datasetId: string, frame: number, line: [number, number][], method?: StereoMatchMethod): Promise<InteractiveTransferLineResponse> {
+  const raw = await post<Record<string, unknown>>('stereo/transfer_line', {
+    datasetId, frame, line, method,
+  });
+  return {
+    success: !!raw.success,
+    error: raw.error as string | undefined,
+    transferredLine: raw.transferred_line as [number, number][] | undefined,
+    length: raw.length as number | undefined,
+    measurement: raw.measurement as InteractiveMeasurement | undefined,
+  };
+}
+
+export function interactiveStereoMeasureLine(
+  datasetId: string,
+  frame: number,
+  leftLine: [number, number][],
+  rightLine: [number, number][],
+  method?: StereoMatchMethod,
+): Promise<{ success: boolean; error?: string; length?: number; measurement?: InteractiveMeasurement }> {
+  return post('stereo/measure_line', {
+    datasetId, frame, leftLine, rightLine, method,
+  });
+}
+
+export function interactiveStereoAggregateLengths(datasetId: string, lengths: number[], method?: string): Promise<{ success: boolean; error?: string; avg_length?: number }> {
+  return post('stereo/aggregate_lengths', { datasetId, lengths, method });
+}
+
+export async function endInteractiveSession(datasetId: string): Promise<void> {
+  await girderRest.delete(`${BASE}/session`, { params: { datasetId } });
+}