diff --git a/docs/source/_static/camera-viz-controls.svg b/docs/source/_static/camera-viz-controls.svg new file mode 100644 index 0000000000..38c6c98b30 --- /dev/null +++ b/docs/source/_static/camera-viz-controls.svg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bb7b720b541ae019f80d60001bb564d9a00dc399d62c197a81c35cbd3a54bce +size 14015 diff --git a/docs/source/getting_started/televiz.rst b/docs/source/getting_started/televiz.rst index 8f186f2b53..cc205aa069 100644 --- a/docs/source/getting_started/televiz.rst +++ b/docs/source/getting_started/televiz.rst @@ -660,6 +660,14 @@ QuadLayer / CylinderLayer / EquirectLayer - ``submit(left, right=None, stream=0)`` — submit a frame (mono: ``left`` only; stereo: both). - ``set_placement(placement)`` / ``placement()`` — placement swap, thread-safe vs the frame loop. +- ``set_stereo_baseline_mm(mm)`` / ``stereo_baseline_mm`` — live per-eye offset, thread-safe vs + the frame loop; applies on the next frame. Inert while the layer is mono, since ``stereo`` is + fixed at construction. Raises ``ValueError`` on a non-finite value. +- ``set_stereo_convergence_deg(deg)`` / ``stereo_convergence_deg`` (``CylinderLayer`` / + ``EquirectLayer``) — the same idea as a per-eye yaw instead of a translation. Prefer it on a + curved surface: translating one gives full disparity dead ahead and less toward the edges, and + none at all at infinite radius, while a rotation is uniform across the arc and works at any + radius. ``QuadLayer`` accepts ``None`` (fullscreen, window mode); the shaped layers validate and raise ``ValueError`` on bad shape parameters. - ``set_visible(visible)`` / ``is_visible()``. diff --git a/docs/source/references/camera_streaming.rst b/docs/source/references/camera_streaming.rst index ab09069ccb..24a1313c1f 100644 --- a/docs/source/references/camera_streaming.rst +++ b/docs/source/references/camera_streaming.rst @@ -230,6 +230,56 @@ In XR, how a plane follows the operator's head is the per-camera ``lock_mode`` u Lazy-mode knobs live under ``placements.``: ``look_away_angle_deg``, ``reposition_distance``, ``reposition_delay_s``, ``transition_duration_s``. +Controller bindings +^^^^^^^^^^^^^^^^^^^ + +In XR the controllers retune the view live, without editing the YAML and restarting. The right +hand changes how the feed looks; the left, what surface it is mapped onto: + +.. figure:: ../_static/camera-viz-controls.svg + :alt: camera_viz controller bindings, left and right + :width: 100% + + Bindings at a glance. The table below is the same thing in words. + +.. list-table:: + :header-rows: 1 + :widths: 22 78 + + * - Input + - Effect + * - Right stick ←/→ + - Stereo plane gap — how far apart the two eyes' planes sit + (``placements..stereo_plane_distance_cm``). Widening it pushes the scene back + instead of packing it into the space in front of the planes. Stereo cameras only; on + ``equirect``, which has no gap to set, the same stick pans the panorama. + * - Right stick click + - Recenter on your view: an ``equirect`` yaws so the middle of the panorama lands dead + ahead, and a placed surface re-snaps its anchor. + * - ``A`` + - Cycle the lock mode: ``world`` → ``head`` → ``gimbal`` → ``lazy``. + * - ``B`` + - Toggle mono / stereo. Stereo cameras only. + * - ``X`` + - Cycle the shape: ``quad`` → ``cylinder`` → ``equirect``. + * - ``Y`` + - Reset everything to the YAML values. + * - Left stick + - Retunes the active shape: ``quad`` size / vertical position, ``cylinder`` arc width / + vertical position, ``equirect`` horizontal / vertical span. + +Changes apply to every camera at once and appear both on the terminal status panel and on a +head-locked panel in the headset that auto-hides shortly after. Neither toggle reallocates: +``B`` sends the left frame to both eyes, and ``X`` flips visibility between shapes built at +startup, so the extra shapes cost VRAM (reported at startup) rather than a stall on the press. + +The stereo gap is bounded below **divergent parallax** — a gap wider than your IPD would need +the eyes to splay outward — and the headset's measured IPD sets that ceiling, not the config. +The HUD suggests a gap derived from the plane distance and that IPD. + +Bindings, rates and limits live under ``display.controls``; see the +:code-file:`README ` for the full set. + Display surfaces ---------------- @@ -252,7 +302,10 @@ panoramic sources look better on a curved surface: set ``shape`` per camera unde visor). * - ``equirect`` - Full 360°×180° sphere around the operator, for equirectangular panorama / VR-video - sources. Lock modes don't apply. + sources. The middle of the texture points along ``equirect_yaw_deg`` (0 = the + direction the headset last recentered on, positive to the left), which is how a camera + mounted facing another way gets aimed. Lock modes don't apply — the sphere is centred + on the operator and follows them everywhere. Curved shapes exist only in XR mode — the viewer exits with an error in window mode. Stereo sources render per-eye textures on the same surface, and ``stereo_baseline_mm`` adds a per-eye @@ -276,6 +329,31 @@ serving. Useful flags: Run ``camera_viz.py --help`` for the rest (install dir, env-config file, WSS proxy toggle). +Runtime settings themselves are declared in the config rather than exported. The viewer ships +defaults (pose wait off, runtime foveation on), and ``display.cloudxr`` overrides them per +deployment — list only what you want to change: + +.. code-block:: yaml + + display: + cloudxr: + NV_DEVICE_PROFILE: apple-vision-pro + NV_ENABLE_POSE_WAIT: null # null drops a viewer default + +The viewer writes these to a generated ``--cloudxr-env-config`` file, which outranks the process +environment — so a value left in your shell (for example after sourcing +``~/.cloudxr/run/cloudxr.env``, as ``--no-launch-cloudxr-runtime`` suggests) cannot silently +override the config, and neither can it override ``--cloudxr-device-profile``. Booleans are +written in the lowercase spelling the runtime's parser recognises; the launcher-computed keys +(``XR_RUNTIME_JSON``, ``NV_CXR_RUNTIME_DIR``, ``NV_CXR_OUTPUT_DIR``, ``XRT_NO_STDIN``) are +rejected by name. Passing ``--cloudxr-env-config`` yourself takes precedence over the generated +file. + +An unknown variable name warns with a suggestion before the runtime launches, and is passed +through regardless — the runtime has more settings than the viewer lists. To confirm what the +runtime resolved, read the settings dump at the top of the newest +``~/.cloudxr/logs/cxr_server.*.log``. + .. _split-mode: Split mode — robot → workstation over RTP @@ -363,8 +441,9 @@ its own plane (and, in split mode, its own RTP port). Abbreviated: lock_mode: lazy # world | head | lazy | gimbal distance: 1.5 # size: [w_m, h_m] - # stereo_baseline_mm: 0 + # stereo_plane_distance_cm: 0 # gap between the eyes' planes # shape: quad # quad | cylinder | equirect (cylinder/equirect are XR-only) + # equirect_yaw_deg: 0.0 # equirect: heading the middle of the feed points at # compositor: openxr # openxr (default) | televiz — quads only # cylinder_radius_m: 2.0 # cylinder_angle_deg: 90 diff --git a/examples/camera_viz/README.md b/examples/camera_viz/README.md index e0b25a2f15..58454b66e3 100644 --- a/examples/camera_viz/README.md +++ b/examples/camera_viz/README.md @@ -129,8 +129,10 @@ display: # camera_viz only offset_x: 0.0 offset_y: 0.0 # size: [w_m, h_m] - # stereo_baseline_mm: 0 # stereo cams: 0 = both eyes share the world quad - # (parallax from the frames); ~65 = virtual IPD push + # stereo_plane_distance_cm: 0 # stereo cams: gap between the left- and + # right-eye planes; 0 = both eyes share + # one plane. See "Stereo plane distance" + # equirect_yaw_deg: 0.0 # equirect: heading the middle of the feed points at # shape: quad # quad (default) | cylinder | equirect — XR only for # the curved shapes # compositor: openxr # openxr (default) | televiz — quads only @@ -169,15 +171,147 @@ display: Lazy knobs under `placements.`: `look_away_angle_deg`, `reposition_distance`, `reposition_delay_s`, `transition_duration_s`. +## Controller bindings (XR) + +Retune the view without editing YAML and restarting. The right hand changes how the feed looks, the left what surface it's on. Quest and Pico report these identically. + +| Input | Effect | +|---|---| +| **Right stick** ←/→ | Stereo plane gap. Stereo cameras only — on `equirect`, pans instead | +| **Right stick click** | Recenter on your view | +| **A** | Lock mode: `world` → `head` → `gimbal` → `lazy` | +| **B** | Mono / stereo. Stereo cameras only | +| **X** | Shape: `quad` → `cylinder` → `equirect` | +| **Y** | Reset everything to the YAML values | +| **Left stick** | Per shape, below | + +| Shape | ←/→ | ↑/↓ | +|---|---|---| +| `quad` | size, aspect preserved | slide up / down | +| `cylinder` | arc width | slide up / down | +| `equirect` | horizontal span | vertical span | + +Recentering reads per shape: an `equirect` yaws so the middle of the panorama lands dead ahead, and a placed surface re-snaps its anchor — the way back from a `world`-locked plane left in another part of the room. + +Changes apply to every camera at once and appear on the status panel and an in-headset HUD that auto-hides ~2.5 s later (`hud: false` to disable). Nothing is written back to the YAML. + +Neither toggle reallocates: **B** sends the left frame to both eyes, **X** flips visibility between shapes all built at startup. The cost is VRAM — two extra layers per camera, reported at startup; `shape_switching: false` keeps only the configured shape. + +A quad's `distance` and a cylinder's `cylinder_radius_m` stay YAML-only. Apparent size is `2·atan((w/2)/d)` and arc width is `radius × angle`, so moving either surface further away enlarges it by the same factor and looks identical — they'd duplicate the size axis. What they *do* change is the real distance to the surface, which the stereo gap works from. + +```yaml +display: + controls: + enabled: true # false disables the bindings + hud: true # in-headset readout + shape_switching: true # keep all 3 shapes resident for X + deadzone: 0.2 # stick rest-position tolerance + plane_distance_rate_cm_per_s: 2.0 # held-stick ramp rates + size_rate_m_per_s: 0.5 + offset_rate_m_per_s: 0.5 + angle_rate_deg_per_s: 40.0 +``` + +Limits — `plane_distance_min_cm` / `plane_distance_max_cm`, `size_range_m`, `offset_y_range_m`, `cylinder_angle_range_deg`, `equirect_h_range_deg`, `equirect_v_half_range_deg` — all default inside what the layers accept. See `configs/zed.yaml`. + +### Stereo plane distance + +A stereo layer draws each eye's image on its own plane; `stereo_plane_distance_cm` is the gap between them. + +```yaml +display: + placements: + zed: + distance: 1.0 # how far away the planes are + stereo_plane_distance_cm: 5.0 # how far apart they are +``` + +At `0` both eyes share one plane, so the whole scene — near objects and far background alike — is packed into the space between you and `distance`. Widening the gap pushes it back and lets it spread out. Applied exactly as given (×10 into the layer's `stereo_baseline_mm`); the stick moves it in 0.1 cm steps. + +The HUD suggests a value from the plane distance and the headset's measured IPD, shown beside the one in use (`5.0/5.2` on the panel). Advice only — the stick sets the value, which is also how you correct a headset whose IPD setting doesn't match your eyes. + +The stick can't reach **divergent parallax**: at a gap equal to your IPD the eyes' rays are parallel, and beyond it they would have to splay outward — the classic cause of stereo eye strain. The ceiling comes from the measured IPD, not the config. + +**B** parks the gap at zero while mono, since one image on two separated planes would only shift its depth, and restores it on the way back. + +Not applicable to `equirect`: the gap shifts each eye's surface, and the sphere sits at infinite radius where translating it does nothing, so the stick skips it there and pans instead. Curved layers have a rotation-based equivalent in Televiz (`stereo_convergence_deg`), which is uniform across the arc and works at any radius. + +> **Not the camera's baseline.** That is the physical gap between the camera's two lenses — fixed in hardware, baked into the pixels, and what sets the scene's depth *scale*. This only moves where that scene sits. + +> Live adjustment needs `Layer.set_stereo_baseline_mm`, newer than the released `isaacteleop` wheel. On an older wheel that one binding disables itself with a notice; A, B, X and Y still work. + +### Aiming an equirect + +The middle of the texture maps to the sphere pose's **−z**, and the middle row to its horizon. With no rotation that is the reference space's forward — wherever the headset faced when it last recentered — so a camera mounted facing another way puts the interesting part of the feed off to one side. + +```yaml +display: + placements: + sky: + shape: equirect + equirect_yaw_deg: -90.0 # heading the middle of the feed points at +``` + +Positive is a turn to the left. The right stick pans it live and the stick click snaps it to your view; **Y** puts it back to the YAML value. The heading in use rides in the panel's `shape` column (`equirect +35°`). + +The sphere has no lock mode: it is centred on you and follows your head everywhere, so `world` / `head` / `lazy` and the `distance` / `size` knobs are all blank for it. This heading is its whole placement. + --- +## CloudXR runtime settings + +The runtime is configured by environment variables, and there are three ways to set one and be silently ignored: a name nothing reads, a stale `source ~/.cloudxr/run/cloudxr.env` in your shell (which outranks both `os.environ` and `--cloudxr-device-profile`), and a boolean spelled `False` — which the runtime's exact-match parser doesn't recognise, so it reads as **true**. + +camera_viz ships defaults in `cloudxr_env.DEFAULT_ENV` — pose wait off, runtime foveation on — so a config that says nothing gets both. `display.cloudxr` is a per-deployment override; list only what you want to change: + +```yaml +display: + cloudxr: + NV_DEVICE_PROFILE: apple-vision-pro + NV_ENABLE_POSE_WAIT: null # null drops a camera_viz default +``` + +camera_viz writes these to a generated `--cloudxr-env-config` file, the one tier that outranks the process environment. Booleans are lowercased for you; the launcher-computed keys (`XR_RUNTIME_JSON`, `NV_CXR_RUNTIME_DIR`, `NV_CXR_OUTPUT_DIR`, `XRT_NO_STDIN`) are refused by name. Passing `--cloudxr-env-config` yourself wins — camera_viz won't overwrite it. + +An unknown variable name warns with a suggestion before anything launches — `NV_CXR_DEVICE_PROFILE` is not a name the runtime reads, and it's one keystroke from one that is — but is still passed through, since the runtime has more knobs than camera_viz lists. + +To confirm what the runtime actually resolved, read the settings dump at the top of the newest `~/.cloudxr/logs/cxr_server.*.log`. Monado's `DEBUG_GET_ONCE` options (`NV_ENABLE_POSE_WAIT` among them) aren't in that dump; run with `XRT_PRINT_OPTIONS=true` and each is logged as `NAME=value (raw)`, where `(nil)` means it never arrived. + +> `NV_DEVICE_PROFILE` is device matching, not tuning. Anything other than `quest3` selects separate-frames packing, and `isFrameClientReconstructed()` bails unless packing is packed-frame — so a non-Quest profile gives up client-reconstructed streaming. + +--- + +## Status panel + +On a terminal, camera_viz redraws a snapshot in place instead of scrolling a log: + +``` +camera_viz xr · local · 1 camera +──────────────────────────────────────────────────────────────────── + render 58.0 fps (target 72) missed 0 gpu 2.1 ms + + camera shape lock eyes size m height m planes cm submit/s + zed cylinder lazy stereo 1.00 +0.00 5.0/5.2 64.0 + + headset IPD 63 mm + + stereo planes: 5.0 cm · suggested: 5.2 cm · IPD: 63 mm +``` + +`planes cm` is the value in use and the suggestion. `-` means the field doesn't apply — equirect has no gap, window mode has no controls. While the panel is live it owns stderr: control messages and source lifecycle lines go through it rather than printing underneath and pushing it out of alignment. When stderr isn't a terminal (piped, or a `deploy`ed systemd unit) it falls back to one line every 5 s with the same numbers. + ## Layout ``` camera_viz/ ├── camera_viz.sh — CLI: setup / loopback / run / deploy / service-* -├── camera_viz.py — receiver / viewer +├── camera_viz.py — receiver / viewer (entrypoint + wiring) ├── camera_streamer.py — robot-side RTP sender (per-camera supervisor) +├── config.py — YAML → SourceEntry: parse + validate, no allocation +├── display.py — VizSession + one layer per surface +├── cloudxr_env.py — display.cloudxr → the runtime's env file +├── dashboard.py — terminal status panel +├── controls/ — XR controller bindings, shapes, stereo geometry, HUD ├── pipeline/ — source ABC + threaded runner ├── placements/ — XR lock-mode strategies ├── sources/ — V4L2 / OAK-D / ZED / synthetic / video replay / rtp_h264 diff --git a/examples/camera_viz/camera_viz.py b/examples/camera_viz/camera_viz.py index 06ebe2d13b..1bc0176226 100755 --- a/examples/camera_viz/camera_viz.py +++ b/examples/camera_viz/camera_viz.py @@ -19,380 +19,141 @@ import argparse import contextlib -import math import signal import sys -from dataclasses import dataclass +import tempfile from pathlib import Path -from typing import List, Optional, Tuple +from typing import List, Optional import yaml -import isaacteleop.viz as viz from isaacteleop.cloudxr import CloudXRLauncher -from pipeline import FrameSource, VizRunner -from placements import PlacementConfig, PlacementStrategy, build as build_placement -from sources import ( - PairedFrameSource, - RtpH264Source, - build_local_camera, - resolve_video_paths, - set_verbose, +import cloudxr_env +import config +import display +from controls import ( + ControllerControls, + ControlTarget, + controls_config_from_yaml, + make_hud, ) +from dashboard import Dashboard +from pipeline import VizRunner +from sources import resolve_video_paths, set_notify_sink, set_verbose -@dataclass -class SourceEntry: - """source + placement + stereo + shape cfg; drives layer construction.""" - - source: FrameSource - placement: Optional[PlacementStrategy] - stereo: bool = False - stereo_baseline_mm: float = 0.0 - # display.placements..shape: quad | cylinder | equirect. - shape: str = "quad" - # Who composites the layer (display.placements..compositor): - # "openxr" (default — the OpenXR runtime) or "televiz" (built-in - # compositor; quads only). - compositor: str = "openxr" - # Cylinder shape parameters (display.placements.). - cylinder_radius_m: float = 2.0 - cylinder_angle_deg: float = 90.0 - - -_VALID_SHAPES = ("quad", "cylinder", "equirect") -_VALID_COMPOSITORS = ("openxr", "televiz") - -# Every key the placements. block understands (lock-mode strategy -# knobs + surface-shape keys). Unknown keys warn instead of silently -# falling back to defaults — a typo'd `cylinder_radius` should not run -# with a 2 m default and no hint. -_KNOWN_PLACEMENT_KEYS = frozenset( - { - "lock_mode", - "distance", - "offset_x", - "offset_y", - "look_away_angle_deg", - "reposition_distance", - "reposition_delay_s", - "transition_duration_s", - "size", - "stereo_baseline_mm", - "shape", - "compositor", - "cylinder_radius_m", - "cylinder_angle_deg", - } -) - - -def _warn_unknown_placement_keys(cam_name: str, pspec: dict) -> None: - import difflib +def _parse_args(argv: Optional[list[str]]): + parser = argparse.ArgumentParser(description="Televiz camera_viz — display side") + parser.add_argument("config", type=Path, help="YAML config file") + parser.add_argument( + "--mode", + choices=("window", "xr"), + default=None, + help="Override display.mode from the config " + "(default: the config's value, or xr when the config omits it).", + ) + CloudXRLauncher.add_launcher_arguments(parser) + return parser.parse_args(argv) - for key in pspec: - if key in _KNOWN_PLACEMENT_KEYS: - continue - hint = difflib.get_close_matches(key, _KNOWN_PLACEMENT_KEYS, n=1) - suggestion = f" (did you mean {hint[0]!r}?)" if hint else "" - print( - f"camera_viz: warning: placements.{cam_name}: unknown key " - f"{key!r}{suggestion} — ignored", - file=sys.stderr, - flush=True, - ) +def _build_display(session, entries, switch_shapes: bool): + """Parallel arrays, one entry per camera stream. -def _shape_for(cam_name: str, placements_cfg: dict) -> Tuple[str, bool, float, float]: - """Per-camera surface config from ``display.placements.``: - ``shape`` (quad | cylinder | equirect, default quad), ``compositor`` - (openxr — the default — or televiz; quads only), ``cylinder_radius_m`` - / ``cylinder_angle_deg`` (cylinder only).""" - pspec = placements_cfg.get(cam_name) or {} - _warn_unknown_placement_keys(cam_name, pspec) - shape = str(pspec.get("shape", "quad")).lower() - if shape not in _VALID_SHAPES: - raise ValueError( - f"camera_viz: placements.{cam_name}.shape must be one of " - f"{'|'.join(_VALID_SHAPES)}, got {shape!r}" - ) - compositor = str(pspec.get("compositor", "openxr")).lower() - if compositor not in _VALID_COMPOSITORS: - raise ValueError( - f"camera_viz: placements.{cam_name}.compositor must be " - f"{'|'.join(_VALID_COMPOSITORS)}, got {compositor!r}" - ) - if compositor == "televiz" and shape != "quad": - raise ValueError( - f"camera_viz: placements.{cam_name}: compositor: televiz only " - f"applies to shape: quad — {shape} layers are composited by the " - "OpenXR runtime always." + They stay parallel because the controls mutate them in place: a shape + switch rewrites ``layers[i]`` and a lock-mode change ``strategies[i]``, + and the runner reads the same lists. + """ + sources, layers, strategies, shape_layers = [], [], [], [] + for entry in entries: + per_shape = display.add_layers(session, entry, switch_shapes) + sources.append(entry.source) + shape_layers.append(per_shape) + layers.append(per_shape[entry.shape]) + # Lock-mode strategies reposition quads AND cylinders (the runner + # adapts the pose to the cylinder's head-anchored center). An + # equirect sphere is centred on the operator with nothing to + # re-snap, so the runner skips it by layer type -- the strategy is + # still kept here, because switching away from equirect needs it. + strategies.append(entry.placement) + return sources, layers, strategies, shape_layers + + +def _build_controls( + session, entries, layers, shape_layers, strategies, controls_cfg, dashboard +): + """Wire the XR controller bindings to the layers they drive. + + ``strategies`` is handed over as-is rather than copied: the controls swap + entries in place on a lock-mode change and the runner reads the same list. + """ + targets = [ + ControlTarget( + name=e.source.spec.name, + layer=layer, + shape=e.shape, + stereo=e.stereo, + plane_distance_cm=e.stereo_plane_distance_cm, + lock_mode=e.lock_mode, + placement_config=e.placement_config, + shape_layers=per_shape, + cylinder_radius_m=e.cylinder_radius_m, + cylinder_angle_deg=e.cylinder_angle_deg, + equirect_yaw_deg=e.equirect_yaw_deg, ) - radius_m = float(pspec.get("cylinder_radius_m", 2.0)) - angle_deg = float(pspec.get("cylinder_angle_deg", 90.0)) - return shape, compositor, radius_m, angle_deg - + for e, layer, per_shape in zip(entries, layers, shape_layers) + ] + # Added last so it composites over the feeds (insertion order is blend + # order for runtime-composited layers). + hud = make_hud(session, controls_cfg.hud) + return ControllerControls( + session, + targets, + strategies, + controls_cfg, + hud=hud, + # The panel owns stderr while it is live; a second writer there lands + # inside it and leaves every repaint misaligned. + log_to_stderr=not dashboard.live, + ) -_VALID_LOCK_MODES = ("world", "head", "lazy", "gimbal") +def _render_extent_note(session, entries) -> List[str]: + """The per-eye extent the runtime asked us to render at. -def _build_placement(spec: Optional[dict], is_xr: bool) -> Optional[PlacementStrategy]: - if spec is not None: - # Validate in every display mode — a typo'd lock_mode shouldn't - # silently become lazy (XR) or pass unnoticed (window). - lock_mode = str(spec.get("lock_mode", "lazy")).lower() - if lock_mode not in _VALID_LOCK_MODES: - raise ValueError( - f"camera_viz: lock_mode must be {'|'.join(_VALID_LOCK_MODES)}, " - f"got {lock_mode!r}" - ) - if not is_xr or spec is None: - return None - cfg_kwargs = {} - if "size" in spec: - cfg_kwargs["size_meters"] = tuple(spec["size"]) - for key in ( - "distance", - "offset_x", - "offset_y", - "look_away_angle_deg", - "reposition_distance", - "reposition_delay_s", - "transition_duration_s", - ): - if key in spec: - cfg_kwargs[key] = spec[key] - cfg = PlacementConfig(**cfg_kwargs) - return build_placement(spec.get("lock_mode", "lazy"), cfg) - - -def _enabled_cameras(cfg: dict) -> List[dict]: - return [c for c in cfg.get("cameras", []) if c.get("enabled", True)] - - -# Default plane width when ``size`` is omitted from a placement block. -# Height is derived from the camera's pixel aspect ratio so the rendered -# plane keeps the picture's shape. -_DEFAULT_PLANE_WIDTH_M = 1.0 - - -def _placement_with_aspect( - spec: Optional[dict], width: int, height: int, is_xr: bool -) -> Optional[PlacementStrategy]: - """Build the placement, filling in ``size`` from the source's aspect - ratio when the YAML doesn't pin it. Width defaults to 1.0 m so a - 16:9 source lands at 1.0 x 0.5625, a 3.55:1 SBS at 1.0 x 0.281.""" - if spec is not None and "size" not in spec: - spec = { - **spec, - "size": [_DEFAULT_PLANE_WIDTH_M, _DEFAULT_PLANE_WIDTH_M * height / width], - } - return _build_placement(spec, is_xr) - - -def _stereo_for(cam: dict, placements_cfg: dict) -> Tuple[bool, float]: - """``cameras..stereo`` (producer toggle) + ``placements..stereo_baseline_mm``.""" - stereo = bool(cam.get("stereo", False)) - pspec = placements_cfg.get(cam["name"]) or {} - baseline_mm = float(pspec.get("stereo_baseline_mm", 0.0)) - return stereo, baseline_mm - - -def _build_local_entries(cfg: dict, is_xr: bool) -> List[SourceEntry]: - """source=local: open each enabled camera directly.""" - placements_cfg = cfg.get("display", {}).get("placements", {}) - entries: List[SourceEntry] = [] - for cam in _enabled_cameras(cfg): - cam_sources = build_local_camera(cam) - # Aspect comes from the built source's spec, not the YAML — video - # sources may omit width/height and size themselves from the file. - first = cam_sources[0].spec - placement = _placement_with_aspect( - placements_cfg.get(cam["name"]), first.width, first.height, is_xr - ) - stereo, baseline_mm = _stereo_for(cam, placements_cfg) - shape, compositor, radius_m, angle_deg = _shape_for(cam["name"], placements_cfg) - for source in cam_sources: - entries.append( - SourceEntry( - source=source, - placement=placement, - stereo=stereo, - stereo_baseline_mm=baseline_mm, - shape=shape, - compositor=compositor, - cylinder_radius_m=radius_m, - cylinder_angle_deg=angle_deg, - ) - ) - return entries - - -def _build_rtp_entries(cfg: dict, is_xr: bool) -> List[SourceEntry]: - """One RTP listener per camera; stereo uses rtp.port + rtp.port_right - and pairs them at the receiver (no wire-level sync — drift OK).""" - placements_cfg = cfg.get("display", {}).get("placements", {}) - entries: List[SourceEntry] = [] - for cam in _enabled_cameras(cfg): - rtp = cam.get("rtp", {}) - if "port" not in rtp: - raise ValueError( - f"camera_viz: camera {cam.get('name')!r} missing rtp.port; " - "required when source: rtp" - ) - if "width" not in cam or "height" not in cam: - raise ValueError( - f"camera_viz: camera {cam.get('name')!r} needs explicit " - "width/height when source: rtp — the receiver sizes its " - "decoder from the YAML, not from the wire" - ) - placement = _placement_with_aspect( - placements_cfg.get(cam["name"]), - int(cam["width"]), - int(cam["height"]), - is_xr, - ) - stereo, baseline_mm = _stereo_for(cam, placements_cfg) - - if stereo: - if "port_right" not in rtp: - raise ValueError( - f"camera_viz: stereo camera {cam.get('name')!r} missing " - "rtp.port_right (required when stereo + source: rtp)" - ) - left = RtpH264Source( - name=f"{cam['name']}.left", - width=int(cam["width"]), - height=int(cam["height"]), - port=int(rtp["port"]), - rtp_buffer_size=int(rtp.get("rtp_buffer_size", 212992)), - gpu_id=int(rtp.get("gpu_id", 0)), - ) - right = RtpH264Source( - name=f"{cam['name']}.right", - width=int(cam["width"]), - height=int(cam["height"]), - port=int(rtp["port_right"]), - rtp_buffer_size=int(rtp.get("rtp_buffer_size", 212992)), - gpu_id=int(rtp.get("gpu_id", 0)), - ) - source: FrameSource = PairedFrameSource( - name=cam["name"], left=left, right=right - ) - else: - source = RtpH264Source( - name=cam["name"], - width=int(cam["width"]), - height=int(cam["height"]), - port=int(rtp["port"]), - rtp_buffer_size=int(rtp.get("rtp_buffer_size", 212992)), - gpu_id=int(rtp.get("gpu_id", 0)), - ) - - shape, compositor, radius_m, angle_deg = _shape_for(cam["name"], placements_cfg) - entries.append( - SourceEntry( - source=source, - placement=placement, - stereo=stereo, - stereo_baseline_mm=baseline_mm, - shape=shape, - compositor=compositor, - cylinder_radius_m=radius_m, - cylinder_angle_deg=angle_deg, - ) - ) - return entries - - -def _make_session(cfg: dict, mode_override: Optional[str] = None) -> viz.VizSession: - display = cfg.get("display", {}) - # --mode overrides display.mode when given. - mode_str = (mode_override or display.get("mode", "xr")).lower() - session_cfg = viz.VizSessionConfig() - if mode_str == "window": - session_cfg.mode = viz.DisplayMode.kWindow - w = display.get("window", {}) - session_cfg.window_width = int(w.get("width", 1280)) - session_cfg.window_height = int(w.get("height", 720)) - elif mode_str == "xr": - session_cfg.mode = viz.DisplayMode.kXr - x = display.get("xr", {}) - session_cfg.xr_near_z = float(x.get("near_z", 0.05)) - session_cfg.xr_far_z = float(x.get("far_z", 100.0)) - else: - raise ValueError( - f"camera_viz: display.mode must be window|xr, got {mode_str!r}" - ) - if "clear_color" in display: - session_cfg.clear_color = tuple(display["clear_color"]) - session_cfg.app_name = display.get("app_name", "camera_viz") - return viz.VizSession.create(session_cfg) - - -def _add_layer(session: viz.VizSession, entry: SourceEntry): - """Register one layer for ``entry`` per its ``shape`` (from - ``display.placements.`` in the YAML). - - quad → QuadLayer, composited by the OpenXR runtime by default - (``compositor: televiz`` opts into the built-in compositor); - the placement strategy positions it per frame. - cylinder → CylinderLayer: the feed wrapped on an arc facing the user - (``cylinder_radius_m`` / ``cylinder_angle_deg``, aspect from - the source). Runtime-composited always. - equirect → EquirectLayer: full 360x180 sphere (the source is expected - to be an equirect panorama). Runtime-composited always. + This is the number that says whether a foveation + NV_CXR_RUNTIME_FOVEATION_UNWARPED_WIDTH override reached the app: the + runtime hands it back through xrEnumerateViewConfigurationViews. Shown + next to the widest source so the ratio is readable -- a feed wider than + the extent is being minified before it is ever encoded, which is what + aliases. """ - spec = entry.source.spec - if entry.shape == "cylinder": - layer_cfg = viz.CylinderLayerConfig() - layer_cfg.name = spec.name - layer_cfg.resolution = viz.Resolution(spec.width, spec.height) - layer_cfg.stereo = entry.stereo - layer_cfg.stereo_baseline_mm = entry.stereo_baseline_mm - # aspect_ratio 0 = derived from the source resolution (square texels). - layer_cfg.placement = viz.CylinderLayerPlacement( - radius_m=entry.cylinder_radius_m, - central_angle_rad=math.radians(entry.cylinder_angle_deg), - ) - return session.add_cylinder_layer(layer_cfg) - if entry.shape == "equirect": - layer_cfg = viz.EquirectLayerConfig() - layer_cfg.name = spec.name - layer_cfg.resolution = viz.Resolution(spec.width, spec.height) - layer_cfg.stereo = entry.stereo - # Baseline only matters at finite sphere radius; harmless at the - # default infinite-radius placement (full 360x180 sphere). - layer_cfg.stereo_baseline_mm = entry.stereo_baseline_mm - return session.add_equirect_layer(layer_cfg) - - layer_cfg = viz.QuadLayerConfig() - layer_cfg.name = spec.name - layer_cfg.resolution = viz.Resolution(spec.width, spec.height) - layer_cfg.format = viz.PixelFormat.kRGBA8 - if entry.stereo: - layer_cfg.stereo = True - layer_cfg.stereo_baseline_mm = entry.stereo_baseline_mm - # OpenXR-runtime composition is the default (kXr only; window mode is - # always composited by Televiz). Requires a placement, which the - # placement strategy applies below. - layer_cfg.openxr_composition = entry.compositor == "openxr" - return session.add_quad_layer(layer_cfg) + get = getattr(session, "get_recommended_resolution", None) + if get is None: # older isaacteleop wheel + return [] + try: + resolution = get() + width, height = resolution.width, resolution.height + except Exception as exc: # noqa: BLE001 -- a note must not be fatal... + # ...but it must not vanish either: swallowing this silently is why + # the line went missing when get_recommended_resolution turned out to + # return a Resolution rather than a tuple. + print(f"camera_viz: render extent unavailable: {exc!r}", file=sys.stderr) + return [] + note = f"render extent {width}x{height} per eye" + widest = max((e.source.spec.width for e in entries), default=0) + if widest: + note += f"; widest source {widest} px ({width / widest:.2f}x)" + # Printed here as well as returned: this is the one number that says + # whether a foveation override reached the app, and the status panel + # shows it dimmed on its last line, where it is easy to miss and gone + # entirely if the run dies later in startup. + print(f"camera_viz: {note}", file=sys.stderr, flush=True) + return [note] def main(argv: Optional[list[str]] = None) -> int: - parser = argparse.ArgumentParser(description="Televiz camera_viz — display side") - parser.add_argument("config", type=Path, help="YAML config file") - parser.add_argument( - "--mode", - choices=("window", "xr"), - default=None, - help="Override display.mode from the config " - "(default: the config's value, or xr when the config omits it).", - ) - CloudXRLauncher.add_launcher_arguments(parser) - args = parser.parse_args(argv) + args = _parse_args(argv) with open(args.config) as f: cfg = yaml.safe_load(f) @@ -411,29 +172,56 @@ def main(argv: Optional[list[str]] = None) -> int: raise ValueError(f"camera_viz: source must be local|rtp, got {source_mode!r}") effective_mode = (args.mode or cfg.get("display", {}).get("mode", "xr")).lower() - # Shaped layers (display.placements..shape) are composited by - # the OpenXR runtime, so they need XR mode and a current isaacteleop — - # fail fast, before launching the runtime, when either is missing. - placements_cfg = cfg.get("display", {}).get("placements", {}) - for cam in _enabled_cameras(cfg): - shape, _, _, _ = _shape_for(cam["name"], placements_cfg) - if shape == "quad": - continue - if effective_mode != "xr": - raise SystemExit( - f"camera_viz: placements.{cam['name']}.shape: {shape} is " - "composited by the OpenXR runtime and requires XR mode; " - "use --mode xr or shape: quad in window mode." - ) - - # In XR mode, attach to the CloudXR runtime (+ WSS proxy for headset - # clients) before creating the session — VizSession's OpenXR instance - # needs XR_RUNTIME_JSON and a live runtime, both of which the launcher - # provides. Window mode never touches a runtime. + config.check_shapes_are_displayable(cfg, effective_mode) + + # CloudXR runtime settings (display.cloudxr) go through a generated + # --cloudxr-env-config file rather than os.environ: an env file is the one + # tier that outranks a stale `source ~/.cloudxr/run/cloudxr.env` in the + # shell, which otherwise beats both a setdefault here and the launcher's + # own --cloudxr-device-profile. See cloudxr_env for the rest of the + # reasoning. No effect under --no-launch-cloudxr-runtime: that runtime + # already started with whatever environment it was given. + cloudxr_settings = {} + if effective_mode == "xr": + cloudxr_settings = cloudxr_env.env_from_yaml(cfg.get("display", {})) + + # In XR mode, launch the in-process CloudXR runtime (+ WSS proxy for + # headset clients) before creating the session — VizSession's OpenXR + # instance needs XR_RUNTIME_JSON + a running service, both of which the + # launcher provides. --no-launch-cloudxr-runtime skips this when a + # runtime is already up (e.g. after sourcing ~/.cloudxr/run/cloudxr.env). + # Window mode never launches a runtime. # Entered manually (not ``with``) so the unclean-stop path below can - # SKIP the teardown: stopping a runtime this process owns while a worker - # thread is still inside session.render() would rip the OpenXR service - # out from under a live xrWaitFrame. + # SKIP the teardown: stopping the runtime while a worker thread is + # still inside session.render() would rip the OpenXR service out from + # under a live xrWaitFrame — the same hazard the skip-destroy + # mitigation exists for. The launcher registers an atexit stop, which + # fires once the stuck (non-daemon) thread finally exits. + settings_stack = contextlib.ExitStack() + if effective_mode == "xr" and not args.launch_cloudxr_runtime: + # Nothing is launched, so nothing carries these to the runtime that is + # already up -- it started with whatever environment it was given. + extra = set(cloudxr_settings) - set(cloudxr_env.DEFAULT_ENV) + if extra or cloudxr_settings != cloudxr_env.DEFAULT_ENV: + print( + "camera_viz: warning: display.cloudxr is ignored under " + "--no-launch-cloudxr-runtime; set those variables in the " + "environment of the runtime process instead", + file=sys.stderr, + flush=True, + ) + if effective_mode == "xr" and args.launch_cloudxr_runtime: + # An explicit --cloudxr-env-config is the operator overriding the + # config file; do not overwrite it with the generated one. + if getattr(args, "cloudxr_env_config", None) is None: + work_dir = Path(settings_stack.enter_context(tempfile.TemporaryDirectory())) + args.cloudxr_env_config = str( + cloudxr_env.write_env_file(cloudxr_settings, work_dir / "cloudxr.env") + ) + # A null context, not the launcher's NoopContext: that landed after the + # isaacteleop this sample pins, and importing it would break the sample + # against its own wheel. Swap it in (and drop the None guard below) once + # the pin catches up. launch_ctx = ( CloudXRLauncher.launch_context(args) if effective_mode == "xr" @@ -442,34 +230,82 @@ def main(argv: Optional[list[str]] = None) -> int: launcher = launch_ctx.__enter__() stop_launcher = True try: - session = _make_session(cfg, mode_override=args.mode) + controls_cfg = controls_config_from_yaml(cfg.get("display", {})) + # Window mode has no controllers, so don't ask for their extensions. + want_controls = controls_cfg.enabled and effective_mode == "xr" + session = display.make_session( + cfg, + mode_override=args.mode, + required_extensions=( + ControllerControls.required_extensions() if want_controls else None + ), + ) is_xr = session.is_xr_mode() if source_mode == "local": - entries = _build_local_entries(cfg, is_xr) + entries = config.build_local_entries(cfg, is_xr) else: - entries = _build_rtp_entries(cfg, is_xr) - - # Build sources, layers, and placement strategies in parallel arrays. - sources, layers, strategies = [], [], [] - for entry in entries: - sources.append(entry.source) - layers.append(_add_layer(session, entry)) - # Placement lock-mode strategies reposition quads AND cylinders - # (the runner adapts the pose to the cylinder's head-anchored - # center); an equirect sphere has nothing to re-snap. - strategies.append( - entry.placement if entry.shape in ("quad", "cylinder") else None + entries = config.build_rtp_entries(cfg, is_xr) + + # Shape switching needs every shape resident, and the shaped layers + # are XR-only, so it is off outside XR regardless of the config. + switch_shapes = want_controls and is_xr and controls_cfg.shape_switching + + sources, layers, strategies, shape_layers = _build_display( + session, entries, switch_shapes + ) + + cameras = f"{len(sources)} camera" + ("s" if len(sources) != 1 else "") + header = f"{effective_mode} · {source_mode} · {cameras}" + notes = _render_extent_note(session, entries) if is_xr else [] + if switch_shapes: + extra = sum( + display.estimate_layer_bytes(e, shape) + for e in entries + for shape in config.VALID_SHAPES + if shape != e.shape + ) + notes.append( + f"shape switching on — {len(config.VALID_SHAPES) - 1} extra layer(s) per " + f"camera, about {extra / (1024 * 1024):.0f} MiB additional VRAM" ) - shapes = ",".join(sorted({e.shape for e in entries})) or "quad" - print( - f"camera_viz: source={source_mode}, mode={effective_mode}, " - f"xr={is_xr}, shapes={shapes}, {len(sources)} layer(s)", - flush=True, + # Built before the controls and the sources' notifications are + # rerouted: while it is live it owns stderr, and a second writer on + # that stream lands inside the panel (see Dashboard.note). + dashboard = Dashboard() + if dashboard.live: + set_notify_sink(dashboard.note) + else: + # Nothing is redrawing, so the header would never be seen. + print(f"camera_viz: {header}", flush=True) + for note in notes: + print(f"camera_viz: {note}", file=sys.stderr, flush=True) + + controls = ( + _build_controls( + session, + entries, + layers, + shape_layers, + strategies, + controls_cfg, + dashboard, + ) + if want_controls and is_xr + else None ) - runner = VizRunner(session, sources, layers, strategies) + runner = VizRunner( + session, + sources, + layers, + strategies, + controls=controls, + dashboard=dashboard, + header=header, + notes=notes, + ) def _on_signal(signum, frame): print(f"camera_viz: stopping (signal {signum})...", flush=True) @@ -478,32 +314,50 @@ def _on_signal(signum, frame): signal.signal(signal.SIGINT, _on_signal) signal.signal(signal.SIGTERM, _on_signal) + # Entered manually rather than with ``with``: teardown is ordered and + # conditional. The controls' trackers borrow the XrInstance / XrSession + # that VizSession owns, so they must detach AFTER the render thread + # stops polling them but BEFORE destroy() frees those handles — and + # neither may happen at all if a worker thread is still alive. + if controls is not None: + controls.__enter__() runner.start() try: runner.wait( health_check=launcher.health_check if launcher is not None else None ) finally: - # Skip session.destroy() when a worker thread is still alive — - # it may be inside session.render() and destroying under it - # would UAF on the Vulkan / CUDA handles. Non-daemon thread - # keeps the process alive; OS reaps at exit. Leave the CloudXR - # runtime up too (see launch_ctx comment above). + # Skip destroy() when a worker thread is still alive — it may be + # inside session.render() and destroying under it would UAF on the + # Vulkan / CUDA handles. Non-daemon thread keeps the process alive; + # OS reaps at exit. Leave the CloudXR runtime up too (see the + # launch_ctx comment above). clean = runner.stop() + # Unhook first: a source thread that outlives stop() would + # otherwise notify into a panel that has closed. + set_notify_sink(None) + dashboard.close() if clean: + if controls is not None: + controls.__exit__(None, None, None) session.destroy() else: stop_launcher = False print( - "camera_viz: worker thread did not exit; leaving VizSession " - "and the CloudXR runtime alive to avoid use-after-free. " - "Process will keep running until the stuck thread completes.", + "camera_viz: worker thread did not exit; leaving VizSession, " + "the controller session and the CloudXR runtime alive to " + "avoid use-after-free. Process will keep running until the " + "stuck thread completes.", file=sys.stderr, flush=True, ) finally: if stop_launcher: launch_ctx.__exit__(None, None, None) + # Held open until here: the launcher reads the generated env file at + # launch, and keeping it on disk for the run makes the settings + # actually in force inspectable while the runtime is up. + settings_stack.close() return 0 diff --git a/examples/camera_viz/cloudxr_env.py b/examples/camera_viz/cloudxr_env.py new file mode 100644 index 0000000000..90502fdce9 --- /dev/null +++ b/examples/camera_viz/cloudxr_env.py @@ -0,0 +1,140 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""CloudXR runtime settings, declared in the YAML instead of the process env. + +The runtime takes its configuration from environment variables, and there are +three ways to set one and be silently wrong: + +* a name nothing reads -- ``NV_CXR_DEVICE_PROFILE`` looks like its neighbours + but the runtime reads ``NV_DEVICE_PROFILE``; +* a stale shell export -- the launcher writes ``~/.cloudxr/run/cloudxr.env`` + and tells you to source it, after which ``os.environ.setdefault`` here and + even ``--cloudxr-device-profile`` both quietly lose to the sourced value; +* a bool spelled ``False``, which the runtime's exact-match parser does not + recognise and therefore reads as **true**. + +None of the three produces an error. So the settings go through the one tier +that outranks the process environment: a ``--cloudxr-env-config`` file, whose +entries beat both a shell export and the launcher's own defaults. Verified +against isaacteleop.cloudxr.env_config._load_resolve_and_apply, which merges +the file last. + +Each of the three has a guard here: an unknown name warns with a suggestion, +the env file outranks the shell, and booleans are written in the one spelling +the parser reads as false. +""" + +from __future__ import annotations + +import difflib +import sys +from pathlib import Path +from typing import Dict + +# Computed by the launcher; an env file that sets them earns a UserWarning and +# is ignored, so reject them here where the message can name the YAML key. +RESERVED_KEYS = frozenset( + {"XR_RUNTIME_JSON", "XRT_NO_STDIN", "NV_CXR_RUNTIME_DIR", "NV_CXR_OUTPUT_DIR"} +) + +# camera_viz's own defaults, overridable per deployment from the YAML. +DEFAULT_ENV: Dict[str, str] = { + # The runtime otherwise blocks each server frame until a fresh client pose + # arrives; the app is not a head-tracked renderer, so it need not wait. + "NV_ENABLE_POSE_WAIT": "false", + # Runtime-side fixed foveation: the composited image is warped before + # encoding, so peripheral pixels cost less bandwidth. Off in the runtime by + # default, and it applies to the layers fast path camera_viz uses. + "NV_CXR_RUNTIME_FOVEATION": "true", +} + +# Names the runtime actually reads, so a typo can be caught before launch +# rather than by silently doing nothing. Not exhaustive -- an unlisted name +# warns and is still passed through, because the runtime has many more knobs +# than camera_viz has opinions about. +KNOWN_KEYS = frozenset( + { + "NV_ENABLE_POSE_WAIT", + "NV_MAX_POSE_WAIT_DURATION_MS", + "NV_CXR_RUNTIME_FOVEATION", + "NV_CXR_RUNTIME_FOVEATION_BLUR", + "NV_CXR_RUNTIME_FOVEATION_WARPED_WIDTH", + "NV_CXR_RUNTIME_FOVEATION_UNWARPED_WIDTH", + "NV_CXR_RUNTIME_FOVEATION_INSET", + "NV_CXR_APP_PQW_INSET", + "NV_DEVICE_PROFILE", + "NV_DISABLE_DEPTH_DILATION", + "NV_IMMEDIATE_COMPOSITOR", + "NV_MAX_FPS", + "NV_SKIP_FRAME_ON_SPIKE", + "XRT_PRINT_OPTIONS", + } +) + + +def env_from_yaml(display: dict) -> Dict[str, str]: + """``display.cloudxr`` merged over :data:`DEFAULT_ENV`. + + Values are stringified here rather than at the call site because the + conversion is where a bug hides: YAML ``false`` reaches Python as ``False`` + and ``str()`` would spell it ``"False"``, which the runtime's parser does + not recognise -- and an unrecognised value reads as *true*, i.e. the exact + opposite of what the config says. + """ + spec = display.get("cloudxr") or {} + if not isinstance(spec, dict): + raise ValueError( + f"camera_viz: display.cloudxr must be a mapping of " + f"NAME: value, got {type(spec).__name__}" + ) + env = dict(DEFAULT_ENV) + for key, value in spec.items(): + name = str(key) + if name in RESERVED_KEYS: + raise ValueError( + f"camera_viz: display.cloudxr.{name} is computed by the " + "CloudXR launcher and cannot be set here" + ) + _warn_if_unknown(name) + if value is None: # explicit null = "drop camera_viz's default" + env.pop(name, None) + continue + env[name] = _as_env_value(value) + return env + + +def _warn_if_unknown(name: str) -> None: + """A name nothing reads is the failure mode this module exists for: + ``NV_CXR_DEVICE_PROFILE`` sits between two variables that are spelled that + way, but the runtime reads ``NV_DEVICE_PROFILE`` and ignores the other + without a word.""" + if name in KNOWN_KEYS: + return + hint = difflib.get_close_matches(name, KNOWN_KEYS, n=1) + suggestion = f" (did you mean {hint[0]!r}?)" if hint else "" + print( + f"camera_viz: warning: display.cloudxr.{name} is not a CloudXR " + f"variable camera_viz knows{suggestion} — passing it through anyway", + file=sys.stderr, + flush=True, + ) + + +def _as_env_value(value) -> str: + if isinstance(value, bool): + return "true" if value else "false" + return str(value) + + +def write_env_file(env: Dict[str, str], path: Path) -> Path: + """Write ``env`` as the launcher's ``KEY=value`` format. + + Deliberately not ``export KEY=value``: that is the shape the launcher + *writes* for humans to source, but its own parser splits on the first + ``=`` without stripping a prefix, so an exported line would arrive as the + key ``"export KEY"`` and be ignored. + """ + lines = ["# Generated by camera_viz from display.cloudxr — do not edit."] + lines += [f"{key}={value}" for key, value in sorted(env.items())] + path.write_text("\n".join(lines) + "\n") + return path diff --git a/examples/camera_viz/config.py b/examples/camera_viz/config.py new file mode 100644 index 0000000000..abbd7df97b --- /dev/null +++ b/examples/camera_viz/config.py @@ -0,0 +1,338 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The YAML, turned into the objects the viewer runs on. + +Everything here reads ``cameras`` and ``display.placements`` and produces +:class:`SourceEntry` values — one per camera stream, carrying its source, its +placement strategy and its surface config. Nothing here touches Vulkan, the +CloudXR runtime or a display: this is the parse-and-validate half, so a +malformed config fails before anything is allocated. + +Unknown keys warn rather than falling back silently — a typo'd +``cylinder_radius`` should not run at the 2 m default with no hint. +""" + +from __future__ import annotations + +import sys +from dataclasses import dataclass +from typing import List, Optional, Tuple + +from pipeline import FrameSource +from placements import PlacementConfig, PlacementStrategy, build as build_placement +from sources import PairedFrameSource, RtpH264Source, build_local_camera + + +@dataclass +class SourceEntry: + """source + placement + stereo + shape cfg; drives layer construction.""" + + source: FrameSource + placement: Optional[PlacementStrategy] + stereo: bool = False + # display.placements..stereo_plane_distance_cm — the gap between + # the left-eye and right-eye planes. + stereo_plane_distance_cm: float = 0.0 + # Kept so the controls can rebuild a strategy when the lock mode is + # cycled at runtime; None outside XR (no placement to rebuild). + lock_mode: str = "lazy" + placement_config: Optional[PlacementConfig] = None + # display.placements..shape: quad | cylinder | equirect. + shape: str = "quad" + # Who composites the layer (display.placements..compositor): + # "openxr" (default — the OpenXR runtime) or "televiz" (built-in + # compositor; quads only). + compositor: str = "openxr" + # Cylinder shape parameters (display.placements.). + cylinder_radius_m: float = 2.0 + cylinder_angle_deg: float = 90.0 + # Equirect shape parameter: heading the middle of the panorama points + # at, degrees about +Y (0 = the reference space's forward, positive to + # the left). The sphere has no lock-mode strategy, so this is how a feed + # whose camera does not face the way the headset started gets aimed. + equirect_yaw_deg: float = 0.0 + + +VALID_SHAPES = ("quad", "cylinder", "equirect") + +_VALID_COMPOSITORS = ("openxr", "televiz") + +# Every key the placements. block understands (lock-mode strategy +# knobs + surface-shape keys). Unknown keys warn instead of silently +# falling back to defaults — a typo'd `cylinder_radius` should not run +# with a 2 m default and no hint. +_KNOWN_PLACEMENT_KEYS = frozenset( + { + "lock_mode", + "distance", + "offset_x", + "offset_y", + "look_away_angle_deg", + "reposition_distance", + "reposition_delay_s", + "transition_duration_s", + "size", + "stereo_plane_distance_cm", + "shape", + "compositor", + "cylinder_radius_m", + "cylinder_angle_deg", + "equirect_yaw_deg", + } +) + + +def _warn_unknown_placement_keys(cam_name: str, pspec: dict) -> None: + import difflib + + for key in pspec: + if key in _KNOWN_PLACEMENT_KEYS: + continue + hint = difflib.get_close_matches(key, _KNOWN_PLACEMENT_KEYS, n=1) + suggestion = f" (did you mean {hint[0]!r}?)" if hint else "" + print( + f"camera_viz: warning: placements.{cam_name}: unknown key " + f"{key!r}{suggestion} — ignored", + file=sys.stderr, + flush=True, + ) + + +def _shape_for( + cam_name: str, placements_cfg: dict +) -> Tuple[str, str, float, float, float]: + """Per-camera surface config from ``display.placements.``: + ``shape`` (quad | cylinder | equirect, default quad), ``compositor`` + (openxr — the default — or televiz; quads only), ``cylinder_radius_m`` + / ``cylinder_angle_deg`` (cylinder only), ``equirect_yaw_deg`` + (equirect only).""" + pspec = placements_cfg.get(cam_name) or {} + _warn_unknown_placement_keys(cam_name, pspec) + shape = str(pspec.get("shape", "quad")).lower() + if shape not in VALID_SHAPES: + raise ValueError( + f"camera_viz: placements.{cam_name}.shape must be one of " + f"{'|'.join(VALID_SHAPES)}, got {shape!r}" + ) + compositor = str(pspec.get("compositor", "openxr")).lower() + if compositor not in _VALID_COMPOSITORS: + raise ValueError( + f"camera_viz: placements.{cam_name}.compositor must be " + f"{'|'.join(_VALID_COMPOSITORS)}, got {compositor!r}" + ) + if compositor == "televiz" and shape != "quad": + raise ValueError( + f"camera_viz: placements.{cam_name}: compositor: televiz only " + f"applies to shape: quad — {shape} layers are composited by the " + "OpenXR runtime always." + ) + radius_m = float(pspec.get("cylinder_radius_m", 2.0)) + angle_deg = float(pspec.get("cylinder_angle_deg", 90.0)) + yaw_deg = float(pspec.get("equirect_yaw_deg", 0.0)) + return shape, compositor, radius_m, angle_deg, yaw_deg + + +_VALID_LOCK_MODES = ("world", "head", "lazy", "gimbal") + + +def _build_placement( + spec: Optional[dict], is_xr: bool +) -> Tuple[Optional[PlacementStrategy], str, Optional[PlacementConfig]]: + """Returns (strategy, lock_mode, config). The last two let the + controls rebuild a strategy when the lock mode changes at runtime.""" + if spec is not None: + # Validate in every display mode — a typo'd lock_mode shouldn't + # silently become lazy (XR) or pass unnoticed (window). + lock_mode = str(spec.get("lock_mode", "lazy")).lower() + if lock_mode not in _VALID_LOCK_MODES: + raise ValueError( + f"camera_viz: lock_mode must be {'|'.join(_VALID_LOCK_MODES)}, " + f"got {lock_mode!r}" + ) + if not is_xr or spec is None: + return None, "lazy", None + cfg_kwargs = {} + if "size" in spec: + cfg_kwargs["size_meters"] = tuple(spec["size"]) + for key in ( + "distance", + "offset_x", + "offset_y", + "look_away_angle_deg", + "reposition_distance", + "reposition_delay_s", + "transition_duration_s", + ): + if key in spec: + cfg_kwargs[key] = spec[key] + cfg = PlacementConfig(**cfg_kwargs) + # The normalised form, not the raw one: the controls cycle it through + # LOCK_MODE_CYCLE.index(), so a config saying `WORLD` would validate here + # and then raise ValueError on the first press of A. + lock_mode = str(spec.get("lock_mode", "lazy")).lower() + return build_placement(lock_mode, cfg), lock_mode, cfg + + +def _enabled_cameras(cfg: dict) -> List[dict]: + return [c for c in cfg.get("cameras", []) if c.get("enabled", True)] + + +# Default plane width when ``size`` is omitted from a placement block. +# Height is derived from the camera's pixel aspect ratio so the rendered +# plane keeps the picture's shape. +_DEFAULT_PLANE_WIDTH_M = 1.0 + + +def _placement_with_aspect( + spec: Optional[dict], width: int, height: int, is_xr: bool +) -> Tuple[Optional[PlacementStrategy], str, Optional[PlacementConfig]]: + """Build the placement, filling in ``size`` from the source's aspect + ratio when the YAML doesn't pin it. Width defaults to 1.0 m so a + 16:9 source lands at 1.0 x 0.5625, a 3.55:1 SBS at 1.0 x 0.281.""" + if spec is not None and "size" not in spec: + spec = { + **spec, + "size": [_DEFAULT_PLANE_WIDTH_M, _DEFAULT_PLANE_WIDTH_M * height / width], + } + return _build_placement(spec, is_xr) + + +def _stereo_for(cam: dict, placements_cfg: dict) -> Tuple[bool, float]: + """``cameras..stereo`` (producer toggle) plus the placement's + ``placements..stereo_plane_distance_cm`` — the gap between the + left-eye and right-eye planes in 3D.""" + stereo = bool(cam.get("stereo", False)) + pspec = placements_cfg.get(cam["name"]) or {} + return stereo, float(pspec.get("stereo_plane_distance_cm", 0.0)) + + +def build_local_entries(cfg: dict, is_xr: bool) -> List[SourceEntry]: + """source=local: open each enabled camera directly.""" + placements_cfg = cfg.get("display", {}).get("placements", {}) + entries: List[SourceEntry] = [] + for cam in _enabled_cameras(cfg): + cam_sources = build_local_camera(cam) + # Aspect comes from the built source's spec, not the YAML — video + # sources may omit width/height and size themselves from the file. + first = cam_sources[0].spec + placement, lock_mode, placement_cfg = _placement_with_aspect( + placements_cfg.get(cam["name"]), first.width, first.height, is_xr + ) + stereo, plane_distance_cm = _stereo_for(cam, placements_cfg) + shape, compositor, radius_m, angle_deg, yaw_deg = _shape_for( + cam["name"], placements_cfg + ) + for source in cam_sources: + entries.append( + SourceEntry( + source=source, + placement=placement, + stereo=stereo, + stereo_plane_distance_cm=plane_distance_cm, + shape=shape, + compositor=compositor, + cylinder_radius_m=radius_m, + cylinder_angle_deg=angle_deg, + equirect_yaw_deg=yaw_deg, + lock_mode=lock_mode, + placement_config=placement_cfg, + ) + ) + return entries + + +def build_rtp_entries(cfg: dict, is_xr: bool) -> List[SourceEntry]: + """One RTP listener per camera; stereo uses rtp.port + rtp.port_right + and pairs them at the receiver (no wire-level sync — drift OK).""" + placements_cfg = cfg.get("display", {}).get("placements", {}) + entries: List[SourceEntry] = [] + for cam in _enabled_cameras(cfg): + rtp = cam.get("rtp", {}) + if "port" not in rtp: + raise ValueError( + f"camera_viz: camera {cam.get('name')!r} missing rtp.port; " + "required when source: rtp" + ) + if "width" not in cam or "height" not in cam: + raise ValueError( + f"camera_viz: camera {cam.get('name')!r} needs explicit " + "width/height when source: rtp — the receiver sizes its " + "decoder from the YAML, not from the wire" + ) + placement, lock_mode, placement_cfg = _placement_with_aspect( + placements_cfg.get(cam["name"]), + int(cam["width"]), + int(cam["height"]), + is_xr, + ) + stereo, plane_distance_cm = _stereo_for(cam, placements_cfg) + + if stereo: + if "port_right" not in rtp: + raise ValueError( + f"camera_viz: stereo camera {cam.get('name')!r} missing " + "rtp.port_right (required when stereo + source: rtp)" + ) + left = RtpH264Source( + name=f"{cam['name']}.left", + width=int(cam["width"]), + height=int(cam["height"]), + port=int(rtp["port"]), + rtp_buffer_size=int(rtp.get("rtp_buffer_size", 212992)), + gpu_id=int(rtp.get("gpu_id", 0)), + ) + right = RtpH264Source( + name=f"{cam['name']}.right", + width=int(cam["width"]), + height=int(cam["height"]), + port=int(rtp["port_right"]), + rtp_buffer_size=int(rtp.get("rtp_buffer_size", 212992)), + gpu_id=int(rtp.get("gpu_id", 0)), + ) + source: FrameSource = PairedFrameSource( + name=cam["name"], left=left, right=right + ) + else: + source = RtpH264Source( + name=cam["name"], + width=int(cam["width"]), + height=int(cam["height"]), + port=int(rtp["port"]), + rtp_buffer_size=int(rtp.get("rtp_buffer_size", 212992)), + gpu_id=int(rtp.get("gpu_id", 0)), + ) + + shape, compositor, radius_m, angle_deg, yaw_deg = _shape_for( + cam["name"], placements_cfg + ) + entries.append( + SourceEntry( + source=source, + placement=placement, + stereo=stereo, + stereo_plane_distance_cm=plane_distance_cm, + shape=shape, + compositor=compositor, + cylinder_radius_m=radius_m, + cylinder_angle_deg=angle_deg, + equirect_yaw_deg=yaw_deg, + lock_mode=lock_mode, + placement_config=placement_cfg, + ) + ) + return entries + + +def check_shapes_are_displayable(cfg: dict, effective_mode: str) -> None: + """Shaped layers are composited by the OpenXR runtime, so they need XR + mode. Checked here, before the runtime is launched, rather than as a + failure part-way through building the session.""" + placements_cfg = cfg.get("display", {}).get("placements", {}) + for cam in _enabled_cameras(cfg): + shape = _shape_for(cam["name"], placements_cfg)[0] + if shape != "quad" and effective_mode != "xr": + raise SystemExit( + f"camera_viz: placements.{cam['name']}.shape: {shape} is " + "composited by the OpenXR runtime and requires XR mode; " + "use --mode xr or shape: quad in window mode." + ) diff --git a/examples/camera_viz/configs/synthetic_stereo.yaml b/examples/camera_viz/configs/synthetic_stereo.yaml index e896ba4ea3..4ccf1f70ef 100644 --- a/examples/camera_viz/configs/synthetic_stereo.yaml +++ b/examples/camera_viz/configs/synthetic_stereo.yaml @@ -40,7 +40,7 @@ display: synth: lock_mode: lazy distance: 1.0 - stereo_baseline_mm: 0.0 # 0 → both eyes see the same world quad + stereo_plane_distance_cm: 0.0 # 0 → both eyes see the same world quad # (parallax purely from the captured frames). # Try 65.0 (typical IPD) to virtually push the # screen further; -30.0 to bring it closer. diff --git a/examples/camera_viz/configs/zed.yaml b/examples/camera_viz/configs/zed.yaml index eb5d973358..7f35c91e74 100644 --- a/examples/camera_viz/configs/zed.yaml +++ b/examples/camera_viz/configs/zed.yaml @@ -21,9 +21,9 @@ cameras: # Per-eye dims pick the SDK preset (HD2K=2208x1242, HD1080=1920x1080, # HD720=1280x720, VGA=672x376). Mismatch errors at startup with the # full list. - width: 1280 - height: 720 - fps: 60 + width: 1920 + height: 1080 + fps: 30 bus_type: usb # usb | gmsl serial_number: 0 # 0 → first available stereo: true @@ -42,15 +42,59 @@ display: near_z: 0.05 far_z: 100.0 clear_color: [0.0, 0.0, 0.0, 0.0] + # CloudXR runtime settings. camera_viz already sets sane defaults + # (cloudxr_env.DEFAULT_ENV: pose wait off, runtime foveation on) — this + # block is a per-deployment override, so list only what you want to change. + # It reaches the runtime as a generated --cloudxr-env-config file, which + # outranks the process environment, so a stale + # `source ~/.cloudxr/run/cloudxr.env` in your shell cannot silently win. + # camera_viz warns if the runtime resolves one differently from what was + # asked. `null` drops a default; booleans are written lowercase, the only + # spelling the runtime's parser reads as false. + # cloudxr: + # NV_DEVICE_PROFILE: apple-vision-pro # default: the launcher's quest3. + # # Non-quest3 profiles use separate-frames + # # packing, which turns off the runtime's + # # client-reconstructed streaming. + # Controller bindings (XR only). + # right stick X : stereo plane gap right A : lock mode right B : mono/stereo + # (equirect: pan) right stick click : recenter on your view + # left stick : shape params left X : shape left Y : reset + controls: + enabled: true + # hud: true # in-headset readout of what each press + # # changed; opaque, auto-hides after ~2.5 s + # shape_switching: true # keep all 3 shapes resident so left X is + # # instant; costs the extra shapes' VRAM + # deadzone: 0.2 # stick rest-position tolerance + # plane_distance_rate_cm_per_s: 2.0 # held-stick ramp rates (0.1 cm steps) + # size_rate_m_per_s: 0.5 + # offset_rate_m_per_s: 0.5 + # angle_rate_deg_per_s: 40.0 + # plane_distance_min_cm: -10.0 # limits + # plane_distance_max_cm: 10.0 + # size_range_m: [0.2, 8.0] + # offset_y_range_m: [-2.0, 2.0] + # cylinder_angle_range_deg: [15.0, 350.0] + # equirect_h_range_deg: [30.0, 360.0] + # equirect_v_half_range_deg: [10.0, 90.0] placements: zed: # size defaults to 1.0 m wide × aspect-derived height lock_mode: lazy distance: 1.0 - stereo_baseline_mm: 50.0 + # Gap between the left-eye and right-eye planes. 0 = both eyes share + # one plane and all depth comes from the captured images, which packs + # the whole scene into the space between you and `distance`. Widening + # the gap pushes the scene back. The HUD suggests a value from the + # plane distance and the measured IPD; the right stick trims it. + stereo_plane_distance_cm: 5.0 # Surface the feed is mapped onto (all shapes are per-camera): # shape: cylinder # quad (default) | cylinder | equirect - # compositor: openxr # openxr (default: the OpenXR runtime + compositor: openxr # openxr (default: the OpenXR runtime # # composites the layer) | televiz (built-in # # compositor; quad shape only) # cylinder_radius_m: 2.0 # cylinder only # cylinder_angle_deg: 45 # cylinder only + # equirect_yaw_deg: 0.0 # equirect only: heading the middle of the + # # feed points at (0 = where the headset last + # # recentered, positive to the left) diff --git a/examples/camera_viz/controls/__init__.py b/examples/camera_viz/controls/__init__.py new file mode 100644 index 0000000000..33c391e760 --- /dev/null +++ b/examples/camera_viz/controls/__init__.py @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""XR controller bindings for camera_viz. + +Four modules behind one door, in the same spirit as :mod:`placements`: + +``bindings`` + What each button and stick does, and the state each camera carries. +``shapes`` + The left stick, per surface shape -- one strategy class per shape. +``stereo`` + The gap-to-perceived-distance geometry, as pure functions. +``hud`` + The head-locked panel that shows what a press just did. + +Import from the package, not the modules: ``from controls import +ControllerControls``. +""" + +from .bindings import ( + DEFAULT_IPD_MM, + FAR_TARGET_M, + LOCK_MODE_CYCLE, + MAX_OFFSET_FRACTION_OF_IPD, + PLANE_DISTANCE_STEP_CM, + SHAPE_CYCLE, + SHAPE_PARAMS, + ControllerControls, + ControlsConfig, + ControlTarget, + controls_config_from_yaml, + summarize, +) +from .hud import make_hud, split_message + +__all__ = [ + "DEFAULT_IPD_MM", + "FAR_TARGET_M", + "LOCK_MODE_CYCLE", + "MAX_OFFSET_FRACTION_OF_IPD", + "PLANE_DISTANCE_STEP_CM", + "SHAPE_CYCLE", + "SHAPE_PARAMS", + "ControlTarget", + "ControllerControls", + "ControlsConfig", + "controls_config_from_yaml", + "make_hud", + "split_message", + "summarize", +] diff --git a/examples/camera_viz/controls/bindings.py b/examples/camera_viz/controls/bindings.py new file mode 100644 index 0000000000..208464b180 --- /dev/null +++ b/examples/camera_viz/controls/bindings.py @@ -0,0 +1,737 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Controller bindings for camera_viz (Quest / Pico). + +Right hand -- where the feed sits and how it reads in depth: + + thumbstick X gap between the eyes' stereo planes, held to ramp; + on equirect, which has no gap to set, it pans the + panorama instead + thumbstick recenter: aim the surface at wherever you are looking + click + A cycle lock mode (world -> head -> lazy) + B toggle mono / stereo + +Left hand -- what surface it is mapped onto: + + X cycle shape (quad -> cylinder -> equirect) + Y reset every parameter to the YAML values + thumbstick shape-dependent, see controls.shapes: + quad X = size, Y = height + cylinder X = arc, Y = height + equirect X = h-span, Y = v-span + +Input arrives through a ``ControllerTracker`` on a ``DeviceIOSession`` that +shares the OpenXR session Televiz owns, so there is one CloudXR connection +for rendering and input both. XR only -- window mode has no controllers. + +Every limit below sits strictly inside what the layers validate, because +``set_placement`` raises on a bad value and this runs on the render thread. +""" + +from __future__ import annotations + +import math +import sys +import threading +from dataclasses import dataclass, field +from typing import Any, List, Optional, Sequence + +from . import shapes, stereo +from .hud import split_message +from placements import ( + PlacementConfig, + PlacementStrategy, + build as build_placement, + heading_deg, +) + +# Re-exported so ``controls`` stays the one import for callers and tests. +DEFAULT_IPD_MM = stereo.DEFAULT_IPD_MM +MAX_OFFSET_FRACTION_OF_IPD = stereo.MAX_FRACTION_OF_IPD +FAR_TARGET_M = stereo.FAR_TARGET_M +PLANE_DISTANCE_STEP_CM = stereo.STEP_CM + +# isaacteleop is imported lazily inside the methods that touch the device +# session: everything else here is policy (deadzone, clamping, cycle order) +# and is unit-testable without the SDK or a headset. + +# Cycle order for the A button. Starts from whatever the YAML set, so the +# first press moves to the next entry after it. +LOCK_MODE_CYCLE = ("world", "head", "gimbal", "lazy") + +# Cycle order for the left X button. +SHAPE_CYCLE = ("quad", "cylinder", "equirect") + +# Per-shape stick bindings, for the log line and the docs. +#: Each shape's two stick axes, from the control that implements them. +SHAPE_PARAMS = {shape: shapes.axes(shape) for shape in SHAPE_CYCLE} + + +@dataclass +class ControlsConfig: + """``display.controls`` in the YAML.""" + + enabled: bool = True + # Held-stick ramp rate. At 2 cm/s that is 20 of the 0.1 cm steps a + # second: fine enough to land on a value, quick enough to cross the + # usable range in about three. + plane_distance_rate_cm_per_s: float = 2.0 + plane_distance_min_cm: float = -10.0 + # The ceiling is also bounded by the measured IPD, so a stick held to the + # stop can never reach divergent parallax. + plane_distance_max_cm: float = 10.0 + # Quest/Pico sticks rest a few percent off centre; below this the axis + # reads as zero so an untouched stick never drifts the plane distance. + deadzone: float = 0.2 + + # In-headset readout of what each press changed. Opaque, head-locked + # below the eyeline, auto-hiding a couple of seconds after the change. + hud: bool = True + + # Keep every shape resident so the left X button is an atomic + # set_visible instead of a layer rebuild. Costs the extra shapes' VRAM; + # camera_viz prints the figure at startup. + shape_switching: bool = True + + # Left-stick rates, per second of full deflection. + size_rate_m_per_s: float = 0.5 + offset_rate_m_per_s: float = 0.5 + angle_rate_deg_per_s: float = 40.0 + + # Limits, all strictly inside the layers' own validation. + size_range_m: tuple = (0.2, 8.0) + offset_y_range_m: tuple = (-2.0, 2.0) + cylinder_angle_range_deg: tuple = (15.0, 350.0) + equirect_h_range_deg: tuple = (30.0, 360.0) + equirect_v_half_range_deg: tuple = (10.0, 90.0) + + +@dataclass +class ControlTarget: + """One controllable layer.""" + + name: str + layer: Any + shape: str + # Camera produces per-eye frames. False disables both the baseline and + # the mono/stereo toggle: neither means anything on a mono source. + stereo: bool + # ``placements..stereo_plane_distance_cm``: the gap between the + # left-eye and right-eye planes in 3D. Applied as-is -- the HUD suggests + # a value rather than computing one behind your back. + plane_distance_cm: float + lock_mode: str + placement_config: Optional[PlacementConfig] = None + # Submit thread reads this; render thread writes it. + force_mono: threading.Event = field(default_factory=threading.Event) + + # {shape: layer} for every resident shape. Just {shape: layer} when + # shape switching is off, which makes the X button a no-op. + shape_layers: dict = field(default_factory=dict) + # Set while mono so the gap can be put back on the way out. + plane_distance_before_mono: Optional[float] = None + # Live shaped-layer params; the quad's live in placement_config, which + # the strategy already reads every frame. + cylinder_radius_m: float = 2.0 + cylinder_angle_deg: float = 90.0 + equirect_h_deg: float = 360.0 + equirect_v_half_deg: float = 90.0 + # Heading the middle of the panorama points at, degrees about +Y. The + # sphere has no strategy to place it, so this is its whole pose. + equirect_yaw_deg: float = 0.0 + + def __post_init__(self) -> None: + # Snapshot for the Y button. Tuples/floats only, so a reset can't be + # aliased by a later edit. + self._initial = { + "shape": self.shape, + "lock_mode": self.lock_mode, + "plane_distance_cm": self.plane_distance_cm, + "cylinder_radius_m": self.cylinder_radius_m, + "cylinder_angle_deg": self.cylinder_angle_deg, + "equirect_h_deg": self.equirect_h_deg, + "equirect_v_half_deg": self.equirect_v_half_deg, + "equirect_yaw_deg": self.equirect_yaw_deg, + "size_meters": ( + tuple(self.placement_config.size_meters) + if self.placement_config is not None + else None + ), + "distance": ( + self.placement_config.distance + if self.placement_config is not None + else None + ), + "offset_y": ( + self.placement_config.offset_y + if self.placement_config is not None + else 0.0 + ), + } + + +class ControllerControls: + """Owns the input session and applies its events to the layers. + + ``strategies`` is the *same list object* the runner iterates, mutated in + place on a lock-mode change. Both that mutation and the read happen on + the render thread, so no lock is needed; the mono/stereo flag crosses to + the submit thread and uses an Event instead. + """ + + def __init__( + self, + viz_session: Any, + targets: Sequence[ControlTarget], + strategies: List[Optional[PlacementStrategy]], + config: Optional[ControlsConfig] = None, + tracker: Optional[Any] = None, + hud: Optional[Any] = None, + log_to_stderr: bool = True, + ) -> None: + """``tracker`` overrides the ControllerTracker; tests inject a fake + so the policy can be exercised without the SDK or a headset. + + ``log_to_stderr`` is off when the status panel is live: it redraws + stderr in place by moving the cursor, and a second writer on the same + stream lands mid-panel and leaves a trail of half-erased copies. The + panel shows the same events on its own line, so nothing is lost. + """ + if len(targets) != len(strategies): + raise ValueError( + f"targets / strategies length mismatch: " + f"{len(targets)} vs {len(strategies)}" + ) + self._session = viz_session + self._targets = list(targets) + self._strategies = strategies + self._cfg = config or ControlsConfig() + self._hud = hud + self._log = _log if log_to_stderr else _discard + if tracker is None: + import isaacteleop.deviceio as deviceio + + tracker = deviceio.ControllerTracker() + self._tracker = tracker + self._device_session: Optional[Any] = None + self._device_ctx: Optional[Any] = None + self._prev_a = False + self._prev_b = False + self._prev_stick = False + self._prev_x = False + self._prev_y = False + # Held-stick messages would otherwise print once per rendered frame. + self._elapsed = 0.0 + self._last_log: dict = {} + self._ipd_mm = DEFAULT_IPD_MM + self._last_event = "" + # Live baseline needs a viz newer than the released wheel. A and B + # are pure Python and work either way, so drop only this binding + # rather than failing the whole run. + self._baseline_supported = all( + hasattr(t.layer, "set_stereo_baseline_mm") for t in self._targets + ) + if not self._baseline_supported and any(t.stereo for t in self._targets): + self._log( + "installed isaacteleop has no Layer.set_stereo_baseline_mm; " + "thumbstick baseline disabled (A and B still work)" + ) + + @staticmethod + def required_extensions() -> List[str]: + """OpenXR extensions to declare on ``VizSessionConfig`` *before* + creating the session -- Televiz owns the XrInstance.""" + import isaacteleop.deviceio as deviceio + + return list( + deviceio.DeviceIOSession.get_required_extensions( + [deviceio.ControllerTracker()] + ) + ) + + def __enter__(self) -> "ControllerControls": + import isaacteleop.deviceio as deviceio + import isaacteleop.oxr as oxr + + handles = self._session.get_oxr_handles() + if handles is None: + raise RuntimeError("camera_viz: controls require XR mode") + self._device_ctx = deviceio.DeviceIOSession.run( + [self._tracker], oxr.OpenXRSessionHandles(*handles) + ) + self._device_session = self._device_ctx.__enter__() + return self + + def __exit__(self, *exc) -> None: + if self._device_ctx is not None: + self._device_ctx.__exit__(*exc) + self._device_ctx = None + self._device_session = None + + def active_layer(self, index: int) -> Any: + """Layer currently visible for source ``index``. The runner calls this + every pass -- it is the single source of truth for the active shape.""" + return self._targets[index].layer + + def strategy(self, index: int) -> Optional[PlacementStrategy]: + """Placement strategy for source ``index``, following A-button swaps.""" + return self._strategies[index] + + def status(self, index: int) -> dict: + """Per-camera state for the status panel.""" + target = self._targets[index] + cfg = target.placement_config + gap = None if target.shape == "equirect" else target.plane_distance_cm + return { + # The heading rides along with the shape: it is the sphere's + # whole placement, and the lock / size / gap columns are all + # blank for it. + "shape": ( + f"{target.shape} {target.equirect_yaw_deg:+.0f}°" + if target.shape == "equirect" + else target.shape + ), + "lock_mode": target.lock_mode, + "stereo": target.stereo and not target.force_mono.is_set(), + "size_m": cfg.size_meters[0] if cfg is not None else None, + "offset_y_m": cfg.offset_y if cfg is not None else None, + "plane_distance_cm": gap, + "suggested_cm": self._suggested_plane_distance_cm(target) + if gap is not None + else None, + } + + @property + def ipd_mm(self) -> float: + return self._ipd_mm + + @property + def last_event(self) -> str: + return self._last_event + + def force_mono(self, index: int) -> bool: + """Submit thread: send the left frame to both eyes for this layer.""" + return self._targets[index].force_mono.is_set() + + def step(self, dt: float, ipd_mm: Optional[float] = None) -> None: + """Render thread: pump the device session and apply one frame of input. + + ``ipd_mm`` is the headset's reported eye separation. It converts + stereo distances and bounds the offset below divergence. + """ + if ipd_mm is not None and abs(ipd_mm - self._ipd_mm) > 0.1: + self._ipd_mm = ipd_mm + self._log(f"headset IPD: {ipd_mm:.1f} mm") + if self._device_session is None: + return + self._elapsed += dt + self._device_session.update() + self._step_right(dt) + self._step_left(dt) + if self._hud is not None: + self._hud.step(dt, self._session.head_pose_now()) + + def _step_right(self, dt: float) -> None: + controller = self._tracker.get_right_controller(self._device_session).data + if controller is None: + # Controller asleep or out of range. Drop the edge state so + # waking it mid-press doesn't fire a phantom transition. + self._prev_a = self._prev_b = self._prev_stick = False + return + + inputs = controller.inputs + a, b = bool(inputs.primary_click), bool(inputs.secondary_click) + click = bool(inputs.thumbstick_click) + if a and not self._prev_a: + self._cycle_lock_modes() + if b and not self._prev_b: + self._toggle_stereo() + if click and not self._prev_stick: + self._recenter() + self._prev_a, self._prev_b, self._prev_stick = a, b, click + + # One stick, two shapes' worth of meaning: a target is either an + # equirect or it isn't, so exactly one of these acts on it. + self._adjust_plane_distance(float(inputs.thumbstick_x), dt) + self._adjust_equirect_yaw(float(inputs.thumbstick_x), dt) + + def _step_left(self, dt: float) -> None: + controller = self._tracker.get_left_controller(self._device_session).data + if controller is None: + self._prev_x = self._prev_y = False + return + + inputs = controller.inputs + x, y = bool(inputs.primary_click), bool(inputs.secondary_click) + if x and not self._prev_x: + self._cycle_shapes() + if y and not self._prev_y: + self._reset_params() + self._prev_x, self._prev_y = x, y + + self._adjust_shape_params( + self._axis(float(inputs.thumbstick_x)), + self._axis(float(inputs.thumbstick_y)), + dt, + ) + + def _axis(self, value: float) -> float: + """Deadzoned, rescaled so the ramp starts at zero just past the edge + rather than jumping to deadzone * rate.""" + if abs(value) < self._cfg.deadzone: + return 0.0 + span = 1.0 - self._cfg.deadzone + return math.copysign((abs(value) - self._cfg.deadzone) / span, value) + + def _surface_distance_m(self, target) -> Optional[float]: + """How far the target's surface sits from the viewer, or None when the + shape has no meaningful one (an infinite sphere).""" + if target.shape == "cylinder": + return target.cylinder_radius_m + if target.shape == "equirect": + return None # centred on the viewer, effectively at infinity + cfg = target.placement_config + return cfg.distance if cfg is not None else None + + def _gap_limits(self) -> tuple: + return (self._cfg.plane_distance_min_cm, self._cfg.plane_distance_max_cm) + + def _max_plane_distance_cm(self) -> float: + return stereo.max_gap_cm(self._ipd_mm, self._cfg.plane_distance_max_cm) + + def _clamp_plane_distance(self, value: float) -> float: + return stereo.clamp_gap_cm(value, self._ipd_mm, self._gap_limits()) + + @staticmethod + def _stepped(value: float) -> float: + return stereo.step(value) + + def _suggested_plane_distance_cm(self, target) -> Optional[float]: + return stereo.suggested_gap_cm( + self._surface_distance_m(target), self._ipd_mm, self._gap_limits() + ) + + def _stereo_distance_cm(self, target) -> Optional[float]: + return stereo.perceived_distance_cm( + self._surface_distance_m(target), self._ipd_mm, target.plane_distance_cm + ) + + def _adjust_plane_distance(self, axis: float, dt: float) -> None: + """Right stick: widen or narrow the gap between the eyes' surfaces.""" + if not self._baseline_supported: + return + scaled = self._axis(axis) + if scaled == 0.0: + return + delta = scaled * self._cfg.plane_distance_rate_cm_per_s * dt + + changed, suggestions = [], [] + for target in self._targets: + # Equirect is skipped, not clamped to nothing: the gap works by + # shifting each eye's surface, and camera_viz's sphere is at + # infinite radius, where translating it changes nothing at all. + if not target.stereo or target.shape == "equirect": + continue + if not self._nudge_gap(target, delta): + continue + changed.append( + (target.name, f"{self._stepped(target.plane_distance_cm):.1f} cm") + ) + suggested = self._suggested_plane_distance_cm(target) + if suggested is not None: + suggestions.append((target.name, f"{suggested:.1f} cm")) + if changed: + self._notify( + f"stereo planes: {summarize(changed)}", + self._gap_detail(suggestions), + log_key="plane_distance", + ) + + def _nudge_gap(self, target, delta: float) -> bool: + """Move one target's gap by ``delta``. False when it stayed on the + same step, in which case the sub-step remainder is kept so a slow ramp + still gets there, but nothing is re-applied or re-announced.""" + new = self._clamp_plane_distance(target.plane_distance_cm + delta) + if self._stepped(new) == self._stepped(target.plane_distance_cm): + target.plane_distance_cm = new + return False + self._set_plane_distance(target, new) + return True + + def _gap_detail(self, suggestions: List[tuple]) -> str: + """Second HUD line: the suggestion, and the IPD it came from -- which + is how a headset whose IPD setting is wrong becomes visible.""" + ipd = f"IPD: {self._ipd_mm:.0f} mm" + if not suggestions: + return ipd + # The count already appears on the first line; repeating it is noise. + values = {value for _, value in suggestions} + shown = suggestions[0][1] if len(values) == 1 else summarize(suggestions) + return f"suggested: {shown} · {ipd}" + + def _adjust_equirect_yaw(self, axis: float, dt: float) -> None: + """Right stick on an equirect: pan the panorama. + + The gap binding skips this shape (an infinite sphere cannot be + translated), so the stick is free here and panning is what an + operator actually reaches for on a 360 feed -- a camera is rarely + mounted facing exactly the way the headset was when it recentered. + """ + scaled = self._axis(axis) + if scaled == 0.0: + return + delta = scaled * self._cfg.angle_rate_deg_per_s * dt + changed = [] + for target in self._targets: + if target.shape != "equirect": + continue + # Stick right pans the view right: the middle of the texture + # swings left, which is +heading. + target.equirect_yaw_deg = _wrap_deg(target.equirect_yaw_deg + delta) + shapes.apply_equirect(target) + changed.append((target.name, f"{target.equirect_yaw_deg:+.0f}°")) + if changed: + self._notify(f"pan: {summarize(changed)}", log_key="equirect_yaw") + + def _recenter(self) -> None: + """Thumbstick click: put the surface back where you are looking. + + One gesture, read per shape: an equirect yaws so the middle of the + panorama lands dead ahead, and anything with a placement re-snaps its + anchor -- which is the way out of a world-locked plane left behind in + another part of the room. + """ + head = self._session.head_pose_now() + if head is None: + return + heading = heading_deg(head.orientation) + for index, target in enumerate(self._targets): + if target.shape == "equirect": + target.equirect_yaw_deg = _wrap_deg(heading) + shapes.apply_equirect(target) + elif target.placement_config is not None: + # A fresh strategy re-snaps on its next update; retuning the + # live one would keep the anchor it is holding. + self._strategies[index] = build_placement( + target.lock_mode, target.placement_config + ) + self._notify("recentered on your view") + + def _cycle_lock_modes(self) -> None: + changed = [] + for i, target in enumerate(self._targets): + # An equirect sphere is centred on the operator; there is no + # placement to re-lock, so it has no strategy to swap. + if self._strategies[i] is None or target.placement_config is None: + continue + nxt = LOCK_MODE_CYCLE[ + (LOCK_MODE_CYCLE.index(target.lock_mode) + 1) % len(LOCK_MODE_CYCLE) + ] + target.lock_mode = nxt + self._strategies[i] = build_placement(nxt, target.placement_config) + changed.append((target.name, nxt)) + if changed: + self._notify("lock mode: " + summarize(changed)) + + def _toggle_stereo(self) -> None: + targets = [t for t in self._targets if t.stereo] + if not targets: + self._notify("mono/stereo: no stereo-capable camera") + return + # One shared decision, so a mixed set can't end up half-toggled: + # if any layer is still stereo, the press collapses all to mono. + to_mono = any(not t.force_mono.is_set() for t in targets) + for target in targets: + if to_mono: + target.force_mono.set() + # Both eyes get the same image now, so a plane gap would only + # shove that flat image to some other depth. Park it at zero + # and put the operator's value back on the way out. + target.plane_distance_before_mono = target.plane_distance_cm + self._set_plane_distance(target, 0.0) + else: + target.force_mono.clear() + if target.plane_distance_before_mono is not None: + self._set_plane_distance(target, target.plane_distance_before_mono) + target.plane_distance_before_mono = None + self._notify("eyes: mono" if to_mono else "eyes: stereo") + + def _set_plane_distance(self, target, value: float) -> None: + """``value`` is the running total; the layer only ever sees whole + steps of it.""" + target.plane_distance_cm = value + if self._baseline_supported and target.stereo: + # viz still speaks millimetres; this is the only conversion. + target.layer.set_stereo_baseline_mm(self._stepped(value) * 10.0) + + # ── Left hand: shape + shape params ─────────────────────────────── + + def _cycle_shapes(self) -> None: + """Swap which shape is visible. Both set_visible calls land before + the render() that follows on this thread, so no frame ever sees the + pair half-applied.""" + changed = [] + for i, target in enumerate(self._targets): + if len(target.shape_layers) < 2: + continue + nxt = SHAPE_CYCLE[(SHAPE_CYCLE.index(target.shape) + 1) % len(SHAPE_CYCLE)] + new_layer = target.shape_layers.get(nxt) + if new_layer is None: + continue + target.shape_layers[target.shape].set_visible(False) + new_layer.set_visible(True) + target.shape = nxt + target.layer = new_layer + # The gap is tracked per camera but applied per layer, so the + # newly visible one is still on whatever the YAML built it with. + self._set_plane_distance(target, target.plane_distance_cm) + changed.append((target.name, nxt)) + if changed: + shape = self._targets[0].shape if self._targets else "" + hint = SHAPE_PARAMS.get(shape) + suffix = f" (stick: X {hint[0]}, Y {hint[1]})" if hint else "" + self._notify("shape: " + summarize(changed) + suffix) + + def _adjust_shape_params(self, ax: float, ay: float, dt: float) -> None: + """Left stick. One entry per camera with its changed parameters joined: + summarize() collapses one value across cameras, so handing it two + parameters from the same camera would read as cameras disagreeing and + prefix every one with the camera's name.""" + if ax == 0.0 and ay == 0.0: + return + changed = [] + for i, target in enumerate(self._targets): + control = shapes.for_shape(target.shape) + if control is None: + continue + parts = control.adjust(target, self._cfg, self._strategies[i], ax, ay, dt) + if parts: + changed.append((target.name, " ".join(parts))) + if changed: + self._notify(summarize(changed), log_key="shape_params") + + def _reset_params(self) -> None: + """Y button: back to the YAML values -- the way out of a demo that has + been knocked out of shape.""" + for index, target in enumerate(self._targets): + self._reset_target(index, target) + self._notify("reset to config defaults") + + def _reset_target(self, index: int, target) -> None: + initial = target._initial + strategy = self._strategies[index] + + if target.shape != initial["shape"] and initial["shape"] in target.shape_layers: + target.shape_layers[target.shape].set_visible(False) + target.shape = initial["shape"] + target.layer = target.shape_layers[target.shape] + target.layer.set_visible(True) + + target.force_mono.clear() + target.plane_distance_before_mono = None + self._set_plane_distance(target, initial["plane_distance_cm"]) + + if target.lock_mode != initial["lock_mode"]: + target.lock_mode = initial["lock_mode"] + if target.placement_config is not None and strategy is not None: + # A fresh strategy is right here: reset is meant to re-snap. + strategy = build_placement(target.lock_mode, target.placement_config) + self._strategies[index] = strategy + + if target.placement_config is not None and initial["size_meters"]: + shapes.retune( + target, + strategy, + size_meters=initial["size_meters"], + distance=initial["distance"], + offset_y=initial["offset_y"], + ) + + target.cylinder_radius_m = initial["cylinder_radius_m"] + target.cylinder_angle_deg = initial["cylinder_angle_deg"] + target.equirect_h_deg = initial["equirect_h_deg"] + target.equirect_v_half_deg = initial["equirect_v_half_deg"] + target.equirect_yaw_deg = initial["equirect_yaw_deg"] + shapes.apply_all(target) + + def _notify( + self, + message: str, + detail: str = "", + log_key: Optional[str] = None, + log_period: float = 0.25, + ) -> None: + """Single place every control message goes: stderr for whoever is at + the workstation, HUD for whoever is wearing the headset. + + The HUD always updates -- it is a live readout, and throttling it made + a value moving in 0.1 cm steps look like it moved in 0.5 cm ones. + ``log_key`` throttles only the stderr line, which is a log and would + otherwise scroll a page per second while a stick is held. + + ``detail`` takes the panel's second line verbatim instead of letting + the message wrap into it. + """ + self._last_event = f"{message} · {detail}" if detail else message + if ( + log_key is None + or self._elapsed - self._last_log.get(log_key, -1e9) >= log_period + ): + if log_key is not None: + self._last_log[log_key] = self._elapsed + self._log(f"{message} | {detail}" if detail else message) + if self._hud is not None: + self._hud.show([message, detail] if detail else split_message(message)) + + +def summarize(items: List[tuple]) -> str: + """Collapse per-camera ``(name, value)`` pairs for the message line. + + Every binding applies to every camera at once, so the values normally + agree -- say it once with a count instead of repeating it N times and + overflowing the HUD. + """ + if not items: + return "" + values = {value for _, value in items} + if len(values) == 1: + value = items[0][1] + return value if len(items) == 1 else f"{value} ({len(items)} cameras)" + return " ".join(f"{name} {value}" for name, value in items) + + +def _clamp(value: float, limits) -> float: + lo, hi = limits + return min(max(value, lo), hi) + + +def _wrap_deg(angle: float) -> float: + """Into (-180, 180], so a panned-all-the-way-round readout stays legible.""" + return (angle + 180.0) % 360.0 - 180.0 + + +def _log(message: str) -> None: + # Operator feedback for a run without the panel: confirmation at the + # workstation that a press registered. + print(f"camera_viz: controls: {message}", file=sys.stderr, flush=True) + + +def _discard(message: str) -> None: + """Sink for when the status panel owns stderr.""" + + +def controls_config_from_yaml(display: dict) -> ControlsConfig: + """Parse ``display.controls``; unknown keys raise rather than silently + doing nothing.""" + spec = display.get("controls", {}) + if not isinstance(spec, dict): + raise ValueError("camera_viz: display.controls must be a mapping") + known = {f.name for f in ControlsConfig.__dataclass_fields__.values()} + unknown = set(spec) - known + if unknown: + raise ValueError( + f"camera_viz: display.controls: unknown key(s) {sorted(unknown)}; " + f"valid keys are {sorted(known)}" + ) + return ControlsConfig(**spec) diff --git a/examples/camera_viz/controls/hud.py b/examples/camera_viz/controls/hud.py new file mode 100644 index 0000000000..21f8e153aa --- /dev/null +++ b/examples/camera_viz/controls/hud.py @@ -0,0 +1,169 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""In-headset readout for the controller bindings. + +Televiz has no screen-space overlay layer yet, so this is an ordinary +``QuadLayer`` head-locked below the operator's eyeline, fed a text bitmap. +Opaque on purpose: an alpha-blended layer drops the frame out of CloudXR's +colour-only reconstructed streaming path, which costs bandwidth all session +to light up a few glyphs. The panel hides itself a couple of seconds after +the last change. +""" + +from __future__ import annotations + +from typing import List, Optional, Sequence + +import numpy as np +from PIL import Image, ImageDraw, ImageFont + +from placements import HeadLocked, PlacementConfig + +_W, _H = 1280, 160 +_PAD, _BAR = 36, 12 # left text inset, accent-bar width + +_BG = (18, 19, 23) +_ACCENT = (118, 185, 0) # NVIDIA green +_PRIMARY = (242, 244, 246) +_SECONDARY = (150, 158, 168) + +_REGULAR = "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf" +_BOLD = "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf" + + +def _font(path: str, size: int): + try: + return ImageFont.truetype(path, size) + except OSError: + # No size argument: it arrived in Pillow 10.1 and the floor here is + # 10.0. The fallback bitmap font ignores size anyway, so asking for + # one only risks a TypeError on the version we claim to support. + return ImageFont.load_default() + + +# One size for both lines: a smaller continuation line reads as a different +# message rather than the same one wrapping. Hierarchy comes from colour. +_TITLE = _font(_BOLD, 44) +_BODY = _font(_REGULAR, 44) +_TEXT_W = _W - _PAD - 24 # usable width for one line + + +def _render(lines: Sequence[str]) -> np.ndarray: + """Panel bitmap as RGBA8. Session- and GPU-free, so directly testable.""" + img = Image.new("RGB", (_W, _H), _BG) + draw = ImageDraw.Draw(img) + draw.rectangle((0, 0, _BAR, _H), fill=_ACCENT) + # A lone line sits centred; a pair straddles the middle. + top = 20 if len(lines) > 1 else 52 + styles = ((_TITLE, _PRIMARY, top), (_BODY, _SECONDARY, top + 62)) + for line, (font, colour, y) in zip(lines[:2], styles): + draw.text((_PAD, y), _fit(line, font), font=font, fill=colour) + # The layer wants RGBA8 and the panel is opaque, so tack on a solid alpha. + return np.ascontiguousarray( + np.dstack([np.asarray(img), np.full((_H, _W), 255, np.uint8)]) + ) + + +def _fit(line: str, font) -> str: + """Ellipsize rather than let a long multi-camera message run off the edge.""" + if font.getlength(line) <= _TEXT_W: + return line + while line and font.getlength(line + "...") > _TEXT_W: + line = line[:-1] + return line + "..." + + +def split_message(message: str, max_lines: int = 2) -> List[str]: + """Wrap a control message at word boundaries. + + Each line is measured with the font it will actually be drawn in -- + the first is the larger title face, so measuring everything with the + body font would let line one overflow and get ellipsized. + """ + lines: List[str] = [] + current = "" + for word in message.split(): + candidate = f"{current} {word}".strip() + font = _TITLE if not lines else _BODY + if current and font.getlength(candidate) > _TEXT_W: + lines.append(current) + current = word + if len(lines) == max_lines: + break + else: + current = candidate + if current and len(lines) < max_lines: + lines.append(current) + return lines[:max_lines] + + +class Hud: + """One head-locked panel. Not thread-safe: the render thread owns it.""" + + def __init__( + self, + session, + *, + distance_m: float = 1.1, + offset_y_m: float = -0.32, + width_m: float = 0.9, + hold_s: float = 2.5, + ) -> None: + import isaacteleop.viz as viz + + self._viz = viz + cfg = viz.QuadLayerConfig() + cfg.name = "controls_hud" + cfg.resolution = viz.Resolution(_W, _H) + cfg.format = viz.PixelFormat.kRGBA8 + # Runtime-composited quads sample the texture directly, so a mip chain + # would be allocated and never read. + cfg.generate_mipmaps = False + cfg.alpha_blend = False + self._layer = session.add_quad_layer(cfg) + self._layer.set_visible(False) + + self._hold_s = hold_s + self._since_show = 0.0 + self._visible = False + self._placement = HeadLocked( + PlacementConfig( + size_meters=(width_m, width_m * _H / _W), + distance=distance_m, + offset_y=offset_y_m, + ) + ) + + def show(self, lines: Sequence[str]) -> None: + """Draw ``lines`` and restart the hold timer.""" + import cupy as cp + + self._layer.submit(cp.asarray(_render(lines))) + self._layer.set_visible(True) + self._visible = True + self._since_show = 0.0 + + def step(self, dt: float, head_pose) -> None: + """Render-thread tick: hold the panel in front of the head, then hide + it once the hold expires.""" + if not self._visible: + return + self._since_show += dt + if self._since_show >= self._hold_s: + self._layer.set_visible(False) + self._visible = False + return + if head_pose is None: + return + placed = self._placement.update(head_pose.position, head_pose.orientation) + self._layer.set_placement( + self._viz.QuadLayerPlacement( + self._viz.Pose3D(placed.position, placed.orientation), + placed.size_meters, + ) + ) + + +def make_hud(session, enabled: bool) -> Optional[Hud]: + """``None`` when disabled, so callers can stay branch-light.""" + return Hud(session) if enabled else None diff --git a/examples/camera_viz/controls/shapes.py b/examples/camera_viz/controls/shapes.py new file mode 100644 index 0000000000..9366d65a36 --- /dev/null +++ b/examples/camera_viz/controls/shapes.py @@ -0,0 +1,202 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""What the left stick does, per surface shape. + +One class per shape, in the same spirit as ``placements.lock_modes``: the two +axis labels live next to the behaviour they describe, so adding a shape means +adding a class rather than editing an if/elif and a separate label table. + +The rule every shape follows: **the two axes must not collapse into one +control.** Apparent size is ``2*atan((w/2)/d)`` and a cylinder's arc width is +``radius * angle``, so moving either surface further away enlarges it by the +same factor and looks identical to resizing it. Distance and radius are +therefore YAML placement choices, not stick axes -- they would be a second +copy of the size axis. What they *do* change is the real distance to the +surface, which :mod:`stereo` works from. +""" + +from __future__ import annotations + +import math +from dataclasses import replace +from typing import List, Optional, Tuple + +from placements import yaw_quat + + +def _clamp(value: float, limits: Tuple[float, float]) -> float: + low, high = limits + return min(max(value, low), high) + + +def retune(target, strategy, **updates) -> None: + """Swap in a modified :class:`PlacementConfig`. + + It is frozen, so retuning means replacing it and handing the replacement + to the live strategy. Rebuilding the strategy instead would drop its lazy + anchor and re-snap the plane on every nudge. + """ + target.placement_config = replace(target.placement_config, **updates) + if strategy is not None: + strategy.retune(target.placement_config) + + +class ShapeControl: + """Maps the left stick's two axes onto one shape's parameters.""" + + #: (X label, Y label), shown when the shape is selected. + AXES: Tuple[str, str] = ("", "") + + def adjust( + self, target, cfg, strategy, ax: float, ay: float, dt: float + ) -> List[str]: + """Apply one frame of stick input; return a description per change.""" + raise NotImplementedError + + # Height is common to the flat and curved surfaces that have a placement. + @staticmethod + def _adjust_height(target, cfg, strategy, ay: float, dt: float) -> List[str]: + placement = target.placement_config + if ay == 0.0 or placement is None: + return [] + offset = _clamp( + placement.offset_y + ay * cfg.offset_rate_m_per_s * dt, cfg.offset_y_range_m + ) + if offset == placement.offset_y: + return [] + retune(target, strategy, offset_y=offset) + return [f"height: {offset:+.2f} m"] + + +class QuadControl(ShapeControl): + AXES = ("size", "height") + + def adjust(self, target, cfg, strategy, ax, ay, dt) -> List[str]: + placement = target.placement_config + if placement is None: + return [] + out: List[str] = [] + if ax != 0.0: + width, height = placement.size_meters + new_width = _clamp( + width + ax * cfg.size_rate_m_per_s * dt, cfg.size_range_m + ) + if new_width != width: + # Aspect preserved: the source's pixels can't be re-shaped + # after the fact, so overall scale is the only sane knob. + retune( + target, + strategy, + size_meters=(new_width, new_width * height / width), + ) + out.append(f"size: {new_width:.2f} m") + return out + self._adjust_height(target, cfg, strategy, ay, dt) + + +class CylinderControl(ShapeControl): + AXES = ("arc", "height") + + def adjust(self, target, cfg, strategy, ax, ay, dt) -> List[str]: + out: List[str] = [] + angle = _clamp( + target.cylinder_angle_deg + ax * cfg.angle_rate_deg_per_s * dt, + cfg.cylinder_angle_range_deg, + ) + if angle != target.cylinder_angle_deg: + target.cylinder_angle_deg = angle + apply_cylinder(target) + out.append(f"arc: {angle:.0f}°") + return out + self._adjust_height(target, cfg, strategy, ay, dt) + + +class EquirectControl(ShapeControl): + AXES = ("h-span", "v-span") + + def adjust(self, target, cfg, strategy, ax, ay, dt) -> List[str]: + horizontal = _clamp( + target.equirect_h_deg + ax * cfg.angle_rate_deg_per_s * dt, + cfg.equirect_h_range_deg, + ) + vertical = _clamp( + target.equirect_v_half_deg + ay * cfg.angle_rate_deg_per_s * dt, + cfg.equirect_v_half_range_deg, + ) + if (horizontal, vertical) == ( + target.equirect_h_deg, + target.equirect_v_half_deg, + ): + return [] + target.equirect_h_deg, target.equirect_v_half_deg = horizontal, vertical + apply_equirect(target) + return [f"span: {horizontal:.0f}° x {2 * vertical:.0f}°"] + + +_CONTROLS = { + "quad": QuadControl(), + "cylinder": CylinderControl(), + "equirect": EquirectControl(), +} + + +def for_shape(shape: str) -> Optional[ShapeControl]: + return _CONTROLS.get(shape) + + +def axes(shape: str) -> Optional[Tuple[str, str]]: + control = _CONTROLS.get(shape) + return control.AXES if control is not None else None + + +# ── Pushing a target's shaped params onto its layer ─────────────────── +# +# Read-modify-write in both cases: the runner rewrites only ``.pose`` each +# frame, so these survive it; writing a fresh placement here would race it. + + +def apply_cylinder(target) -> None: + layer = target.shape_layers.get("cylinder", target.layer) + if layer is None: + return + placement = layer.placement() + placement.radius_m = target.cylinder_radius_m + placement.central_angle_rad = math.radians(target.cylinder_angle_deg) + layer.set_placement(placement) + + +def apply_equirect(target) -> None: + layer = target.shape_layers.get("equirect", target.layer) + if layer is None: + return + placement = layer.placement() + # The texture's horizontal center maps to the pose's -z, so yawing the + # pose is what aims the middle of the panorama. This is the sphere's only + # useful pose knob: its radius is infinite, where translation does + # nothing. Vertically the center sits on the pose's horizon. + placement.pose.orientation = yaw_quat(math.radians(target.equirect_yaw_deg)) + placement.central_horizontal_angle_rad = math.radians(target.equirect_h_deg) + # Kept symmetric about the horizon, which also keeps upper > lower. + placement.upper_vertical_angle_rad = math.radians(target.equirect_v_half_deg) + placement.lower_vertical_angle_rad = -math.radians(target.equirect_v_half_deg) + layer.set_placement(placement) + + +def apply_all(target) -> None: + """Push every shaped param onto its layer (used by the reset button).""" + if "cylinder" in target.shape_layers or target.shape == "cylinder": + apply_cylinder(target) + if "equirect" in target.shape_layers or target.shape == "equirect": + apply_equirect(target) + + +__all__ = [ + "CylinderControl", + "EquirectControl", + "QuadControl", + "ShapeControl", + "apply_all", + "apply_cylinder", + "apply_equirect", + "axes", + "for_shape", + "retune", +] diff --git a/examples/camera_viz/controls/stereo.py b/examples/camera_viz/controls/stereo.py new file mode 100644 index 0000000000..490ccbf3ae --- /dev/null +++ b/examples/camera_viz/controls/stereo.py @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Geometry for the stereo plane gap. + +A stereo layer draws each eye's image on its own surface. Separating the two +by ``gap`` moves where zero-disparity content appears:: + + D = Z * ipd / (ipd - gap) for a surface Z away + +which inverts to ``gap = ipd * (1 - Z / D)``. The stick ramps the gap rather +than the distance because the gap is linear in vergence angle, which is what +the eyes actually track; the distance is what gets displayed, because that is +what the operator means. + +The camera's own baseline is deliberately absent from all of this. It is +baked into the captured pixels and sets the scene's depth *scale*, which no +viewer-side knob can change; the gap only sets where that scene sits. + +Pure functions of (surface distance, IPD, limits) — no session, no layers, so +the maths is testable on its own. +""" + +from __future__ import annotations + +from typing import Optional, Tuple + +# Fallback until the runtime reports eye poses. Adult mean. +DEFAULT_IPD_MM = 63.0 + +# At a gap equal to the IPD the eyes' rays are parallel and the feed sits at +# infinity; past it they would have to diverge, which eyes cannot do and which +# is the classic cause of stereo eye strain. Stay clear of the cliff. +MAX_FRACTION_OF_IPD = 0.9 + +# Where the suggestion aims the far end of the scene. Far enough that the +# scene spreads out, near enough that the suggestion keeps clear of the +# ceiling instead of pinning to it -- advice that is always "the maximum" is +# not advice. +FAR_TARGET_M = 6.0 + +# The gap is quantised to this, so held-stick values land on clean numbers +# and match what the readout prints. +STEP_CM = 0.1 + + +def step(value: float) -> float: + """Snap to :data:`STEP_CM`. + + Apply on the way out, never to the running total: one frame at 60 Hz + moves less than half a step, so rounding the total every frame would + round it straight back and the stick would never move at all. + """ + return round(value / STEP_CM) * STEP_CM + + +def max_gap_cm(ipd_mm: float, configured_max_cm: float) -> float: + """Ceiling: the configured one, or the IPD-derived one if tighter.""" + return min(configured_max_cm, ipd_mm / 10.0 * MAX_FRACTION_OF_IPD) + + +def clamp_gap_cm(value: float, ipd_mm: float, limits: Tuple[float, float]) -> float: + minimum, maximum = limits + return min(max(value, minimum), max_gap_cm(ipd_mm, maximum)) + + +def suggested_gap_cm( + surface_m: Optional[float], ipd_mm: float, limits: Tuple[float, float] +) -> Optional[float]: + """A starting gap for a surface ``surface_m`` away, or None when the shape + has no meaningful distance (an infinite sphere). + + Stepped as well as clamped: a suggestion you cannot dial in exactly is no + use. + """ + if surface_m is None: + return None + ideal = ipd_mm / 10.0 * (1.0 - surface_m / FAR_TARGET_M) + return step(clamp_gap_cm(ideal, ipd_mm, limits)) + + +def perceived_distance_cm( + surface_m: Optional[float], ipd_mm: float, gap_cm: float +) -> Optional[float]: + """Where zero-disparity content currently appears, in cm. + + None when the geometry does not define it: an infinite sphere, or a gap + at the IPD where the rays are parallel and the answer is infinity. + """ + if surface_m is None: + return None + denominator = ipd_mm - gap_cm * 10.0 + if denominator <= 1e-3: + return None + return surface_m * ipd_mm / denominator * 100.0 diff --git a/examples/camera_viz/dashboard.py b/examples/camera_viz/dashboard.py new file mode 100644 index 0000000000..e0f3b99a0f --- /dev/null +++ b/examples/camera_viz/dashboard.py @@ -0,0 +1,231 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Live status panel, redrawn in place. + +A scrolling log makes you reconstruct the current state from the last few +lines; a panel just shows it. Everything here is a snapshot of *now* -- +timings, per-camera settings, and the last thing a button did. + +Falls back to one line per period when stderr is not a terminal, because +redrawing in a systemd journal or a piped log produces nothing but escape +codes. +""" + +from __future__ import annotations + +import shutil +import sys +import threading +from dataclasses import dataclass, field +from typing import List, Optional, Sequence + +_ACCENT = "\033[38;5;148m" # NVIDIA-ish green +_DIM = "\033[2m" +_BOLD = "\033[1m" +_RESET = "\033[0m" + +# Units live in the header so the rows stay numbers. +_COLUMNS = ( + ("camera", 12), + ("shape", 15), # equirect carries its heading here + ("lock", 8), + ("eyes", 7), + ("size m", 8), + ("height m", 10), + ("planes cm", 12), + ("submit/s", 9), +) + + +@dataclass +class CameraRow: + """One camera's line in the panel.""" + + name: str + shape: str + lock_mode: str + stereo: bool + size_m: Optional[float] + offset_y_m: Optional[float] + plane_distance_cm: Optional[float] + suggested_cm: Optional[float] + submit_fps: float + + def cells(self) -> List[str]: + def opt(value, fmt): + return format(value, fmt) if value is not None else "-" + + planes = opt(self.plane_distance_cm, ".1f") + if self.plane_distance_cm is not None and self.suggested_cm is not None: + planes = f"{planes}/{self.suggested_cm:.1f}" + return [ + self.name, + self.shape, + self.lock_mode, + "stereo" if self.stereo else "mono", + opt(self.size_m, ".2f"), + opt(self.offset_y_m, "+.2f"), + planes, + f"{self.submit_fps:.1f}", + ] + + +@dataclass +class Snapshot: + """Everything the panel draws.""" + + header: str + render: str + rows: Sequence[CameraRow] = () + ipd_mm: Optional[float] = None + notes: List[str] = field(default_factory=list) + last_event: str = "" + + +class Dashboard: + """Redraws a fixed block of lines in place. + + The render thread drives :meth:`show`, but a capture thread can call + :meth:`note` at any moment, and two interleaved writes would tear an + escape sequence in half -- so the stream is held under a lock. + """ + + def __init__(self, stream=None, colour: Optional[bool] = None) -> None: + self._out = stream if stream is not None else sys.stderr + self._lock = threading.Lock() + self._painted = 0 + self._columns = 0 + self._closed = False + self._live = self._out.isatty() if hasattr(self._out, "isatty") else False + self._colour = self._live if colour is None else colour + + @property + def live(self) -> bool: + """True when redrawing in place; False means one line per period.""" + return self._live + + def show(self, snapshot: Snapshot) -> None: + with self._lock: + self._show(snapshot) + + def _show(self, snapshot: Snapshot) -> None: + if not self._live: + self._out.write(self._one_line(snapshot) + "\n") + self._out.flush() + return + size = shutil.get_terminal_size((100, 24)) + if size.columns != self._columns: + # Rewrapped underneath us: the old panel's height no longer says + # how far up its top is. Start a fresh one below the wreckage + # rather than erase whatever is there now. + self._columns, self._painted = size.columns, 0 + lines = self._compose(snapshot) + # One row per line is the whole premise of the cursor arithmetic, so + # a panel taller than the terminal cannot be redrawn in place: the + # top scrolls off and every repaint leaves a copy behind. + if len(lines) >= size.lines: + self._painted = 0 + self._out.write(self._one_line(snapshot) + "\n") + self._out.flush() + return + if self._painted: + self._out.write(f"\033[{self._painted}A") + for line in lines: + self._out.write("\033[2K" + line + "\n") + # A shorter panel than last time would leave stale rows below. + for _ in range(max(0, self._painted - len(lines))): + self._out.write("\033[2K\n") + self._out.flush() + self._painted = max(len(lines), self._painted) + + def note(self, text: str) -> None: + """Print a line above the panel, keeping the panel intact. + + The panel owns the cursor while it is live, so anything else writing + to the same stream lands inside it and every later repaint is off by + however many lines that write scrolled -- which is how you end up + with a column of half-erased headers. Route lifecycle messages here + and they scroll away above the panel like an ordinary log. + """ + with self._lock: + self._note(text) + + def _note(self, text: str) -> None: + if self._closed: + # A source thread can outlive runner.stop() and notify into a + # panel that has already handed the cursor back. + return + if self._live and self._painted: + self._out.write(f"\033[{self._painted}A") + for _ in range(self._painted): + self._out.write("\033[2K\n") + self._out.write(f"\033[{self._painted}A") + self._painted = 0 + self._out.write(text + "\n") + self._out.flush() + + def close(self) -> None: + """Leave the cursor below the panel so a later print doesn't land in + the middle of it. Idempotent, and silences later notes.""" + with self._lock: + self._closed = True + if self._live and self._painted: + self._out.write("\n") + self._out.flush() + self._painted = 0 + + # ── composition ────────────────────────────────────────────────── + + def _paint(self, text: str, width: int, style: str = "") -> str: + """Truncate first, then colour. Doing it the other way round can cut a + line mid-escape and leave the terminal stuck in that style.""" + text = text[:width] + return f"{style}{text}{_RESET}" if style and self._colour else text + + def _width(self) -> int: + """Widest line we will draw. + + One short of the terminal: a line that fills the last column leaves + the cursor wrap-pending, and the newline after it costs two rows on + some terminals instead of one -- which slides the panel down a row + per repaint and strands a copy of the header each time. + """ + columns = shutil.get_terminal_size((100, 24)).columns + return max(20, min(columns - 1, 100)) + + def _compose(self, snapshot: Snapshot) -> List[str]: + width = self._width() + lines = [ + self._paint(f"camera_viz {snapshot.header}", width, _BOLD), + self._paint("─" * width, width, _DIM), + self._paint(f" {snapshot.render}", width), + "", + self._paint(" " + _header_row(), width, _DIM), + ] + lines += [ + self._paint(" " + _format_row(r.cells()), width) for r in snapshot.rows + ] + + if snapshot.ipd_mm is not None: + lines += ["", self._paint(f" headset IPD {snapshot.ipd_mm:.0f} mm", width)] + lines += [self._paint(f" {note}", width, _DIM) for note in snapshot.notes] + if snapshot.last_event: + lines += ["", self._paint(f" {snapshot.last_event}", width, _ACCENT)] + return lines + + def _one_line(self, snapshot: Snapshot) -> str: + """Log-friendly form: the same numbers, one line, no escape codes.""" + cameras = " | ".join( + f"{r.name} {r.submit_fps:.1f} submit/s" for r in snapshot.rows + ) + return f"camera_viz: stats: {snapshot.render} | {cameras}" + + +def _header_row() -> str: + return "".join(name.ljust(width) for name, width in _COLUMNS) + + +def _format_row(cells: Sequence[str]) -> str: + return "".join( + str(cell)[: width - 1].ljust(width) for cell, (_, width) in zip(cells, _COLUMNS) + ) diff --git a/examples/camera_viz/display.py b/examples/camera_viz/display.py new file mode 100644 index 0000000000..5586d00f25 --- /dev/null +++ b/examples/camera_viz/display.py @@ -0,0 +1,144 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Building the display side: the VizSession and one layer per surface. + +Everything here consumes the parsed :class:`~config.SourceEntry` values and +allocates — a session in the configured display mode, then a layer per shape. +Kept apart from the parsing so the config half stays free of Vulkan. +""" + +from __future__ import annotations + +import math +from typing import List, Optional + +import isaacteleop.viz as viz + +from config import SourceEntry, VALID_SHAPES +from placements import yaw_quat + +# ImageLayerBase::kSlotCount (kMaxFramesInFlight + 2). Only used to +# report the VRAM that shape switching adds. +_MAILBOX_SLOTS = 7 + + +def make_session( + cfg: dict, + mode_override: Optional[str] = None, + required_extensions: Optional[List[str]] = None, +) -> viz.VizSession: + display = cfg.get("display", {}) + # --mode overrides display.mode when given. + mode_str = (mode_override or display.get("mode", "xr")).lower() + session_cfg = viz.VizSessionConfig() + if mode_str == "window": + session_cfg.mode = viz.DisplayMode.kWindow + w = display.get("window", {}) + session_cfg.window_width = int(w.get("width", 1280)) + session_cfg.window_height = int(w.get("height", 720)) + elif mode_str == "xr": + session_cfg.mode = viz.DisplayMode.kXr + x = display.get("xr", {}) + session_cfg.xr_near_z = float(x.get("near_z", 0.05)) + session_cfg.xr_far_z = float(x.get("far_z", 100.0)) + else: + raise ValueError( + f"camera_viz: display.mode must be window|xr, got {mode_str!r}" + ) + if "clear_color" in display: + session_cfg.clear_color = tuple(display["clear_color"]) + session_cfg.app_name = display.get("app_name", "camera_viz") + # Televiz creates the XrInstance, so anything downstream needs (here the + # controller tracker's action-context extension) has to be declared now. + if required_extensions: + session_cfg.required_extensions = list(required_extensions) + return viz.VizSession.create(session_cfg) + + +def _build_layer(session: viz.VizSession, entry: SourceEntry, shape: str): + """One layer of ``shape`` for ``entry``. + + quad → QuadLayer, composited by the OpenXR runtime by default + (``compositor: televiz`` opts into the built-in compositor); + the placement strategy positions it per frame. + cylinder → CylinderLayer: the feed wrapped on an arc facing the user + (``cylinder_radius_m`` / ``cylinder_angle_deg``, aspect from + the source). Runtime-composited always. + equirect → EquirectLayer: full 360x180 sphere (the source is expected + to be an equirect panorama), aimed by ``equirect_yaw_deg``. + Runtime-composited always. + """ + spec = entry.source.spec + if shape == "cylinder": + layer_cfg = viz.CylinderLayerConfig() + layer_cfg.name = spec.name + layer_cfg.resolution = viz.Resolution(spec.width, spec.height) + layer_cfg.stereo = entry.stereo + layer_cfg.stereo_baseline_mm = entry.stereo_plane_distance_cm * 10.0 + # aspect_ratio 0 = derived from the source resolution (square texels). + layer_cfg.placement = viz.CylinderLayerPlacement( + radius_m=entry.cylinder_radius_m, + central_angle_rad=math.radians(entry.cylinder_angle_deg), + ) + return session.add_cylinder_layer(layer_cfg) + if shape == "equirect": + layer_cfg = viz.EquirectLayerConfig() + layer_cfg.name = spec.name + layer_cfg.resolution = viz.Resolution(spec.width, spec.height) + layer_cfg.stereo = entry.stereo + # Baseline only matters at finite sphere radius; harmless at the + # default infinite-radius placement (full 360x180 sphere). + layer_cfg.stereo_baseline_mm = entry.stereo_plane_distance_cm * 10.0 + # Set explicitly rather than leaning on the default so the controls + # have a known starting point to adjust from and reset to. The pose's + # -z is where the middle of the panorama lands, so yawing it aims the + # feed; position is irrelevant on the default infinite-radius sphere. + layer_cfg.placement = viz.EquirectLayerPlacement( + pose=viz.Pose3D( + (0.0, 0.0, 0.0), yaw_quat(math.radians(entry.equirect_yaw_deg)) + ) + ) + return session.add_equirect_layer(layer_cfg) + + layer_cfg = viz.QuadLayerConfig() + layer_cfg.name = spec.name + layer_cfg.resolution = viz.Resolution(spec.width, spec.height) + layer_cfg.format = viz.PixelFormat.kRGBA8 + if entry.stereo: + layer_cfg.stereo = True + layer_cfg.stereo_baseline_mm = entry.stereo_plane_distance_cm * 10.0 + # OpenXR-runtime composition is the default (kXr only; window mode is + # always composited by Televiz). Requires a placement, which the + # placement strategy applies below. + layer_cfg.openxr_composition = entry.compositor == "openxr" + return session.add_quad_layer(layer_cfg) + + +def add_layers( + session: viz.VizSession, entry: SourceEntry, all_shapes: bool +) -> "dict[str, object]": + """Returns ``{shape: layer}`` for ``entry``. + + With ``all_shapes`` every shape is built up front and all but the + configured one start hidden, so switching later is just an atomic + ``set_visible`` — no reallocation and no ``vkDeviceWaitIdle`` mid-demo, + which is what removing and re-adding a layer would cost. + """ + shapes = VALID_SHAPES if all_shapes else (entry.shape,) + layers = {} + for shape in shapes: + layer = _build_layer(session, entry, shape) + layer.set_visible(shape == entry.shape) + layers[shape] = layer + return layers + + +def estimate_layer_bytes(entry: SourceEntry, shape: str) -> int: + """Rough VRAM for one layer's mailbox: kSlotCount images, doubled for + stereo, plus the quad's mip chain.""" + spec = entry.source.spec + per_image = spec.width * spec.height * 4 + total = per_image * _MAILBOX_SLOTS * (2 if entry.stereo else 1) + if shape == "quad": + total = int(total * 4 / 3) # capped mip chain ≈ +33% + return total diff --git a/examples/camera_viz/pipeline/runner.py b/examples/camera_viz/pipeline/runner.py index 561991320d..e1c551fba3 100644 --- a/examples/camera_viz/pipeline/runner.py +++ b/examples/camera_viz/pipeline/runner.py @@ -16,18 +16,36 @@ from __future__ import annotations import logging -import sys +import math import threading import time from typing import Callable, Optional, Sequence import isaacteleop.viz as viz +from dashboard import CameraRow, Dashboard, Snapshot + from .interface import FrameSource logger = logging.getLogger(__name__) +def _measure_ipd_mm(info) -> Optional[float]: + """Headset IPD from the per-eye view poses, or None outside XR stereo. + + This is what the runtime believes the lens separation to be. It drives + the stereo-distance maths, and a user whose headset IPD setting is wrong + trims the difference out by hand on the stick. + """ + views = getattr(info, "views", None) + if views is None or len(views) < 2: + return None + left, right = views[0].pose.position, views[1].pose.position + ipd = math.dist(tuple(left), tuple(right)) * 1000.0 + # A degenerate value means the runtime has not placed the eyes yet. + return ipd if ipd > 1.0 else None + + # Submit thread poll interval when no source has new data. SUBMIT_POLL_S = 0.001 @@ -39,6 +57,9 @@ # deadlines / GPU time). STATS_PERIOD_S = 5.0 +# The panel is a snapshot, so it refreshes far more often than a log would. +LIVE_STATS_PERIOD_S = 0.5 + # Window-mode stop-check granularity. stop() calls cond.notify_all() # so this is normally a safety net, not a hot path — value isn't a # render rate, it's "how long until Ctrl-C is honored if a notify @@ -63,6 +84,10 @@ def __init__( sources: Sequence[FrameSource], layers: Sequence[viz.QuadLayer], placement_strategies: Optional[Sequence[Optional[object]]] = None, + controls: Optional[object] = None, + dashboard: Optional[Dashboard] = None, + header: str = "", + notes: Optional[Sequence[str]] = None, ) -> None: if len(sources) != len(layers): raise ValueError( @@ -84,9 +109,17 @@ def __init__( if placement_strategies is not None else [None] * len(layers) ) + # Polled on the render thread once per XR frame. It owns the live + # shape / lock-mode state, so the active layer and strategy are + # resolved through it rather than read from the lists above. + self._controls = controls + self._dashboard = dashboard if dashboard is not None else Dashboard() + self._header = header + self._notes = list(notes or ()) self._stop = threading.Event() self._submit_thread: Optional[threading.Thread] = None self._render_thread: Optional[threading.Thread] = None + self._stats_thread: Optional[threading.Thread] = None # Submit thread bumps the version + notifies after each publish; # render thread compares versions under the lock, so wakeups # can't be lost. @@ -100,7 +133,6 @@ def __init__( # Per-source submit counters (submit thread writes, stats print # reads — plain ints under the GIL, approximate reads are fine). self._submit_counts = [0] * len(self._layers) - self._stats_t0 = 0.0 def start(self) -> None: if self._submit_thread is not None or self._render_thread is not None: @@ -128,6 +160,11 @@ def start(self) -> None: target=self._render_loop, name="camera_viz_render", daemon=False ) self._render_thread.start() + # Daemon: it only draws, so it must never hold up an exit. + self._stats_thread = threading.Thread( + target=self._stats_loop, name="camera_viz_stats", daemon=True + ) + self._stats_thread.start() def stop(self) -> bool: """Returns True iff both worker threads exited within the join budget. @@ -210,6 +247,25 @@ def _record_error(self, exc: BaseException, where: str) -> None: # ── Submit thread ────────────────────────────────────────────────── + def _active_layer(self, index: int): + """Layer to submit to / place this frame. + + Resolved through the controls, never from ``self._layers``: this + class copies the lists it is handed, so a shape switch made by the + controls would otherwise be invisible here and the runner would keep + feeding the layer that just went hidden. + """ + if self._controls is not None: + return self._controls.active_layer(index) + return self._layers[index] + + def _active_strategy(self, index: int): + """Placement strategy for this frame; same reasoning as above (the + A button swaps it).""" + if self._controls is not None: + return self._controls.strategy(index) + return self._strategies[index] + def _submit_loop(self) -> None: try: self._submit_loop_inner() @@ -219,20 +275,42 @@ def _submit_loop(self) -> None: def _submit_loop_inner(self) -> None: # Pin to the source's GPU on the first frame. device_pinned = False - self._stats_t0 = time.monotonic() + # Which layer each source was last submitted to, and the frame it + # sent, so a shape switch can re-send immediately (below). + last_layers = [None] * len(self._layers) + last_frames = [None] * len(self._layers) while not self._stop.is_set(): published_any = False - for i, (layer, source) in enumerate(zip(self._layers, self._sources)): + for i, source in enumerate(self._sources): + # Re-read per pass: the controls swap this on a shape change. + layer = self._active_layer(i) frame = source.latest() if frame is None: - continue + # No new frame. If the shape just changed, re-send the + # last one so the newly visible layer isn't blank until + # the camera produces the next (up to 1/fps away). The + # source may have recycled the buffer, so the worst case + # is one torn frame on the switch, not a stall. + if layer is last_layers[i] or last_frames[i] is None: + continue + frame = last_frames[i] if not device_pinned: self._pin_to_device(frame) device_pinned = True if frame.image_right is not None: - layer.submit(frame.image, frame.image_right, stream=frame.stream) + # A stereo layer must always be fed two buffers, so the + # mono override ships the left frame to both eyes rather + # than dropping to the one-arg form (which would throw). + right = ( + frame.image + if self._controls is not None and self._controls.force_mono(i) + else frame.image_right + ) + layer.submit(frame.image, right, stream=frame.stream) else: layer.submit(frame.image, stream=frame.stream) + last_layers[i] = layer + last_frames[i] = frame self._submit_counts[i] += 1 published_any = True if published_any: @@ -241,34 +319,76 @@ def _submit_loop_inner(self) -> None: self._data_cond.notify() else: self._stop.wait(timeout=SUBMIT_POLL_S) + + # ── Stats thread ─────────────────────────────────────────────────── + + def _stats_loop(self) -> None: + """The panel on its own thread, never the submit thread. + + Writing it from the submit thread put a blocking write() between the + camera and the layer: a terminal that stops draining (a paused ssh + session, tmux scrollback, ^S) fills the pty buffer in about 25 paints + and then the feed stops until someone scrolls back down. A paint costs + 0.035 ms on a local pty -- it is the tail that matters, not the median. + """ + period = LIVE_STATS_PERIOD_S if self._dashboard.live else STATS_PERIOD_S + last = time.monotonic() + while not self._stop.wait(timeout=period): now = time.monotonic() - if now - self._stats_t0 >= STATS_PERIOD_S: - self._print_stats(now - self._stats_t0) - self._stats_t0 = now + try: + self._print_stats(now - last) + except Exception: # noqa: BLE001 — a broken pipe must not stop the feed + logger.debug("status panel write failed", exc_info=True) + last = now def _print_stats(self, elapsed: float) -> None: - """One stderr line per STATS_PERIOD_S: per-source submit rate + - the session's render-side numbers.""" - parts = [] + """Push a snapshot of everything to the status panel. + + The panel redraws in place on a terminal and falls back to one line + per period when it isn't one, so this is the only stats path. + """ + rows = [] for i, source in enumerate(self._sources): rate = self._submit_counts[i] / elapsed if elapsed > 0 else 0.0 self._submit_counts[i] = 0 - parts.append(f"{source.spec.name} {rate:.1f} submit/s") + state = self._controls.status(i) if self._controls is not None else {} + rows.append( + CameraRow( + name=source.spec.name, + shape=state.get("shape", "quad"), + lock_mode=state.get("lock_mode", "-"), + stereo=bool(state.get("stereo", False)), + size_m=state.get("size_m"), + offset_y_m=state.get("offset_y_m"), + plane_distance_cm=state.get("plane_distance_cm"), + suggested_cm=state.get("suggested_cm"), + submit_fps=rate, + ) + ) + self._dashboard.show( + Snapshot( + header=self._header, + render=self._render_summary(), + rows=rows, + ipd_mm=self._controls.ipd_mm if self._controls is not None else None, + notes=self._notes, + last_event=self._controls.last_event + if self._controls is not None + else "", + ) + ) + + def _render_summary(self) -> str: try: t = self._session.get_frame_timing_stats() - render = ( - f"render {t.render_fps:.1f} fps" - + (f" (target {t.target_fps:.0f})" if t.target_fps else "") - + f", missed {t.missed_frames}" - + (f", gpu {t.gpu_time_ms:.1f} ms" if t.gpu_time_ms else "") - + (f", stale {t.stale_layers}" if t.stale_layers else "") - ) except Exception: - render = "render n/a" - print( - "camera_viz: stats: " + render + " | " + " | ".join(parts), - file=sys.stderr, - flush=True, + return "render n/a" + return ( + f"render {t.render_fps:.1f} fps" + + (f" (target {t.target_fps:.0f})" if t.target_fps else "") + + f" missed {t.missed_frames}" + + (f" gpu {t.gpu_time_ms:.1f} ms" if t.gpu_time_ms else "") + + (f" stale {t.stale_layers}" if t.stale_layers else "") ) def _pin_to_device(self, frame) -> None: @@ -305,9 +425,20 @@ def _render_loop_inner(self) -> None: self._render_loop_window() def _render_loop_xr(self) -> None: + last = time.monotonic() + ipd_mm = None while not self._stop.is_set(): + now = time.monotonic() + dt, last = now - last, now + if self._controls is not None: + # Before placements: a lock-mode change this frame should + # take effect on this frame's pose, not the next one. The IPD + # is one frame stale, which is irrelevant -- it is fixed by + # the headset's lens separation. + self._controls.step(dt, ipd_mm) self._apply_xr_placements() - self._session.render() + info = self._session.render() + ipd_mm = _measure_ipd_mm(info) or ipd_mm if self._session.should_close(): self._stop.set() @@ -328,14 +459,20 @@ def _render_loop_window(self) -> None: self._stop.set() def _apply_xr_placements(self) -> None: - if not any(s is not None for s in self._strategies): + strategies = [self._active_strategy(i) for i in range(len(self._layers))] + if not any(s is not None for s in strategies): return head = self._session.head_pose_now() if head is None: return - for layer, strategy in zip(self._layers, self._strategies): + for i, strategy in enumerate(strategies): if strategy is None: continue + layer = self._active_layer(i) + # An equirect sphere is centred on the operator: no pose to + # re-lock, and its placement type isn't a QuadLayerPlacement. + if isinstance(layer, viz.EquirectLayer): + continue placement = strategy.update(head.position, head.orientation) if isinstance(layer, viz.CylinderLayer): # The cylinder's pose is the strategy's head anchor (its arc diff --git a/examples/camera_viz/placements/__init__.py b/examples/camera_viz/placements/__init__.py index 09fe7af8ca..65cdcb12d1 100644 --- a/examples/camera_viz/placements/__init__.py +++ b/examples/camera_viz/placements/__init__.py @@ -6,6 +6,9 @@ example, not in viz_layers. """ +# Re-exported: the same yaw maths positions an equirect sphere, which has no +# lock-mode strategy of its own (see controls.shapes). +from ._math import heading_deg, yaw_quat from .lock_modes import ( HeadLocked, LazyLocked, @@ -24,4 +27,6 @@ "PlacementStrategy", "WorldLocked", "build", + "heading_deg", + "yaw_quat", ] diff --git a/examples/camera_viz/placements/_math.py b/examples/camera_viz/placements/_math.py index 6faba85fa0..989cb70d15 100644 --- a/examples/camera_viz/placements/_math.py +++ b/examples/camera_viz/placements/_math.py @@ -52,6 +52,17 @@ def yaw_quat(yaw: float) -> Quat: return _rot_to_wxyz(Rotation.from_euler("y", yaw)) +def heading_deg(q: Quat) -> float: + """Which way ``q`` faces about +Y, in degrees. + + 0 looks down -Z (the reference space's forward), +90 down +X's opposite, + i.e. positive is a turn to the left. Inverse of :func:`yaw_quat` up to + the projection, so ``heading_deg(yaw_quat(t)) == degrees(t)``. + """ + fx, _fy, fz = project_forward_xz(q) + return math.degrees(math.atan2(-fx, -fz)) + + def project_forward_xz(q: Quat) -> Vec3: """OpenXR head forward (-Z in head local) projected onto XZ, normalized. diff --git a/examples/camera_viz/placements/lock_modes.py b/examples/camera_viz/placements/lock_modes.py index 159944e787..50e04d8c25 100644 --- a/examples/camera_viz/placements/lock_modes.py +++ b/examples/camera_viz/placements/lock_modes.py @@ -16,7 +16,6 @@ angle_between_xz_deg, normalize_angle, project_forward_xz, - quat_mul, rotate_vec, smoothstep, yaw_quat, @@ -51,8 +50,8 @@ class Placement: local −z points at the plane — the pose a curved surface centered on the viewer (``viz.CylinderLayerPlacement``) should use. Each strategy computes the anchor itself because the facing conventions differ - (world/lazy build yaw-only quats whose +z faces the head; head-locked - uses the full head orientation flipped 180°). + (world/lazy/gimbal build yaw-only quats whose +z faces the head; + head-locked uses the full head orientation, pitch and roll included). """ position: Vec3 @@ -70,6 +69,16 @@ class PlacementStrategy(ABC): @abstractmethod def update(self, head_pos: Vec3, head_orientation: Quat) -> Placement: ... + def retune(self, config: PlacementConfig) -> None: + """Swap the tuning without resetting the strategy's live state. + + ``PlacementConfig`` is frozen, so a caller adjusting size or + distance at runtime builds a replacement with ``dataclasses.replace`` + and hands it over here. Rebuilding the strategy instead would drop + the lazy anchor and re-snap the plane on every change. + """ + self._config = config + def _target_position(head_pos: Vec3, forward_xz: Vec3, cfg: PlacementConfig) -> Vec3: """Place the quad ``distance`` ahead, with right-vector / up-vector offsets. @@ -83,6 +92,10 @@ def _target_position(head_pos: Vec3, forward_xz: Vec3, cfg: PlacementConfig) -> ) +def _shift_y(position: Vec3, delta: float) -> Vec3: + return (position[0], position[1] + delta, position[2]) + + def _yaw_to_face(target: Vec3, plane_pos: Vec3) -> float: """Yaw rotation that aims the plane's front at ``target``. Mirrors ``CameraPlane::compute_yaw_to_face``.""" @@ -95,39 +108,39 @@ class WorldLocked(PlacementStrategy): def __init__(self, config: PlacementConfig) -> None: self._config = config - self._cached: Optional[Placement] = None + # The head pose the plane was placed around, frozen on first update. + # Everything else is recomputed from it each frame so a retune (size, + # height) takes effect at once; caching the finished Placement instead + # made this mode ignore retuning entirely. + self._anchor_head: Optional[Vec3] = None + self._anchor_forward_xz: Optional[Vec3] = None def update(self, head_pos: Vec3, head_orientation: Quat) -> Placement: - if self._cached is None: - forward_xz = project_forward_xz(head_orientation) - position = _target_position(head_pos, forward_xz, self._config) - yaw = _yaw_to_face(head_pos, position) - orientation = yaw_quat(yaw) - # Anchor = the head point the plane was placed around: push the - # plane back along its local +z (which faces the head). - back = rotate_vec(orientation, (0.0, 0.0, 1.0)) - anchor = tuple( - position[i] + back[i] * self._config.distance for i in range(3) - ) - self._cached = Placement( - position, - orientation, - self._config.size_meters, - self._config.distance, - anchor, - orientation, - ) - return self._cached - + if self._anchor_head is None: + self._anchor_head = head_pos + self._anchor_forward_xz = project_forward_xz(head_orientation) -# 180° about +Y: rotates the quad to face back at the user. -_ROT_Y_180: Quat = (0.0, 0.0, 1.0, 0.0) + position = _target_position( + self._anchor_head, self._anchor_forward_xz, self._config + ) + orientation = yaw_quat(_yaw_to_face(self._anchor_head, position)) + # Anchor = the head point the plane was placed around: push the plane + # back along its local +z (which faces the head). + back = rotate_vec(orientation, (0.0, 0.0, 1.0)) + anchor = tuple(position[i] + back[i] * self._config.distance for i in range(3)) + return Placement( + position, + orientation, + self._config.size_meters, + self._config.distance, + anchor, + orientation, + ) class HeadLocked(PlacementStrategy): - """Follow the head every frame, full 6-DoF. - Mirrors ``CameraPlane::update_head`` + the ``head_rotation * rotY(π)`` - in ``CameraPlane::rotation``.""" + """Follow the head every frame, full 6-DoF (pitch and roll included, + unlike the yaw-only world / gimbal / lazy modes).""" def __init__(self, config: PlacementConfig) -> None: self._config = config @@ -142,11 +155,13 @@ def update(self, head_pos: Vec3, head_orientation: Quat) -> Placement: head_pos[1] + forward[1] * d + right[1] * ox + up[1] * oy, head_pos[2] + forward[2] * d + right[2] * ox + up[2] * oy, ) - orientation = quat_mul(head_orientation, _ROT_Y_180) + # No flip: every mode returns the rotation that faces the plane back + # at the viewer, which is identity for a level head. Rotating by 180 + # deg here pointed the quad away, and an OpenXR quad layer is + # single-sided, so the feed went black. + orientation = head_orientation # Anchor = the head itself (plus the configured offsets, already # baked into ``position``): pull the plane back by ``distance``. - # Orientation is the UNflipped head pose so the anchor's local −z - # (where a cylinder arc bows out) tracks the gaze direction. anchor = ( position[0] - forward[0] * d, position[1] - forward[1] * d, @@ -218,6 +233,23 @@ def __init__(self, config: PlacementConfig) -> None: self._target_position: Vec3 = (0.0, 0.0, 0.0) self._target_yaw = 0.0 + def retune(self, config: PlacementConfig) -> None: + """Carry a height change onto the position already placed. + + Unlike the other modes this one cannot recompute from an anchor -- its + position is wherever the last re-snap and transition left it. Without + this, a height change sat unapplied until the next re-snap, which is + the whole point of lazy mode not happening. + """ + delta_y = config.offset_y - self._config.offset_y + super().retune(config) + if delta_y: + self._position = _shift_y(self._position, delta_y) + self._target_position = _shift_y(self._target_position, delta_y) + self._transition_start_position = _shift_y( + self._transition_start_position, delta_y + ) + def update(self, head_pos: Vec3, head_orientation: Quat) -> Placement: now = time.monotonic() forward_xz = project_forward_xz(head_orientation) diff --git a/examples/camera_viz/scripts/_install_deps.sh b/examples/camera_viz/scripts/_install_deps.sh index 0ffd9205dc..a41a62082c 100755 --- a/examples/camera_viz/scripts/_install_deps.sh +++ b/examples/camera_viz/scripts/_install_deps.sh @@ -538,7 +538,8 @@ fi # girepository-1.0 build path, and Ubuntu 22.04 only ships 1.0 # (libgirepository1.0-dev). 3.50.x supports both. Source-builds against # the C deps installed in ensure_apt_deps(); pycairo is a transitive dep. -PKGS=("pyyaml>=6.0" "$target_cupy" "numpy>=1.23" "scipy>=1.15") +# pillow renders the XR controls HUD (examples/camera_viz/hud.py). +PKGS=("pyyaml>=6.0" "$target_cupy" "numpy>=1.23" "scipy>=1.15" "pillow>=10.0") [[ "$MODE" == full ]] && PKGS=("$ISAACTELEOP_PKG" "${PKGS[@]}") $WITH_V4L2 && PKGS+=("opencv-python>=4.5") $WITH_OAKD && PKGS+=("depthai>=3.0") diff --git a/examples/camera_viz/sources/__init__.py b/examples/camera_viz/sources/__init__.py index defdadbb3e..89132a9c81 100644 --- a/examples/camera_viz/sources/__init__.py +++ b/examples/camera_viz/sources/__init__.py @@ -13,7 +13,7 @@ from pipeline import FrameSource -from ._helpers import PairedFrameSource, set_verbose +from ._helpers import PairedFrameSource, set_notify_sink, set_verbose from .oakd import OakdSource from .rtp_h264 import RtpH264Source from .synthetic import SyntheticSource, SyntheticStereoSource @@ -32,6 +32,7 @@ "ZedSource", "build_local_camera", "resolve_video_paths", + "set_notify_sink", "set_verbose", ] diff --git a/examples/camera_viz/sources/_helpers.py b/examples/camera_viz/sources/_helpers.py index 48f777b1da..5ae6e3f67e 100644 --- a/examples/camera_viz/sources/_helpers.py +++ b/examples/camera_viz/sources/_helpers.py @@ -46,10 +46,28 @@ from pipeline import Frame, FrameSource, SourceSpec +_SINK = None + + +def set_notify_sink(sink) -> None: + """Send notifications somewhere other than stderr. + + The status panel redraws stderr in place, so it hands its own writer in + here: a lifecycle message printed underneath it would land mid-panel and + leave every later repaint misaligned. + """ + global _SINK + _SINK = sink + + def notify(tag: str, msg: str) -> None: # Stderr-direct so it shows without a configured Python logger. # Reserved for lifecycle events; periodic stats use notify_verbose. - print(f"[{tag}] {msg}", file=sys.stderr, flush=True) + line = f"[{tag}] {msg}" + if _SINK is not None: + _SINK(line) + return + print(line, file=sys.stderr, flush=True) _VERBOSE = False @@ -75,7 +93,7 @@ def _verbose_enabled() -> bool: def notify_verbose(tag: str, msg: str) -> None: """Periodic stats; gated by YAML ``verbose:`` or CAMERA_VIZ_VERBOSE.""" if _verbose_enabled(): - print(f"[{tag}] {msg}", file=sys.stderr, flush=True) + notify(tag, msg) logger = logging.getLogger(__name__) diff --git a/src/viz/layers/cpp/cylinder_layer.cpp b/src/viz/layers/cpp/cylinder_layer.cpp index 0dffe11992..102586cb42 100644 --- a/src/viz/layers/cpp/cylinder_layer.cpp +++ b/src/viz/layers/cpp/cylinder_layer.cpp @@ -42,6 +42,10 @@ const CylinderLayer::Config& validate_config(const CylinderLayer::Config& config { throw std::invalid_argument("CylinderLayer: stereo_baseline_mm must be finite"); } + if (!std::isfinite(config.stereo_convergence_deg)) + { + throw std::invalid_argument("CylinderLayer: stereo_convergence_deg must be finite"); + } validate_placement(config.placement); return config; } @@ -56,7 +60,9 @@ CylinderLayer::CylinderLayer(const VkContext& ctx, Config config) config.format, config.stereo, /*mip_levels=*/1), - config_(std::move(config)) + config_(std::move(config)), + stereo_baseline_mm_(config_.stereo_baseline_mm), + stereo_convergence_deg_(config_.stereo_convergence_deg) { placement_ = config_.placement; } @@ -108,7 +114,8 @@ std::optional CylinderLayer::acquire_native_layer(uint32_t in_f v.color_right = config_.stereo ? slots_right_[cur]->vk_image() : VK_NULL_HANDLE; v.extent = resolution(); v.pose = placement.pose; - v.stereo_baseline_mm = config_.stereo ? config_.stereo_baseline_mm : 0.0f; + v.stereo_baseline_mm = config_.stereo ? stereo_baseline_mm_.load(std::memory_order_relaxed) : 0.0f; + v.stereo_convergence_deg = config_.stereo ? stereo_convergence_deg_.load(std::memory_order_relaxed) : 0.0f; v.alpha_blend = config_.alpha_blend; v.radius = placement.radius_m; v.central_angle = placement.central_angle_rad; @@ -134,4 +141,32 @@ CylinderLayer::Config::Placement CylinderLayer::placement() const noexcept return placement_; } +void CylinderLayer::set_stereo_baseline_mm(float baseline_mm) +{ + if (!std::isfinite(baseline_mm)) + { + throw std::invalid_argument("CylinderLayer: stereo_baseline_mm must be finite"); + } + stereo_baseline_mm_.store(baseline_mm, std::memory_order_relaxed); +} + +float CylinderLayer::stereo_baseline_mm() const noexcept +{ + return stereo_baseline_mm_.load(std::memory_order_relaxed); +} + +void CylinderLayer::set_stereo_convergence_deg(float convergence_deg) +{ + if (!std::isfinite(convergence_deg)) + { + throw std::invalid_argument("CylinderLayer: stereo_convergence_deg must be finite"); + } + stereo_convergence_deg_.store(convergence_deg, std::memory_order_relaxed); +} + +float CylinderLayer::stereo_convergence_deg() const noexcept +{ + return stereo_convergence_deg_.load(std::memory_order_relaxed); +} + } // namespace viz diff --git a/src/viz/layers/cpp/equirect_layer.cpp b/src/viz/layers/cpp/equirect_layer.cpp index 3a17ff148e..bc35175982 100644 --- a/src/viz/layers/cpp/equirect_layer.cpp +++ b/src/viz/layers/cpp/equirect_layer.cpp @@ -49,6 +49,10 @@ const EquirectLayer::Config& validate_config(const EquirectLayer::Config& config { throw std::invalid_argument("EquirectLayer: stereo_baseline_mm must be finite"); } + if (!std::isfinite(config.stereo_convergence_deg)) + { + throw std::invalid_argument("EquirectLayer: stereo_convergence_deg must be finite"); + } validate_placement(config.placement); return config; } @@ -63,7 +67,9 @@ EquirectLayer::EquirectLayer(const VkContext& ctx, Config config) config.format, config.stereo, /*mip_levels=*/1), - config_(std::move(config)) + config_(std::move(config)), + stereo_baseline_mm_(config_.stereo_baseline_mm), + stereo_convergence_deg_(config_.stereo_convergence_deg) { placement_ = config_.placement; } @@ -115,7 +121,8 @@ std::optional EquirectLayer::acquire_native_layer(uint32_t in_f v.color_right = config_.stereo ? slots_right_[cur]->vk_image() : VK_NULL_HANDLE; v.extent = resolution(); v.pose = placement.pose; - v.stereo_baseline_mm = config_.stereo ? config_.stereo_baseline_mm : 0.0f; + v.stereo_baseline_mm = config_.stereo ? stereo_baseline_mm_.load(std::memory_order_relaxed) : 0.0f; + v.stereo_convergence_deg = config_.stereo ? stereo_convergence_deg_.load(std::memory_order_relaxed) : 0.0f; v.alpha_blend = config_.alpha_blend; v.radius = placement.radius_m; v.central_horizontal_angle = placement.central_horizontal_angle_rad; @@ -138,4 +145,32 @@ EquirectLayer::Config::Placement EquirectLayer::placement() const noexcept return placement_; } +void EquirectLayer::set_stereo_baseline_mm(float baseline_mm) +{ + if (!std::isfinite(baseline_mm)) + { + throw std::invalid_argument("EquirectLayer: stereo_baseline_mm must be finite"); + } + stereo_baseline_mm_.store(baseline_mm, std::memory_order_relaxed); +} + +float EquirectLayer::stereo_baseline_mm() const noexcept +{ + return stereo_baseline_mm_.load(std::memory_order_relaxed); +} + +void EquirectLayer::set_stereo_convergence_deg(float convergence_deg) +{ + if (!std::isfinite(convergence_deg)) + { + throw std::invalid_argument("EquirectLayer: stereo_convergence_deg must be finite"); + } + stereo_convergence_deg_.store(convergence_deg, std::memory_order_relaxed); +} + +float EquirectLayer::stereo_convergence_deg() const noexcept +{ + return stereo_convergence_deg_.load(std::memory_order_relaxed); +} + } // namespace viz diff --git a/src/viz/layers/cpp/inc/viz/layers/cylinder_layer.hpp b/src/viz/layers/cpp/inc/viz/layers/cylinder_layer.hpp index 615712ea3a..c55aa84cdd 100644 --- a/src/viz/layers/cpp/inc/viz/layers/cylinder_layer.hpp +++ b/src/viz/layers/cpp/inc/viz/layers/cylinder_layer.hpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -57,6 +58,14 @@ class CylinderLayer : public ImageLayerBase // false. float stereo_baseline_mm = 0.0f; + // Per-eye yaw in degrees; the left eye's surface rotates +half, the + // right −half. This is the knob to reach for on a curved surface: + // translating one (stereo_baseline_mm) gives full disparity dead + // ahead and progressively less toward the edges, and does nothing + // at all at infinite radius. A rotation is uniform across the arc + // and works at any radius. Positive pushes content away. + float stereo_convergence_deg = 0.0f; + // Composite honoring the texture's alpha channel // (XR_COMPOSITION_LAYER_BLEND_TEXTURE_SOURCE_ALPHA_BIT). Off by // default: camera feeds are opaque, and alpha-free layers keep @@ -116,10 +125,24 @@ class CylinderLayer : public ImageLayerBase void set_placement(const Config::Placement& placement); Config::Placement placement() const noexcept; + // Live stereo baseline, thread-safe vs the render thread; takes effect + // next frame. No-op while the layer is mono (Config::stereo == false). + // Throws std::invalid_argument on a non-finite value. + void set_stereo_baseline_mm(float baseline_mm); + float stereo_baseline_mm() const noexcept; + + // Live per-eye yaw, thread-safe vs the render thread; takes effect next + // frame. No-op while the layer is mono. Throws std::invalid_argument on + // a non-finite value. + void set_stereo_convergence_deg(float convergence_deg); + float stereo_convergence_deg() const noexcept; + private: Config config_; mutable std::mutex placement_mutex_; + std::atomic stereo_baseline_mm_; + std::atomic stereo_convergence_deg_; Config::Placement placement_{}; }; diff --git a/src/viz/layers/cpp/inc/viz/layers/equirect_layer.hpp b/src/viz/layers/cpp/inc/viz/layers/equirect_layer.hpp index ac140ecd86..20a320b190 100644 --- a/src/viz/layers/cpp/inc/viz/layers/equirect_layer.hpp +++ b/src/viz/layers/cpp/inc/viz/layers/equirect_layer.hpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -65,6 +66,14 @@ class EquirectLayer : public ImageLayerBase // Ignored when stereo is false. float stereo_baseline_mm = 0.0f; + // Per-eye yaw in degrees; the left eye's surface rotates +half, the + // right −half. This is the knob to reach for on a curved surface: + // translating one (stereo_baseline_mm) gives full disparity dead + // ahead and progressively less toward the edges, and does nothing + // at all at infinite radius. A rotation is uniform across the arc + // and works at any radius. Positive pushes content away. + float stereo_convergence_deg = 0.0f; + // Composite honoring the texture's alpha channel // (XR_COMPOSITION_LAYER_BLEND_TEXTURE_SOURCE_ALPHA_BIT). Off by // default: opaque panoramas keep the frame eligible for the @@ -124,10 +133,24 @@ class EquirectLayer : public ImageLayerBase void set_placement(const Config::Placement& placement); Config::Placement placement() const noexcept; + // Live stereo baseline, thread-safe vs the render thread; takes effect + // next frame. No-op while the layer is mono (Config::stereo == false). + // Throws std::invalid_argument on a non-finite value. + void set_stereo_baseline_mm(float baseline_mm); + float stereo_baseline_mm() const noexcept; + + // Live per-eye yaw, thread-safe vs the render thread; takes effect next + // frame. No-op while the layer is mono. Throws std::invalid_argument on + // a non-finite value. + void set_stereo_convergence_deg(float convergence_deg); + float stereo_convergence_deg() const noexcept; + private: Config config_; mutable std::mutex placement_mutex_; + std::atomic stereo_baseline_mm_; + std::atomic stereo_convergence_deg_; Config::Placement placement_{}; }; diff --git a/src/viz/layers/cpp/inc/viz/layers/quad_layer.hpp b/src/viz/layers/cpp/inc/viz/layers/quad_layer.hpp index 3e67c3dbd3..c88611338d 100644 --- a/src/viz/layers/cpp/inc/viz/layers/quad_layer.hpp +++ b/src/viz/layers/cpp/inc/viz/layers/quad_layer.hpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -167,6 +168,13 @@ class QuadLayer : public ImageLayerBase void set_placement(std::optional placement); std::optional placement() const noexcept; + // Live stereo baseline, thread-safe vs record(); takes effect next + // frame. No-op while the layer is mono (Config::stereo == false) -- + // a baseline needs two eyes to separate. Throws std::invalid_argument + // on a non-finite value, matching construction. + void set_stereo_baseline_mm(float baseline_mm); + float stereo_baseline_mm() const noexcept; + protected: // Native quad: first GPU read is the backend's copy into the quad // swapchain (TRANSFER). Composited-with-mips: the mip-gen blit chain @@ -214,6 +222,10 @@ class QuadLayer : public ImageLayerBase // Live placement; lock for set_placement / record() snapshot. mutable std::mutex placement_mutex_; + + // Lock-free: record() snapshots it once per frame so both eyes + // shift by the same amount even if a setter lands mid-record. + std::atomic stereo_baseline_mm_; std::optional placement_; }; diff --git a/src/viz/layers/cpp/quad_layer.cpp b/src/viz/layers/cpp/quad_layer.cpp index 6e02f37e02..c8f41cda6e 100644 --- a/src/viz/layers/cpp/quad_layer.cpp +++ b/src/viz/layers/cpp/quad_layer.cpp @@ -125,7 +125,8 @@ QuadLayer::QuadLayer(const VkContext& ctx, VkRenderPass render_pass, Config conf resolve_mip_levels(config)), render_pass_(render_pass), config_(std::move(config)), - mip_levels_(resolve_mip_levels(config_)) + mip_levels_(resolve_mip_levels(config_)), + stereo_baseline_mm_(config_.stereo_baseline_mm) { placement_ = config_.placement; init(); @@ -350,7 +351,7 @@ std::optional QuadLayer::acquire_native_layer(uint32_t in_fligh v.extent = config_.resolution; v.pose = placement->pose; v.size_meters = placement->size_meters; - v.stereo_baseline_mm = config_.stereo ? config_.stereo_baseline_mm : 0.0f; + v.stereo_baseline_mm = config_.stereo ? stereo_baseline_mm_.load(std::memory_order_relaxed) : 0.0f; v.alpha_blend = config_.alpha_blend; v.source_id = this; return v; @@ -425,7 +426,8 @@ void QuadLayer::record(VkCommandBuffer cmd, // constant. Skipped when the layer is mono (baseline doesn't apply) // OR outside kXr (both eyes converge to a single view, no signed // disambiguation possible). Zero baseline elides to the mono MVP. - const bool apply_baseline = xr_mode && config_.stereo && config_.stereo_baseline_mm != 0.0f; + const float baseline_mm = stereo_baseline_mm_.load(std::memory_order_relaxed); + const bool apply_baseline = xr_mode && config_.stereo && baseline_mm != 0.0f; glm::vec3 baseline_axis_ws{ 0.0f }; if (apply_baseline) { @@ -457,7 +459,7 @@ void QuadLayer::record(VkCommandBuffer cmd, const float sign = (view_idx == 0) ? -1.0f : +1.0f; // 0.5 to halve the disparity per eye, 0.001 to convert mm → m // (placement.pose.position is in world meters). - eye_placement.pose.position += sign * (config_.stereo_baseline_mm * 0.0005f) * baseline_axis_ws; + eye_placement.pose.position += sign * (baseline_mm * 0.0005f) * baseline_axis_ws; } const glm::mat4 mvp = placement_mvp(eye_placement, view); std::memcpy(data.mvp, &mvp[0][0], sizeof(data.mvp)); @@ -482,6 +484,20 @@ std::optional QuadLayer::placement() const noexcep return placement_; } +void QuadLayer::set_stereo_baseline_mm(float baseline_mm) +{ + if (!std::isfinite(baseline_mm)) + { + throw std::invalid_argument("QuadLayer: stereo_baseline_mm must be finite"); + } + stereo_baseline_mm_.store(baseline_mm, std::memory_order_relaxed); +} + +float QuadLayer::stereo_baseline_mm() const noexcept +{ + return stereo_baseline_mm_.load(std::memory_order_relaxed); +} + VkPipelineStageFlags QuadLayer::first_read_stage() const noexcept { // Native quad: first GPU read is the backend's copy into the quad diff --git a/src/viz/python/layers_bindings.cpp b/src/viz/python/layers_bindings.cpp index ed13d8c54e..eaa00f2d11 100644 --- a/src/viz/python/layers_bindings.cpp +++ b/src/viz/python/layers_bindings.cpp @@ -198,6 +198,10 @@ numpy on a CUDA device pointer); the binding converts it on the fly. "Update placement at runtime (validated; raises ValueError on non-positive " "size_meters). None switches to fullscreen (window mode only).") .def("placement", &viz::QuadLayer::placement) + .def("set_stereo_baseline_mm", &viz::QuadLayer::set_stereo_baseline_mm, "baseline_mm"_a, + "Live stereo baseline in millimeters. Takes effect on the next frame; " + "inert while the layer is mono. Raises ValueError on a non-finite value.") + .def_property_readonly("stereo_baseline_mm", &viz::QuadLayer::stereo_baseline_mm) // Bind via concrete-type lambdas: is_visible/set_visible live on // LayerBase, which isn't a registered pybind base of QuadLayer, so a // direct &QuadLayer::is_visible resolves to LayerBase and rejects a @@ -246,6 +250,13 @@ numpy on a CUDA device pointer); the binding converts it on the fly. "along the placement's local +x axis — same convention as QuadLayer. " "0 (default) → both eyes see the same world cylinder and depth comes " "from the image pair. Ignored unless stereo.") + .def_readwrite("stereo_convergence_deg", &viz::CylinderLayer::Config::stereo_convergence_deg, + "Per-eye yaw in degrees; the left eye's surface rotates +half, the right " + "-half. Prefer this over stereo_baseline_mm on a curved surface: a " + "translation gives full disparity dead ahead and less toward the " + "edges, and none at all at infinite radius, while a rotation is " + "uniform across the arc. Positive pushes content away. Ignored " + "unless stereo.") .def_readwrite( "alpha_blend", &viz::CylinderLayer::Config::alpha_blend, "Composite honoring the texture's alpha channel (XR_COMPOSITION_LAYER_BLEND_TEXTURE_SOURCE_ALPHA_BIT). Off by default: opaque content composites fully within the layer bounds (passthrough still shows outside them) and alpha-free layers keep the frame eligible for CloudXR's client-reconstructed streaming, which excludes source-alpha layers. Turn on for translucent content (HUDs, overlays).") @@ -266,7 +277,17 @@ order). Same submit contract as QuadLayer. cylinder_cls .def("set_placement", &viz::CylinderLayer::set_placement, "placement"_a, "Update placement at runtime (validated; raises ValueError on bad shape params).") - .def("placement", &viz::CylinderLayer::placement); + .def("placement", &viz::CylinderLayer::placement) + .def("set_stereo_baseline_mm", &viz::CylinderLayer::set_stereo_baseline_mm, "baseline_mm"_a, + "Live stereo baseline in millimeters. Takes effect on the next frame; " + "inert while the layer is mono. Raises ValueError on a non-finite value.") + .def_property_readonly("stereo_baseline_mm", &viz::CylinderLayer::stereo_baseline_mm) + .def("set_stereo_convergence_deg", &viz::CylinderLayer::set_stereo_convergence_deg, "convergence_deg"_a, + "Live per-eye yaw in degrees. The knob for curved surfaces: rotating them " + "gives uniform disparity across the arc and works at infinite radius, where " + "translating (stereo_baseline_mm) does nothing. Raises ValueError on a " + "non-finite value.") + .def_property_readonly("stereo_convergence_deg", &viz::CylinderLayer::stereo_convergence_deg); bind_image_layer_common(cylinder_cls, "CylinderLayer.submit"); // ── EquirectLayer (native-only: XrCompositionLayerEquirect2KHR) ─── @@ -315,6 +336,13 @@ order). Same submit contract as QuadLayer. "R changes by ~shift/R, which is zero for the infinite (0/+inf) sphere. " "0 (default) keeps the VR180/VR360 shared-sphere convention. Ignored " "unless stereo.") + .def_readwrite("stereo_convergence_deg", &viz::EquirectLayer::Config::stereo_convergence_deg, + "Per-eye yaw in degrees; the left eye's surface rotates +half, the right " + "-half. Prefer this over stereo_baseline_mm on a curved surface: a " + "translation gives full disparity dead ahead and less toward the " + "edges, and none at all at infinite radius, while a rotation is " + "uniform across the arc. Positive pushes content away. Ignored " + "unless stereo.") .def_readwrite( "alpha_blend", &viz::EquirectLayer::Config::alpha_blend, "Composite honoring the texture's alpha channel (XR_COMPOSITION_LAYER_BLEND_TEXTURE_SOURCE_ALPHA_BIT). Off by default: opaque content composites fully within the layer bounds (passthrough still shows outside them) and alpha-free layers keep the frame eligible for CloudXR's client-reconstructed streaming, which excludes source-alpha layers. Turn on for translucent content (HUDs, overlays).") @@ -336,7 +364,17 @@ FIRST when it acts as a background. Same submit contract as QuadLayer. equirect_cls .def("set_placement", &viz::EquirectLayer::set_placement, "placement"_a, "Update placement at runtime (validated; raises ValueError on bad shape params).") - .def("placement", &viz::EquirectLayer::placement); + .def("placement", &viz::EquirectLayer::placement) + .def("set_stereo_baseline_mm", &viz::EquirectLayer::set_stereo_baseline_mm, "baseline_mm"_a, + "Live stereo baseline in millimeters. Takes effect on the next frame; " + "inert while the layer is mono. Raises ValueError on a non-finite value.") + .def_property_readonly("stereo_baseline_mm", &viz::EquirectLayer::stereo_baseline_mm) + .def("set_stereo_convergence_deg", &viz::EquirectLayer::set_stereo_convergence_deg, "convergence_deg"_a, + "Live per-eye yaw in degrees. The knob for curved surfaces: rotating them " + "gives uniform disparity across the arc and works at infinite radius, where " + "translating (stereo_baseline_mm) does nothing. Raises ValueError on a " + "non-finite value.") + .def_property_readonly("stereo_convergence_deg", &viz::EquirectLayer::stereo_convergence_deg); bind_image_layer_common(equirect_cls, "EquirectLayer.submit"); // ── ProjectionLayer ──────────────────────────────────────────────── diff --git a/src/viz/session/cpp/inc/viz/session/layer_base.hpp b/src/viz/session/cpp/inc/viz/session/layer_base.hpp index ba8012e905..b2bd267eeb 100644 --- a/src/viz/session/cpp/inc/viz/session/layer_base.hpp +++ b/src/viz/session/cpp/inc/viz/session/layer_base.hpp @@ -94,6 +94,15 @@ struct NativeLayerView // equirect sphere is unchanged by construction, so it's a no-op there). float stereo_baseline_mm = 0.0f; + // Per-eye yaw about the placement's local +y (degrees); the left eye's + // layer pose rotates +half, the right −half. The curved-surface + // equivalent of stereo_baseline_mm: on a surface wrapped around the + // viewer a horizontal image shift IS a rotation, so this gives uniform + // disparity across the whole arc where a translation falls off toward + // the edges — and it still works at infinite radius, where translating + // the surface does nothing at all. Ignored mono. + float stereo_convergence_deg = 0.0f; + // Composite this layer honoring its texture's alpha channel // (XR_COMPOSITION_LAYER_BLEND_TEXTURE_SOURCE_ALPHA_BIT). False = // opaque within the layer's bounds — the right setting for camera diff --git a/src/viz/session/cpp/xr_backend.cpp b/src/viz/session/cpp/xr_backend.cpp index e6c96a2c32..30a72d16bc 100644 --- a/src/viz/session/cpp/xr_backend.cpp +++ b/src/viz/session/cpp/xr_backend.cpp @@ -1035,6 +1035,14 @@ void XrBackend::record_native_layers(VkCommandBuffer cmd, const Frame& /*frame*/ baseline_axis_ws = glm::mat3_cast(view.pose.orientation) * glm::vec3(1.0f, 0.0f, 0.0f); } + // Curved surfaces converge by rotation instead: see + // NativeLayerView::stereo_convergence_deg. Positive pushes content + // away, matching the baseline's sign — a +yaw swings the surface's + // −z (and with it the image) toward −x, so the left eye takes the + // positive half. + const bool apply_convergence = stereo && view.stereo_convergence_deg != 0.0f; + const float half_convergence_rad = glm::radians(view.stereo_convergence_deg) * 0.5f; + for (uint32_t e = 0; e < eye_count; ++e) { ViewSwapchain& sw = qs.eyes[e]; @@ -1087,6 +1095,12 @@ void XrBackend::record_native_layers(VkCommandBuffer cmd, const Frame& /*frame*/ // 0.5 halves the disparity per eye; 0.001 converts mm → m. eye_pose.position += sign * (view.stereo_baseline_mm * 0.0005f) * baseline_axis_ws; } + if (apply_convergence) + { + const float sign = (e == 0) ? +1.0f : -1.0f; + const glm::quat yaw = glm::angleAxis(sign * half_convergence_rad, glm::vec3(0.0f, 1.0f, 0.0f)); + eye_pose.orientation = glm::normalize(eye_pose.orientation * yaw); + } NativeLayerSubmit submit{}; submit.swapchain = sw.handle; diff --git a/tests/cpp/viz/layers/test_quad_layer.cpp b/tests/cpp/viz/layers/test_quad_layer.cpp index d8d285f866..9222e47552 100644 --- a/tests/cpp/viz/layers/test_quad_layer.cpp +++ b/tests/cpp/viz/layers/test_quad_layer.cpp @@ -417,6 +417,56 @@ TEST_CASE("QuadLayer ctor rejects non-finite stereo_baseline_mm", "[gpu][quad_la CHECK_THROWS_AS(QuadLayer(ctx, target->render_pass(), cfg), std::invalid_argument); } +TEST_CASE("QuadLayer::set_stereo_baseline_mm round-trips and rejects non-finite", "[gpu][quad_layer][stereo]") +{ + if (!is_gpu_available()) + { + SKIP("No Vulkan-capable GPU available"); + } + auto& ctx = shared_vk_context(); + auto target = RenderTarget::create(ctx, RenderTarget::Config{ Resolution{ 64, 64 } }); + + QuadLayer::Config cfg; + cfg.resolution = { 64, 64 }; + cfg.stereo = true; + cfg.stereo_baseline_mm = 65.0f; + QuadLayer layer(ctx, target->render_pass(), cfg); + + // Seeded from Config, then live-settable including negative (planes cross). + CHECK(layer.stereo_baseline_mm() == 65.0f); + layer.set_stereo_baseline_mm(0.0f); + CHECK(layer.stereo_baseline_mm() == 0.0f); + layer.set_stereo_baseline_mm(-42.5f); + CHECK(layer.stereo_baseline_mm() == -42.5f); + + // Same finite-check as construction, and a rejected value must not land. + CHECK_THROWS_AS(layer.set_stereo_baseline_mm(std::numeric_limits::quiet_NaN()), std::invalid_argument); + CHECK_THROWS_AS(layer.set_stereo_baseline_mm(std::numeric_limits::infinity()), std::invalid_argument); + CHECK(layer.stereo_baseline_mm() == -42.5f); +} + +// Mono layers accept the setter (apps toggle stereo without branching) but +// the value stays inert -- record() and the native path both gate on +// Config::stereo, which is fixed at construction. +TEST_CASE("QuadLayer::set_stereo_baseline_mm is accepted but inert while mono", "[gpu][quad_layer][stereo]") +{ + if (!is_gpu_available()) + { + SKIP("No Vulkan-capable GPU available"); + } + auto& ctx = shared_vk_context(); + auto target = RenderTarget::create(ctx, RenderTarget::Config{ Resolution{ 64, 64 } }); + + QuadLayer::Config cfg; + cfg.resolution = { 64, 64 }; + cfg.stereo = false; + QuadLayer layer(ctx, target->render_pass(), cfg); + + layer.set_stereo_baseline_mm(65.0f); + CHECK(layer.stereo_baseline_mm() == 65.0f); + CHECK(layer.device_image_right(0) == nullptr); +} + TEST_CASE("QuadLayer stereo allocates paired DeviceImages for every slot", "[gpu][quad_layer][stereo]") { if (!is_gpu_available()) diff --git a/tests/cpp/viz/layers/test_shaped_layers.cpp b/tests/cpp/viz/layers/test_shaped_layers.cpp index 723613b0bd..dc17257643 100644 --- a/tests/cpp/viz/layers/test_shaped_layers.cpp +++ b/tests/cpp/viz/layers/test_shaped_layers.cpp @@ -217,6 +217,157 @@ TEST_CASE("CylinderLayer acquire promotes the mailbox and carries shape params", CHECK_THROWS_AS(layer.acquire_native_layer(0), std::logic_error); } +// The value that matters is the one the backend reads, so assert through +// acquire_native_layer() rather than the getter: a setter that updated only +// Config would pass a round-trip check and still never reach the runtime. +// Curved surfaces converge by rotating, not translating: a translation gives +// full disparity dead ahead and less toward the arc edges, and none at all at +// the infinite radius an equirect sphere defaults to. +TEST_CASE("Shaped layers carry a live stereo convergence into the native view", + "[gpu][cylinder_layer][equirect_layer][native][stereo]") +{ + if (!is_gpu_available()) + { + SKIP("No Vulkan-capable GPU available"); + } + auto& ctx = shared_vk_context(); + + void* left = alloc_device_pixels(64, 32); + CudaFree guard_l{ left }; + void* right = alloc_device_pixels(64, 32); + CudaFree guard_r{ right }; + + SECTION("CylinderLayer") + { + CylinderLayer::Config cfg; + cfg.resolution = { 64, 32 }; + cfg.stereo = true; + cfg.stereo_convergence_deg = 1.5f; + CylinderLayer layer(ctx, cfg); + CHECK(layer.stereo_convergence_deg() == 1.5f); + + layer.set_stereo_convergence_deg(-0.75f); + layer.submit(device_buffer(left, 64, 32), device_buffer(right, 64, 32)); + const auto view = layer.acquire_native_layer(0); + REQUIRE(view.has_value()); + CHECK(view->stereo_convergence_deg == -0.75f); + + CHECK_THROWS_AS(layer.set_stereo_convergence_deg(std::numeric_limits::infinity()), std::invalid_argument); + CHECK(layer.stereo_convergence_deg() == -0.75f); + } + + SECTION("EquirectLayer") + { + EquirectLayer::Config cfg; + cfg.resolution = { 64, 32 }; + cfg.stereo = true; + EquirectLayer layer(ctx, cfg); + + layer.set_stereo_convergence_deg(2.0f); + layer.submit(device_buffer(left, 64, 32), device_buffer(right, 64, 32)); + const auto view = layer.acquire_native_layer(0); + REQUIRE(view.has_value()); + CHECK(view->stereo_convergence_deg == 2.0f); + } + + SECTION("mono stays inert") + { + CylinderLayer::Config cfg; + cfg.resolution = { 64, 32 }; + cfg.stereo = false; + CylinderLayer layer(ctx, cfg); + + layer.set_stereo_convergence_deg(2.0f); + layer.submit(device_buffer(left, 64, 32)); + const auto view = layer.acquire_native_layer(0); + REQUIRE(view.has_value()); + CHECK(view->stereo_convergence_deg == 0.0f); + } +} + +TEST_CASE("Shaped layer ctors reject a non-finite stereo convergence", "[gpu][cylinder_layer][equirect_layer][stereo]") +{ + if (!is_gpu_available()) + { + SKIP("No Vulkan-capable GPU available"); + } + auto& ctx = shared_vk_context(); + + CylinderLayer::Config cyl; + cyl.resolution = { 64, 32 }; + cyl.stereo_convergence_deg = std::numeric_limits::quiet_NaN(); + CHECK_THROWS_AS(CylinderLayer(ctx, cyl), std::invalid_argument); + + EquirectLayer::Config eq; + eq.resolution = { 64, 32 }; + eq.stereo_convergence_deg = std::numeric_limits::infinity(); + CHECK_THROWS_AS(EquirectLayer(ctx, eq), std::invalid_argument); +} + +TEST_CASE("Shaped layers carry a live stereo baseline into the native view", + "[gpu][cylinder_layer][equirect_layer][native][stereo]") +{ + if (!is_gpu_available()) + { + SKIP("No Vulkan-capable GPU available"); + } + auto& ctx = shared_vk_context(); + + void* left = alloc_device_pixels(64, 32); + CudaFree guard_l{ left }; + void* right = alloc_device_pixels(64, 32); + CudaFree guard_r{ right }; + + SECTION("CylinderLayer") + { + CylinderLayer::Config cfg; + cfg.resolution = { 64, 32 }; + cfg.stereo = true; + cfg.stereo_baseline_mm = 63.0f; + CylinderLayer layer(ctx, cfg); + + layer.set_stereo_baseline_mm(30.0f); + layer.submit(device_buffer(left, 64, 32), device_buffer(right, 64, 32)); + const auto view = layer.acquire_native_layer(0); + REQUIRE(view.has_value()); + CHECK(view->stereo_baseline_mm == 30.0f); + + CHECK_THROWS_AS(layer.set_stereo_baseline_mm(std::numeric_limits::quiet_NaN()), std::invalid_argument); + CHECK(layer.stereo_baseline_mm() == 30.0f); + } + + SECTION("EquirectLayer") + { + EquirectLayer::Config cfg; + cfg.resolution = { 64, 32 }; + cfg.stereo = true; + cfg.stereo_baseline_mm = 63.0f; + EquirectLayer layer(ctx, cfg); + + layer.set_stereo_baseline_mm(-12.5f); + layer.submit(device_buffer(left, 64, 32), device_buffer(right, 64, 32)); + const auto view = layer.acquire_native_layer(0); + REQUIRE(view.has_value()); + CHECK(view->stereo_baseline_mm == -12.5f); + } + + // Mono keeps the native view at zero however the setter is driven -- + // the gate is Config::stereo, which construction fixes. + SECTION("mono stays inert") + { + CylinderLayer::Config cfg; + cfg.resolution = { 64, 32 }; + cfg.stereo = false; + CylinderLayer layer(ctx, cfg); + + layer.set_stereo_baseline_mm(65.0f); + layer.submit(device_buffer(left, 64, 32)); + const auto view = layer.acquire_native_layer(0); + REQUIRE(view.has_value()); + CHECK(view->stereo_baseline_mm == 0.0f); + } +} + TEST_CASE("EquirectLayer stereo acquire pairs both eyes on one sphere", "[gpu][equirect_layer][native]") { if (!is_gpu_available()) diff --git a/tests/python/examples/camera_viz/pyproject.toml b/tests/python/examples/camera_viz/pyproject.toml index 0882fbd286..9e615a0d74 100644 --- a/tests/python/examples/camera_viz/pyproject.toml +++ b/tests/python/examples/camera_viz/pyproject.toml @@ -19,6 +19,10 @@ dev = [ "pyyaml>=6.0", # Encodes + decodes the synthetic clips in test_video_file_source.py. "opencv-python>=4.5", + # placements/_math.py (pulled in by test_controls.py) uses Rotation. + "scipy>=1.15", + # hud.py draws the controls panel with PIL. + "pillow>=10.0", ] gpu = [ "cupy-cuda12x", diff --git a/tests/python/examples/camera_viz/test_cloudxr_env.py b/tests/python/examples/camera_viz/test_cloudxr_env.py new file mode 100644 index 0000000000..db19adc459 --- /dev/null +++ b/tests/python/examples/camera_viz/test_cloudxr_env.py @@ -0,0 +1,126 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""CloudXR runtime settings: YAML -> env file -> verification. + +Every case here is one of the three ways a runtime setting was silently +ignored before this module existed. +""" + +from __future__ import annotations + +import pytest + +import cloudxr_env + + +# ── YAML -> env ─────────────────────────────────────────────────────── + + +def test_defaults_apply_without_any_yaml(): + env = cloudxr_env.env_from_yaml({}) + assert env["NV_ENABLE_POSE_WAIT"] == "false" + assert env["NV_CXR_RUNTIME_FOVEATION"] == "true" + + +def test_yaml_overrides_the_default(): + env = cloudxr_env.env_from_yaml({"cloudxr": {"NV_CXR_RUNTIME_FOVEATION": False}}) + assert env["NV_CXR_RUNTIME_FOVEATION"] == "false" + + +def test_booleans_are_lowercased(): + """The runtime matches "false" by exact strcmp and reads anything it does + not recognise as TRUE -- so Python's "False" would silently mean true, the + exact opposite of the config.""" + env = cloudxr_env.env_from_yaml({"cloudxr": {"NV_ENABLE_POSE_WAIT": False}}) + assert env["NV_ENABLE_POSE_WAIT"] == "false" + assert "False" not in env.values() + + +def test_null_drops_a_camera_viz_default(): + env = cloudxr_env.env_from_yaml({"cloudxr": {"NV_ENABLE_POSE_WAIT": None}}) + assert "NV_ENABLE_POSE_WAIT" not in env + + +def test_numbers_survive_as_strings(): + env = cloudxr_env.env_from_yaml({"cloudxr": {"NV_MAX_FPS": 72}}) + assert env["NV_MAX_FPS"] == "72" + + +def test_a_launcher_computed_key_is_refused(): + """The launcher ignores these with a UserWarning buried in the output; + refusing here names the YAML key instead.""" + with pytest.raises(ValueError, match="computed by the CloudXR launcher"): + cloudxr_env.env_from_yaml({"cloudxr": {"XR_RUNTIME_JSON": "/tmp/x.json"}}) + + +def test_a_non_mapping_block_is_refused(): + with pytest.raises(ValueError, match="must be a mapping"): + cloudxr_env.env_from_yaml({"cloudxr": ["NV_MAX_FPS=72"]}) + + +# ── env file ────────────────────────────────────────────────────────── + + +def test_env_file_is_written_without_export(tmp_path): + """The launcher writes `export KEY=value` for humans to source, but its own + parser splits on the first `=` without stripping the prefix -- an exported + line arrives as the key "export KEY" and is dropped.""" + path = cloudxr_env.write_env_file({"NV_MAX_FPS": "72"}, tmp_path / "cloudxr.env") + body = path.read_text() + assert "NV_MAX_FPS=72" in body + assert "export" not in body + + +def test_the_launcher_can_parse_what_we_write(tmp_path): + """Round-trip through the real parser rather than trusting the format.""" + from isaacteleop.cloudxr.env_config import EnvConfig + + env = {"NV_DEVICE_PROFILE": "apple-vision-pro", "NV_ENABLE_POSE_WAIT": "false"} + path = cloudxr_env.write_env_file(env, tmp_path / "cloudxr.env") + assert EnvConfig._load_env_file(path) == env + + +def test_an_env_file_entry_beats_a_stale_shell_export(tmp_path, monkeypatch): + """The whole reason this module exists: a sourced ~/.cloudxr/run/cloudxr.env + leaves NV_DEVICE_PROFILE in the shell, which beats both os.environ + .setdefault and --cloudxr-device-profile. Only the env file outranks it.""" + from isaacteleop.cloudxr.env_config import EnvConfig + + monkeypatch.setenv("NV_DEVICE_PROFILE", "Quest3") + path = cloudxr_env.write_env_file( + {"NV_DEVICE_PROFILE": "apple-vision-pro"}, tmp_path / "cloudxr.env" + ) + EnvConfig._instance = None # singleton: do not inherit another test's state + EnvConfig.from_args( + str(tmp_path / "install"), + path, + launcher_defaults={"NV_DEVICE_PROFILE": "Quest3"}, + ) + assert EnvConfig._instance.resolved("NV_DEVICE_PROFILE") == "apple-vision-pro" + EnvConfig._instance = None + + +# ── a name nothing reads ────────────────────────────────────────────── + + +def test_an_unknown_name_warns_with_a_suggestion(capsys): + """The failure this module exists for: NV_CXR_DEVICE_PROFILE sits between + two variables spelled that way, and the runtime ignores it in silence.""" + env = cloudxr_env.env_from_yaml( + {"cloudxr": {"NV_CXR_DEVICE_PROFILE": "apple-vision-pro"}} + ) + err = capsys.readouterr().err + assert "NV_CXR_DEVICE_PROFILE" in err + assert "did you mean 'NV_DEVICE_PROFILE'?" in err + # Warned, not dropped: the runtime has more knobs than camera_viz lists. + assert env["NV_CXR_DEVICE_PROFILE"] == "apple-vision-pro" + + +def test_a_known_name_is_silent(capsys): + cloudxr_env.env_from_yaml({"cloudxr": {"NV_DEVICE_PROFILE": "quest3"}}) + assert capsys.readouterr().err == "" + + +def test_every_default_is_a_known_name(): + """A default that is not in KNOWN_KEYS would warn on every single run.""" + assert set(cloudxr_env.DEFAULT_ENV) <= cloudxr_env.KNOWN_KEYS diff --git a/tests/python/examples/camera_viz/test_config.py b/tests/python/examples/camera_viz/test_config.py new file mode 100644 index 0000000000..ca323d4465 --- /dev/null +++ b/tests/python/examples/camera_viz/test_config.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""YAML -> SourceEntry parsing and validation.""" + +from __future__ import annotations + +import pytest + +import config + + +def test_shape_config_carries_the_equirect_heading(): + shape, _, _, _, yaw = config._shape_for( + "sky", {"sky": {"shape": "equirect", "equirect_yaw_deg": -90.0}} + ) + assert (shape, yaw) == ("equirect", -90.0) + assert config._shape_for("cam", {})[4] == 0.0 + + +def test_a_curved_shape_is_refused_before_the_runtime_starts(): + """Also pins the shape-config tuple against its callers: an extra field + once slipped past a positional unpack here.""" + cfg = { + "cameras": [{"name": "sky", "enabled": True}], + "display": {"placements": {"sky": {"shape": "equirect"}}}, + } + config.check_shapes_are_displayable(cfg, "xr") + with pytest.raises(SystemExit, match="requires XR mode"): + config.check_shapes_are_displayable(cfg, "window") diff --git a/tests/python/examples/camera_viz/test_controls.py b/tests/python/examples/camera_viz/test_controls.py new file mode 100644 index 0000000000..bb943230ce --- /dev/null +++ b/tests/python/examples/camera_viz/test_controls.py @@ -0,0 +1,1090 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for the right-controller bindings. + +The input side is faked: these exercise the policy (edge detection, +deadzone, clamping, cycle order, mixed-camera toggling), which is where +the bugs live. Driving a real ControllerTracker needs a headset. +""" + +from __future__ import annotations + +import math +from dataclasses import replace as dataclasses_replace + +import pytest + +from controls import ( + LOCK_MODE_CYCLE, + SHAPE_CYCLE, + ControllerControls, + ControlsConfig, + ControlTarget, +) +from controls import controls_config_from_yaml +from placements import ( + PlacementConfig, + build as build_placement, + heading_deg, + yaw_quat, +) + + +class FakeLayer: + def __init__(self) -> None: + self.baseline_mm = 0.0 + + def set_stereo_baseline_mm(self, value: float) -> None: + self.baseline_mm = value + + +class FakeInputs: + def __init__(self, a=False, b=False, x=0.0, y=0.0, click=False) -> None: + self.primary_click = a + self.secondary_click = b + self.thumbstick_x = x + self.thumbstick_y = y + self.thumbstick_click = click + + +class FakeSnapshot: + def __init__(self, inputs) -> None: + self.inputs = inputs + + +class FakeTracked: + def __init__(self, data) -> None: + self.data = data + + +class FakeTracker: + """Stands in for ControllerTracker; per-hand state is swapped per step.""" + + def __init__(self) -> None: + self.right = FakeInputs() + self.left = FakeInputs() + + def get_right_controller(self, _session): + return FakeTracked(None if self.right is None else FakeSnapshot(self.right)) + + def get_left_controller(self, _session): + return FakeTracked(None if self.left is None else FakeSnapshot(self.left)) + + +class FakeSession: + """DeviceIOSession stub: step() only calls update().""" + + def update(self) -> None: + pass + + +class FakePose3D: + def __init__(self, orientation) -> None: + self.position = (0.0, 0.0, 0.0) + self.orientation = orientation + + +class FakeVizSession: + """Only the bit the controls touch per frame.""" + + def __init__(self, head=None) -> None: + self.head = head + + def head_pose_now(self): + return self.head + + +def _make(targets, config=None): + """Real ControllerControls with only the device boundary faked, so the + policy under test is the shipping code path.""" + # Real strategies: the quad path calls retune() on them. + strategies = [ + build_placement(t.lock_mode, t.placement_config) + if t.placement_config is not None + else None + for t in targets + ] + controls = ControllerControls( + FakeVizSession(), + targets, + strategies, + config or ControlsConfig(), + tracker=FakeTracker(), + ) + controls._device_session = FakeSession() + return controls, strategies + + +def _stereo_target(name="cam", plane_distance=0.0, lock_mode="lazy"): + return ControlTarget( + name=name, + layer=FakeLayer(), + shape="quad", + stereo=True, + plane_distance_cm=plane_distance, + lock_mode=lock_mode, + placement_config=PlacementConfig(), + ) + + +def _press(controls, dt=1.0, **kwargs): + """Right hand: a, b, x (stick), click (stick press).""" + controls._tracker.right = FakeInputs(**kwargs) + controls._tracker.left = FakeInputs() + controls.step(dt) + + +def _press_left(controls, dt=1.0, **kwargs): + """Left hand: a = X button, b = Y button, x / y = stick.""" + controls._tracker.right = FakeInputs() + controls._tracker.left = FakeInputs(**kwargs) + controls.step(dt) + + +# ── A: lock mode ────────────────────────────────────────────────────── + + +def test_a_cycles_lock_mode_in_order(): + target = _stereo_target(lock_mode="world") + controls, strategies = _make([target]) + before = strategies[0] + + _press(controls, a=True) + assert target.lock_mode == "head" + # A new strategy object must land in the shared list the runner reads. + assert strategies[0] is not before + + _press(controls, a=False) + _press(controls, a=True) + assert target.lock_mode == "gimbal" + + +def test_a_wraps_around_the_cycle(): + target = _stereo_target(lock_mode=LOCK_MODE_CYCLE[-1]) + controls, _ = _make([target]) + _press(controls, a=True) + assert target.lock_mode == LOCK_MODE_CYCLE[0] + + +def test_held_a_fires_once(): + """Rising edge only — a held button must not cycle every frame.""" + target = _stereo_target(lock_mode="world") + controls, _ = _make([target]) + for _ in range(5): + _press(controls, a=True) + assert target.lock_mode == "head" + + +def test_equirect_without_strategy_is_skipped(): + target = ControlTarget( + name="sky", + layer=FakeLayer(), + shape="equirect", + stereo=True, + plane_distance_cm=0.0, + lock_mode="lazy", + placement_config=None, + ) + controls, strategies = _make([target]) + _press(controls, a=True) + assert strategies[0] is None + assert target.lock_mode == "lazy" + + +# ── B: mono / stereo ────────────────────────────────────────────────── + + +def test_b_toggles_mono_then_back(): + target = _stereo_target() + controls, _ = _make([target]) + + _press(controls, b=True) + assert target.force_mono.is_set() + + _press(controls, b=False) + _press(controls, b=True) + assert not target.force_mono.is_set() + + +def test_b_is_a_noop_for_mono_cameras(): + target = ControlTarget( + name="mono", + layer=FakeLayer(), + shape="quad", + stereo=False, + plane_distance_cm=0.0, + lock_mode="lazy", + placement_config=PlacementConfig(), + ) + controls, _ = _make([target]) + _press(controls, b=True) + assert not target.force_mono.is_set() + + +def test_b_keeps_mixed_cameras_in_step(): + """One press must not leave half the cameras mono and half stereo.""" + a, b = _stereo_target("a"), _stereo_target("b") + b.force_mono.set() + controls, _ = _make([a, b]) + + # 'a' is still stereo, so the press collapses everything to mono. + _press(controls, b=True) + assert a.force_mono.is_set() and b.force_mono.is_set() + + _press(controls, b=False) + _press(controls, b=True) + assert not a.force_mono.is_set() and not b.force_mono.is_set() + + +# ── Thumbstick: baseline ────────────────────────────────────────────── + + +def test_stick_inside_deadzone_does_not_drift(): + target = _stereo_target(plane_distance=1.0) + controls, _ = _make([target], ControlsConfig(deadzone=0.2)) + _press(controls, x=0.19) + assert target.plane_distance_cm == 1.0 + assert target.layer.baseline_mm == 0.0 # setter never called + + +def test_stick_ramps_and_reaches_the_layer(): + target = _stereo_target(plane_distance=0.0) + controls, _ = _make( + [target], ControlsConfig(deadzone=0.0, plane_distance_rate_cm_per_s=4.0) + ) + _press(controls, x=1.0) # dt == 1.0 + assert target.plane_distance_cm == pytest.approx(4.0) + # viz still speaks millimetres, so the layer sees 10x the cm value. + assert target.layer.baseline_mm == pytest.approx(40.0) + + _press(controls, x=-1.0) + assert target.plane_distance_cm == pytest.approx(0.0) + + +def test_deadzone_rescale_starts_the_ramp_at_zero(): + """Just past the deadzone should crawl, not jump to deadzone * rate.""" + target = _stereo_target() + controls, _ = _make( + [target], ControlsConfig(deadzone=0.5, plane_distance_rate_cm_per_s=10.0) + ) + _press(controls, x=0.51) + assert target.plane_distance_cm == pytest.approx(0.2, abs=0.05) + + +def test_offset_can_never_reach_divergent_parallax(): + """At an offset equal to the IPD the eyes' rays are parallel; past it they + must diverge, which they physically cannot do. The ceiling is derived from + the measured IPD, so a stick held to the stop stays clear of it however + generous the configured limit is.""" + from controls import MAX_OFFSET_FRACTION_OF_IPD + + target = _stereo_target(plane_distance=0.0) + controls, _ = _make( + [target], + ControlsConfig( + deadzone=0.0, plane_distance_rate_cm_per_s=1e5, plane_distance_max_cm=1000.0 + ), + ) + ipd_mm = 63.0 + for _ in range(5): + _press(controls, dt=1.0, x=1.0) + controls.step(0.0, ipd_mm) + ipd_cm = ipd_mm / 10.0 + assert target.plane_distance_cm == pytest.approx( + ipd_cm * MAX_OFFSET_FRACTION_OF_IPD + ) + assert target.plane_distance_cm < ipd_cm + + +def test_offset_clamps_to_the_configured_range_when_it_is_the_tighter_one(): + target = _stereo_target(plane_distance=0.0) + controls, _ = _make( + [target], + ControlsConfig( + deadzone=0.0, plane_distance_rate_cm_per_s=100.0, plane_distance_max_cm=2.0 + ), + ) + _press(controls, x=1.0) + assert target.plane_distance_cm == pytest.approx(2.0) + # Already clamped: no further change, and no redundant setter call. + target.layer.baseline_mm = -1.0 + _press(controls, x=1.0) + assert target.plane_distance_cm == pytest.approx(2.0) + assert target.layer.baseline_mm == -1.0 + + +def test_mono_camera_baseline_untouched(): + target = ControlTarget( + name="mono", + layer=FakeLayer(), + shape="quad", + stereo=False, + plane_distance_cm=0.0, + lock_mode="lazy", + placement_config=PlacementConfig(), + ) + controls, _ = _make([target], ControlsConfig(deadzone=0.0)) + _press(controls, x=1.0) + assert target.plane_distance_cm == 0.0 + + +# ── Controller loss ─────────────────────────────────────────────────── + + +def test_absent_controller_does_not_fire_a_phantom_press(): + """Dropping out mid-press then returning must not read as a new press.""" + target = _stereo_target(lock_mode="world") + controls, _ = _make([target]) + + _press(controls, a=True) + assert target.lock_mode == "head" + + controls._tracker.right = None # controller asleep + controls.step(1.0) + + _press(controls, a=True) # still held on return + assert target.lock_mode == "gimbal" + + +# ── Config parsing ──────────────────────────────────────────────────── + + +def test_baseline_disabled_when_viz_lacks_the_setter(): + """An older installed isaacteleop must degrade, not raise every frame.""" + target = _stereo_target(plane_distance=0.0) + + class OldLayer: # released wheel: no set_stereo_baseline_mm + pass + + target.layer = OldLayer() + controls, _ = _make([target], ControlsConfig(deadzone=0.0)) + # Detected by the constructor's probe, not forced by the test. + assert controls._baseline_supported is False + _press(controls, x=1.0) + assert target.plane_distance_cm == 0.0 + + # A still works on the same wheel. + _press(controls, a=True) + assert target.lock_mode == "world" + + +def test_controls_config_defaults_and_overrides(): + assert controls_config_from_yaml({}).enabled is True + cfg = controls_config_from_yaml({"controls": {"enabled": False, "deadzone": 0.3}}) + assert cfg.enabled is False and cfg.deadzone == 0.3 + + +def test_controls_config_rejects_unknown_keys(): + with pytest.raises(ValueError, match="unknown key"): + controls_config_from_yaml({"controls": {"deadzoneee": 0.3}}) + + +# ── Left X: shape cycling ───────────────────────────────────────────── + + +class FakeShapedLayer(FakeLayer): + """Layer with visibility + a mutable placement, like the shaped layers.""" + + def __init__(self, placement) -> None: + super().__init__() + self.visible = False + self._placement = placement + + def set_visible(self, visible: bool) -> None: + self.visible = bool(visible) + + def placement(self): + return self._placement + + def set_placement(self, placement) -> None: + self._placement = placement + + +class FakeCylinderPlacement: + def __init__(self) -> None: + self.radius_m = 2.0 + self.central_angle_rad = math.radians(90.0) + + +class FakePose: + """viz.Pose3D's surface: the placement's ``pose`` is a live reference, so + writing ``pose.orientation`` reaches the placement.""" + + def __init__(self) -> None: + self.position = (0.0, 0.0, 0.0) + self.orientation = (1.0, 0.0, 0.0, 0.0) + + +class FakeEquirectPlacement: + def __init__(self) -> None: + self.pose = FakePose() + self.central_horizontal_angle_rad = math.radians(360.0) + self.upper_vertical_angle_rad = math.radians(90.0) + self.lower_vertical_angle_rad = math.radians(-90.0) + + +def _switchable_target(shape="quad", name="cam"): + shape_layers = { + "quad": FakeShapedLayer(None), + "cylinder": FakeShapedLayer(FakeCylinderPlacement()), + "equirect": FakeShapedLayer(FakeEquirectPlacement()), + } + shape_layers[shape].visible = True + return ControlTarget( + name=name, + layer=shape_layers[shape], + shape=shape, + stereo=True, + plane_distance_cm=0.0, + lock_mode="lazy", + placement_config=PlacementConfig(), + shape_layers=shape_layers, + ) + + +def test_left_x_cycles_shape_and_swaps_visibility(): + target = _switchable_target("quad") + controls, _ = _make([target]) + + _press_left(controls, a=True) + assert target.shape == "cylinder" + assert target.shape_layers["cylinder"].visible + assert not target.shape_layers["quad"].visible + # This is what the runner asks for each pass. + assert controls.active_layer(0) is target.shape_layers["cylinder"] + + _press_left(controls, a=False) + _press_left(controls, a=True) + assert target.shape == "equirect" + + +def test_left_x_wraps_and_holding_fires_once(): + target = _switchable_target(SHAPE_CYCLE[-1]) + controls, _ = _make([target]) + for _ in range(4): + _press_left(controls, a=True) # held + assert target.shape == SHAPE_CYCLE[0] + + +def test_left_x_is_a_noop_without_shape_switching(): + """Only the configured shape is resident, so there is nothing to cycle.""" + target = _stereo_target() + controls, _ = _make([target]) + _press_left(controls, a=True) + assert target.shape == "quad" + + +def test_exactly_one_shape_visible_through_a_full_cycle(): + target = _switchable_target("quad") + controls, _ = _make([target]) + for _ in range(len(SHAPE_CYCLE) + 1): + visible = [s for s, layer in target.shape_layers.items() if layer.visible] + assert visible == [target.shape] + _press_left(controls, a=False) + _press_left(controls, a=True) + + +# ── Left stick: per-shape parameters ────────────────────────────────── + + +def test_quad_x_sizes_the_plane_without_touching_its_distance(): + """Width in metres, matching the YAML's own unit. Distance is fixed at + runtime, so width maps 1:1 to how big it looks.""" + target = _switchable_target("quad") + d0 = target.placement_config.distance + w0, h0 = target.placement_config.size_meters + controls, strategies = _make( + [target], ControlsConfig(deadzone=0.0, size_rate_m_per_s=1.0) + ) + + _press_left(controls, x=1.0, dt=0.5) + cfg = target.placement_config + w1, h1 = cfg.size_meters + assert cfg.distance == d0 # X does not move the plane + assert w1 == pytest.approx(w0 + 0.5) + assert w1 / h1 == pytest.approx(w0 / h0) # aspect preserved + assert strategies[0]._config is cfg + + +def test_quad_y_slides_the_plane_up_and_down(): + """The axis distance used to occupy, doing something distance cannot.""" + target = _switchable_target("quad") + size0 = target.placement_config.size_meters + controls, _ = _make([target], ControlsConfig(deadzone=0.0, offset_rate_m_per_s=1.0)) + + _press_left(controls, y=1.0, dt=0.25) + cfg = target.placement_config + assert cfg.offset_y == pytest.approx(0.25) + assert cfg.size_meters == size0 # and it does not resize + + +def test_cylinder_x_widens_the_arc_and_y_slides_it(): + """Radius is not a stick axis: arc width is radius*angle, so growing the + radius grows the surface by the same factor and nothing looks different.""" + target = _switchable_target("cylinder") + radius0 = target.cylinder_radius_m + controls, _ = _make( + [target], + ControlsConfig( + deadzone=0.0, angle_rate_deg_per_s=40.0, offset_rate_m_per_s=1.0 + ), + ) + + _press_left(controls, x=-1.0, dt=1.0) + assert target.cylinder_angle_deg == pytest.approx(50.0) + assert target.layer.placement().central_angle_rad == pytest.approx( + math.radians(50.0) + ) + assert target.cylinder_radius_m == radius0 # untouched + + _press_left(controls, y=1.0, dt=0.25) + assert target.placement_config.offset_y == pytest.approx(0.25) + + +def test_equirect_stick_drives_spans_symmetrically(): + target = _switchable_target("equirect") + controls, _ = _make( + [target], ControlsConfig(deadzone=0.0, angle_rate_deg_per_s=40.0) + ) + _press_left(controls, x=-1.0, y=-1.0, dt=1.0) + assert target.equirect_h_deg == pytest.approx(320.0) + assert target.equirect_v_half_deg == pytest.approx(50.0) + placement = target.layer.placement() + assert placement.upper_vertical_angle_rad == pytest.approx( + -placement.lower_vertical_angle_rad + ) + + +@pytest.mark.parametrize( + "shape,axis,attr,limit_name,to_max", + [ + ("cylinder", "x", "cylinder_angle_deg", "cylinder_angle_range_deg", True), + ("cylinder", "x", "cylinder_angle_deg", "cylinder_angle_range_deg", False), + ("equirect", "x", "equirect_h_deg", "equirect_h_range_deg", True), + ("equirect", "y", "equirect_v_half_deg", "equirect_v_half_range_deg", False), + ], +) +def test_shape_params_clamp_inside_layer_validation( + shape, axis, attr, limit_name, to_max +): + """Out-of-range values make set_placement raise on the render thread, so + the clamp is what keeps a held stick from killing the run.""" + target = _switchable_target(shape) + cfg = ControlsConfig(deadzone=0.0, size_rate_m_per_s=1e4, angle_rate_deg_per_s=1e4) + controls, _ = _make([target], cfg) + kwargs = {axis: 1.0 if to_max else -1.0} + _press_left(controls, dt=1.0, **kwargs) + lo, hi = getattr(cfg, limit_name) + assert getattr(target, attr) == pytest.approx(hi if to_max else lo) + + +def test_default_limits_sit_inside_the_layer_specs(): + """cylinder needs (0, 2pi) exclusive; equirect (0, 2pi] and |v| <= pi/2.""" + cfg = ControlsConfig() + lo, hi = cfg.cylinder_angle_range_deg + assert lo > 0.0 and hi < 360.0 + lo_h, hi_h = cfg.equirect_h_range_deg + assert lo_h > 0.0 and hi_h <= 360.0 + lo_v, hi_v = cfg.equirect_v_half_range_deg + assert lo_v > 0.0 and hi_v <= 90.0 + + +# ── Left Y: reset ───────────────────────────────────────────────────── + + +def test_left_y_resets_shape_params_and_modes(): + target = _switchable_target("quad") + cfg = target.placement_config + controls, _ = _make( + [target], + ControlsConfig( + deadzone=0.0, + size_rate_m_per_s=1.0, + plane_distance_rate_cm_per_s=10.0, + ), + ) + size0, distance0 = cfg.size_meters, cfg.distance + + _press_left(controls, a=True) # -> cylinder + _press(controls, a=True) # lock mode moves on + _press(controls, b=True) # -> mono + _press(controls, x=1.0, dt=1.0) # baseline drifts + _press_left(controls, x=1.0, dt=1.0) + assert target.shape == "cylinder" + + _press_left(controls, b=True) + assert target.shape == "quad" + assert target.shape_layers["quad"].visible + assert not target.shape_layers["cylinder"].visible + assert target.lock_mode == "lazy" + assert target.plane_distance_cm == 0.0 + assert not target.force_mono.is_set() + assert cfg.size_meters == size0 and cfg.distance == distance0 + assert target.cylinder_radius_m == pytest.approx(2.0) + + +# ── Logging ─────────────────────────────────────────────────────────── + + +class RecordingHud: + def __init__(self): + self.lines = [] + + def show(self, lines): + self.lines.append(lines[0]) + + def step(self, dt, head_pose): + pass + + +def test_hud_shows_every_step_not_the_log_rate(): + """The readout is live. Throttling it made a value moving in 0.1 cm steps + look like it moved in whatever the log period times the rate came to.""" + target = _stereo_target(plane_distance=0.0) + controls, _ = _make( + [target], ControlsConfig(deadzone=0.0, plane_distance_rate_cm_per_s=2.0) + ) + hud = RecordingHud() + controls._hud = hud + for _ in range(60): # one second at 60 Hz + _press(controls, x=1.0, dt=1.0 / 60.0) + + values = [float(line.split()[2]) for line in hud.lines] + assert len(values) == 20 # 2 cm/s in 0.1 cm steps + assert {round(b - a, 3) for a, b in zip(values, values[1:])} == {0.1} + + +def test_held_stick_log_is_throttled(capsys): + """At display rate an unthrottled log would print ~90 lines a second.""" + target = _stereo_target() + controls, _ = _make( + [target], ControlsConfig(deadzone=0.0, plane_distance_rate_cm_per_s=1.0) + ) + for _ in range(60): + _press(controls, x=1.0, dt=1.0 / 60.0) + lines = [ln for ln in capsys.readouterr().err.splitlines() if "stereo planes" in ln] + assert 1 <= len(lines) <= 6 + + +# ── Message summarising (multi-camera) ──────────────────────────────── + + +def test_summarize_says_a_shared_value_once(): + """Bindings apply to every camera at once, so values normally agree — + repeating them per camera is what overflowed the HUD bar.""" + from controls import summarize + + assert summarize([("zed", "52.0 mm")]) == "52.0 mm" + assert summarize([("a", "52.0 mm"), ("b", "52.0 mm")]) == "52.0 mm (2 cameras)" + + +def test_summarize_lists_cameras_that_disagree(): + from controls import summarize + + out = summarize([("front", "52.0 mm"), ("rear", "10.0 mm")]) + assert "front 52.0 mm" in out and "rear 10.0 mm" in out + assert "cameras)" not in out + + +def test_summarize_handles_nothing_changing(): + from controls import summarize + + assert summarize([]) == "" + + +def test_multi_camera_baseline_message_stays_one_line(capsys): + """End-to-end: three cameras must not produce three repeated values.""" + from controls.hud import _TEXT_W, _TITLE, split_message + + targets = [_stereo_target(n) for n in ("front", "left", "right")] + controls, _ = _make(targets, ControlsConfig(deadzone=0.0)) + _press(controls, x=1.0, dt=1.0) + + line = [ln for ln in capsys.readouterr().err.splitlines() if "stereo planes" in ln][ + -1 + ] + # stderr joins the panel's two lines with " | "; the HUD gets them + # separately, so the headline is what has to fit. + headline = line.split("controls: ", 1)[1].split(" | ")[0] + assert "(3 cameras)" in headline + assert len(split_message(headline)) == 1 + assert _TITLE.getlength(headline) <= _TEXT_W + + +# ── Suggested plane distance ────────────────────────────────────────── + + +def test_suggestion_comes_from_the_plane_distance_and_ipd(): + """d = ipd * (1 - Z / FAR_TARGET_M): a 1 m plane at a 63 mm IPD suggests + 6.3 * 0.9 = 5.67 cm.""" + from controls import FAR_TARGET_M + + target = _stereo_target() + target.placement_config = PlacementConfig(distance=1.0) + controls, _ = _make([target]) + controls.step(0.0, 63.0) + from controls import PLANE_DISTANCE_STEP_CM + + raw = 6.3 * (1.0 - 1.0 / FAR_TARGET_M) + expected = round(raw / PLANE_DISTANCE_STEP_CM) * PLANE_DISTANCE_STEP_CM + # Stepped, so it is a value the stick can actually land on. + assert controls._suggested_plane_distance_cm(target) == pytest.approx(expected) + + +def test_suggestion_moves_with_the_plane_distance(): + """A nearer plane needs a wider gap to reach the same far end.""" + near, far = _stereo_target("near"), _stereo_target("far") + near.placement_config = PlacementConfig(distance=0.8) + far.placement_config = PlacementConfig(distance=3.0) + controls, _ = _make([near, far]) + controls.step(0.0, 63.0) + assert controls._suggested_plane_distance_cm( + near + ) > controls._suggested_plane_distance_cm(far) + + +def test_suggestion_moves_with_the_ipd(): + target = _stereo_target() + target.placement_config = PlacementConfig(distance=1.0) + controls, _ = _make([target]) + controls.step(0.0, 58.0) + narrow = controls._suggested_plane_distance_cm(target) + controls.step(0.0, 70.0) + assert controls._suggested_plane_distance_cm(target) > narrow + + +def test_suggestion_never_exceeds_the_divergence_ceiling(): + target = _stereo_target() + target.placement_config = PlacementConfig(distance=0.05) # absurdly near + controls, _ = _make([target]) + controls.step(0.0, 63.0) + from controls import PLANE_DISTANCE_STEP_CM + + # Rounding to a step can land half a step above the raw ceiling. + assert controls._suggested_plane_distance_cm(target) <= ( + controls._max_plane_distance_cm() + PLANE_DISTANCE_STEP_CM / 2 + ) + + +def test_suggestion_is_only_advice_never_applied(): + """The value in use stays whatever was configured or trimmed to.""" + target = _stereo_target(plane_distance=0.0) + target.placement_config = PlacementConfig(distance=1.0) + controls, _ = _make([target]) + for _ in range(5): + controls.step(0.0, 63.0) + assert target.plane_distance_cm == 0.0 + + +def test_suggestion_is_shown_next_to_the_value(capsys): + target = _stereo_target() + target.placement_config = PlacementConfig(distance=1.0) + controls, _ = _make([target], ControlsConfig(deadzone=0.0)) + _press(controls, x=1.0, dt=0.5) + err = capsys.readouterr().err + assert "stereo planes" in err and "suggested" in err and "IPD" in err + + +def test_equirect_has_no_suggestion(): + """A sphere centred on the viewer has no plane distance to work from.""" + target = _switchable_target("equirect") + controls, _ = _make([target]) + assert controls._suggested_plane_distance_cm(target) is None + + +def test_stick_still_moves_at_display_frame_rates(): + """One 60 Hz frame moves less than half a step, so rounding the running + total each frame would pin the value at its start forever.""" + target = _stereo_target(plane_distance=0.0) + controls, _ = _make( + [target], ControlsConfig(deadzone=0.0, plane_distance_rate_cm_per_s=2.0) + ) + for _ in range(60): # one second at 60 Hz + _press(controls, x=1.0, dt=1.0 / 60.0) + assert target.plane_distance_cm == pytest.approx(2.0, abs=0.05) + + +def test_the_layer_only_ever_sees_whole_steps(): + from controls import PLANE_DISTANCE_STEP_CM + + target = _stereo_target(plane_distance=0.0) + controls, _ = _make( + [target], ControlsConfig(deadzone=0.0, plane_distance_rate_cm_per_s=2.0) + ) + for _ in range(60): + _press(controls, x=1.0, dt=1.0 / 60.0) + mm = target.layer.baseline_mm + steps = mm / (PLANE_DISTANCE_STEP_CM * 10.0) + assert steps == pytest.approx(round(steps), abs=1e-6), mm + + +# ── Mono / stereo and the plane gap ─────────────────────────────────── + + +def test_mono_parks_the_gap_and_stereo_puts_it_back(): + """Both eyes see one image in mono, so a gap would only shove that flat + image to another depth.""" + target = _stereo_target(plane_distance=5.0) + controls, _ = _make([target]) + + _press(controls, b=True) # -> mono + assert target.plane_distance_cm == 0.0 + assert target.layer.baseline_mm == 0.0 + + _press(controls, b=False) + _press(controls, b=True) # -> stereo + assert target.plane_distance_cm == pytest.approx(5.0) + assert target.layer.baseline_mm == pytest.approx(50.0) + + +def test_a_gap_set_while_mono_is_not_clobbered_on_the_way_out(): + target = _stereo_target(plane_distance=5.0) + controls, _ = _make( + [target], ControlsConfig(deadzone=0.0, plane_distance_rate_cm_per_s=2.0) + ) + _press(controls, b=True) + _press(controls, b=False) + _press(controls, b=True) + assert target.plane_distance_cm == pytest.approx(5.0) + + +def test_reset_clears_the_remembered_mono_gap(): + target = _stereo_target(plane_distance=5.0) + controls, _ = _make([target]) + _press(controls, b=True) # -> mono, gap parked + _press_left(controls, b=True) # Y: reset + assert target.plane_distance_cm == pytest.approx(5.0) + assert target.plane_distance_before_mono is None + assert not target.force_mono.is_set() + + +def test_equirect_gap_is_skipped_because_it_does_nothing(): + """The gap shifts each eye's surface, and camera_viz's sphere is at + infinite radius — translating that changes nothing, so the stick must not + pretend otherwise.""" + target = _switchable_target("equirect") + controls, _ = _make([target], ControlsConfig(deadzone=0.0)) + _press(controls, x=1.0, dt=1.0) + assert target.plane_distance_cm == 0.0 + assert target.layer.baseline_mm == 0.0 + + +def test_quad_and_cylinder_axes_do_different_things(): + """The point of the rework: each shape's two axes must not collapse into + one control.""" + from controls import SHAPE_PARAMS + + for shape, (x_name, y_name) in SHAPE_PARAMS.items(): + assert x_name != y_name, shape + + +# ── Retuning must land immediately in every lock mode ───────────────── + + +@pytest.mark.parametrize("lock_mode", ["world", "head", "gimbal", "lazy"]) +def test_height_change_moves_the_plane_now_not_at_the_next_resnap(lock_mode): + """world cached the finished placement and ignored retuning outright; + lazy only recomputed on a re-snap, so a height change sat unapplied until + you happened to look away.""" + from placements import PlacementConfig, build + + head_pos, head_q = (0.0, 1.5, 0.0), (1.0, 0.0, 0.0, 0.0) + cfg = PlacementConfig(distance=1.0) + strategy = build(lock_mode, cfg) + before = strategy.update(head_pos, head_q).position[1] + + strategy.retune(dataclasses_replace(cfg, offset_y=0.4)) + after = strategy.update(head_pos, head_q).position[1] + assert after == pytest.approx(before + 0.4, abs=1e-6), lock_mode + + +@pytest.mark.parametrize("lock_mode", ["world", "head", "gimbal", "lazy"]) +def test_size_change_lands_immediately_too(lock_mode): + from placements import PlacementConfig, build + + head_pos, head_q = (0.0, 1.5, 0.0), (1.0, 0.0, 0.0, 0.0) + cfg = PlacementConfig(distance=1.0) + strategy = build(lock_mode, cfg) + strategy.update(head_pos, head_q) + + strategy.retune(dataclasses_replace(cfg, size_meters=(2.0, 1.125))) + assert strategy.update(head_pos, head_q).size_meters == (2.0, 1.125), lock_mode + + +def test_world_lock_still_pins_the_plane_across_a_retune(): + """Retuning must not become an excuse to re-place a world-locked plane.""" + from placements import PlacementConfig, build + + cfg = PlacementConfig(distance=1.0) + strategy = build("world", cfg) + placed = strategy.update((0.0, 1.5, 0.0), (1.0, 0.0, 0.0, 0.0)).position + + strategy.retune(dataclasses_replace(cfg, size_meters=(2.0, 1.125))) + # Head has since moved; the plane must not follow it. + after = strategy.update((3.0, 1.5, -2.0), (1.0, 0.0, 0.0, 0.0)).position + assert after == pytest.approx(placed) + + +def test_the_plane_gap_follows_a_shape_switch(): + """It is tracked per camera but applied per layer, so a switch has to + carry it onto the layer that just became visible.""" + target = _switchable_target("quad") + target.plane_distance_cm = 5.0 + controls, _ = _make([target], ControlsConfig(deadzone=0.0)) + for layer in target.shape_layers.values(): + layer.baseline_mm = 99.0 # as if built from a different YAML value + + _press_left(controls, a=True) # -> cylinder + assert target.layer.baseline_mm == pytest.approx(50.0) + + +def test_a_shape_switch_while_mono_keeps_the_gap_parked(): + target = _switchable_target("quad") + target.plane_distance_cm = 5.0 + controls, _ = _make([target]) + for layer in target.shape_layers.values(): + layer.baseline_mm = 99.0 + + _press(controls, b=True) # -> mono, gap parked at 0 + _press_left(controls, a=True) # -> cylinder + assert target.layer.baseline_mm == 0.0 + + _press(controls, b=False) + _press(controls, b=True) # -> stereo again + assert target.layer.baseline_mm == pytest.approx(50.0) + + +def test_reset_restores_height_too(): + """It restored size and distance but not offset_y, so a nudged plane + stayed nudged.""" + target = _switchable_target("quad") + controls, _ = _make([target], ControlsConfig(deadzone=0.0, offset_rate_m_per_s=1.0)) + _press_left(controls, y=1.0, dt=0.5) + assert target.placement_config.offset_y == pytest.approx(0.5) + + _press_left(controls, b=True) # Y: reset + assert target.placement_config.offset_y == pytest.approx(0.0) + + +# ── Equirect aiming: pan + recenter ─────────────────────────────────── + + +def _looking(yaw_deg: float) -> FakeVizSession: + return FakeVizSession(FakePose3D(yaw_quat(math.radians(yaw_deg)))) + + +def test_right_stick_pans_an_equirect_instead_of_setting_a_gap(): + """The sphere has no gap to set, so the stick is free -- and panning is + what a 360 feed actually needs, since the camera rarely faces the way the + headset did when it recentered.""" + target = _switchable_target("equirect") + controls, _ = _make( + [target], ControlsConfig(deadzone=0.0, angle_rate_deg_per_s=40.0) + ) + + _press(controls, x=1.0, dt=0.5) + assert target.equirect_yaw_deg == pytest.approx(20.0) + placement = target.shape_layers["equirect"].placement() + assert placement.pose.orientation == pytest.approx(yaw_quat(math.radians(20.0))) + + +def test_panning_wraps_instead_of_running_off(): + target = _switchable_target("equirect") + target.equirect_yaw_deg = 170.0 + controls, _ = _make( + [target], ControlsConfig(deadzone=0.0, angle_rate_deg_per_s=40.0) + ) + _press(controls, x=1.0, dt=1.0) + assert target.equirect_yaw_deg == pytest.approx(-150.0) + + +def test_the_stick_does_not_pan_a_shape_that_has_a_gap(): + target = _switchable_target("quad") + controls, _ = _make([target], ControlsConfig(deadzone=0.0)) + _press(controls, x=1.0, dt=1.0) + assert target.equirect_yaw_deg == 0.0 + + +def test_stick_click_aims_the_panorama_where_you_are_looking(): + target = _switchable_target("equirect") + controls, _ = _make([target]) + controls._session = _looking(35.0) + + _press(controls, click=True) + assert target.equirect_yaw_deg == pytest.approx(35.0, abs=1e-3) + + +def test_stick_click_resnaps_a_placed_layer(): + """Same gesture, read per shape: a world-locked plane left in another + part of the room comes back.""" + target = _switchable_target("quad", name="cam") + target.lock_mode = "world" + controls, strategies = _make([target]) + controls._session = _looking(0.0) + before = strategies[0] + + _press(controls, click=True) + assert strategies[0] is not before + + +def test_recenter_is_a_press_not_a_hold(): + target = _switchable_target("equirect") + controls, _ = _make([target]) + controls._session = _looking(35.0) + _press(controls, click=True) + target.equirect_yaw_deg = 0.0 + _press(controls, click=True) # still held + assert target.equirect_yaw_deg == 0.0 + + +def test_recenter_does_nothing_without_a_head_pose(): + """Tracking loss must not slam the panorama to a default heading.""" + target = _switchable_target("equirect") + target.equirect_yaw_deg = 90.0 + controls, _ = _make([target]) # FakeVizSession head is None + _press(controls, click=True) + assert target.equirect_yaw_deg == 90.0 + + +def test_reset_restores_the_panorama_heading(): + target = _switchable_target("equirect") + controls, _ = _make( + [target], ControlsConfig(deadzone=0.0, angle_rate_deg_per_s=40.0) + ) + _press(controls, x=1.0, dt=1.0) + assert target.equirect_yaw_deg != 0.0 + + _press_left(controls, b=True) # Y: reset + assert target.equirect_yaw_deg == pytest.approx(0.0) + placement = target.shape_layers["equirect"].placement() + assert placement.pose.orientation == pytest.approx((1.0, 0.0, 0.0, 0.0)) + + +def test_heading_is_the_inverse_of_the_yaw_it_is_built_from(): + for degrees in (-179.0, -90.0, 0.0, 45.0, 179.0): + assert heading_deg(yaw_quat(math.radians(degrees))) == pytest.approx( + degrees, abs=1e-3 + ) + + +# ── The panel owns stderr while it is live ──────────────────────────── + + +def test_control_messages_stay_off_stderr_when_the_panel_owns_it(capsys): + """Two writers on one stream is what left a column of half-erased + headers scrolling up the terminal.""" + target = _switchable_target("quad") + controls = ControllerControls( + FakeVizSession(), + [target], + [build_placement(target.lock_mode, target.placement_config)], + ControlsConfig(), + tracker=FakeTracker(), + log_to_stderr=False, + ) + controls._device_session = FakeSession() + _press(controls, a=True) + + assert capsys.readouterr().err == "" + # The event still reaches the panel, which is what draws it. + assert "lock" in controls.last_event diff --git a/tests/python/examples/camera_viz/test_dashboard.py b/tests/python/examples/camera_viz/test_dashboard.py new file mode 100644 index 0000000000..e89b1ef9e8 --- /dev/null +++ b/tests/python/examples/camera_viz/test_dashboard.py @@ -0,0 +1,202 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Status panel: layout, redraw bookkeeping, and the non-terminal fallback.""" + +from __future__ import annotations + +import io +import os +import re + +from dashboard import CameraRow, Dashboard, Snapshot + +ESCAPE = re.compile(r"\x1b\[[0-9;]*[A-Za-z]") + + +def _snapshot(rows=None, **kwargs): + defaults = dict( + header="xr · local · 1 camera", + render="render 58.0 fps missed 0", + rows=rows + if rows is not None + else [CameraRow("zed", "cylinder", "lazy", True, 1.0, 0.0, 5.0, 5.2, 64.0)], + ) + defaults.update(kwargs) + return Snapshot(**defaults) + + +class FakeTTY(io.StringIO): + def isatty(self): + return True + + +def test_not_a_terminal_falls_back_to_one_line(): + """Redrawing into a journal or a pipe produces nothing but escape codes.""" + out = io.StringIO() # StringIO.isatty() is False + dash = Dashboard(stream=out) + assert not dash.live + dash.show(_snapshot()) + text = out.getvalue() + assert text.count("\n") == 1 + assert "\x1b[" not in text + assert "zed 64.0 submit/s" in text + + +def test_live_panel_redraws_in_place(): + out = FakeTTY() + dash = Dashboard(stream=out, colour=False) + assert dash.live + dash.show(_snapshot()) + first = out.getvalue() + assert "\x1b[2K" in first # clears each line + assert "\x1b[1A" not in first # nothing to move up over yet + + out.truncate(0), out.seek(0) + dash.show(_snapshot()) + # Second paint walks back up over the first. + assert re.search(r"\x1b\[\d+A", out.getvalue()) + + +def test_a_shorter_panel_clears_the_rows_it_left_behind(): + out = FakeTTY() + dash = Dashboard(stream=out, colour=False) + two = [ + CameraRow("a", "quad", "lazy", True, 1.0, 0.0, 5.0, 5.2, 60.0), + CameraRow("b", "quad", "lazy", True, 1.0, 0.0, 5.0, 5.2, 60.0), + ] + dash.show(_snapshot(rows=two)) + tall = dash._painted + + out.truncate(0), out.seek(0) + dash.show(_snapshot(rows=two[:1])) + # Same number of lines written, so the vacated row is blanked not stale. + assert out.getvalue().count("\x1b[2K") == tall + + +def test_lines_fit_the_width_and_never_cut_an_escape(): + """Truncating after colouring would clip the reset and leave the terminal + stuck in that style.""" + dash = Dashboard(stream=FakeTTY(), colour=True) + snap = _snapshot(notes=["x" * 300], last_event="y" * 300, ipd_mm=63.0) + for line in dash._compose(snap): + assert len(ESCAPE.sub("", line)) <= 100 + if "\x1b[" in line: + assert line.endswith("\x1b[0m") + + +def test_row_shows_the_suggestion_beside_the_value(): + dash = Dashboard(stream=FakeTTY(), colour=False) + body = "\n".join(dash._compose(_snapshot())) + assert "5.0/5.2" in body + + +def test_row_renders_missing_values_as_a_dash(): + """equirect has no plane gap and window mode has no lock mode.""" + row = CameraRow("sky", "equirect", "-", True, None, None, None, None, 30.0) + cells = row.cells() + assert cells[4] == "-" and cells[6] == "-" + + +def test_close_leaves_the_cursor_below_the_panel(): + out = FakeTTY() + dash = Dashboard(stream=out, colour=False) + dash.show(_snapshot()) + out.truncate(0), out.seek(0) + dash.close() + assert out.getvalue() == "\n" + assert dash._painted == 0 + + +# ── Staying aligned with the terminal ───────────────────────────────── +# +# Every one of these is the same failure seen from a different angle: one +# logical line stopped costing exactly one terminal row, the cursor +# arithmetic drifted, and each repaint stranded another copy of the header. + + +def test_lines_stop_one_short_of_the_last_column(monkeypatch): + """A line that fills the last column leaves the cursor wrap-pending, and + the newline after it costs two rows on some terminals.""" + monkeypatch.setattr( + "shutil.get_terminal_size", lambda fallback=None: os.terminal_size((64, 24)) + ) + dash = Dashboard(stream=FakeTTY(), colour=False) + lines = dash._compose(_snapshot()) + assert max(len(line) for line in lines) == 63 + + +def test_a_panel_taller_than_the_terminal_gives_up_on_redrawing(monkeypatch): + monkeypatch.setattr( + "shutil.get_terminal_size", lambda fallback=None: os.terminal_size((100, 5)) + ) + out = FakeTTY() + dash = Dashboard(stream=out, colour=False) + dash.show(_snapshot()) + text = out.getvalue() + assert text.count("\n") == 1 + assert "\x1b[" not in text + assert dash._painted == 0 + + +def test_a_resize_starts_a_fresh_panel(monkeypatch): + """After a rewrap the old panel's height no longer says how far up its + top is, so erasing that many lines would eat unrelated output.""" + columns = [100] + monkeypatch.setattr( + "shutil.get_terminal_size", + lambda fallback=None: os.terminal_size((columns[0], 40)), + ) + out = FakeTTY() + dash = Dashboard(stream=out, colour=False) + dash.show(_snapshot()) + columns[0] = 70 + out.truncate(0), out.seek(0) + dash.show(_snapshot()) + assert "\x1b[" in out.getvalue() # still painting + assert not re.search(r"\x1b\[\d+A", out.getvalue()) # but not upwards + + +def test_a_note_scrolls_above_the_panel_instead_of_into_it(): + out = FakeTTY() + dash = Dashboard(stream=out, colour=False) + dash.show(_snapshot()) + painted = dash._painted + out.truncate(0), out.seek(0) + + dash.note("[zed] reconnected") + text = out.getvalue() + # Panel erased, cursor back at its top, note written there: the next + # show() paints below it rather than half over it. + assert text.startswith(f"\x1b[{painted}A") + assert text.endswith("[zed] reconnected\n") + assert dash._painted == 0 + + +def test_a_note_without_a_panel_is_just_a_line(): + out = io.StringIO() + dash = Dashboard(stream=out) + dash.note("[zed] reconnected") + assert out.getvalue() == "[zed] reconnected\n" + + +def test_a_note_after_close_is_dropped(): + """Source threads can outlive runner.stop() and notify into a panel that + has already handed the cursor back.""" + out = FakeTTY() + dash = Dashboard(stream=out, colour=False) + dash.show(_snapshot()) + dash.close() + out.truncate(0), out.seek(0) + + dash.note("[zed] stopped") + assert out.getvalue() == "" + + +def test_close_is_idempotent(): + out = FakeTTY() + dash = Dashboard(stream=out, colour=False) + dash.show(_snapshot()) + dash.close() + out.truncate(0), out.seek(0) + dash.close() + assert out.getvalue() == "" diff --git a/tests/python/examples/camera_viz/test_hud.py b/tests/python/examples/camera_viz/test_hud.py new file mode 100644 index 0000000000..bafc68a462 --- /dev/null +++ b/tests/python/examples/camera_viz/test_hud.py @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""HUD bitmap + wrapping. Drawing is deliberately session-free, so it can be +checked without a GPU or a headset.""" + +from __future__ import annotations + +import numpy as np + +from controls import hud +from controls.hud import ( + _BODY, + _H, + _TITLE, + _W, + _fit, + _render, + split_message, +) + + +def test_render_shape_and_opacity(): + img = _render(["shape: cylinder"]) + assert img.shape == (_H, _W, 4) + assert img.dtype == np.uint8 + assert img.flags["C_CONTIGUOUS"] # submit() needs a contiguous buffer + # Fully opaque is what keeps the frame eligible for CloudXR's + # colour-only streaming. + assert (img[..., 3] == 255).all() + + +def test_render_draws_text_and_the_accent_bar(): + assert not np.array_equal(_render([""]), _render(["shape: cylinder"])) + # Left edge is the accent, not the background. + assert tuple(_render([""])[_H // 2, 0][:3]) == hud._ACCENT + + +def test_channel_order_is_rgb(): + """A red-ish background must come back red-first; a swapped buffer would + ship silently otherwise.""" + original = hud._BG + try: + hud._BG = (200, 0, 0) + px = hud._render([""])[_H // 2, _W // 2] + finally: + hud._BG = original + assert tuple(int(c) for c in px) == (200, 0, 0, 255) + + +def test_only_two_lines_are_drawn(): + assert np.array_equal(_render(["one", "two"]), _render(["one", "two", "three"])) + + +def test_long_line_is_ellipsized_to_the_panel(): + fitted = _fit("x" * 500, _TITLE) + assert fitted.endswith("...") + assert _TITLE.getlength(fitted) <= hud._TEXT_W + assert _fit("shape: cylinder", _TITLE) == "shape: cylinder" + + +def test_wrapped_lines_fit_the_font_each_will_be_drawn_in(): + """Line one renders in the larger title face; measuring it with the body + font let it overflow and get ellipsized.""" + lines = split_message("zed radius=2.50m arc=90deg") + assert _TITLE.getlength(lines[0]) <= hud._TEXT_W + if len(lines) > 1: + assert _BODY.getlength(lines[1]) <= hud._TEXT_W + # And nothing was silently dropped. + assert " ".join(lines) == "zed radius=2.50m arc=90deg" + + +def test_split_message_keeps_words_whole_and_caps_at_two_lines(): + lines = split_message(" ".join(f"cam{i}=52.0mm" for i in range(40))) + assert len(lines) == 2 + assert all(not w.startswith("=") for line in lines for w in line.split()) + + +def test_short_message_is_one_line(): + assert split_message("mono") == ["mono"] + + +def test_single_line_is_positioned_differently_from_a_pair(): + """A lone line centres instead of sitting under an empty second row.""" + assert not np.array_equal(_render(["mono"]), _render(["mono", ""])) diff --git a/tests/python/examples/camera_viz/test_placements.py b/tests/python/examples/camera_viz/test_placements.py new file mode 100644 index 0000000000..fd937cb4a4 --- /dev/null +++ b/tests/python/examples/camera_viz/test_placements.py @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Lock-mode placement invariants. + +The one that matters is facing: an OpenXR quad layer is single-sided, so a +plane rotated away from the viewer renders nothing at all. ``head`` shipped +with a 180-degree flip carried over from the old CameraPlane renderer and +showed a black feed. +""" + +from __future__ import annotations + +import math + +import pytest + +from placements import PlacementConfig, build +from placements._math import rotate_vec + +MODES = ("world", "head", "gimbal", "lazy") + +# Level, 45 deg right, 30 deg up, 45 deg right + 20 deg down. +HEAD_POSES = [ + (1.0, 0.0, 0.0, 0.0), + (math.cos(math.radians(22.5)), 0.0, math.sin(math.radians(22.5)), 0.0), + (math.cos(math.radians(15.0)), math.sin(math.radians(15.0)), 0.0, 0.0), + (0.9186, -0.1622, 0.3564, 0.0629), +] + + +def _dot(a, b): + return sum(x * y for x, y in zip(a, b)) + + +@pytest.mark.parametrize("mode", MODES) +@pytest.mark.parametrize("head_q", HEAD_POSES) +def test_plane_faces_the_viewer(mode, head_q): + """The plane's +z (its front) must point back toward the head.""" + head_pos = (0.0, 1.5, 0.0) + placement = build(mode, PlacementConfig(distance=1.0)).update(head_pos, head_q) + + front = rotate_vec(placement.orientation, (0.0, 0.0, 1.0)) + to_head = tuple(head_pos[i] - placement.position[i] for i in range(3)) + norm = math.sqrt(_dot(to_head, to_head)) + assert norm > 0.1, "plane landed on top of the head" + to_head = tuple(c / norm for c in to_head) + + # > 0 is "not facing away"; the strategies aim it squarely, so require + # the front within 45 deg of the head direction. + assert _dot(front, to_head) > math.cos(math.radians(45.0)), ( + f"{mode} faces away from the viewer: front={front} to_head={to_head}" + ) + + +@pytest.mark.parametrize("mode", MODES) +def test_plane_sits_the_configured_distance_away(mode): + head_pos = (0.0, 1.5, 0.0) + placement = build(mode, PlacementConfig(distance=1.25)).update( + head_pos, (1.0, 0.0, 0.0, 0.0) + ) + d = math.dist(placement.position, head_pos) + assert d == pytest.approx(1.25, abs=1e-4) + + +def test_head_locked_tracks_pitch_but_the_yaw_only_modes_do_not(): + """head is the 6-DoF mode; the others deliberately stay level.""" + head_pos = (0.0, 1.5, 0.0) + pitched = (math.cos(math.radians(15.0)), math.sin(math.radians(15.0)), 0.0, 0.0) + + head_y = build("head", PlacementConfig(distance=1.0)).update(head_pos, pitched) + assert head_y.position[1] != pytest.approx(1.5, abs=1e-3) + + for mode in ("world", "gimbal", "lazy"): + p = build(mode, PlacementConfig(distance=1.0)).update(head_pos, pitched) + assert p.position[1] == pytest.approx(1.5, abs=1e-6) diff --git a/tests/python/examples/camera_viz/test_runner_controls.py b/tests/python/examples/camera_viz/test_runner_controls.py new file mode 100644 index 0000000000..dfd2f0a5aa --- /dev/null +++ b/tests/python/examples/camera_viz/test_runner_controls.py @@ -0,0 +1,154 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""VizRunner <-> controls integration. + +These cover the seam the per-module tests structurally cannot: VizRunner +copies the lists it is handed, so a controls-side shape or lock-mode swap +only lands if the runner resolves it live. Without this, switching to +cylinder kept feeding the hidden quad and the feed went black. +""" + +from __future__ import annotations + +from pipeline.runner import VizRunner + + +class FakeLayer: + def __init__(self, name) -> None: + self.name = name + self.submitted = 0 + + def submit(self, *args, **kwargs) -> None: + self.submitted += 1 + + +class FakeSpec: + name = "cam" + + +class FakeSource: + spec = FakeSpec() + + def latest(self): + return None + + def start(self) -> None: ... + + def stop(self) -> None: ... + + +class FakeSession: + def is_xr_mode(self) -> bool: + return True + + +class FakeControls: + """Stands in for ControllerControls' resolver surface.""" + + def __init__(self, layer, strategy=None) -> None: + self.layer = layer + self._strategy = strategy + + def active_layer(self, index): + return self.layer + + def strategy(self, index): + return self._strategy + + def force_mono(self, index) -> bool: + return False + + +def _runner(controls=None, layers=None, strategies=None): + layers = layers or [FakeLayer("quad")] + return VizRunner( + FakeSession(), [FakeSource()], layers, strategies, controls=controls + ) + + +def test_runner_follows_a_shape_switch(): + quad, cylinder = FakeLayer("quad"), FakeLayer("cylinder") + controls = FakeControls(quad) + runner = _runner(controls, [quad]) + + assert runner._active_layer(0) is quad + controls.layer = cylinder # what pressing X does + assert runner._active_layer(0) is cylinder + + +def test_runner_follows_a_lock_mode_swap(): + layer = FakeLayer("quad") + first, second = object(), object() + controls = FakeControls(layer, first) + runner = _runner(controls, [layer], [first]) + + assert runner._active_strategy(0) is first + controls._strategy = second # what pressing A does + assert runner._active_strategy(0) is second + + +def test_runner_copies_stay_authoritative_without_controls(): + """No controls: the runner's own lists are the source of truth.""" + layer = FakeLayer("quad") + strategy = object() + runner = _runner(None, [layer], [strategy]) + assert runner._active_layer(0) is layer + assert runner._active_strategy(0) is strategy + + +def test_mutating_the_caller_list_does_not_reach_the_runner(): + """Pins down why the resolver exists: VizRunner copies, so the old + shared-list approach could not work.""" + layer, other = FakeLayer("quad"), FakeLayer("cylinder") + caller_layers = [layer] + runner = _runner(None, caller_layers) + caller_layers[0] = other + assert runner._active_layer(0) is layer + + +# ── The panel must not sit between the camera and the layer ─────────── + + +def test_the_panel_is_drawn_off_the_submit_thread(): + """A terminal that stops draining (paused ssh, tmux scrollback, ^S) fills + the pty buffer in ~25 paints and blocks the writer. On the submit thread + that stops the video feed, so the panel gets its own thread.""" + import inspect + + from pipeline.runner import VizRunner + + submit_src = inspect.getsource(VizRunner._submit_loop_inner) + assert "_print_stats" not in submit_src + assert "_print_stats" in inspect.getsource(VizRunner._stats_loop) + + +def test_a_broken_panel_write_does_not_kill_the_stats_thread(monkeypatch): + """stderr can go away mid-run (the terminal closes, the pipe breaks). The + panel is decoration; losing it must not take the stats thread with it.""" + import threading + + from pipeline import runner as runner_mod + + monkeypatch.setattr(runner_mod, "LIVE_STATS_PERIOD_S", 0.005) + + calls = [] + barrier = threading.Event() + + class ExplodingDashboard: + live = True + + def show(self, snapshot): + calls.append(1) + if len(calls) >= 3: + barrier.set() + raise OSError("terminal went away") + + runner = _runner() + runner._dashboard = ExplodingDashboard() + thread = threading.Thread(target=runner._stats_loop, daemon=True) + thread.start() + # It raised on every call and still came back for the next period. + assert barrier.wait(timeout=3.0), f"stats thread stopped after {len(calls)} calls" + runner._stop.set() + thread.join(timeout=2.0) + assert not thread.is_alive()