From d204750eca076833b0a8ec69580f8138f60455e9 Mon Sep 17 00:00:00 2001
From: Griswald Brooks <griswald.brooks@gmail.com>
Date: Wed, 5 Aug 2026 08:25:49 -0400
Subject: [PATCH] fix(objectives): switch SAM2 point-query Objectives to the
 model bundle manifest

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 .../objectives/segment_image_from_point.xml    |  4 ++--
 .../segment_point_cloud_from_clicked_point.xml |  4 ++--
 .../objectives/generate_graspable_object.xml   |  4 ++--
 .../objectives/grasp_object_from_point.xml     |  4 ++--
 .../objectives/segment_image_from_point.xml    |  4 ++--
 .../segment_point_cloud_from_point.xml         |  4 ++--
 ..._segment_point_cloud_from_clicked_point.xml |  4 ++--
 .../segment_image_from_point_subtree.xml       | 18 ++++++------------
 ..._point_cloud_from_clicked_point_subtree.xml | 15 +++++++++------
 9 files changed, 29 insertions(+), 32 deletions(-)

diff --git a/src/hangar_sim/objectives/segment_image_from_point.xml b/src/hangar_sim/objectives/segment_image_from_point.xml
index ccbfc3c97..9d4668035 100644
--- a/src/hangar_sim/objectives/segment_image_from_point.xml
+++ b/src/hangar_sim/objectives/segment_image_from_point.xml
@@ -10,8 +10,8 @@
       <SubTree
         ID="Segment Image from Point Subtree"
         _collapsed="true"
-        decoder_model_path="models/sam2_decoder.onnx"
-        encoder_model_path="models/sam2_hiera_large_encoder.onnx"
+        model_bundle_manifest="models/model.yaml"
+        runtime_id="onnxruntime"
         image_topic_name="/wrist_camera/color"
         masks_visualization_topic="/masks_visualization"
         model_package="moveit_pro_sam2"
diff --git a/src/hangar_sim/objectives/segment_point_cloud_from_clicked_point.xml b/src/hangar_sim/objectives/segment_point_cloud_from_clicked_point.xml
index da3e6cd21..f6c275c78 100644
--- a/src/hangar_sim/objectives/segment_point_cloud_from_clicked_point.xml
+++ b/src/hangar_sim/objectives/segment_point_cloud_from_clicked_point.xml
@@ -13,8 +13,8 @@
       <SubTree
         ID="Segment Point Cloud from Clicked Point Subtree"
         camera_topic_name="/wrist_camera/camera_info"
-        decoder_model_path="models/sam2_decoder.onnx"
-        encoder_model_path="models/sam2_hiera_large_encoder.onnx"
+        model_bundle_manifest="models/model.yaml"
+        runtime_id="onnxruntime"
         image_topic_name="/wrist_camera/color"
         masks_visualization_topic="/masks_visualization"
         model_package="moveit_pro_sam2"
diff --git a/src/kitchen_sim/objectives/generate_graspable_object.xml b/src/kitchen_sim/objectives/generate_graspable_object.xml
index 52563b3ef..7c2e2d11c 100644
--- a/src/kitchen_sim/objectives/generate_graspable_object.xml
+++ b/src/kitchen_sim/objectives/generate_graspable_object.xml
@@ -27,8 +27,8 @@
         image="{image}"
         masks2d="{masks2d}"
         pixel_coords="{pixel_coords}"
-        decoder_model_path="models/sam2_decoder.onnx"
-        encoder_model_path="models/sam2_hiera_large_encoder.onnx"
+        model_bundle_manifest="models/model.yaml"
+        runtime_id="onnxruntime"
         model_package="moveit_pro_sam2"
       />
       <Action
diff --git a/src/kitchen_sim/objectives/grasp_object_from_point.xml b/src/kitchen_sim/objectives/grasp_object_from_point.xml
index 58569b0da..25c3bfd28 100644
--- a/src/kitchen_sim/objectives/grasp_object_from_point.xml
+++ b/src/kitchen_sim/objectives/grasp_object_from_point.xml
@@ -26,8 +26,8 @@
         image="{image}"
         masks2d="{masks2d}"
         pixel_coords="{pixel_coords}"
-        decoder_model_path="models/sam2_decoder.onnx"
-        encoder_model_path="models/sam2_hiera_large_encoder.onnx"
+        model_bundle_manifest="models/model.yaml"
+        runtime_id="onnxruntime"
         model_package="moveit_pro_sam2"
       />
       <Action
diff --git a/src/kitchen_sim/objectives/segment_image_from_point.xml b/src/kitchen_sim/objectives/segment_image_from_point.xml
index 96a642332..b0aac37d8 100644
--- a/src/kitchen_sim/objectives/segment_image_from_point.xml
+++ b/src/kitchen_sim/objectives/segment_image_from_point.xml
@@ -24,8 +24,8 @@
         image="{image}"
         masks2d="{masks2d}"
         pixel_coords="{pixel_coords}"
-        decoder_model_path="models/sam2_decoder.onnx"
-        encoder_model_path="models/sam2_hiera_large_encoder.onnx"
+        model_bundle_manifest="models/model.yaml"
+        runtime_id="onnxruntime"
         model_package="moveit_pro_sam2"
       />
       <Action
diff --git a/src/kitchen_sim/objectives/segment_point_cloud_from_point.xml b/src/kitchen_sim/objectives/segment_point_cloud_from_point.xml
index 12b34722e..096487ad6 100644
--- a/src/kitchen_sim/objectives/segment_point_cloud_from_point.xml
+++ b/src/kitchen_sim/objectives/segment_point_cloud_from_point.xml
@@ -25,8 +25,8 @@
         image="{image}"
         masks2d="{masks2d}"
         pixel_coords="{pixel_coords}"
-        decoder_model_path="models/sam2_decoder.onnx"
-        encoder_model_path="models/sam2_hiera_large_encoder.onnx"
+        model_bundle_manifest="models/model.yaml"
+        runtime_id="onnxruntime"
         model_package="moveit_pro_sam2"
       />
       <Action
diff --git a/src/lab_sim/objectives/ml_segment_point_cloud_from_clicked_point.xml b/src/lab_sim/objectives/ml_segment_point_cloud_from_clicked_point.xml
index 53386823f..0d6d8093d 100644
--- a/src/lab_sim/objectives/ml_segment_point_cloud_from_clicked_point.xml
+++ b/src/lab_sim/objectives/ml_segment_point_cloud_from_clicked_point.xml
@@ -14,8 +14,8 @@
         ID="Segment Point Cloud from Clicked Point Subtree"
         _collapsed="true"
         camera_topic_name="/wrist_camera/camera_info"
-        decoder_model_path="models/sam2_decoder.onnx"
-        encoder_model_path="models/sam2_hiera_large_encoder.onnx"
+        model_bundle_manifest="models/model.yaml"
+        runtime_id="onnxruntime"
         image_topic_name="/wrist_camera/color"
         masks_visualization_topic="/masks_visualization"
         model_package="moveit_pro_sam2"
diff --git a/src/moveit_pro_ur_configs/picknik_ur_base_config/objectives/segment_image_from_point_subtree.xml b/src/moveit_pro_ur_configs/picknik_ur_base_config/objectives/segment_image_from_point_subtree.xml
index 5d636e648..447bd53c3 100644
--- a/src/moveit_pro_ur_configs/picknik_ur_base_config/objectives/segment_image_from_point_subtree.xml
+++ b/src/moveit_pro_ur_configs/picknik_ur_base_config/objectives/segment_image_from_point_subtree.xml
@@ -5,8 +5,8 @@
     ID="Segment Image from Point Subtree"
     _description="Captures an image and segments it using three user provided prompt points."
     _favorite="false"
-    decoder_model_path="models/sam2_decoder.onnx"
-    encoder_model_path="models/sam2_hiera_large_encoder.onnx"
+    model_bundle_manifest="models/model.yaml"
+    runtime_id="onnxruntime"
     image_topic_name="/wrist_camera/color"
     masks_visualization_topic="/masks_visualization"
     model_package="moveit_pro_sam2"
@@ -32,8 +32,8 @@
         image="{image}"
         pixel_coords="{pixel_coords}"
         masks2d="{masks2d}"
-        decoder_model_path="{decoder_model_path}"
-        encoder_model_path="{encoder_model_path}"
+        model_bundle_manifest="{model_bundle_manifest}"
+        runtime_id="{runtime_id}"
         model_package="{model_package}"
       />
       <Action
@@ -51,14 +51,8 @@
         <Metadata runnable="false" />
         <Metadata subcategory="Perception - ML" />
       </MetadataFields>
-      <inout_port
-        name="decoder_model_path"
-        default="models/sam2_decoder.onnx"
-      />
-      <inout_port
-        name="encoder_model_path"
-        default="models/sam2_hiera_large_encoder.onnx"
-      />
+      <inout_port name="model_bundle_manifest" default="models/model.yaml" />
+      <inout_port name="runtime_id" default="onnxruntime" />
       <inout_port name="image_topic_name" default="/wrist_camera/color" />
       <inout_port
         name="masks_visualization_topic"
diff --git a/src/moveit_pro_ur_configs/picknik_ur_base_config/objectives/segment_point_cloud_from_clicked_point_subtree.xml b/src/moveit_pro_ur_configs/picknik_ur_base_config/objectives/segment_point_cloud_from_clicked_point_subtree.xml
index 60687b585..62caf05c6 100644
--- a/src/moveit_pro_ur_configs/picknik_ur_base_config/objectives/segment_point_cloud_from_clicked_point_subtree.xml
+++ b/src/moveit_pro_ur_configs/picknik_ur_base_config/objectives/segment_point_cloud_from_clicked_point_subtree.xml
@@ -9,8 +9,8 @@
     _description="Captures a point cloud and requests the user to click an object in the image to be segmented. The point cloud is then filtered to only include the selected object."
     _favorite="false"
     camera_topic_name="{camera_topic_name}"
-    decoder_model_path="{decoder_model_path}"
-    encoder_model_path="{encoder_model_path}"
+    model_bundle_manifest="{model_bundle_manifest}"
+    runtime_id="{runtime_id}"
     image_topic_name="{image_topic_name}"
     masks_visualization_topic="{masks_visualization_topic}"
     model_package="{model_package}"
@@ -21,8 +21,8 @@
       <SubTree
         ID="Segment Image from Point Subtree"
         _collapsed="true"
-        decoder_model_path="{decoder_model_path}"
-        encoder_model_path="{encoder_model_path}"
+        model_bundle_manifest="{model_bundle_manifest}"
+        runtime_id="{runtime_id}"
         image_topic="{image_topic_name}"
         masks_visualization_topic="{masks_visualization_topic}"
         model_package="{model_package}"
@@ -60,8 +60,11 @@
   <TreeNodesModel>
     <SubTree ID="Segment Point Cloud from Clicked Point Subtree">
       <inout_port name="camera_topic_name" default="{camera_topic_name}" />
-      <inout_port name="decoder_model_path" default="{decoder_model_path}" />
-      <inout_port name="encoder_model_path" default="{encoder_model_path}" />
+      <inout_port
+        name="model_bundle_manifest"
+        default="{model_bundle_manifest}"
+      />
+      <inout_port name="runtime_id" default="{runtime_id}" />
       <inout_port name="image_topic_name" default="{image_topic_name}" />
       <inout_port
         name="masks_visualization_topic"