From 1152e6cea44c6cc34b6e6abaabacb8681fc80916 Mon Sep 17 00:00:00 2001
From: Griswald Brooks <griswald.brooks@gmail.com>
Date: Wed, 22 Jul 2026 22:13:50 -0400
Subject: [PATCH 1/2] feat: switch SAM3 objectives to the model bundle manifest

---
 ...e_from_no_negative_text_prompt_subtree.xml | 21 +------------------
 .../segment_point_cloud_from_text.xml         | 20 ------------------
 .../objectives/calibrate_sam3_mask_areas.xml  |  5 +----
 ...ve_boxes_to_loading_zone_from_waypoint.xml |  5 +----
 .../objectives/segment_image_from_prompt.xml  |  5 +----
 ...d_bottles_on_table_from_image_exemplar.xml |  5 +----
 .../ml_segment_bottles_from_file.xml          |  5 +----
 .../objectives/segment_bottle_subtree.xml     |  5 +----
 ...e_from_no_negative_text_prompt_subtree.xml | 21 +------------------
 src/moveit_pro_sam3                           |  2 +-
 ...e_from_no_negative_text_prompt_subtree.xml | 21 +------------------
 ...segment_image_from_text_prompt_subtree.xml | 21 +------------------
 ...t_point_cloud_from_text_prompt_subtree.xml | 20 ------------------
 13 files changed, 11 insertions(+), 145 deletions(-)

diff --git a/src/dual_arm_sim/objectives/segment_image_from_no_negative_text_prompt_subtree.xml b/src/dual_arm_sim/objectives/segment_image_from_no_negative_text_prompt_subtree.xml
index d7f8d366c..edf643b2b 100644
--- a/src/dual_arm_sim/objectives/segment_image_from_no_negative_text_prompt_subtree.xml
+++ b/src/dual_arm_sim/objectives/segment_image_from_no_negative_text_prompt_subtree.xml
@@ -19,10 +19,7 @@
       <Action
         ID="GetMasks2DFromExemplar"
         model_package="{model_package}"
-        encoder_model_path="{encoder_model_path}"
-        decoder_model_path="{decoder_model_path}"
-        geometry_encoder_model_path="{geometry_encoder_model_path}"
-        text_encoder_model_path="{text_encoder_model_path}"
+        model_bundle_manifest="models/model.yaml"
         confidence_threshold="{confidence_threshold}"
         target_image="{image}"
         text_prompt="{text_prompt}"
@@ -45,18 +42,6 @@
         <Metadata subcategory="Perception - ML" />
       </MetadataFields>
       <inout_port name="confidence_threshold" default="0.5" />
-      <inout_port
-        name="decoder_model_path"
-        default="models/sam3_decoder.onnx"
-      />
-      <inout_port
-        name="encoder_model_path"
-        default="models/sam3_vision_encoder.onnx"
-      />
-      <inout_port
-        name="geometry_encoder_model_path"
-        default="models/sam3_geometry_encoder.onnx"
-      />
       <inout_port name="image_topic_name" default="{image_topic_name}" />
       <inout_port
         name="masks_visualization_topic"
@@ -64,10 +49,6 @@
       />
       <inout_port name="masks2d" default="{masks2d}" />
       <inout_port name="model_package" default="moveit_pro_sam3" />
-      <inout_port
-        name="text_encoder_model_path"
-        default="models/sam3_text_encoder.onnx"
-      />
       <inout_port name="text_prompt" default="{text_prompt}" />
     </SubTree>
   </TreeNodesModel>
diff --git a/src/dual_arm_sim/objectives/segment_point_cloud_from_text.xml b/src/dual_arm_sim/objectives/segment_point_cloud_from_text.xml
index 68fd1f38c..e1486591b 100644
--- a/src/dual_arm_sim/objectives/segment_point_cloud_from_text.xml
+++ b/src/dual_arm_sim/objectives/segment_point_cloud_from_text.xml
@@ -25,10 +25,6 @@
         _collapsed="true"
         masks_visualization_topic="{masks_visualization_topic}"
         model_package="{model_package}"
-        encoder_model_path="{encoder_model_path}"
-        decoder_model_path="{decoder_model_path}"
-        geometry_encoder_model_path="{geometry_encoder_model_path}"
-        text_encoder_model_path="{text_encoder_model_path}"
         text_prompt="{text_prompt}"
         confidence_threshold="{confidence_threshold}"
         image_topic_name="{image_topic_name}"
@@ -86,18 +82,6 @@
         default="/wrist_camera/camera_info"
       />
       <inout_port name="confidence_threshold" default="0.5" />
-      <inout_port
-        name="decoder_model_path"
-        default="models/sam3_decoder.onnx"
-      />
-      <inout_port
-        name="encoder_model_path"
-        default="models/sam3_vision_encoder.onnx"
-      />
-      <inout_port
-        name="geometry_encoder_model_path"
-        default="models/sam3_geometry_encoder.onnx"
-      />
       <inout_port name="image_topic_name" default="/wrist_camera/color" />
       <inout_port
         name="masks_visualization_topic"
@@ -108,10 +92,6 @@
       <inout_port name="point_cloud" default="{point_cloud}" />
       <inout_port name="point_cloud_vector" default="{point_cloud_vector}" />
       <inout_port name="points_topic_name" default="/wrist_camera/points" />
-      <inout_port
-        name="text_encoder_model_path"
-        default="models/sam3_text_encoder.onnx"
-      />
       <inout_port name="text_prompt" default="{text_prompt}" />
       <inout_port name="use_latest_transform" default="false" />
     </SubTree>
diff --git a/src/hangar_sim/objectives/calibrate_sam3_mask_areas.xml b/src/hangar_sim/objectives/calibrate_sam3_mask_areas.xml
index faa05849e..916cfdf0a 100644
--- a/src/hangar_sim/objectives/calibrate_sam3_mask_areas.xml
+++ b/src/hangar_sim/objectives/calibrate_sam3_mask_areas.xml
@@ -39,10 +39,7 @@
       <Action
         ID="GetMasks2DFromExemplar"
         model_package="moveit_pro_sam3"
-        encoder_model_path="models/sam3_vision_encoder.onnx"
-        decoder_model_path="models/sam3_decoder.onnx"
-        geometry_encoder_model_path="models/sam3_geometry_encoder.onnx"
-        text_encoder_model_path="models/sam3_text_encoder.onnx"
+        model_bundle_manifest="models/model.yaml"
         confidence_threshold="{threshold}"
         target_image="{image}"
         text_prompt="{prompt}"
diff --git a/src/hangar_sim/objectives/move_boxes_to_loading_zone_from_waypoint.xml b/src/hangar_sim/objectives/move_boxes_to_loading_zone_from_waypoint.xml
index 19f06ab46..160e5a307 100644
--- a/src/hangar_sim/objectives/move_boxes_to_loading_zone_from_waypoint.xml
+++ b/src/hangar_sim/objectives/move_boxes_to_loading_zone_from_waypoint.xml
@@ -108,10 +108,7 @@
         <Action
           ID="GetMasks2DFromExemplar"
           model_package="moveit_pro_sam3"
-          encoder_model_path="models/sam3_vision_encoder.onnx"
-          decoder_model_path="models/sam3_decoder.onnx"
-          geometry_encoder_model_path="models/sam3_geometry_encoder.onnx"
-          text_encoder_model_path="models/sam3_text_encoder.onnx"
+          model_bundle_manifest="models/model.yaml"
           confidence_threshold="{confidence_threshold}"
           target_image="{image}"
           text_prompt="{text_prompt}"
diff --git a/src/kitchen_sim/objectives/segment_image_from_prompt.xml b/src/kitchen_sim/objectives/segment_image_from_prompt.xml
index 4cb272eb4..72a868b53 100644
--- a/src/kitchen_sim/objectives/segment_image_from_prompt.xml
+++ b/src/kitchen_sim/objectives/segment_image_from_prompt.xml
@@ -17,10 +17,7 @@
       <Action
         ID="GetMasks2DFromExemplar"
         model_package="moveit_pro_sam3"
-        encoder_model_path="models/sam3_vision_encoder.onnx"
-        decoder_model_path="models/sam3_decoder.onnx"
-        geometry_encoder_model_path="models/sam3_geometry_encoder.onnx"
-        text_encoder_model_path="models/sam3_text_encoder.onnx"
+        model_bundle_manifest="models/model.yaml"
         confidence_threshold="0.5"
         target_image="{image}"
         text_prompt="a stove circular burner"
diff --git a/src/lab_sim/objectives/ml_find_bottles_on_table_from_image_exemplar.xml b/src/lab_sim/objectives/ml_find_bottles_on_table_from_image_exemplar.xml
index 157645bad..d67f5ead0 100644
--- a/src/lab_sim/objectives/ml_find_bottles_on_table_from_image_exemplar.xml
+++ b/src/lab_sim/objectives/ml_find_bottles_on_table_from_image_exemplar.xml
@@ -31,10 +31,7 @@
       <Action
         ID="GetMasks2DFromExemplar"
         model_package="moveit_pro_sam3"
-        encoder_model_path="models/sam3_vision_encoder.onnx"
-        decoder_model_path="models/sam3_decoder.onnx"
-        geometry_encoder_model_path="models/sam3_geometry_encoder.onnx"
-        text_encoder_model_path="models/sam3_text_encoder.onnx"
+        model_bundle_manifest="models/model.yaml"
         confidence_threshold="0.6"
         confidence_scores_str="{confidence_scores_str}"
         target_image="{wrist_camera_image}"
diff --git a/src/lab_sim/objectives/ml_segment_bottles_from_file.xml b/src/lab_sim/objectives/ml_segment_bottles_from_file.xml
index e1d02f8c0..00da12055 100644
--- a/src/lab_sim/objectives/ml_segment_bottles_from_file.xml
+++ b/src/lab_sim/objectives/ml_segment_bottles_from_file.xml
@@ -14,10 +14,7 @@
       <Action
         ID="GetMasks2DFromExemplar"
         model_package="moveit_pro_sam3"
-        encoder_model_path="models/sam3_vision_encoder.onnx"
-        decoder_model_path="models/sam3_decoder.onnx"
-        geometry_encoder_model_path="models/sam3_geometry_encoder.onnx"
-        text_encoder_model_path="models/sam3_text_encoder.onnx"
+        model_bundle_manifest="models/model.yaml"
         confidence_threshold="0.5"
         confidence_scores_str="{confidence_scores_str}"
         target_image="{image}"
diff --git a/src/lab_sim/objectives/segment_bottle_subtree.xml b/src/lab_sim/objectives/segment_bottle_subtree.xml
index b2d0f3364..5e212f7a5 100644
--- a/src/lab_sim/objectives/segment_bottle_subtree.xml
+++ b/src/lab_sim/objectives/segment_bottle_subtree.xml
@@ -60,14 +60,11 @@
           confidence_scores="{confidence_scores}"
           target_image="{image}"
           text_prompt="{object_prompt}"
-          geometry_encoder_model_path="models/sam3_geometry_encoder.onnx"
           confidence_scores_str="{confidence_scores_str}"
           mask_count="{mask_count}"
           masks2d="{refined_masks}"
           confidence_threshold="0.5"
-          text_encoder_model_path="models/sam3_text_encoder.onnx"
-          encoder_model_path="models/sam3_vision_encoder.onnx"
-          decoder_model_path="models/sam3_decoder.onnx"
+          model_bundle_manifest="models/model.yaml"
           model_package="moveit_pro_sam3"
           exemplar_image="{exemplar_image}"
           target_bboxes="{target_bboxes}"
diff --git a/src/moveit_pro_franka_configs/franka_base_config/objectives/segment_image_from_no_negative_text_prompt_subtree.xml b/src/moveit_pro_franka_configs/franka_base_config/objectives/segment_image_from_no_negative_text_prompt_subtree.xml
index d7f8d366c..edf643b2b 100644
--- a/src/moveit_pro_franka_configs/franka_base_config/objectives/segment_image_from_no_negative_text_prompt_subtree.xml
+++ b/src/moveit_pro_franka_configs/franka_base_config/objectives/segment_image_from_no_negative_text_prompt_subtree.xml
@@ -19,10 +19,7 @@
       <Action
         ID="GetMasks2DFromExemplar"
         model_package="{model_package}"
-        encoder_model_path="{encoder_model_path}"
-        decoder_model_path="{decoder_model_path}"
-        geometry_encoder_model_path="{geometry_encoder_model_path}"
-        text_encoder_model_path="{text_encoder_model_path}"
+        model_bundle_manifest="models/model.yaml"
         confidence_threshold="{confidence_threshold}"
         target_image="{image}"
         text_prompt="{text_prompt}"
@@ -45,18 +42,6 @@
         <Metadata subcategory="Perception - ML" />
       </MetadataFields>
       <inout_port name="confidence_threshold" default="0.5" />
-      <inout_port
-        name="decoder_model_path"
-        default="models/sam3_decoder.onnx"
-      />
-      <inout_port
-        name="encoder_model_path"
-        default="models/sam3_vision_encoder.onnx"
-      />
-      <inout_port
-        name="geometry_encoder_model_path"
-        default="models/sam3_geometry_encoder.onnx"
-      />
       <inout_port name="image_topic_name" default="{image_topic_name}" />
       <inout_port
         name="masks_visualization_topic"
@@ -64,10 +49,6 @@
       />
       <inout_port name="masks2d" default="{masks2d}" />
       <inout_port name="model_package" default="moveit_pro_sam3" />
-      <inout_port
-        name="text_encoder_model_path"
-        default="models/sam3_text_encoder.onnx"
-      />
       <inout_port name="text_prompt" default="{text_prompt}" />
     </SubTree>
   </TreeNodesModel>
diff --git a/src/moveit_pro_sam3 b/src/moveit_pro_sam3
index 0d6463ee2..4b3874d9b 160000
--- a/src/moveit_pro_sam3
+++ b/src/moveit_pro_sam3
@@ -1 +1 @@
-Subproject commit 0d6463ee293adefddadc80c97aa80096dd38a21e
+Subproject commit 4b3874d9b718bc0713b7bc598abe2f4a4c3ca88b
diff --git a/src/moveit_pro_ur_configs/picknik_ur_base_config/objectives/segment_image_from_no_negative_text_prompt_subtree.xml b/src/moveit_pro_ur_configs/picknik_ur_base_config/objectives/segment_image_from_no_negative_text_prompt_subtree.xml
index d7f8d366c..edf643b2b 100644
--- a/src/moveit_pro_ur_configs/picknik_ur_base_config/objectives/segment_image_from_no_negative_text_prompt_subtree.xml
+++ b/src/moveit_pro_ur_configs/picknik_ur_base_config/objectives/segment_image_from_no_negative_text_prompt_subtree.xml
@@ -19,10 +19,7 @@
       <Action
         ID="GetMasks2DFromExemplar"
         model_package="{model_package}"
-        encoder_model_path="{encoder_model_path}"
-        decoder_model_path="{decoder_model_path}"
-        geometry_encoder_model_path="{geometry_encoder_model_path}"
-        text_encoder_model_path="{text_encoder_model_path}"
+        model_bundle_manifest="models/model.yaml"
         confidence_threshold="{confidence_threshold}"
         target_image="{image}"
         text_prompt="{text_prompt}"
@@ -45,18 +42,6 @@
         <Metadata subcategory="Perception - ML" />
       </MetadataFields>
       <inout_port name="confidence_threshold" default="0.5" />
-      <inout_port
-        name="decoder_model_path"
-        default="models/sam3_decoder.onnx"
-      />
-      <inout_port
-        name="encoder_model_path"
-        default="models/sam3_vision_encoder.onnx"
-      />
-      <inout_port
-        name="geometry_encoder_model_path"
-        default="models/sam3_geometry_encoder.onnx"
-      />
       <inout_port name="image_topic_name" default="{image_topic_name}" />
       <inout_port
         name="masks_visualization_topic"
@@ -64,10 +49,6 @@
       />
       <inout_port name="masks2d" default="{masks2d}" />
       <inout_port name="model_package" default="moveit_pro_sam3" />
-      <inout_port
-        name="text_encoder_model_path"
-        default="models/sam3_text_encoder.onnx"
-      />
       <inout_port name="text_prompt" default="{text_prompt}" />
     </SubTree>
   </TreeNodesModel>
diff --git a/src/moveit_pro_ur_configs/picknik_ur_base_config/objectives/segment_image_from_text_prompt_subtree.xml b/src/moveit_pro_ur_configs/picknik_ur_base_config/objectives/segment_image_from_text_prompt_subtree.xml
index e3b173efb..3e87a2a56 100644
--- a/src/moveit_pro_ur_configs/picknik_ur_base_config/objectives/segment_image_from_text_prompt_subtree.xml
+++ b/src/moveit_pro_ur_configs/picknik_ur_base_config/objectives/segment_image_from_text_prompt_subtree.xml
@@ -19,10 +19,7 @@
       <Action
         ID="GetMasks2DFromExemplar"
         model_package="{model_package}"
-        encoder_model_path="{encoder_model_path}"
-        decoder_model_path="{decoder_model_path}"
-        geometry_encoder_model_path="{geometry_encoder_model_path}"
-        text_encoder_model_path="{text_encoder_model_path}"
+        model_bundle_manifest="models/model.yaml"
         confidence_threshold="{confidence_threshold}"
         target_image="{image}"
         text_prompt="{text_prompt}"
@@ -43,18 +40,6 @@
         <Metadata subcategory="Perception - ML" />
       </MetadataFields>
       <inout_port name="confidence_threshold" default="0.5" />
-      <inout_port
-        name="decoder_model_path"
-        default="models/sam3_decoder.onnx"
-      />
-      <inout_port
-        name="encoder_model_path"
-        default="models/sam3_vision_encoder.onnx"
-      />
-      <inout_port
-        name="geometry_encoder_model_path"
-        default="models/sam3_geometry_encoder.onnx"
-      />
       <inout_port name="image_topic_name" default="{image_topic_name}" />
       <inout_port
         name="masks_visualization_topic"
@@ -62,10 +47,6 @@
       />
       <inout_port name="masks2d" default="{masks2d}" />
       <inout_port name="model_package" default="moveit_pro_sam3" />
-      <inout_port
-        name="text_encoder_model_path"
-        default="models/sam3_text_encoder.onnx"
-      />
       <inout_port name="text_prompt" default="{text_prompt}" />
     </SubTree>
   </TreeNodesModel>
diff --git a/src/moveit_pro_ur_configs/picknik_ur_base_config/objectives/segment_point_cloud_from_text_prompt_subtree.xml b/src/moveit_pro_ur_configs/picknik_ur_base_config/objectives/segment_point_cloud_from_text_prompt_subtree.xml
index e6c9ab383..09e840528 100644
--- a/src/moveit_pro_ur_configs/picknik_ur_base_config/objectives/segment_point_cloud_from_text_prompt_subtree.xml
+++ b/src/moveit_pro_ur_configs/picknik_ur_base_config/objectives/segment_point_cloud_from_text_prompt_subtree.xml
@@ -25,10 +25,6 @@
         _collapsed="true"
         masks_visualization_topic="{masks_visualization_topic}"
         model_package="{model_package}"
-        encoder_model_path="{encoder_model_path}"
-        decoder_model_path="{decoder_model_path}"
-        geometry_encoder_model_path="{geometry_encoder_model_path}"
-        text_encoder_model_path="{text_encoder_model_path}"
         text_prompt="{text_prompt}"
         confidence_threshold="{confidence_threshold}"
         image_topic_name="{image_topic_name}"
@@ -85,18 +81,6 @@
         default="/wrist_camera/camera_info"
       />
       <inout_port name="confidence_threshold" default="0.5" />
-      <inout_port
-        name="decoder_model_path"
-        default="models/sam3_decoder.onnx"
-      />
-      <inout_port
-        name="encoder_model_path"
-        default="models/sam3_vision_encoder.onnx"
-      />
-      <inout_port
-        name="geometry_encoder_model_path"
-        default="models/sam3_geometry_encoder.onnx"
-      />
       <inout_port name="image_topic_name" default="/wrist_camera/color" />
       <inout_port
         name="masks_visualization_topic"
@@ -107,10 +91,6 @@
       <inout_port name="point_cloud" default="{point_cloud}" />
       <inout_port name="point_cloud_vector" default="{point_cloud_vector}" />
       <inout_port name="points_topic_name" default="/wrist_camera/points" />
-      <inout_port
-        name="text_encoder_model_path"
-        default="models/sam3_text_encoder.onnx"
-      />
       <inout_port name="text_prompt" default="{text_prompt}" />
     </SubTree>
   </TreeNodesModel>

From df2dd26bb88eaf798d0600c78aab697fec8975a2 Mon Sep 17 00:00:00 2001
From: Griswald Brooks <griswald.brooks@gmail.com>
Date: Wed, 29 Jul 2026 11:35:21 -0400
Subject: [PATCH 2/2] chore: pin moveit_pro_sam3 to the rebased bundle-manifest
 commit

---
 src/moveit_pro_sam3 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/moveit_pro_sam3 b/src/moveit_pro_sam3
index 4b3874d9b..e9bb655e4 160000
--- a/src/moveit_pro_sam3
+++ b/src/moveit_pro_sam3
@@ -1 +1 @@
-Subproject commit 4b3874d9b718bc0713b7bc598abe2f4a4c3ca88b
+Subproject commit e9bb655e4210f9d6c416e5fceb71f26ceba50c14