Skip to content

Commit 3ce862f

Browse files
nvcycnvcyc
authored
Add support for rosidl::Buffer in rosidl Python path for rclpy (#250)
* Add support for rosidl::Buffer in rosidl Python path for rclpy Signed-off-by: CY Chen <cyc@nvidia.com> * Add missing rosidl_buffer_py dependency Signed-off-by: CY Chen <cyc@nvidia.com> * Add owns_rosidl_buffer assignment Signed-off-by: CY Chen <cyc@nvidia.com> --------- Signed-off-by: CY Chen <cyc@nvidia.com>
1 parent 01af609 commit 3ce862f

3 files changed

Lines changed: 167 additions & 54 deletions

File tree

‎rosidl_generator_py/package.xml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
<exec_depend>rosidl_cli</exec_depend>
4646
<exec_depend>rosidl_generator_c</exec_depend>
4747
<exec_depend>rosidl_parser</exec_depend>
48+
<exec_depend>rosidl_buffer_py</exec_depend>
4849
<exec_depend>rosidl_runtime_c</exec_depend>
4950
<exec_depend>rpyutils</exec_depend>
5051

@@ -55,6 +56,7 @@
5556
<test_depend>python3-numpy</test_depend>
5657
<test_depend>python3-pytest</test_depend>
5758
<test_depend>rmw</test_depend>
59+
<test_depend>rosidl_buffer_py</test_depend>
5860
<test_depend>rosidl_cmake</test_depend>
5961

6062
<!--

‎rosidl_generator_py/resource/_msg.py.em‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -430,11 +430,13 @@ if isinstance(type_, AbstractNestedType):
430430
if len(field) == 0:
431431
fieldstr = '[]'
432432
else:
433-
if self._check_fields:
434-
assert fieldstr.startswith('array(')
435-
prefix = "array('X', "
436-
suffix = ')'
437-
fieldstr = fieldstr[len(prefix):-len(suffix)]
433+
from rosidl_buffer import Buffer as _RosidlBuffer
434+
if not isinstance(field, _RosidlBuffer):
435+
if self._check_fields:
436+
assert fieldstr.startswith('array(')
437+
prefix = "array('X', "
438+
suffix = ')'
439+
fieldstr = fieldstr[len(prefix):-len(suffix)]
438440
args.append(s + '=' + fieldstr)
439441
return '%s(%s)' % ('.'.join(typename), ', '.join(args))
440442

@@ -491,6 +493,16 @@ if isinstance(member.type, (Array, AbstractSequence)):
491493
' please use a subclass of collections.abc.Sequence like list',
492494
DeprecationWarning)
493495
@[ end if]@
496+
@# Buffer type dispatch for uint8[] fields must run unconditionally (not behind _check_fields)
497+
@# because it is a type dispatch, not a validation check.
498+
@[ if isinstance(member.type, AbstractNestedType) and isinstance(member.type.value_type, BasicType) and member.type.value_type.typename in SPECIAL_NESTED_BASIC_TYPES]@
499+
@[ if isinstance(member.type, AbstractSequence) and isinstance(member.type, UnboundedSequence) and member.type.value_type.typename == 'uint8']@
500+
from rosidl_buffer import Buffer as _RosidlBuffer
501+
if isinstance(value, _RosidlBuffer):
502+
self._@(member.name) = value # type: ignore[assignment]
503+
return
504+
@[ end if]@
505+
@[ end if]@
494506
@[ if isinstance(type_, NamespacedType)]@
495507
@[ if (
496508
type_.name.endswith(ACTION_GOAL_SUFFIX) or
@@ -521,7 +533,11 @@ TEMPLATE(
521533
@[ else]@
522534
if isinstance(value, list):
523535
@[ end if]@
536+
@[ if isinstance(member.type, UnboundedSequence) and isinstance(member.type.value_type, BasicType) and member.type.value_type.typename == 'uint8']@
537+
self._@(member.name) = value # type: ignore[assignment]
538+
@[ else]@
524539
self._@(member.name) = value
540+
@[ end if]@
525541
return
526542
@[ end if]@
527543
@[ if isinstance(member.type, AbstractNestedType)]@

‎rosidl_generator_py/resource/_msg_support.c.em‎

Lines changed: 144 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ from rosidl_parser.definition import Array
1111
from rosidl_parser.definition import BasicType
1212
from rosidl_parser.definition import EMPTY_STRUCTURE_REQUIRED_MEMBER_NAME
1313
from rosidl_parser.definition import NamespacedType
14+
from rosidl_parser.definition import UnboundedSequence
1415
from rosidl_parser.definition import SERVICE_RESPONSE_MESSAGE_SUFFIX
1516
from rosidl_parser.definition import SERVICE_REQUEST_MESSAGE_SUFFIX
1617

@@ -28,13 +29,25 @@ def primitive_msg_type_to_c(type_):
2829
return BASIC_IDL_TYPES_TO_C[type_.typename]
2930

3031

32+
# Check if this message has any uint8[] buffer fields
33+
has_buffer_fields = False
34+
for member in message.structure.members:
35+
if isinstance(member.type, UnboundedSequence) and isinstance(member.type.value_type, BasicType) and member.type.value_type.typename == 'uint8':
36+
has_buffer_fields = True
37+
break
38+
39+
3140
include_parts = [package_name] + list(interface_path.parents[0].parts) + [
3241
'detail', convert_camel_case_to_lower_case_underscore(interface_path.stem)]
3342
include_base = '/'.join(include_parts)
3443

3544
header_files = [
3645
'Python.h',
3746
'stdbool.h',
47+
]
48+
if has_buffer_fields:
49+
header_files.append('stdint.h')
50+
header_files += [
3851
'numpy/ndarrayobject.h',
3952
'rosidl_runtime_c/visibility_control.h',
4053
include_base + '__struct.h',
@@ -68,6 +81,7 @@ repeated_header_file = header_file in include_directives
6881
#endif
6982
@[ end if]@
7083
@[end for]@
84+
@# Buffer-backed uint8[] fields use the is_rosidl_buffer flag on the sequence struct.
7185

7286
@{
7387
have_not_included_primitive_arrays = True
@@ -250,6 +264,43 @@ nested_type = '__'.join(type_.namespaced_name())
250264
@[ end if]@
251265
@[ elif isinstance(member.type, AbstractNestedType)]@
252266
@[ if isinstance(member.type, AbstractSequence) and isinstance(member.type.value_type, BasicType)]@
267+
@[ if isinstance(member.type, UnboundedSequence) and member.type.value_type.typename == 'uint8']@
268+
// Check if the field is an rosidl_buffer.Buffer with a non-CPU backend
269+
{
270+
PyObject * backend_attr = PyObject_GetAttrString(field, "backend_type");
271+
if (backend_attr != NULL) {
272+
const char * backend_str = PyUnicode_AsUTF8(backend_attr);
273+
if (backend_str != NULL && strcmp(backend_str, "cpu") != 0) {
274+
// Non-CPU backend: set is_rosidl_buffer flag instead of copying data
275+
PyObject * rosidl_buffer_mod = PyImport_ImportModule("rosidl_buffer");
276+
if (rosidl_buffer_mod != NULL) {
277+
PyObject * get_ptr_func = PyObject_GetAttrString(rosidl_buffer_mod, "_get_buffer_ptr");
278+
if (get_ptr_func != NULL) {
279+
PyObject * ptr_result = PyObject_CallFunctionObjArgs(get_ptr_func, field, NULL);
280+
if (ptr_result != NULL) {
281+
uintptr_t buffer_ptr = (uintptr_t)PyLong_AsUnsignedLongLong(ptr_result);
282+
ros_message->@(member.name).data = (uint8_t *)buffer_ptr;
283+
ros_message->@(member.name).size = 0;
284+
ros_message->@(member.name).capacity = 0;
285+
ros_message->@(member.name).is_rosidl_buffer = true;
286+
ros_message->@(member.name).owns_rosidl_buffer = false;
287+
Py_DECREF(ptr_result);
288+
}
289+
Py_DECREF(get_ptr_func);
290+
}
291+
Py_DECREF(rosidl_buffer_mod);
292+
}
293+
Py_DECREF(backend_attr);
294+
Py_DECREF(field);
295+
// Sentinel is set, skip normal conversion for this field
296+
goto @(member.name)__done;
297+
}
298+
Py_DECREF(backend_attr);
299+
} else {
300+
PyErr_Clear();
301+
}
302+
}
303+
@[ end if]@
253304
if (PyObject_CheckBuffer(field)) {
254305
// Optimization for converting arrays of primitives
255306
Py_buffer view;
@@ -513,6 +564,10 @@ nested_type = '__'.join(type_.namespaced_name())
513564
@[ end if]@
514565
Py_DECREF(field);
515566
}
567+
@[ if isinstance(member.type, UnboundedSequence) and isinstance(member.type.value_type, BasicType) and member.type.value_type.typename == 'uint8']@
568+
@(member.name)__done:
569+
;
570+
@[ end if]@
516571
@[end for]@
517572

518573
return true;
@@ -569,60 +624,100 @@ if isinstance(type_, AbstractNestedType):
569624
memcpy(dst, src, @(member.type.size) * sizeof(@primitive_msg_type_to_c(member.type.value_type)));
570625
Py_DECREF(field);
571626
@[ elif isinstance(member.type, AbstractSequence)]@
572-
field = PyObject_GetAttrString(_pymessage, "@(member.name)");
573-
if (!field) {
574-
return NULL;
575-
}
576-
assert(field->ob_type != NULL);
577-
assert(field->ob_type->tp_name != NULL);
578-
assert(strcmp(field->ob_type->tp_name, "array.array") == 0);
579-
// ensure that itemsize matches the sizeof of the ROS message field
580-
PyObject * itemsize_attr = PyObject_GetAttrString(field, "itemsize");
581-
assert(itemsize_attr != NULL);
582-
size_t itemsize = PyLong_AsSize_t(itemsize_attr);
583-
Py_DECREF(itemsize_attr);
584-
if (itemsize != sizeof(@primitive_msg_type_to_c(member.type.value_type))) {
585-
PyErr_SetString(PyExc_RuntimeError, "itemsize doesn't match expectation");
586-
Py_DECREF(field);
587-
return NULL;
588-
}
589-
// clear the array, poor approach to remove potential default values
590-
Py_ssize_t length = PyObject_Length(field);
591-
if (-1 == length) {
592-
Py_DECREF(field);
593-
return NULL;
594-
}
595-
if (length > 0) {
596-
PyObject * pop = PyObject_GetAttrString(field, "pop");
597-
assert(pop != NULL);
598-
for (Py_ssize_t i = 0; i < length; ++i) {
599-
PyObject * ret = PyObject_CallFunctionObjArgs(pop, NULL);
600-
if (!ret) {
601-
Py_DECREF(pop);
602-
Py_DECREF(field);
603-
return NULL;
627+
@[ if isinstance(member.type, UnboundedSequence) and member.type.value_type.typename == 'uint8']@
628+
if (ros_message->@(member.name).is_rosidl_buffer) {
629+
// The RMW deserialized into a vendor-backed buffer — wrap it in a Python Buffer.
630+
// All C++ operations go through the rosidl_buffer._rosidl_buffer_py module since this
631+
// file is compiled as C.
632+
PyObject * rosidl_buffer_internal = PyImport_ImportModule("rosidl_buffer");
633+
if (rosidl_buffer_internal != NULL) {
634+
PyObject * take_func = PyObject_GetAttrString(rosidl_buffer_internal, "_take_buffer_from_ptr");
635+
if (take_func != NULL) {
636+
PyObject * ptr_arg = PyLong_FromUnsignedLongLong(
637+
(uint64_t)(uintptr_t)ros_message->@(member.name).data);
638+
field = PyObject_CallFunctionObjArgs(take_func, ptr_arg, NULL);
639+
Py_XDECREF(ptr_arg);
640+
Py_DECREF(take_func);
641+
if (field != NULL) {
642+
// Ownership transferred to Python — clear the C sequence immediately
643+
ros_message->@(member.name).data = NULL;
644+
ros_message->@(member.name).size = 0;
645+
ros_message->@(member.name).capacity = 0;
646+
ros_message->@(member.name).is_rosidl_buffer = false;
647+
ros_message->@(member.name).owns_rosidl_buffer = false;
648+
}
604649
}
605-
Py_DECREF(ret);
650+
Py_DECREF(rosidl_buffer_internal);
606651
}
607-
Py_DECREF(pop);
608-
}
609-
if (ros_message->@(member.name).size > 0) {
610-
// populating the array.array using the frombytes method
611-
PyObject * frombytes = PyObject_GetAttrString(field, "frombytes");
612-
assert(frombytes != NULL);
613-
@primitive_msg_type_to_c(member.type.value_type) * src = &(ros_message->@(member.name).data[0]);
614-
PyObject * data = PyBytes_FromStringAndSize((const char *)src, ros_message->@(member.name).size * sizeof(@primitive_msg_type_to_c(member.type.value_type)));
615-
assert(data != NULL);
616-
PyObject * ret = PyObject_CallFunctionObjArgs(frombytes, data, NULL);
617-
Py_DECREF(data);
618-
Py_DECREF(frombytes);
619-
if (!ret) {
652+
if (field == NULL) {
653+
return NULL;
654+
}
655+
// Set the Buffer on the Python message object
656+
if (PyObject_SetAttrString(_pymessage, "@(member.name)", field) == -1) {
620657
Py_DECREF(field);
621658
return NULL;
622659
}
623-
Py_DECREF(ret);
624-
}
625-
Py_DECREF(field);
660+
Py_DECREF(field);
661+
} else {
662+
@[ end if]@
663+
@{bi = ' ' if (isinstance(member.type, UnboundedSequence) and member.type.value_type.typename == 'uint8') else ''}@
664+
@(bi) field = PyObject_GetAttrString(_pymessage, "@(member.name)");
665+
@(bi) if (!field) {
666+
@(bi) return NULL;
667+
@(bi) }
668+
@(bi) assert(field->ob_type != NULL);
669+
@(bi) assert(field->ob_type->tp_name != NULL);
670+
@(bi) assert(strcmp(field->ob_type->tp_name, "array.array") == 0);
671+
@(bi) // ensure that itemsize matches the sizeof of the ROS message field
672+
@(bi) PyObject * itemsize_attr = PyObject_GetAttrString(field, "itemsize");
673+
@(bi) assert(itemsize_attr != NULL);
674+
@(bi) size_t itemsize = PyLong_AsSize_t(itemsize_attr);
675+
@(bi) Py_DECREF(itemsize_attr);
676+
@(bi) if (itemsize != sizeof(@primitive_msg_type_to_c(member.type.value_type))) {
677+
@(bi) PyErr_SetString(PyExc_RuntimeError, "itemsize doesn't match expectation");
678+
@(bi) Py_DECREF(field);
679+
@(bi) return NULL;
680+
@(bi) }
681+
@(bi) // clear the array, poor approach to remove potential default values
682+
@(bi) Py_ssize_t length = PyObject_Length(field);
683+
@(bi) if (-1 == length) {
684+
@(bi) Py_DECREF(field);
685+
@(bi) return NULL;
686+
@(bi) }
687+
@(bi) if (length > 0) {
688+
@(bi) PyObject * pop = PyObject_GetAttrString(field, "pop");
689+
@(bi) assert(pop != NULL);
690+
@(bi) for (Py_ssize_t i = 0; i < length; ++i) {
691+
@(bi) PyObject * ret = PyObject_CallFunctionObjArgs(pop, NULL);
692+
@(bi) if (!ret) {
693+
@(bi) Py_DECREF(pop);
694+
@(bi) Py_DECREF(field);
695+
@(bi) return NULL;
696+
@(bi) }
697+
@(bi) Py_DECREF(ret);
698+
@(bi) }
699+
@(bi) Py_DECREF(pop);
700+
@(bi) }
701+
@(bi) if (ros_message->@(member.name).size > 0) {
702+
@(bi) // populating the array.array using the frombytes method
703+
@(bi) PyObject * frombytes = PyObject_GetAttrString(field, "frombytes");
704+
@(bi) assert(frombytes != NULL);
705+
@(bi) @primitive_msg_type_to_c(member.type.value_type) * src = &(ros_message->@(member.name).data[0]);
706+
@(bi) PyObject * data = PyBytes_FromStringAndSize((const char *)src, ros_message->@(member.name).size * sizeof(@primitive_msg_type_to_c(member.type.value_type)));
707+
@(bi) assert(data != NULL);
708+
@(bi) PyObject * ret = PyObject_CallFunctionObjArgs(frombytes, data, NULL);
709+
@(bi) Py_DECREF(data);
710+
@(bi) Py_DECREF(frombytes);
711+
@(bi) if (!ret) {
712+
@(bi) Py_DECREF(field);
713+
@(bi) return NULL;
714+
@(bi) }
715+
@(bi) Py_DECREF(ret);
716+
@(bi) }
717+
@(bi) Py_DECREF(field);
718+
@[ if isinstance(member.type, UnboundedSequence) and member.type.value_type.typename == 'uint8']@
719+
} // end else (non-buffer path)
720+
@[ end if]@
626721
@[ end if]@
627722
@[ else]@
628723
@[ if isinstance(type_, NamespacedType)]@

0 commit comments

Comments
 (0)