Fix py::cast(std::shared_ptr<T>) for shared_ptr-compatible custom holders

include/pybind11/cast.h

@@ -1026,6 +1026,15 @@ struct copyable_holder_caster<
             return type_caster_base<type>::cast_holder(srcs, &src);
         }
 
+        if (tinfo != nullptr && tinfo->holder_enum_v == holder_enum_t::custom_holder) {
+            auto aliasing_owner
+                = std::shared_ptr<void>(src, const_cast<void *>(srcs.result.cppobj));
+            detail::init_instance_with_shared_ptr shared_ptr_payload(std::move(aliasing_owner));
+            detail::init_instance_with_shared_ptr_guard shared_ptr_payload_guard(
+                &shared_ptr_payload);
+            return type_caster_base<type>::cast_holder(srcs, &shared_ptr_payload);
+        }
+
         if (parent) {
             return type_caster_generic::cast_non_owning(
                 srcs, return_value_policy::reference_internal, parent);

include/pybind11/detail/type_caster_base.h

@@ -101,6 +101,50 @@ class loader_life_support {
     }
 };
 
+struct init_instance_with_shared_ptr {
+    std::shared_ptr<void> aliasing_owner;
+    mutable const init_instance_with_shared_ptr *previous = nullptr;
+
+    explicit init_instance_with_shared_ptr(std::shared_ptr<void> aliasing_owner_)
+        : aliasing_owner(std::move(aliasing_owner_)) {}
+};
+
+class init_instance_with_shared_ptr_guard {
+private:
+    static const init_instance_with_shared_ptr *&tls_current_payload() {
+        static thread_local const init_instance_with_shared_ptr *payload_ptr = nullptr;
+        return payload_ptr;
+    }
+
+    const init_instance_with_shared_ptr *payload;
+
+public:
+    explicit init_instance_with_shared_ptr_guard(const init_instance_with_shared_ptr *payload_)
+        : payload(payload_) {
+        auto &current = tls_current_payload();
+        payload->previous = current;
+        current = payload;
+    }
+
+    ~init_instance_with_shared_ptr_guard() {
+        auto &current = tls_current_payload();
+        if (current != payload) {
+            pybind11_fail("init_instance_with_shared_ptr_guard: internal error");
+        }
+        current = payload->previous;
+    }
+
+    static const init_instance_with_shared_ptr *lookup(const void *holder_ptr) {
+        for (auto *current = tls_current_payload(); current != nullptr;
+             current = current->previous) {
+            if (current == holder_ptr) {
+                return current;
+            }
+        }
+        return nullptr;
+    }
+};
+
 // Gets the cache entry for the given type, creating it if necessary.  The return value is the pair
 // returned by emplace, i.e. an iterator for the entry and a bool set to `true` if the entry was
 // just created.

include/pybind11/pybind11.h

@@ -2768,6 +2768,25 @@ class class_ : public detail::generic_type {
             holder_type(std::move(*const_cast<holder_type *>(holder_ptr)));
     }
 
+    template <typename H = holder_type,
+              detail::enable_if_t<std::is_constructible<H, std::shared_ptr<type>>::value, int> = 0>
+    static bool
+    init_holder_from_shared_ptr(const detail::value_and_holder &v_h,
+                                const detail::init_instance_with_shared_ptr &shared_ptr_payload) {
+        auto shared_ptr
+            = std::shared_ptr<type>(shared_ptr_payload.aliasing_owner, v_h.value_ptr<type>());
+        new (std::addressof(v_h.holder<holder_type>())) holder_type(std::move(shared_ptr));
+        return true;
+    }
+
+    template <typename H = holder_type,
+              detail::enable_if_t<!std::is_constructible<H, std::shared_ptr<type>>::value, int>
+              = 0>
+    static bool init_holder_from_shared_ptr(const detail::value_and_holder &,
+                                            const detail::init_instance_with_shared_ptr &) {
+        return false;
+    }
+
     /// Initialize holder object, variant 2: try to construct from existing holder object, if
     /// possible
     static void init_holder(detail::instance *inst,
@@ -2795,6 +2814,16 @@ class class_ : public detail::generic_type {
             register_instance(inst, v_h.value_ptr(), v_h.type);
             v_h.set_instance_registered();
         }
+        if (auto *shared_ptr_payload
+            = detail::init_instance_with_shared_ptr_guard::lookup(holder_ptr)) {
+            if (init_holder_from_shared_ptr(v_h, *shared_ptr_payload)) {
+                v_h.set_holder_constructed();
+                return;
+            }
+            throw cast_error("Unable to convert std::shared_ptr<T> to Python when the bound type "
+                             "does not use std::shared_ptr or py::smart_holder as its holder "
+                             "type");
+        }
         init_holder(inst, v_h, (const holder_type *) holder_ptr, v_h.value_ptr<type>());
     }
 

tests/test_smart_ptr.cpp

@@ -83,6 +83,26 @@ class const_only_shared_ptr {
     // for demonstration purpose only, this imitates smart pointer with a const-only pointer
 };
 
+template <typename T>
+class shared_ptr_as_custom_holder {
+    std::shared_ptr<T> ptr_;
+
+public:
+    using element_type = T;
+
+    shared_ptr_as_custom_holder() = default;
+
+    explicit shared_ptr_as_custom_holder(T *) {
+        throw std::runtime_error("invalid shared_ptr_as_custom_holder constructor call");
+    }
+
+    explicit shared_ptr_as_custom_holder(std::shared_ptr<T> ptr) : ptr_(std::move(ptr)) {}
+
+    T *get() const { return ptr_.get(); }
+
+    operator std::shared_ptr<T>() const { return ptr_; }
+};
+
 // Custom object with builtin reference counting (see 'object.h' for the implementation)
 class MyObject1 : public Object {
 public:
@@ -239,6 +259,25 @@ struct SharedFromThisRef {
     std::shared_ptr<B> shared = std::make_shared<B>();
 };
 
+class PrivateDtorWithCustomHolder {
+public:
+    static std::shared_ptr<PrivateDtorWithCustomHolder> create(int value) {
+        return {new PrivateDtorWithCustomHolder(value),
+                [](PrivateDtorWithCustomHolder *ptr) { delete ptr; }};
+    }
+
+    int value = 0;
+
+private:
+    explicit PrivateDtorWithCustomHolder(int value_) : value(value_) {}
+    ~PrivateDtorWithCustomHolder() = default;
+};
+
+std::shared_ptr<PrivateDtorWithCustomHolder> &private_dtor_with_custom_holder_singleton() {
+    static auto singleton = PrivateDtorWithCustomHolder::create(17);
+    return singleton;
+}
+
 // Issue #865: shared_from_this doesn't work with virtual inheritance
 struct SharedFromThisVBase : std::enable_shared_from_this<SharedFromThisVBase> {
     SharedFromThisVBase() = default;
@@ -341,6 +380,7 @@ struct holder_helper<ref<T>> {
 // Make pybind aware of the ref-counted wrapper type (s):
 PYBIND11_DECLARE_HOLDER_TYPE(T, ref<T>, true)
 PYBIND11_DECLARE_HOLDER_TYPE(T, const_only_shared_ptr<T>, true)
+PYBIND11_DECLARE_HOLDER_TYPE(T, shared_ptr_as_custom_holder<T>)
 // The following is not required anymore for std::shared_ptr, but it should compile without error:
 PYBIND11_DECLARE_HOLDER_TYPE(T, std::shared_ptr<T>)
 PYBIND11_DECLARE_HOLDER_TYPE(T, huge_unique_ptr<T>)
@@ -501,6 +541,20 @@ TEST_SUBMODULE(smart_ptr, m) {
         .def(py::init([](const std::string &value) { return MyObject6::createObject(value); }))
         .def_property_readonly("value", &MyObject6::value);
 
+    py::class_<PrivateDtorWithCustomHolder,
+               shared_ptr_as_custom_holder<PrivateDtorWithCustomHolder>>(
+        m, "PrivateDtorWithCustomHolder")
+        .def_property(
+            "value",
+            [](const PrivateDtorWithCustomHolder &self) { return self.value; },
+            [](PrivateDtorWithCustomHolder &self, int value) { self.value = value; })
+        .def_static("get_singleton_holder", []() {
+            return shared_ptr_as_custom_holder<PrivateDtorWithCustomHolder>(
+                private_dtor_with_custom_holder_singleton());
+        });
+    m.def("get_private_dtor_with_custom_holder_shared_ptr",
+          []() { return private_dtor_with_custom_holder_singleton(); });
+
     // test_shared_ptr_and_references
     using A = SharedPtrRef::A;
     py::class_<A, std::shared_ptr<A>>(m, "A");

tests/test_smart_ptr.py

@@ -368,3 +368,13 @@ def test_move_only_holder_caster_shared_ptr_with_smart_holder_support_enabled():
 def test_const_only_holder():
     o = m.MyObject6("my_data")
     assert o.value == "my_data"
+
+
+def test_shared_ptr_cast_for_custom_holder_with_private_dtor():
+    holder_obj = m.PrivateDtorWithCustomHolder.get_singleton_holder()
+    shared_ptr_obj = m.get_private_dtor_with_custom_holder_shared_ptr()
+
+    holder_obj.value = 23
+
+    assert shared_ptr_obj.value == 23
+    assert m.get_private_dtor_with_custom_holder_shared_ptr().value == 23