From 3e79f8d9ac64e12057f38d42befdc460a4ee29af Mon Sep 17 00:00:00 2001
From: Jiayuan Chen <jiayuan.chen@linux.dev>
Date: Sat, 11 Jul 2026 12:55:49 +0800
Subject: [PATCH] Fix SIGSEGV in Python stub on non-graceful (parent death)
 termination

When the parent (tritonserver) process dies abruptly -- SIGKILL after the
Kubernetes termination grace period, an OOM-kill, or a crash -- it cannot
send a Finalize command, so the stub's health-monitoring background thread
takes the "Non-graceful termination detected" path. That path called
Stub::DestroyInstance() and exit(1). Both unmap the shared-memory region
(DestroyInstance via ~SharedMemoryManager -> ~mapped_region -> munmap, and
exit() via static destructors) while the RunCommand loop and the model
instance threads are still operating on that region and its process-shared
mutexes.

Unmapping the region out from under those still-running threads is a data
race: a thread that touches a shm mutex after the region is unmapped faults
with SIGSEGV inside libpthread (SEGV_MAPERR on the lock word). Because a
whole pod's stubs share one parent, they all take this path at once and a
batch of them crash simultaneously at the same instruction.

The background thread cannot safely tear the region down: it cannot join the
main thread (which in turn joins it), and the main thread is almost always
either blocked on, or actively locking, a shm object. Since the process is
exiting anyway and the region is owned and reclaimed by the parent (the
kernel reclaims all mappings on process exit), terminate immediately with
_exit() so that no destructors run.

Signed-off-by: Jiayuan Chen <jiayuan.chen@linux.dev>
---
 src/pb_stub.cc | 36 +++++++++++++++++-------------------
 1 file changed, 17 insertions(+), 19 deletions(-)

diff --git a/src/pb_stub.cc b/src/pb_stub.cc
index cd19f3ef..e966d313 100644
--- a/src/pb_stub.cc
+++ b/src/pb_stub.cc
@@ -61,6 +61,7 @@
 #include <windows.h>
 #else
 #include <sys/wait.h>
+#include <unistd.h>  // _exit
 #endif
 
 #ifdef TRITON_ENABLE_GPU
@@ -76,8 +77,6 @@ using cudaStream_t = void*;
 
 namespace triton { namespace backend { namespace python {
 
-std::atomic<bool> non_graceful_exit = {false};
-
 void
 SignalHandler(int signum)
 {
@@ -2162,7 +2161,7 @@ main(int argc, char** argv)
 #endif
   std::atomic<bool> background_thread_running = {true};
   std::thread background_thread =
-      std::thread([stub, &parent_pid, &background_thread_running, &logger] {
+      std::thread([stub, &parent_pid, &background_thread_running] {
         // Send a dummy message after the stub process is launched to notify the
         // parent process that the health thread has started.
         std::unique_ptr<IPCMessage> ipc_message = IPCMessage::Create(
@@ -2179,23 +2178,22 @@ main(int argc, char** argv)
           stub->UpdateHealth();
 
           if (!ParentProcessActive(parent_pid)) {
-            // When unhealthy, we should stop attempting to send
-            // messages to the backend ASAP.
-            if (stub->StubToParentServiceActive()) {
-              stub->TerminateStubToParentQueueMonitor();
-            }
-            if (stub->ParentToStubServiceActive()) {
-              stub->TerminateParentToStubQueueMonitor();
-            }
-            // Destroy Stub
+            // The parent process (tritonserver) died abruptly (e.g. SIGKILL
+            // after the termination grace period, an OOM-kill, or a crash) and
+            // could not send a Finalize command. The RunCommand loop and the
+            // model instance threads are still operating on the shared memory
+            // region and its process-shared mutexes. Running the normal
+            // teardown from this background thread is unsafe: DestroyInstance()
+            // unmaps the region (~SharedMemoryManager -> ~mapped_region ->
+            // munmap), and exit() would unmap it as well via static
+            // destructors. Doing so while those threads are still using the
+            // region is a data race that faults them with SIGSEGV inside the
+            // shm mutex (SEGV_MAPERR on the lock word). Since the process is
+            // going away and the region is owned and reclaimed by the parent
+            // (the kernel reclaims all mappings on process exit), terminate
+            // immediately with _exit() so that no destructors run.
             LOG_INFO << "Non-graceful termination detected. ";
-            background_thread_running = false;
-            non_graceful_exit = true;
-
-            // Destroy stub and exit.
-            logger.reset();
-            Stub::DestroyInstance();
-            exit(1);
+            _exit(1);
           }
         }
       });