From ac09800163dd595efaa2577668f198c3c868f007 Mon Sep 17 00:00:00 2001 From: Sicong Liu Date: Mon, 27 Jul 2026 11:15:26 -0700 Subject: [PATCH 1/3] Build Linux extensions with GCC 13 within the RHEL 9 ABI floor The Linux extensions are built on Ubuntu 24.04 with GCC 13 and must load on RHEL 9, which ships glibc 2.34 and libstdc++ 3.4.29. Rather than change where they are built, this makes the produced binaries meet that floor. GCC 13 emits a reference to std::ios_base_library_init()@GLIBCXX_3.4.32 into every translation unit that includes , and that single symbol was the only libstdc++ requirement above the floor. The shipped R sources therefore no longer include it: the includes in RExtension.cpp and RTypeUtils.cpp were unused, and Logger.cpp writes through stdio. The Rcpp and RInside headers also include and cannot be changed here, so the symbol is additionally defined locally. R must not link libstdc++ statically: libR.so already links libstdc++.so.6, and a second static copy puts two C++ runtimes in one process, which aborts partway through script execution. language-extensions/common/linux/AbiFloorCompat_linux.cpp collects the local definitions needed to stay within the floor: std::ios_base_library_init 3.4.32 , via Rcpp/RInside _dl_find_object 2.35 libgcc.a unwinder arc4random 2.36 libstdc++.a __isoc23_strtoul 2.38 libstdc++.a __isoc23_strtoll/strtoull 2.38 C23 redirect arc4random is backed by getrandom(2) with a /dev/urandom fallback and aborts rather than returning predictable bytes. _dl_find_object reports "no information", which makes the libgcc unwinder use its dl_iterate_phdr path - the same path taken on every glibc older than 2.35. All definitions are hidden, so the extensions never interpose on glibc for the rest of the host process. Also in this change: - restore-packages.sh asserts the GCC major version so the toolchain cannot silently vary between languages. - The Java extension links libstdc++/libgcc statically (it does not load libR, so there is no duplicate-runtime hazard) and uses the modern JDK lib/server layout instead of the Java 8 jre/lib/amd64/server path. - Test fixes: a dangling c_str() in the R result-column tests, and UTF-16 literals and raw byte comparisons in the Python output-parameter tests, which were relying on std::wstring under -fshort-wchar. --- language-extensions/R/src/CMakeLists.txt | 37 +++- language-extensions/R/src/Logger.cpp | 7 +- language-extensions/R/src/RExtension.cpp | 1 - language-extensions/R/src/RTypeUtils.cpp | 1 - .../src/RExtensionGetResultColumnTests.cpp | 3 +- .../common/linux/AbiFloorCompat_linux.cpp | 182 ++++++++++++++++++ language-extensions/java/src/CMakeLists.txt | 24 ++- language-extensions/python/src/CMakeLists.txt | 26 ++- .../python/src/PythonLibrarySession.cpp | 18 ++ .../test/include/PythonExtensionApiTests.h | 4 +- .../test/src/PythonGetOutputParamTests.cpp | 52 +++-- restore-packages.sh | 19 +- 12 files changed, 315 insertions(+), 59 deletions(-) create mode 100644 language-extensions/common/linux/AbiFloorCompat_linux.cpp diff --git a/language-extensions/R/src/CMakeLists.txt b/language-extensions/R/src/CMakeLists.txt index 32f511c1..6381ba0e 100644 --- a/language-extensions/R/src/CMakeLists.txt +++ b/language-extensions/R/src/CMakeLists.txt @@ -31,12 +31,27 @@ file(GLOB REXTENSION_SOURCE_FILES ${REXTENSION_HOME}/common/src/*.cpp ${REXTENSION_SRC_DIR}/${PLATFORM}/*.cpp) +# RHEL 9 ABI floor shims. Required because libstdc++.a and libgcc.a from the +# Ubuntu 24.04 toolchain reference glibc symbols newer than 2.34. See +# language-extensions/common/linux/AbiFloorCompat_linux.cpp. +# +if (${PLATFORM} STREQUAL linux) + list(APPEND REXTENSION_SOURCE_FILES ${ENL_ROOT}/language-extensions/common/linux/AbiFloorCompat_linux.cpp) +endif() + # Create the target library # add_library(RExtension SHARED ${REXTENSION_SOURCE_FILES} ) +# R links libstdc++ dynamically, so RHEL9's 3.4.29 copy will not provide +# std::ios_base_library_init(). Ask the shared compat unit to define it. +# +if (${PLATFORM} STREQUAL linux) + target_compile_definitions(RExtension PRIVATE ABI_FLOOR_PROVIDE_IOS_BASE_LIBRARY_INIT) +endif() + set_target_properties(RExtension PROPERTIES VERSION ${REXTENSION_VERSION} @@ -47,17 +62,29 @@ if (${PLATFORM} STREQUAL linux) target_compile_options(RExtension PRIVATE -Wall -Wextra -g -fPIC -Werror -std=c++17 -Wno-unused-parameter -Wno-cast-function-type -Wno-format-overflow -Wno-deprecated-copy -Wno-sign-conversion -Wno-conversion -Wno-maybe-uninitialized) + # RHEL9 ABI floor (GLIBCXX <= 3.4.29): GCC 13 emits a reference to + # std::ios_base_library_init()@GLIBCXX_3.4.32 into any translation unit that + # includes . The RExtension sources no longer include it directly, + # but the Rcpp and RInside headers do, so the reference cannot be removed at + # the source level. + # + # libstdc++ is deliberately NOT linked statically here. libR.so already links + # libstdc++.so.6, so a static copy would put two C++ runtimes in one process + # and the R extension aborts (SIGABRT) part-way through a script execution. + # -Wl,-z,nodelete does not help: the fault is duplication, not unload order. + # The single offending symbol is instead defined locally in + # language-extensions/common/linux/AbiFloorCompat_linux.cpp. set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--as-needed \ -Wl,--export-dynamic -fopenmp -Wl,-rpath,'${ORIGIN}'") set(CMAKE_FIND_LIBRARY_SUFFIXES "${CMAKE_FIND_LIBRARY_SUFFIXES};.so.1;.so.3;.so.5") - set(USR_LIB_PATH "/usr/lib/x86_64-linux-gnu") + set(USR_LIB_PATH "/usr/lib64") # Find dependent libraries with fallback paths # - find_library(PCRE_LIB NAMES pcre libpcre PATHS ${USR_LIB_PATH} /usr/lib /lib NO_DEFAULT_PATH) - find_library(LZMA_LIB NAMES lzma liblzma PATHS ${USR_LIB_PATH} /usr/lib /lib NO_DEFAULT_PATH) - find_library(BZ2_LIB NAMES bz2 libbz2 PATHS ${USR_LIB_PATH} /usr/lib /lib NO_DEFAULT_PATH) - find_library(Z_LIB NAMES z libz PATHS ${USR_LIB_PATH} /usr/lib /lib NO_DEFAULT_PATH) + find_library(PCRE_LIB NAMES pcre libpcre PATHS ${USR_LIB_PATH} /usr/lib /lib /usr/lib/x86_64-linux-gnu NO_DEFAULT_PATH) + find_library(LZMA_LIB NAMES lzma liblzma PATHS ${USR_LIB_PATH} /usr/lib /lib /usr/lib/x86_64-linux-gnu NO_DEFAULT_PATH) + find_library(BZ2_LIB NAMES bz2 libbz2 PATHS ${USR_LIB_PATH} /usr/lib /lib /usr/lib/x86_64-linux-gnu NO_DEFAULT_PATH) + find_library(Z_LIB NAMES z libz PATHS ${USR_LIB_PATH} /usr/lib /lib /usr/lib/x86_64-linux-gnu NO_DEFAULT_PATH) find_library(RT_LIB rt ${USR_LIB_PATH}) find_library(DL_LIB dl ${USR_LIB_PATH}) find_library(M_LIB m ${USR_LIB_PATH}) diff --git a/language-extensions/R/src/Logger.cpp b/language-extensions/R/src/Logger.cpp index 8a36d2b7..b3abf799 100644 --- a/language-extensions/R/src/Logger.cpp +++ b/language-extensions/R/src/Logger.cpp @@ -27,7 +27,6 @@ //************************************************************************************************** #include -#include #include #include "Common.h" @@ -75,7 +74,7 @@ void Logger::Log(const string &msg) { #if defined(_DEBUG) || defined(_VERBOSE) string msgWithTimestamp = string(GetCurrentTimestamp()) + msg + "\n"; - cout << msgWithTimestamp; + fputs(msgWithTimestamp.c_str(), stdout); #endif } @@ -136,9 +135,9 @@ const char* Logger::GetCurrentTimestamp() // // Description: // Logs the given message to stderr; if R is initialized uses its error printing function, -// else uses std::cerr. +// else writes directly to stderr. // void Logger::LogToStdErr(const string &errorMsgWithTimestamp) { - cerr << errorMsgWithTimestamp; + fputs(errorMsgWithTimestamp.c_str(), stderr); } diff --git a/language-extensions/R/src/RExtension.cpp b/language-extensions/R/src/RExtension.cpp index 36e02c87..d97c81d7 100644 --- a/language-extensions/R/src/RExtension.cpp +++ b/language-extensions/R/src/RExtension.cpp @@ -29,7 +29,6 @@ #include "Common.h" #include -#include #include #include diff --git a/language-extensions/R/src/RTypeUtils.cpp b/language-extensions/R/src/RTypeUtils.cpp index ceaba9e7..58b295ea 100644 --- a/language-extensions/R/src/RTypeUtils.cpp +++ b/language-extensions/R/src/RTypeUtils.cpp @@ -28,7 +28,6 @@ #include "Common.h" #include #include -#include #include "RTypeUtils.h" #include "Unicode.h" diff --git a/language-extensions/R/test/src/RExtensionGetResultColumnTests.cpp b/language-extensions/R/test/src/RExtensionGetResultColumnTests.cpp index eec0d263..3f6f3c47 100644 --- a/language-extensions/R/test/src/RExtensionGetResultColumnTests.cpp +++ b/language-extensions/R/test/src/RExtensionGetResultColumnTests.cpp @@ -475,8 +475,9 @@ namespace ExtensionApiTest // vector chineseBytes = { -28, -67, -96, -27, -91, -67 }; string chineseString = string(chineseBytes.data(), 6); + string worldChineseString = "World" + chineseString; - vector charCol1{ "Hello", "test", "data", ("World" + chineseString).c_str(), + vector charCol1{ "Hello", "test", "data", worldChineseString.c_str(), chineseString.c_str() }; vector charCol2{ "", 0, nullptr, "verify", "-1" }; diff --git a/language-extensions/common/linux/AbiFloorCompat_linux.cpp b/language-extensions/common/linux/AbiFloorCompat_linux.cpp new file mode 100644 index 00000000..650e3ce0 --- /dev/null +++ b/language-extensions/common/linux/AbiFloorCompat_linux.cpp @@ -0,0 +1,182 @@ +//************************************************************************************************** +// @File: AbiFloorCompat_linux.cpp +// +// Purpose: +// RHEL 9 ABI floor compatibility shims for Linux language extensions. +// +// The extensions are built on Ubuntu 24.04 (glibc 2.38, GCC 13) but must load +// on RHEL 9 (glibc 2.34). A small number of symbols above that floor get pulled +// in by the toolchain rather than by our own code, so they cannot be removed at +// the call site: +// +// __isoc23_strtoll @GLIBC_2.38 glibc 2.38 redirects the strtol +// __isoc23_strtoull @GLIBC_2.38 family to C23 variants; the references come +// __isoc23_strtoul @GLIBC_2.38 from vendored headers and from libstdc++.a +// arc4random @GLIBC_2.36 used by libstdc++.a +// _dl_find_object @GLIBC_2.35 used by the libgcc.a unwinder +// +// Each shim is defined under a distinct C++ name carrying an __asm__ label, so +// the emitted symbol is the glibc name. Defining a function literally named +// __isoc23_strtoll does NOT work: has already declared strtoll with +// that same assembler name, and the resulting conflict leaves the reference +// unresolved (observed as "UND __isoc23_strtoll@GLIBC_2.38" in .dynsym). +// +// Every shim is hidden. They satisfy references inside this shared object only +// and are not exported, so this library never interposes on glibc for the rest +// of the host process. +// +// Enforced by build/linux/validate-native-elf.sh (MAX GLIBC <= 2.34). +//************************************************************************************************** + +#if defined(__linux__) + +// (pulled in by any libc header) must be included before testing +// __GLIBC__: the macro is defined by glibc's headers, not by the compiler. +// Guarding on __GLIBC__ before any include silently compiles this whole file to +// nothing, which is exactly how an earlier revision of these shims ended up +// linked but inert. +#include +#include +#include +#include +#include +#include +#include + +#if defined(__GLIBC__) + +#define ABI_FLOOR_HIDDEN __attribute__((visibility("hidden"))) + +extern "C" +{ + // Bind to the classic, always-present glibc entry points. The explicit + // assembler labels bypass the C23 redirect, so these cannot + // recurse back into the shims below. + long long int AbiFloorClassicStrtoll(const char *nptr, char **endptr, int base) __asm__("strtoll"); + unsigned long long int AbiFloorClassicStrtoull(const char *nptr, char **endptr, int base) __asm__("strtoull"); + unsigned long int AbiFloorClassicStrtoul(const char *nptr, char **endptr, int base) __asm__("strtoul"); + + ABI_FLOOR_HIDDEN long long int AbiFloorIsoc23Strtoll(const char *nptr, char **endptr, int base) __asm__("__isoc23_strtoll"); + ABI_FLOOR_HIDDEN unsigned long long int AbiFloorIsoc23Strtoull(const char *nptr, char **endptr, int base) __asm__("__isoc23_strtoull"); + ABI_FLOOR_HIDDEN unsigned long int AbiFloorIsoc23Strtoul(const char *nptr, char **endptr, int base) __asm__("__isoc23_strtoul"); + ABI_FLOOR_HIDDEN uint32_t AbiFloorArc4Random(void) __asm__("arc4random"); + ABI_FLOOR_HIDDEN int AbiFloorDlFindObject(void *address, void *result) __asm__("_dl_find_object"); + + // The only behavioural difference between the C23 and classic strtol forms + // is that the C23 forms also accept a "0b" binary prefix when base is 0 or + // 2. No caller in these extensions parses binary literals. + ABI_FLOOR_HIDDEN long long int AbiFloorIsoc23Strtoll(const char *nptr, char **endptr, int base) + { + return AbiFloorClassicStrtoll(nptr, endptr, base); + } + + ABI_FLOOR_HIDDEN unsigned long long int AbiFloorIsoc23Strtoull(const char *nptr, char **endptr, int base) + { + return AbiFloorClassicStrtoull(nptr, endptr, base); + } + + ABI_FLOOR_HIDDEN unsigned long int AbiFloorIsoc23Strtoul(const char *nptr, char **endptr, int base) + { + return AbiFloorClassicStrtoul(nptr, endptr, base); + } + + // arc4random is a cryptographically secure generator, so this must not + // degrade to a weak source. getrandom(2) is the kernel CSPRNG and has been + // available since glibc 2.25, well under the RHEL 9 floor. /dev/urandom is + // the fallback for seccomp-restricted environments. If neither can produce + // bytes we abort rather than return predictable output. + ABI_FLOOR_HIDDEN uint32_t AbiFloorArc4Random(void) + { + uint32_t value = 0; + unsigned char *out = reinterpret_cast(&value); + size_t filled = 0; + + while (filled < sizeof(value)) + { + ssize_t received = getrandom(out + filled, sizeof(value) - filled, 0); + if (received < 0) + { + if (errno == EINTR) + { + continue; + } + break; + } + filled += static_cast(received); + } + + if (filled < sizeof(value)) + { + int fd = open("/dev/urandom", O_RDONLY | O_CLOEXEC); + if (fd >= 0) + { + while (filled < sizeof(value)) + { + ssize_t received = read(fd, out + filled, sizeof(value) - filled); + if (received <= 0) + { + if (received < 0 && errno == EINTR) + { + continue; + } + break; + } + filled += static_cast(received); + } + close(fd); + } + } + + if (filled < sizeof(value)) + { + abort(); + } + + return value; + } + + // _dl_find_object is the glibc 2.35+ fast path the libgcc unwinder uses to + // locate exception-handling frames. A non-zero return means "no information + // available", which makes libgcc fall back to its dl_iterate_phdr path. That + // fallback is the same code path used on every glibc older than 2.35, so + // unwinding stays correct - only marginally slower on the throw path. + ABI_FLOOR_HIDDEN int AbiFloorDlFindObject(void *address, void *result) + { + (void)address; + (void)result; + return -1; + } + + // std::ios_base_library_init() is GLIBCXX_3.4.32, i.e. GCC 13's libstdc++. + // GCC 13 emits a reference to it from every translation unit that includes + // , and RHEL9's libstdc++ 3.4.29 does not provide it. + // + // This is opt-in via ABI_FLOOR_PROVIDE_IOS_BASE_LIBRARY_INIT because it must + // only be defined when libstdc++ is linked *dynamically*. libstdc++.a already + // contains this symbol in ios_init.o, so defining it alongside a static + // libstdc++ fails the link with "multiple definition of + // std::ios_base_library_init()". + // + // The R extension is the case that needs it: it cannot link libstdc++ + // statically (libR.so already links libstdc++.so.6, and two C++ runtimes in + // one process abort at runtime), yet the include is unavoidable because it + // arrives through the Rcpp and RInside headers. + // + // Defining it as a no-op is safe here: the symbol only drives iostream + // initialisation ordering, the standard streams are still initialised by + // libstdc++'s own machinery, and none of the shipped sources use + // std::cout/std::cerr any more (they log through stdio). The definition is + // hidden, so it satisfies our own translation units without changing + // iostream initialisation for libR or anything else in the host process. +#ifdef ABI_FLOOR_PROVIDE_IOS_BASE_LIBRARY_INIT + ABI_FLOOR_HIDDEN void AbiFloorIosBaseLibraryInit(void) __asm__("_ZSt21ios_base_library_initv"); + ABI_FLOOR_HIDDEN void AbiFloorIosBaseLibraryInit(void) + { + } +#endif +} + +#else +#error "AbiFloorCompat_linux.cpp is compiled for Linux but __GLIBC__ is not defined; the shims would be silently empty." +#endif // __GLIBC__ +#endif // __linux__ diff --git a/language-extensions/java/src/CMakeLists.txt b/language-extensions/java/src/CMakeLists.txt index 928cc9be..d62fc749 100644 --- a/language-extensions/java/src/CMakeLists.txt +++ b/language-extensions/java/src/CMakeLists.txt @@ -32,6 +32,14 @@ file(TO_CMAKE_PATH ${ENL_ROOT}/extension-host/include EXTENSION_API_INCLUDE_DIR) # file(GLOB JAVAEXTENSION_SOURCE_FILES ${JAVAEXTENSION_SRC_DIR}/*.cpp ${JAVAEXTENSION_SRC_DIR}/${PLATFORM}/*.cpp) +# RHEL 9 ABI floor shims. Required because libstdc++.a and libgcc.a from the +# Ubuntu 24.04 toolchain reference glibc symbols newer than 2.34. See +# language-extensions/common/linux/AbiFloorCompat_linux.cpp. +# +if (${PLATFORM} STREQUAL linux) + list(APPEND JAVAEXTENSION_SOURCE_FILES ${ENL_ROOT}/language-extensions/common/linux/AbiFloorCompat_linux.cpp) +endif() + # Create the target library # add_library(JavaExtension SHARED @@ -45,7 +53,11 @@ set_target_properties(JavaExtension ) if (${PLATFORM} STREQUAL linux) - file(TO_CMAKE_PATH ${JAVA_HOME}/jre/lib/amd64/server JRE_ROOT) + if(EXISTS "${JAVA_HOME}/lib/server") + file(TO_CMAKE_PATH ${JAVA_HOME}/lib/server JRE_ROOT) + else() + file(TO_CMAKE_PATH ${JAVA_HOME}/jre/lib/amd64/server JRE_ROOT) + endif() file(TO_CMAKE_PATH ${JAVA_HOME}/include JDK_INCLUDE) file(TO_CMAKE_PATH ${JDK_INCLUDE}/linux JDK_INCLUDE_LINUX) @@ -63,14 +75,8 @@ if (${PLATFORM} STREQUAL linux) target_compile_options(JavaExtension PRIVATE ${CXXFLAGS}) - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -shared -L${OUTPUT_DIRECTORY}/lib -L${LIBS} -Wl,-rpath,${JRE_ROOT}:/opt/mssql-extensibility/lib") - set(USR_LIB_PATH "/usr/local/lib") - - find_library(CXX_LIB c++ ${USR_LIB_PATH}) - find_library(ABI_LIB c++abi ${USR_LIB_PATH}) - find_library(DL_LIB dl ${USR_LIB_PATH}) - - target_link_libraries(JavaExtension ${CXX_LIB} ${ABI_LIB} ${DL_LIB}) + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -shared -static-libstdc++ -static-libgcc -Wl,--exclude-libs,ALL -Wl,-rpath,${JRE_ROOT}:/opt/mssql-extensibility/lib") + target_link_libraries(JavaExtension dl) set(ADDITIONAL_INCLUDES ${JDK_INCLUDE} ${JDK_INCLUDE_LINUX}) elseif(${PLATFORM} STREQUAL windows) diff --git a/language-extensions/python/src/CMakeLists.txt b/language-extensions/python/src/CMakeLists.txt index fd9112c4..a62ff8e0 100644 --- a/language-extensions/python/src/CMakeLists.txt +++ b/language-extensions/python/src/CMakeLists.txt @@ -33,6 +33,14 @@ file(TO_CMAKE_PATH ${ENL_ROOT}/build-output/pythonextension/${PLATFORM} PYTHONEX # file(GLOB PYTHONEXTENSION_SOURCE_FILES ${PYTHONEXTENSION_SRC_DIR}/*.cpp ${PYTHONEXTENSION_SRC_DIR}/${PLATFORM}/*.cpp) +# RHEL 9 ABI floor shims. Required because libstdc++.a and libgcc.a from the +# Ubuntu 24.04 toolchain reference glibc symbols newer than 2.34. See +# language-extensions/common/linux/AbiFloorCompat_linux.cpp. +# +if (${PLATFORM} STREQUAL linux) + list(APPEND PYTHONEXTENSION_SOURCE_FILES ${ENL_ROOT}/language-extensions/common/linux/AbiFloorCompat_linux.cpp) +endif() + # Create the target library # add_library(PythonExtension SHARED @@ -46,6 +54,12 @@ set_target_properties(PythonExtension ) if (${PLATFORM} STREQUAL linux) + # Python links libstdc++ dynamically, so RHEL 9's 3.4.29 copy will not + # provide std::ios_base_library_init(). Ask the shared compat unit to + # define it. + # + target_compile_definitions(PythonExtension PRIVATE ABI_FLOOR_PROVIDE_IOS_BASE_LIBRARY_INIT) + # Python runtime version used in the linux environment differs from the one used in Windows at this time. # This declaration should no longer be needed once the Windows version is updated to 3.12. # @@ -54,7 +68,17 @@ if (${PLATFORM} STREQUAL linux) target_compile_options(PythonExtension PRIVATE -Wall -Wextra -g -O2 -fPIC -Werror -std=c++17 -Wno-unused-parameter -Wno-maybe-uninitialized -fshort-wchar) - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--as-needed -Wl,--export-dynamic -fopenmp") + # The shipped extension must not depend on Boost at runtime: RHEL 9 has no + # libboost_python312, so Boost is built link=static and absorbed here. + # + # libstdc++ is deliberately NOT linked statically. The embedded interpreter + # imports numpy and pandas, whose C extensions link libstdc++.so.6, so a + # static copy puts two C++ runtimes in one process and the test binary + # aborts (SIGABRT) during GetOutputParam. RHEL 9 ships libstdc++ 3.4.29, so + # depending on it is fine as long as nothing above that version is + # referenced - the one offending symbol is defined locally in + # language-extensions/common/linux/AbiFloorCompat_linux.cpp. + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--as-needed -Wl,--export-dynamic -fopenmp") # Prefer shared libraries over static libraries set(CMAKE_FIND_LIBRARY_SUFFIXES .so .so.1.0 ${CMAKE_FIND_LIBRARY_SUFFIXES}) diff --git a/language-extensions/python/src/PythonLibrarySession.cpp b/language-extensions/python/src/PythonLibrarySession.cpp index 2d8f0275..225c0fcc 100644 --- a/language-extensions/python/src/PythonLibrarySession.cpp +++ b/language-extensions/python/src/PythonLibrarySession.cpp @@ -105,6 +105,24 @@ SQLRETURN PythonLibrarySession::InstallLibrary( "external library must be a python package inside a zip."); } + // Older package fixtures and customer packages created on Windows can contain backslashes in + // ZIP entry names. Modern pip treats those as literal characters on Linux, so setup.py is found + // but its package directory is not. Normalize the inner package ZIP before passing it to pip. + // + if (fs::path(installPath).extension().generic_string() == ".zip") + { + string normalizedInstallPath = + (fs::path(tempFolder) / ("normalized-" + fs::path(installPath).filename().string())).generic_string(); + string normalizeScript = "import zipfile\n" + "with zipfile.ZipFile('" + installPath + "', 'r') as source_zip:\n" + " with zipfile.ZipFile('" + normalizedInstallPath + "', 'w') as normalized_zip:\n" + " for entry in source_zip.infolist():\n" + " entry.filename = entry.filename.replace('\\\\', '/')\n" + " normalized_zip.writestr(entry, source_zip.read(entry))"; + bp::exec(normalizeScript.c_str(), m_mainNamespace); + installPath = normalizedInstallPath; + } + string pathToPython = PythonExtensionUtils::GetPathToPython(); // Set the TMPDIR so that pip uses our destination as temp. This allows us to use a diff --git a/language-extensions/python/test/include/PythonExtensionApiTests.h b/language-extensions/python/test/include/PythonExtensionApiTests.h index d1083a80..8137172e 100644 --- a/language-extensions/python/test/include/PythonExtensionApiTests.h +++ b/language-extensions/python/test/include/PythonExtensionApiTests.h @@ -329,8 +329,8 @@ namespace ExtensionApiTest // Test output string param value and strLenOrInd is as expected. // void TestGetWStringOutputParam( - std::vector expectedParamValueVector, - std::vector expectedStrLenOrIndVector); + std::vector expectedParamValueVector, + std::vector expectedStrLenOrIndVector); // Test output raw param value and strLenOrInd is as expected. // diff --git a/language-extensions/python/test/src/PythonGetOutputParamTests.cpp b/language-extensions/python/test/src/PythonGetOutputParamTests.cpp index 9a58f2e4..b6190092 100644 --- a/language-extensions/python/test/src/PythonGetOutputParamTests.cpp +++ b/language-extensions/python/test/src/PythonGetOutputParamTests.cpp @@ -643,38 +643,38 @@ namespace ExtensionApiTest ASSERT_EQ(result, SQL_SUCCESS); EXPECT_EQ(outputSchemaColumnsNumber, 0); - wstring wideStrOfValue7 = wstring(value7.begin(), value7.end()); - const wchar_t* wideCStrOfValue7 = wideStrOfValue7.c_str(); - vector expectedParamValues = { + u16string wideStrOfValue7(value7.begin(), value7.end()); + const char16_t* wideCStrOfValue7 = wideStrOfValue7.c_str(); + vector expectedParamValues = { // Test simple CHAR(5) value with exact string length as the type allows i.e. here 5. // - L"HELLO", + u"HELLO", // Test VARCHAR(6) value with string length more than the type allows - expected truncation. // Above python script sets the parameter to "PyExtension" but we only expect "PyExte". // - L"PyExte", + u"PyExte", // Test a 0 length string // - L"" , + u"", // Empty value. // Test CHAR(10) value with string length less than the type allows. // - L"WORLD", + u"WORLD", // Test a Unicode string // - L"你好", + u"你好", nullptr, nullptr, wideCStrOfValue7}; vector expectedStrLenOrInd = { - static_cast(5 * sizeof(wchar_t)), - static_cast(6 * sizeof(wchar_t)), - static_cast(0 * sizeof(wchar_t)), - static_cast(5 * sizeof(wchar_t)), - static_cast(2 * sizeof(wchar_t)), + static_cast(5 * sizeof(char16_t)), + static_cast(6 * sizeof(char16_t)), + static_cast(0 * sizeof(char16_t)), + static_cast(5 * sizeof(char16_t)), + static_cast(2 * sizeof(char16_t)), SQL_NULL_DATA, SQL_NULL_DATA, - static_cast(128000 * sizeof(wchar_t)), + static_cast(128000 * sizeof(char16_t)), }; @@ -1102,8 +1102,8 @@ namespace ExtensionApiTest // Test wstring output param value and strLenOrInd is as expected. // void PythonExtensionApiTests::TestGetWStringOutputParam( - vector expectedParamValues, - vector expectedStrLenOrInd) + vector expectedParamValues, + vector expectedStrLenOrInd) { ASSERT_EQ(expectedParamValues.size(), expectedStrLenOrInd.size()); @@ -1127,20 +1127,16 @@ namespace ExtensionApiTest { EXPECT_NE(paramValue, nullptr); - wstring paramValueString(static_cast(paramValue), - strLen_or_Ind / sizeof(wchar_t)); - wstring expectedParamValueString(expectedParamValues[paramNumber], - expectedStrLenOrInd[paramNumber] / sizeof(wchar_t)); + // SQLWCHAR data is UTF-16. Compare the returned buffer directly instead of + // constructing std::wstring under -fshort-wchar; libstdc++ itself is built + // for the platform's native 4-byte wchar_t ABI. + const char *paramBytes = reinterpret_cast(paramValue); + const char *expectedParamBytes = + reinterpret_cast(expectedParamValues[paramNumber]); - // Compare the two wstrings byte by byte because EXPECT_STREQ and EXPECT_EQ - // don't work properly for wstrings in Linux with -fshort-wchar - // - const char *paramBytes = reinterpret_cast(paramValueString.c_str()); - const char *expectedParamBytes = reinterpret_cast(expectedParamValueString.c_str()); - - for(SQLINTEGER i=0; i Date: Tue, 28 Jul 2026 13:39:44 -0700 Subject: [PATCH 2/3] Actually initialise the standard streams in the ios_base_library_init shim The ABI-floor shim defined std::ios_base_library_init() as an empty function. That was wrong, and it crashed libPythonExtension on Ubuntu 22.04 and RHEL 9. GCC 13 changed how the standard streams are constructed. Before 13, every translation unit including emitted its own static std::ios_base::Init object. GCC 13 emits a call to std::ios_base_library_init() instead and performs the construction inside libstdc++. Stubbing that call out therefore means the streams are never constructed at all, and the first write to std::cout or std::cerr faults. It only shows up on hosts whose libstdc++ predates the change: Ubuntu 22.04 libstdc++.so.6.0.30 no self-init -> crash RHEL 9 GCC 11/12 era no self-init -> crash Ubuntu 24.04 GCC 13+ library self-initialises -> fine RHEL 10 GCC 13+ library self-initialises -> fine AzureLinux 3 GCC 13+ library self-initialises -> fine which matches the PVS board exactly: the Python satellite core-dumped on Ubuntu 22.04 and RHEL 9 immediately after receiving its first data chunk, while all three newer distros passed. libPythonExtension.so does reference _ZSt4cout@GLIBCXX_3.4 and _ZSt4cerr@GLIBCXX_3.4, so the comment claiming no shipped source used the standard streams was simply incorrect. R was unaffected because libR.so is built by the distro toolchain and its own translation units still emit std::ios_base::Init; libpython3.12 is C and initialises nothing, so the Python extension had no other initialiser. Construct a function-local static std::ios_base::Init instead. The constructor is refcounted and idempotent, the local static guard runs it exactly once no matter how many translation units call in, and std::ios_base::Init::Init is GLIBCXX_3.4 - present in every libstdc++ we target - so the ABI floor is unchanged. --- .../common/linux/AbiFloorCompat_linux.cpp | 39 ++++++++++++++----- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/language-extensions/common/linux/AbiFloorCompat_linux.cpp b/language-extensions/common/linux/AbiFloorCompat_linux.cpp index 650e3ce0..13fc8f35 100644 --- a/language-extensions/common/linux/AbiFloorCompat_linux.cpp +++ b/language-extensions/common/linux/AbiFloorCompat_linux.cpp @@ -45,6 +45,12 @@ #if defined(__GLIBC__) +#ifdef ABI_FLOOR_PROVIDE_IOS_BASE_LIBRARY_INIT +// For std::ios_base::Init, which is what actually constructs the standard +// streams. See AbiFloorIosBaseLibraryInit below. +#include +#endif + #define ABI_FLOOR_HIDDEN __attribute__((visibility("hidden"))) extern "C" @@ -157,21 +163,34 @@ extern "C" // libstdc++ fails the link with "multiple definition of // std::ios_base_library_init()". // - // The R extension is the case that needs it: it cannot link libstdc++ - // statically (libR.so already links libstdc++.so.6, and two C++ runtimes in - // one process abort at runtime), yet the include is unavoidable because it - // arrives through the Rcpp and RInside headers. + // The R and Python extensions are the cases that need it: neither can link + // libstdc++ statically (libR.so already links libstdc++.so.6, and numpy's and + // pandas' C extensions do too - two C++ runtimes in one process abort at + // runtime), yet the include is unavoidable because it arrives through the + // Rcpp/RInside and Boost.Python headers. + // + // This MUST NOT be a no-op. GCC 13 changed how the standard streams are + // constructed: before 13, every TU including emitted its own static + // std::ios_base::Init object; GCC 13 emits a call to this function instead and + // performs the construction inside libstdc++. So on a host whose libstdc++ is + // 3.4.32 or newer the library initialises the streams itself and a stub here + // looks harmless - but on Ubuntu 22.04 (libstdc++.so.6.0.30) and RHEL 9 nothing + // initialises them, std::cout/std::cerr stay unconstructed, and the first write + // crashes the satellite process. That is precisely how libPythonExtension died + // on Ubuntu 22.04 and RHEL 9 while passing on Ubuntu 24.04, RHEL 10 and + // AzureLinux 3, which ship newer libstdc++. // - // Defining it as a no-op is safe here: the symbol only drives iostream - // initialisation ordering, the standard streams are still initialised by - // libstdc++'s own machinery, and none of the shipped sources use - // std::cout/std::cerr any more (they log through stdio). The definition is - // hidden, so it satisfies our own translation units without changing - // iostream initialisation for libR or anything else in the host process. + // Constructing a function-local static std::ios_base::Init performs the pre-13 + // initialisation explicitly. Its constructor is refcounted and idempotent, the + // local static guard runs it exactly once however many translation units call + // in, and std::ios_base::Init::Init is GLIBCXX_3.4 - present in every libstdc++ + // we target, so this does not raise the ABI floor. #ifdef ABI_FLOOR_PROVIDE_IOS_BASE_LIBRARY_INIT ABI_FLOOR_HIDDEN void AbiFloorIosBaseLibraryInit(void) __asm__("_ZSt21ios_base_library_initv"); ABI_FLOOR_HIDDEN void AbiFloorIosBaseLibraryInit(void) { + static std::ios_base::Init iosBaseInit; + (void)iosBaseInit; } #endif } From ee0f33790064491d537f0fc52b9ba6e212e9ea10 Mon Sep 17 00:00:00 2001 From: Sicong Liu Date: Tue, 28 Jul 2026 22:11:16 -0700 Subject: [PATCH 3/3] Log through stdio in the Python extension instead of std::cout/std::cerr The Python satellite segfaulted on Ubuntu 22.04 and RHEL 9 while passing on Ubuntu 24.04, RHEL 10 and AzureLinux 3. Two core dumps place the fault inside the host's own libstdc++, not in our code: round 1 Ubuntu 22.04 libstdc++.so.6.0.30 SIGSEGV at +0xa1f8a round 2 RHEL 9 libstdc++.so.6.0.29 SIGSEGV at +0xa16ea Same function, two different builds of the library. The extension is compiled with GCC 13, so the stream insertion operators are inlined from the GCC 13 headers into this library and then execute against stream objects owned by an older runtime. Ubuntu 24.04, RHEL 10 and AzureLinux 3 ship a GCC 13-era libstdc++ and are unaffected. The satellite log pins it exactly: a passing session's first line after the handshake is the 'Init' trace emitted by Logger::Log, and on the failing distros the log stops immediately before it - the process dies on its very first log call. The symbol-version ABI gate cannot catch this. Every symbol referenced is within GLIBCXX_3.4.29 and the gate passes; the dependency is inlined code relying on newer libstdc++ internals, which no version check can see. Route the three remaining stream users through stdio, which has no such coupling: Logger::LogError/LogException/Log and the relay of the user script's captured stdout/stderr in PythonSession. EKM and the ONNX extension were fixed the same way earlier. R was never affected because it does not reference the standard streams, and Java is safe because it links libstdc++ statically. Also hardens the ZIP normalization added for external libraries, which had two real defects even though it is not the cause of this crash (an uncaught Python exception would raise SIGABRT, and both cores are SIGSEGV): entry data was read after mutating entry.filename, which makes ZipFile.open compare the directory name against the local header and raise BadZipFile on exactly the archives the code exists to fix; and bp::exec was unguarded, so any Python exception would escape into a process where that is fatal. It now reads before renaming, only rewrites archives that actually contain backslashes, and falls back to the original package on any failure. --- language-extensions/python/src/Logger.cpp | 22 ++++++++++-- .../python/src/PythonLibrarySession.cpp | 36 +++++++++++++++---- .../python/src/PythonSession.cpp | 11 ++++-- 3 files changed, 58 insertions(+), 11 deletions(-) diff --git a/language-extensions/python/src/Logger.cpp b/language-extensions/python/src/Logger.cpp index 324fc7cc..53a92f2a 100644 --- a/language-extensions/python/src/Logger.cpp +++ b/language-extensions/python/src/Logger.cpp @@ -13,6 +13,8 @@ #include "Logger.h" +#include + #define TIMESTAMP_LENGTH 35 using namespace std; @@ -24,9 +26,21 @@ char Logger::sm_timestampBuffer[TIMESTAMP_LENGTH] = { 0 }; // Description: // Log an error to stderr with format "TIMESTAMP Error: ". // +// Note: +// These write through stdio rather than std::cerr/std::cout on purpose. The extension is compiled +// with GCC 13 but is loaded into hosts whose libstdc++ is older - RHEL 9 ships +// libstdc++.so.6.0.29 and Ubuntu 22.04 ships 6.0.30 - and the stream insertion operators are +// inlined from the GCC 13 headers into this library, so they execute against stream objects owned +// by the older runtime. That combination segfaults inside libstdc++ on the very first log call, +// which is why the Python satellite core-dumped on those two distros while passing on Ubuntu +// 24.04, RHEL 10 and AzureLinux 3. The symbol-version ABI gate cannot catch this: every symbol we +// reference is within GLIBCXX_3.4.29, the problem is inlined code depending on newer internals. +// stdio has no such coupling. EKM and the ONNX extension were fixed the same way. +// void Logger::LogError(const string &errorMsg) { - cerr << GetCurrentTimestamp() << "Error: " << errorMsg << endl; + fprintf(stderr, "%sError: %s\n", GetCurrentTimestamp().c_str(), errorMsg.c_str()); + fflush(stderr); } //------------------------------------------------------------------------------------------------- @@ -38,7 +52,8 @@ void Logger::LogError(const string &errorMsg) // void Logger::LogException(const exception &e) { - cerr << GetCurrentTimestamp() << "Exception occurred: " << e.what() << endl; + fprintf(stderr, "%sException occurred: %s\n", GetCurrentTimestamp().c_str(), e.what()); + fflush(stderr); } //------------------------------------------------------------------------------------------------- @@ -50,7 +65,8 @@ void Logger::LogException(const exception &e) void Logger::Log(const string &msg) { #if defined(_DEBUG) - cout << GetCurrentTimestamp() << msg << endl; + fprintf(stdout, "%s%s\n", GetCurrentTimestamp().c_str(), msg.c_str()); + fflush(stdout); #endif } diff --git a/language-extensions/python/src/PythonLibrarySession.cpp b/language-extensions/python/src/PythonLibrarySession.cpp index 225c0fcc..6be78177 100644 --- a/language-extensions/python/src/PythonLibrarySession.cpp +++ b/language-extensions/python/src/PythonLibrarySession.cpp @@ -109,18 +109,42 @@ SQLRETURN PythonLibrarySession::InstallLibrary( // ZIP entry names. Modern pip treats those as literal characters on Linux, so setup.py is found // but its package directory is not. Normalize the inner package ZIP before passing it to pip. // + // This is best effort on purpose. It runs while installing an external library inside the + // satellite process, where an uncaught exception terminates the process rather than failing the + // statement, so every failure path has to stay inside Python and report back through a flag: + // + // - the entry data must be read BEFORE rewriting entry.filename. ZipFile.open() compares the + // central-directory name against the local header and raises BadZipFile when they differ, so + // reading after the rewrite throws on exactly the archives this code exists to fix. + // - the archive is only rewritten when an entry actually contains a backslash, so well-formed + // packages are passed through untouched. + // - any failure (unreadable source, unwritable temp folder, malformed archive) leaves + // normalized False and the original path is used, which is what happened before this + // normalization existed. + // if (fs::path(installPath).extension().generic_string() == ".zip") { string normalizedInstallPath = (fs::path(tempFolder) / ("normalized-" + fs::path(installPath).filename().string())).generic_string(); string normalizeScript = "import zipfile\n" - "with zipfile.ZipFile('" + installPath + "', 'r') as source_zip:\n" - " with zipfile.ZipFile('" + normalizedInstallPath + "', 'w') as normalized_zip:\n" - " for entry in source_zip.infolist():\n" - " entry.filename = entry.filename.replace('\\\\', '/')\n" - " normalized_zip.writestr(entry, source_zip.read(entry))"; + "normalized = False\n" + "try:\n" + " with zipfile.ZipFile('" + installPath + "', 'r') as source_zip:\n" + " if any('\\\\' in n for n in source_zip.namelist()):\n" + " with zipfile.ZipFile('" + normalizedInstallPath + "', 'w') as normalized_zip:\n" + " for entry in source_zip.infolist():\n" + " data = source_zip.read(entry)\n" + " entry.filename = entry.filename.replace('\\\\', '/')\n" + " normalized_zip.writestr(entry, data)\n" + " normalized = True\n" + "except Exception:\n" + " normalized = False"; bp::exec(normalizeScript.c_str(), m_mainNamespace); - installPath = normalizedInstallPath; + + if (bp::extract(m_mainNamespace["normalized"])) + { + installPath = normalizedInstallPath; + } } string pathToPython = PythonExtensionUtils::GetPathToPython(); diff --git a/language-extensions/python/src/PythonSession.cpp b/language-extensions/python/src/PythonSession.cpp index 6cf9637f..b9e1496d 100644 --- a/language-extensions/python/src/PythonSession.cpp +++ b/language-extensions/python/src/PythonSession.cpp @@ -211,8 +211,15 @@ void PythonSession::ExecuteWorkflow( string pyStdOut = bp::extract(m_mainNamespace["_temp_out_"]); string pyStdErr = bp::extract(m_mainNamespace["_temp_err_"]); - cout << pyStdOut << endl; - cerr << pyStdErr << endl; + // Relayed through stdio rather than std::cout/std::cerr. The stream insertion operators are + // inlined from the GCC 13 headers but run against the host's older libstdc++ (RHEL 9 ships + // 6.0.29, Ubuntu 22.04 ships 6.0.30), which segfaults inside libstdc++. See the note in + // Logger.cpp. + // + fprintf(stdout, "%s\n", pyStdOut.c_str()); + fflush(stdout); + fprintf(stderr, "%s\n", pyStdErr.c_str()); + fflush(stderr); // In case of streaming clean up the previous stream batch's output buffers //