diff --git a/language-extensions/R/src/CMakeLists.txt b/language-extensions/R/src/CMakeLists.txt index 32f511c1..6381ba0e 100644 --- a/language-extensions/R/src/CMakeLists.txt +++ b/language-extensions/R/src/CMakeLists.txt @@ -31,12 +31,27 @@ file(GLOB REXTENSION_SOURCE_FILES ${REXTENSION_HOME}/common/src/*.cpp ${REXTENSION_SRC_DIR}/${PLATFORM}/*.cpp) +# RHEL 9 ABI floor shims. Required because libstdc++.a and libgcc.a from the +# Ubuntu 24.04 toolchain reference glibc symbols newer than 2.34. See +# language-extensions/common/linux/AbiFloorCompat_linux.cpp. +# +if (${PLATFORM} STREQUAL linux) + list(APPEND REXTENSION_SOURCE_FILES ${ENL_ROOT}/language-extensions/common/linux/AbiFloorCompat_linux.cpp) +endif() + # Create the target library # add_library(RExtension SHARED ${REXTENSION_SOURCE_FILES} ) +# R links libstdc++ dynamically, so RHEL9's 3.4.29 copy will not provide +# std::ios_base_library_init(). Ask the shared compat unit to define it. +# +if (${PLATFORM} STREQUAL linux) + target_compile_definitions(RExtension PRIVATE ABI_FLOOR_PROVIDE_IOS_BASE_LIBRARY_INIT) +endif() + set_target_properties(RExtension PROPERTIES VERSION ${REXTENSION_VERSION} @@ -47,17 +62,29 @@ if (${PLATFORM} STREQUAL linux) target_compile_options(RExtension PRIVATE -Wall -Wextra -g -fPIC -Werror -std=c++17 -Wno-unused-parameter -Wno-cast-function-type -Wno-format-overflow -Wno-deprecated-copy -Wno-sign-conversion -Wno-conversion -Wno-maybe-uninitialized) + # RHEL9 ABI floor (GLIBCXX <= 3.4.29): GCC 13 emits a reference to + # std::ios_base_library_init()@GLIBCXX_3.4.32 into any translation unit that + # includes . The RExtension sources no longer include it directly, + # but the Rcpp and RInside headers do, so the reference cannot be removed at + # the source level. + # + # libstdc++ is deliberately NOT linked statically here. libR.so already links + # libstdc++.so.6, so a static copy would put two C++ runtimes in one process + # and the R extension aborts (SIGABRT) part-way through a script execution. + # -Wl,-z,nodelete does not help: the fault is duplication, not unload order. + # The single offending symbol is instead defined locally in + # language-extensions/common/linux/AbiFloorCompat_linux.cpp. set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--as-needed \ -Wl,--export-dynamic -fopenmp -Wl,-rpath,'${ORIGIN}'") set(CMAKE_FIND_LIBRARY_SUFFIXES "${CMAKE_FIND_LIBRARY_SUFFIXES};.so.1;.so.3;.so.5") - set(USR_LIB_PATH "/usr/lib/x86_64-linux-gnu") + set(USR_LIB_PATH "/usr/lib64") # Find dependent libraries with fallback paths # - find_library(PCRE_LIB NAMES pcre libpcre PATHS ${USR_LIB_PATH} /usr/lib /lib NO_DEFAULT_PATH) - find_library(LZMA_LIB NAMES lzma liblzma PATHS ${USR_LIB_PATH} /usr/lib /lib NO_DEFAULT_PATH) - find_library(BZ2_LIB NAMES bz2 libbz2 PATHS ${USR_LIB_PATH} /usr/lib /lib NO_DEFAULT_PATH) - find_library(Z_LIB NAMES z libz PATHS ${USR_LIB_PATH} /usr/lib /lib NO_DEFAULT_PATH) + find_library(PCRE_LIB NAMES pcre libpcre PATHS ${USR_LIB_PATH} /usr/lib /lib /usr/lib/x86_64-linux-gnu NO_DEFAULT_PATH) + find_library(LZMA_LIB NAMES lzma liblzma PATHS ${USR_LIB_PATH} /usr/lib /lib /usr/lib/x86_64-linux-gnu NO_DEFAULT_PATH) + find_library(BZ2_LIB NAMES bz2 libbz2 PATHS ${USR_LIB_PATH} /usr/lib /lib /usr/lib/x86_64-linux-gnu NO_DEFAULT_PATH) + find_library(Z_LIB NAMES z libz PATHS ${USR_LIB_PATH} /usr/lib /lib /usr/lib/x86_64-linux-gnu NO_DEFAULT_PATH) find_library(RT_LIB rt ${USR_LIB_PATH}) find_library(DL_LIB dl ${USR_LIB_PATH}) find_library(M_LIB m ${USR_LIB_PATH}) diff --git a/language-extensions/R/src/Logger.cpp b/language-extensions/R/src/Logger.cpp index 8a36d2b7..b3abf799 100644 --- a/language-extensions/R/src/Logger.cpp +++ b/language-extensions/R/src/Logger.cpp @@ -27,7 +27,6 @@ //************************************************************************************************** #include -#include #include #include "Common.h" @@ -75,7 +74,7 @@ void Logger::Log(const string &msg) { #if defined(_DEBUG) || defined(_VERBOSE) string msgWithTimestamp = string(GetCurrentTimestamp()) + msg + "\n"; - cout << msgWithTimestamp; + fputs(msgWithTimestamp.c_str(), stdout); #endif } @@ -136,9 +135,9 @@ const char* Logger::GetCurrentTimestamp() // // Description: // Logs the given message to stderr; if R is initialized uses its error printing function, -// else uses std::cerr. +// else writes directly to stderr. // void Logger::LogToStdErr(const string &errorMsgWithTimestamp) { - cerr << errorMsgWithTimestamp; + fputs(errorMsgWithTimestamp.c_str(), stderr); } diff --git a/language-extensions/R/src/RExtension.cpp b/language-extensions/R/src/RExtension.cpp index 36e02c87..d97c81d7 100644 --- a/language-extensions/R/src/RExtension.cpp +++ b/language-extensions/R/src/RExtension.cpp @@ -29,7 +29,6 @@ #include "Common.h" #include -#include #include #include diff --git a/language-extensions/R/src/RTypeUtils.cpp b/language-extensions/R/src/RTypeUtils.cpp index ceaba9e7..58b295ea 100644 --- a/language-extensions/R/src/RTypeUtils.cpp +++ b/language-extensions/R/src/RTypeUtils.cpp @@ -28,7 +28,6 @@ #include "Common.h" #include #include -#include #include "RTypeUtils.h" #include "Unicode.h" diff --git a/language-extensions/R/test/src/RExtensionGetResultColumnTests.cpp b/language-extensions/R/test/src/RExtensionGetResultColumnTests.cpp index eec0d263..3f6f3c47 100644 --- a/language-extensions/R/test/src/RExtensionGetResultColumnTests.cpp +++ b/language-extensions/R/test/src/RExtensionGetResultColumnTests.cpp @@ -475,8 +475,9 @@ namespace ExtensionApiTest // vector chineseBytes = { -28, -67, -96, -27, -91, -67 }; string chineseString = string(chineseBytes.data(), 6); + string worldChineseString = "World" + chineseString; - vector charCol1{ "Hello", "test", "data", ("World" + chineseString).c_str(), + vector charCol1{ "Hello", "test", "data", worldChineseString.c_str(), chineseString.c_str() }; vector charCol2{ "", 0, nullptr, "verify", "-1" }; diff --git a/language-extensions/common/linux/AbiFloorCompat_linux.cpp b/language-extensions/common/linux/AbiFloorCompat_linux.cpp new file mode 100644 index 00000000..13fc8f35 --- /dev/null +++ b/language-extensions/common/linux/AbiFloorCompat_linux.cpp @@ -0,0 +1,201 @@ +//************************************************************************************************** +// @File: AbiFloorCompat_linux.cpp +// +// Purpose: +// RHEL 9 ABI floor compatibility shims for Linux language extensions. +// +// The extensions are built on Ubuntu 24.04 (glibc 2.38, GCC 13) but must load +// on RHEL 9 (glibc 2.34). A small number of symbols above that floor get pulled +// in by the toolchain rather than by our own code, so they cannot be removed at +// the call site: +// +// __isoc23_strtoll @GLIBC_2.38 glibc 2.38 redirects the strtol +// __isoc23_strtoull @GLIBC_2.38 family to C23 variants; the references come +// __isoc23_strtoul @GLIBC_2.38 from vendored headers and from libstdc++.a +// arc4random @GLIBC_2.36 used by libstdc++.a +// _dl_find_object @GLIBC_2.35 used by the libgcc.a unwinder +// +// Each shim is defined under a distinct C++ name carrying an __asm__ label, so +// the emitted symbol is the glibc name. Defining a function literally named +// __isoc23_strtoll does NOT work: has already declared strtoll with +// that same assembler name, and the resulting conflict leaves the reference +// unresolved (observed as "UND __isoc23_strtoll@GLIBC_2.38" in .dynsym). +// +// Every shim is hidden. They satisfy references inside this shared object only +// and are not exported, so this library never interposes on glibc for the rest +// of the host process. +// +// Enforced by build/linux/validate-native-elf.sh (MAX GLIBC <= 2.34). +//************************************************************************************************** + +#if defined(__linux__) + +// (pulled in by any libc header) must be included before testing +// __GLIBC__: the macro is defined by glibc's headers, not by the compiler. +// Guarding on __GLIBC__ before any include silently compiles this whole file to +// nothing, which is exactly how an earlier revision of these shims ended up +// linked but inert. +#include +#include +#include +#include +#include +#include +#include + +#if defined(__GLIBC__) + +#ifdef ABI_FLOOR_PROVIDE_IOS_BASE_LIBRARY_INIT +// For std::ios_base::Init, which is what actually constructs the standard +// streams. See AbiFloorIosBaseLibraryInit below. +#include +#endif + +#define ABI_FLOOR_HIDDEN __attribute__((visibility("hidden"))) + +extern "C" +{ + // Bind to the classic, always-present glibc entry points. The explicit + // assembler labels bypass the C23 redirect, so these cannot + // recurse back into the shims below. + long long int AbiFloorClassicStrtoll(const char *nptr, char **endptr, int base) __asm__("strtoll"); + unsigned long long int AbiFloorClassicStrtoull(const char *nptr, char **endptr, int base) __asm__("strtoull"); + unsigned long int AbiFloorClassicStrtoul(const char *nptr, char **endptr, int base) __asm__("strtoul"); + + ABI_FLOOR_HIDDEN long long int AbiFloorIsoc23Strtoll(const char *nptr, char **endptr, int base) __asm__("__isoc23_strtoll"); + ABI_FLOOR_HIDDEN unsigned long long int AbiFloorIsoc23Strtoull(const char *nptr, char **endptr, int base) __asm__("__isoc23_strtoull"); + ABI_FLOOR_HIDDEN unsigned long int AbiFloorIsoc23Strtoul(const char *nptr, char **endptr, int base) __asm__("__isoc23_strtoul"); + ABI_FLOOR_HIDDEN uint32_t AbiFloorArc4Random(void) __asm__("arc4random"); + ABI_FLOOR_HIDDEN int AbiFloorDlFindObject(void *address, void *result) __asm__("_dl_find_object"); + + // The only behavioural difference between the C23 and classic strtol forms + // is that the C23 forms also accept a "0b" binary prefix when base is 0 or + // 2. No caller in these extensions parses binary literals. + ABI_FLOOR_HIDDEN long long int AbiFloorIsoc23Strtoll(const char *nptr, char **endptr, int base) + { + return AbiFloorClassicStrtoll(nptr, endptr, base); + } + + ABI_FLOOR_HIDDEN unsigned long long int AbiFloorIsoc23Strtoull(const char *nptr, char **endptr, int base) + { + return AbiFloorClassicStrtoull(nptr, endptr, base); + } + + ABI_FLOOR_HIDDEN unsigned long int AbiFloorIsoc23Strtoul(const char *nptr, char **endptr, int base) + { + return AbiFloorClassicStrtoul(nptr, endptr, base); + } + + // arc4random is a cryptographically secure generator, so this must not + // degrade to a weak source. getrandom(2) is the kernel CSPRNG and has been + // available since glibc 2.25, well under the RHEL 9 floor. /dev/urandom is + // the fallback for seccomp-restricted environments. If neither can produce + // bytes we abort rather than return predictable output. + ABI_FLOOR_HIDDEN uint32_t AbiFloorArc4Random(void) + { + uint32_t value = 0; + unsigned char *out = reinterpret_cast(&value); + size_t filled = 0; + + while (filled < sizeof(value)) + { + ssize_t received = getrandom(out + filled, sizeof(value) - filled, 0); + if (received < 0) + { + if (errno == EINTR) + { + continue; + } + break; + } + filled += static_cast(received); + } + + if (filled < sizeof(value)) + { + int fd = open("/dev/urandom", O_RDONLY | O_CLOEXEC); + if (fd >= 0) + { + while (filled < sizeof(value)) + { + ssize_t received = read(fd, out + filled, sizeof(value) - filled); + if (received <= 0) + { + if (received < 0 && errno == EINTR) + { + continue; + } + break; + } + filled += static_cast(received); + } + close(fd); + } + } + + if (filled < sizeof(value)) + { + abort(); + } + + return value; + } + + // _dl_find_object is the glibc 2.35+ fast path the libgcc unwinder uses to + // locate exception-handling frames. A non-zero return means "no information + // available", which makes libgcc fall back to its dl_iterate_phdr path. That + // fallback is the same code path used on every glibc older than 2.35, so + // unwinding stays correct - only marginally slower on the throw path. + ABI_FLOOR_HIDDEN int AbiFloorDlFindObject(void *address, void *result) + { + (void)address; + (void)result; + return -1; + } + + // std::ios_base_library_init() is GLIBCXX_3.4.32, i.e. GCC 13's libstdc++. + // GCC 13 emits a reference to it from every translation unit that includes + // , and RHEL9's libstdc++ 3.4.29 does not provide it. + // + // This is opt-in via ABI_FLOOR_PROVIDE_IOS_BASE_LIBRARY_INIT because it must + // only be defined when libstdc++ is linked *dynamically*. libstdc++.a already + // contains this symbol in ios_init.o, so defining it alongside a static + // libstdc++ fails the link with "multiple definition of + // std::ios_base_library_init()". + // + // The R and Python extensions are the cases that need it: neither can link + // libstdc++ statically (libR.so already links libstdc++.so.6, and numpy's and + // pandas' C extensions do too - two C++ runtimes in one process abort at + // runtime), yet the include is unavoidable because it arrives through the + // Rcpp/RInside and Boost.Python headers. + // + // This MUST NOT be a no-op. GCC 13 changed how the standard streams are + // constructed: before 13, every TU including emitted its own static + // std::ios_base::Init object; GCC 13 emits a call to this function instead and + // performs the construction inside libstdc++. So on a host whose libstdc++ is + // 3.4.32 or newer the library initialises the streams itself and a stub here + // looks harmless - but on Ubuntu 22.04 (libstdc++.so.6.0.30) and RHEL 9 nothing + // initialises them, std::cout/std::cerr stay unconstructed, and the first write + // crashes the satellite process. That is precisely how libPythonExtension died + // on Ubuntu 22.04 and RHEL 9 while passing on Ubuntu 24.04, RHEL 10 and + // AzureLinux 3, which ship newer libstdc++. + // + // Constructing a function-local static std::ios_base::Init performs the pre-13 + // initialisation explicitly. Its constructor is refcounted and idempotent, the + // local static guard runs it exactly once however many translation units call + // in, and std::ios_base::Init::Init is GLIBCXX_3.4 - present in every libstdc++ + // we target, so this does not raise the ABI floor. +#ifdef ABI_FLOOR_PROVIDE_IOS_BASE_LIBRARY_INIT + ABI_FLOOR_HIDDEN void AbiFloorIosBaseLibraryInit(void) __asm__("_ZSt21ios_base_library_initv"); + ABI_FLOOR_HIDDEN void AbiFloorIosBaseLibraryInit(void) + { + static std::ios_base::Init iosBaseInit; + (void)iosBaseInit; + } +#endif +} + +#else +#error "AbiFloorCompat_linux.cpp is compiled for Linux but __GLIBC__ is not defined; the shims would be silently empty." +#endif // __GLIBC__ +#endif // __linux__ diff --git a/language-extensions/java/src/CMakeLists.txt b/language-extensions/java/src/CMakeLists.txt index 928cc9be..d62fc749 100644 --- a/language-extensions/java/src/CMakeLists.txt +++ b/language-extensions/java/src/CMakeLists.txt @@ -32,6 +32,14 @@ file(TO_CMAKE_PATH ${ENL_ROOT}/extension-host/include EXTENSION_API_INCLUDE_DIR) # file(GLOB JAVAEXTENSION_SOURCE_FILES ${JAVAEXTENSION_SRC_DIR}/*.cpp ${JAVAEXTENSION_SRC_DIR}/${PLATFORM}/*.cpp) +# RHEL 9 ABI floor shims. Required because libstdc++.a and libgcc.a from the +# Ubuntu 24.04 toolchain reference glibc symbols newer than 2.34. See +# language-extensions/common/linux/AbiFloorCompat_linux.cpp. +# +if (${PLATFORM} STREQUAL linux) + list(APPEND JAVAEXTENSION_SOURCE_FILES ${ENL_ROOT}/language-extensions/common/linux/AbiFloorCompat_linux.cpp) +endif() + # Create the target library # add_library(JavaExtension SHARED @@ -45,7 +53,11 @@ set_target_properties(JavaExtension ) if (${PLATFORM} STREQUAL linux) - file(TO_CMAKE_PATH ${JAVA_HOME}/jre/lib/amd64/server JRE_ROOT) + if(EXISTS "${JAVA_HOME}/lib/server") + file(TO_CMAKE_PATH ${JAVA_HOME}/lib/server JRE_ROOT) + else() + file(TO_CMAKE_PATH ${JAVA_HOME}/jre/lib/amd64/server JRE_ROOT) + endif() file(TO_CMAKE_PATH ${JAVA_HOME}/include JDK_INCLUDE) file(TO_CMAKE_PATH ${JDK_INCLUDE}/linux JDK_INCLUDE_LINUX) @@ -63,14 +75,8 @@ if (${PLATFORM} STREQUAL linux) target_compile_options(JavaExtension PRIVATE ${CXXFLAGS}) - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -shared -L${OUTPUT_DIRECTORY}/lib -L${LIBS} -Wl,-rpath,${JRE_ROOT}:/opt/mssql-extensibility/lib") - set(USR_LIB_PATH "/usr/local/lib") - - find_library(CXX_LIB c++ ${USR_LIB_PATH}) - find_library(ABI_LIB c++abi ${USR_LIB_PATH}) - find_library(DL_LIB dl ${USR_LIB_PATH}) - - target_link_libraries(JavaExtension ${CXX_LIB} ${ABI_LIB} ${DL_LIB}) + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -shared -static-libstdc++ -static-libgcc -Wl,--exclude-libs,ALL -Wl,-rpath,${JRE_ROOT}:/opt/mssql-extensibility/lib") + target_link_libraries(JavaExtension dl) set(ADDITIONAL_INCLUDES ${JDK_INCLUDE} ${JDK_INCLUDE_LINUX}) elseif(${PLATFORM} STREQUAL windows) diff --git a/language-extensions/python/src/CMakeLists.txt b/language-extensions/python/src/CMakeLists.txt index fd9112c4..a62ff8e0 100644 --- a/language-extensions/python/src/CMakeLists.txt +++ b/language-extensions/python/src/CMakeLists.txt @@ -33,6 +33,14 @@ file(TO_CMAKE_PATH ${ENL_ROOT}/build-output/pythonextension/${PLATFORM} PYTHONEX # file(GLOB PYTHONEXTENSION_SOURCE_FILES ${PYTHONEXTENSION_SRC_DIR}/*.cpp ${PYTHONEXTENSION_SRC_DIR}/${PLATFORM}/*.cpp) +# RHEL 9 ABI floor shims. Required because libstdc++.a and libgcc.a from the +# Ubuntu 24.04 toolchain reference glibc symbols newer than 2.34. See +# language-extensions/common/linux/AbiFloorCompat_linux.cpp. +# +if (${PLATFORM} STREQUAL linux) + list(APPEND PYTHONEXTENSION_SOURCE_FILES ${ENL_ROOT}/language-extensions/common/linux/AbiFloorCompat_linux.cpp) +endif() + # Create the target library # add_library(PythonExtension SHARED @@ -46,6 +54,12 @@ set_target_properties(PythonExtension ) if (${PLATFORM} STREQUAL linux) + # Python links libstdc++ dynamically, so RHEL 9's 3.4.29 copy will not + # provide std::ios_base_library_init(). Ask the shared compat unit to + # define it. + # + target_compile_definitions(PythonExtension PRIVATE ABI_FLOOR_PROVIDE_IOS_BASE_LIBRARY_INIT) + # Python runtime version used in the linux environment differs from the one used in Windows at this time. # This declaration should no longer be needed once the Windows version is updated to 3.12. # @@ -54,7 +68,17 @@ if (${PLATFORM} STREQUAL linux) target_compile_options(PythonExtension PRIVATE -Wall -Wextra -g -O2 -fPIC -Werror -std=c++17 -Wno-unused-parameter -Wno-maybe-uninitialized -fshort-wchar) - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--as-needed -Wl,--export-dynamic -fopenmp") + # The shipped extension must not depend on Boost at runtime: RHEL 9 has no + # libboost_python312, so Boost is built link=static and absorbed here. + # + # libstdc++ is deliberately NOT linked statically. The embedded interpreter + # imports numpy and pandas, whose C extensions link libstdc++.so.6, so a + # static copy puts two C++ runtimes in one process and the test binary + # aborts (SIGABRT) during GetOutputParam. RHEL 9 ships libstdc++ 3.4.29, so + # depending on it is fine as long as nothing above that version is + # referenced - the one offending symbol is defined locally in + # language-extensions/common/linux/AbiFloorCompat_linux.cpp. + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--as-needed -Wl,--export-dynamic -fopenmp") # Prefer shared libraries over static libraries set(CMAKE_FIND_LIBRARY_SUFFIXES .so .so.1.0 ${CMAKE_FIND_LIBRARY_SUFFIXES}) diff --git a/language-extensions/python/src/Logger.cpp b/language-extensions/python/src/Logger.cpp index 324fc7cc..53a92f2a 100644 --- a/language-extensions/python/src/Logger.cpp +++ b/language-extensions/python/src/Logger.cpp @@ -13,6 +13,8 @@ #include "Logger.h" +#include + #define TIMESTAMP_LENGTH 35 using namespace std; @@ -24,9 +26,21 @@ char Logger::sm_timestampBuffer[TIMESTAMP_LENGTH] = { 0 }; // Description: // Log an error to stderr with format "TIMESTAMP Error: ". // +// Note: +// These write through stdio rather than std::cerr/std::cout on purpose. The extension is compiled +// with GCC 13 but is loaded into hosts whose libstdc++ is older - RHEL 9 ships +// libstdc++.so.6.0.29 and Ubuntu 22.04 ships 6.0.30 - and the stream insertion operators are +// inlined from the GCC 13 headers into this library, so they execute against stream objects owned +// by the older runtime. That combination segfaults inside libstdc++ on the very first log call, +// which is why the Python satellite core-dumped on those two distros while passing on Ubuntu +// 24.04, RHEL 10 and AzureLinux 3. The symbol-version ABI gate cannot catch this: every symbol we +// reference is within GLIBCXX_3.4.29, the problem is inlined code depending on newer internals. +// stdio has no such coupling. EKM and the ONNX extension were fixed the same way. +// void Logger::LogError(const string &errorMsg) { - cerr << GetCurrentTimestamp() << "Error: " << errorMsg << endl; + fprintf(stderr, "%sError: %s\n", GetCurrentTimestamp().c_str(), errorMsg.c_str()); + fflush(stderr); } //------------------------------------------------------------------------------------------------- @@ -38,7 +52,8 @@ void Logger::LogError(const string &errorMsg) // void Logger::LogException(const exception &e) { - cerr << GetCurrentTimestamp() << "Exception occurred: " << e.what() << endl; + fprintf(stderr, "%sException occurred: %s\n", GetCurrentTimestamp().c_str(), e.what()); + fflush(stderr); } //------------------------------------------------------------------------------------------------- @@ -50,7 +65,8 @@ void Logger::LogException(const exception &e) void Logger::Log(const string &msg) { #if defined(_DEBUG) - cout << GetCurrentTimestamp() << msg << endl; + fprintf(stdout, "%s%s\n", GetCurrentTimestamp().c_str(), msg.c_str()); + fflush(stdout); #endif } diff --git a/language-extensions/python/src/PythonLibrarySession.cpp b/language-extensions/python/src/PythonLibrarySession.cpp index 2d8f0275..6be78177 100644 --- a/language-extensions/python/src/PythonLibrarySession.cpp +++ b/language-extensions/python/src/PythonLibrarySession.cpp @@ -105,6 +105,48 @@ SQLRETURN PythonLibrarySession::InstallLibrary( "external library must be a python package inside a zip."); } + // Older package fixtures and customer packages created on Windows can contain backslashes in + // ZIP entry names. Modern pip treats those as literal characters on Linux, so setup.py is found + // but its package directory is not. Normalize the inner package ZIP before passing it to pip. + // + // This is best effort on purpose. It runs while installing an external library inside the + // satellite process, where an uncaught exception terminates the process rather than failing the + // statement, so every failure path has to stay inside Python and report back through a flag: + // + // - the entry data must be read BEFORE rewriting entry.filename. ZipFile.open() compares the + // central-directory name against the local header and raises BadZipFile when they differ, so + // reading after the rewrite throws on exactly the archives this code exists to fix. + // - the archive is only rewritten when an entry actually contains a backslash, so well-formed + // packages are passed through untouched. + // - any failure (unreadable source, unwritable temp folder, malformed archive) leaves + // normalized False and the original path is used, which is what happened before this + // normalization existed. + // + if (fs::path(installPath).extension().generic_string() == ".zip") + { + string normalizedInstallPath = + (fs::path(tempFolder) / ("normalized-" + fs::path(installPath).filename().string())).generic_string(); + string normalizeScript = "import zipfile\n" + "normalized = False\n" + "try:\n" + " with zipfile.ZipFile('" + installPath + "', 'r') as source_zip:\n" + " if any('\\\\' in n for n in source_zip.namelist()):\n" + " with zipfile.ZipFile('" + normalizedInstallPath + "', 'w') as normalized_zip:\n" + " for entry in source_zip.infolist():\n" + " data = source_zip.read(entry)\n" + " entry.filename = entry.filename.replace('\\\\', '/')\n" + " normalized_zip.writestr(entry, data)\n" + " normalized = True\n" + "except Exception:\n" + " normalized = False"; + bp::exec(normalizeScript.c_str(), m_mainNamespace); + + if (bp::extract(m_mainNamespace["normalized"])) + { + installPath = normalizedInstallPath; + } + } + string pathToPython = PythonExtensionUtils::GetPathToPython(); // Set the TMPDIR so that pip uses our destination as temp. This allows us to use a diff --git a/language-extensions/python/src/PythonSession.cpp b/language-extensions/python/src/PythonSession.cpp index 6cf9637f..b9e1496d 100644 --- a/language-extensions/python/src/PythonSession.cpp +++ b/language-extensions/python/src/PythonSession.cpp @@ -211,8 +211,15 @@ void PythonSession::ExecuteWorkflow( string pyStdOut = bp::extract(m_mainNamespace["_temp_out_"]); string pyStdErr = bp::extract(m_mainNamespace["_temp_err_"]); - cout << pyStdOut << endl; - cerr << pyStdErr << endl; + // Relayed through stdio rather than std::cout/std::cerr. The stream insertion operators are + // inlined from the GCC 13 headers but run against the host's older libstdc++ (RHEL 9 ships + // 6.0.29, Ubuntu 22.04 ships 6.0.30), which segfaults inside libstdc++. See the note in + // Logger.cpp. + // + fprintf(stdout, "%s\n", pyStdOut.c_str()); + fflush(stdout); + fprintf(stderr, "%s\n", pyStdErr.c_str()); + fflush(stderr); // In case of streaming clean up the previous stream batch's output buffers // diff --git a/language-extensions/python/test/include/PythonExtensionApiTests.h b/language-extensions/python/test/include/PythonExtensionApiTests.h index d1083a80..8137172e 100644 --- a/language-extensions/python/test/include/PythonExtensionApiTests.h +++ b/language-extensions/python/test/include/PythonExtensionApiTests.h @@ -329,8 +329,8 @@ namespace ExtensionApiTest // Test output string param value and strLenOrInd is as expected. // void TestGetWStringOutputParam( - std::vector expectedParamValueVector, - std::vector expectedStrLenOrIndVector); + std::vector expectedParamValueVector, + std::vector expectedStrLenOrIndVector); // Test output raw param value and strLenOrInd is as expected. // diff --git a/language-extensions/python/test/src/PythonGetOutputParamTests.cpp b/language-extensions/python/test/src/PythonGetOutputParamTests.cpp index 9a58f2e4..b6190092 100644 --- a/language-extensions/python/test/src/PythonGetOutputParamTests.cpp +++ b/language-extensions/python/test/src/PythonGetOutputParamTests.cpp @@ -643,38 +643,38 @@ namespace ExtensionApiTest ASSERT_EQ(result, SQL_SUCCESS); EXPECT_EQ(outputSchemaColumnsNumber, 0); - wstring wideStrOfValue7 = wstring(value7.begin(), value7.end()); - const wchar_t* wideCStrOfValue7 = wideStrOfValue7.c_str(); - vector expectedParamValues = { + u16string wideStrOfValue7(value7.begin(), value7.end()); + const char16_t* wideCStrOfValue7 = wideStrOfValue7.c_str(); + vector expectedParamValues = { // Test simple CHAR(5) value with exact string length as the type allows i.e. here 5. // - L"HELLO", + u"HELLO", // Test VARCHAR(6) value with string length more than the type allows - expected truncation. // Above python script sets the parameter to "PyExtension" but we only expect "PyExte". // - L"PyExte", + u"PyExte", // Test a 0 length string // - L"" , + u"", // Empty value. // Test CHAR(10) value with string length less than the type allows. // - L"WORLD", + u"WORLD", // Test a Unicode string // - L"你好", + u"你好", nullptr, nullptr, wideCStrOfValue7}; vector expectedStrLenOrInd = { - static_cast(5 * sizeof(wchar_t)), - static_cast(6 * sizeof(wchar_t)), - static_cast(0 * sizeof(wchar_t)), - static_cast(5 * sizeof(wchar_t)), - static_cast(2 * sizeof(wchar_t)), + static_cast(5 * sizeof(char16_t)), + static_cast(6 * sizeof(char16_t)), + static_cast(0 * sizeof(char16_t)), + static_cast(5 * sizeof(char16_t)), + static_cast(2 * sizeof(char16_t)), SQL_NULL_DATA, SQL_NULL_DATA, - static_cast(128000 * sizeof(wchar_t)), + static_cast(128000 * sizeof(char16_t)), }; @@ -1102,8 +1102,8 @@ namespace ExtensionApiTest // Test wstring output param value and strLenOrInd is as expected. // void PythonExtensionApiTests::TestGetWStringOutputParam( - vector expectedParamValues, - vector expectedStrLenOrInd) + vector expectedParamValues, + vector expectedStrLenOrInd) { ASSERT_EQ(expectedParamValues.size(), expectedStrLenOrInd.size()); @@ -1127,20 +1127,16 @@ namespace ExtensionApiTest { EXPECT_NE(paramValue, nullptr); - wstring paramValueString(static_cast(paramValue), - strLen_or_Ind / sizeof(wchar_t)); - wstring expectedParamValueString(expectedParamValues[paramNumber], - expectedStrLenOrInd[paramNumber] / sizeof(wchar_t)); + // SQLWCHAR data is UTF-16. Compare the returned buffer directly instead of + // constructing std::wstring under -fshort-wchar; libstdc++ itself is built + // for the platform's native 4-byte wchar_t ABI. + const char *paramBytes = reinterpret_cast(paramValue); + const char *expectedParamBytes = + reinterpret_cast(expectedParamValues[paramNumber]); - // Compare the two wstrings byte by byte because EXPECT_STREQ and EXPECT_EQ - // don't work properly for wstrings in Linux with -fshort-wchar - // - const char *paramBytes = reinterpret_cast(paramValueString.c_str()); - const char *expectedParamBytes = reinterpret_cast(expectedParamValueString.c_str()); - - for(SQLINTEGER i=0; i