diff --git a/.github/workflows/build-runai-model-streamer-gcs.yml b/.github/workflows/build-runai-model-streamer-gcs.yml new file mode 100644 index 00000000000..e7233951ddf --- /dev/null +++ b/.github/workflows/build-runai-model-streamer-gcs.yml @@ -0,0 +1,386 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +# +# This workflow is based on: https://github.com/run-ai/runai-model-streamer/blob/master/Makefile +# and https://github.com/run-ai/runai-model-streamer/blob/master/tests/Makefile +--- +name: Build runai-model-streamer-gcs wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'Version glob to (re)build; empty builds every version of docs/packages/runai-model-streamer-gcs.yaml not released yet' + required: false + default: '' + pull_request: + branches: [main] + paths: + - '.github/workflows/build-runai-model-streamer-gcs.yml' + - 'docs/packages/runai-model-streamer-gcs.yaml' + - 'patches/runai-model-streamer-gcs/**' + push: + branches: [main] + paths: + - '.github/workflows/build-runai-model-streamer-gcs.yml' + - 'docs/packages/runai-model-streamer-gcs.yaml' + - 'patches/runai-model-streamer-gcs/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +env: + MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 + BAZEL_VERSION: '7.6.1' + RULES_PYTHON_VERSION: '0.33.2' + RULES_JAVA_VERSION: '7.6.5' + # Pinned by upstream's .devcontainer/Dockerfile, which downloads the amd64/arm64 release binary. + FAKE_GCS_SERVER_VERSION: 'v1.52.2' + +jobs: + setup: + uses: $/.github/workflows/_setup.yml + with: + package: runai-model-streamer-gcs + version: ${{ inputs.version }} + + bazel: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + name: Bootstrap bazel (riscv64) + runs-on: ubuntu-24.04-riscv + timeout-minutes: 120 + + steps: + - name: Restore bazel binary + id: cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: bazel-bin + key: bazel-${{ env.BAZEL_VERSION }}-manylinux_riscv64 + + - name: Bootstrap bazel ${{ env.BAZEL_VERSION }} + if: steps.cache.outputs.cache-hit != 'true' + run: | + mkdir -p bazel-bin + docker run --rm -i --network=host \ + -v "${GITHUB_WORKSPACE}:/work" \ + -w /work \ + -e BAZEL_VERSION="${BAZEL_VERSION}" \ + -e RULES_PYTHON_VERSION="${RULES_PYTHON_VERSION}" \ + -e RULES_JAVA_VERSION="${RULES_JAVA_VERSION}" \ + "${MANYLINUX_RISCV64_IMAGE}" \ + bash <<'SCRIPT' + set -eux + + dnf install -y --setopt=install_weak_deps=False java-21-openjdk-devel zip unzip + JAVA_HOME="$(dirname "$(dirname "$(readlink -f "$(command -v javac)")")")" + export JAVA_HOME + + # rules_python 0.33.2 has no riscv64 PLATFORMS entry (bazelbuild/bazel#23018). + mkdir -p /tmp/rules_python + curl -fsSLo /tmp/rules_python.tar.gz "https://github.com/bazel-contrib/rules_python/releases/download/${RULES_PYTHON_VERSION}/rules_python-${RULES_PYTHON_VERSION}.tar.gz" + tar -xzf /tmp/rules_python.tar.gz -C /tmp/rules_python --strip-components=1 + sed -i 's|fail("No platform declared for host OS {} on arch {}".format(os_name, arch))|return "x86_64-unknown-linux-gnu"|' \ + /tmp/rules_python/python/private/toolchains_repo.bzl + + # rules_java 7.x maps riscv64 to a stray-colon include path for jni_md.h. + mkdir -p /tmp/rules_java + curl -fsSLo /tmp/rules_java.tar.gz "https://github.com/bazelbuild/rules_java/releases/download/${RULES_JAVA_VERSION}/rules_java-${RULES_JAVA_VERSION}.tar.gz" + tar -xzf /tmp/rules_java.tar.gz -C /tmp/rules_java + sed -i 's|\[":include/linux"\]|["include/linux"]|g' /tmp/rules_java/toolchains/BUILD + + mkdir -p /tmp/bazel-src + cd /tmp/bazel-src + curl -fsSLo dist.zip "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-dist.zip" + unzip -q dist.zip + + EXTRA_BAZEL_ARGS="--tool_java_runtime_version=local_jdk \ + --override_module=rules_python=/tmp/rules_python \ + --override_module=rules_java=/tmp/rules_java" \ + bash ./compile.sh + install -m 0755 output/bazel /work/bazel-bin/bazel + SCRIPT + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: bazel-${{ env.BAZEL_VERSION }}-riscv64 + path: bazel-bin/bazel + if-no-files-found: error + + fake_gcs_server: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + name: Build fake-gcs-server (riscv64) + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: 'stable' + cache: false + + - name: Build fake-gcs-server ${{ env.FAKE_GCS_SERVER_VERSION }} + env: + CGO_ENABLED: '0' + GOOS: linux + GOARCH: riscv64 + run: | + go install "github.com/fsouza/fake-gcs-server@${FAKE_GCS_SERVER_VERSION}" + mkdir -p fake-gcs-server + cp "$(go env GOPATH)/bin/linux_riscv64/fake-gcs-server" fake-gcs-server/fake-gcs-server + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: fake-gcs-server-riscv64 + path: fake-gcs-server/fake-gcs-server + if-no-files-found: error + + build_wheel: + name: Build runai-model-streamer-gcs ${{ matrix.version }} py3-none-manylinux_riscv64 + needs: [setup, bazel] + if: needs.setup.outputs.versions != '[]' + runs-on: ubuntu-24.04-riscv + timeout-minutes: 720 + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + + env: + RUNAI_MODEL_STREAMER_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout runai-model-streamer ${{ env.RUNAI_MODEL_STREAMER_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: run-ai/runai-model-streamer + ref: ${{ env.RUNAI_MODEL_STREAMER_VERSION }} + path: runai-model-streamer + persist-credentials: false + + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: python-wheels + persist-credentials: false + + - name: Patch runai-model-streamer source + working-directory: runai-model-streamer + run: git apply -v ../python-wheels/patches/runai-model-streamer-gcs/${{ env.RUNAI_MODEL_STREAMER_VERSION }}/*.patch + + - name: Download bazel + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: bazel-${{ env.BAZEL_VERSION }}-riscv64 + path: bazel-bin + + - name: Build the GCS streamer library and package the wheel + run: | + mkdir -p dist + docker run --rm -i --network=host \ + -v "${GITHUB_WORKSPACE}:/work" \ + -w /work/runai-model-streamer \ + -e PACKAGE_VERSION="${RUNAI_MODEL_STREAMER_VERSION}" \ + "${MANYLINUX_RISCV64_IMAGE}" \ + bash <<'SCRIPT' + set -eux + + # cpp/toolchain links -l:libstdc++.a, which on this image is its own CRB package. + dnf install -y --setopt=install_weak_deps=False java-21-openjdk-devel libstdc++-static + JAVA_HOME="$(dirname "$(dirname "$(readlink -f "$(command -v javac)")")")" + export JAVA_HOME + install -m 0755 /work/bazel-bin/bazel /usr/local/bin/bazel + git config --global --add safe.directory '*' + + # cpp/toolchain/rules.bzl expects the Debian cross-toolchain names; this image's gcc is native. + for tool in gcc g++ ar ld cpp gcov nm objdump strip; do + native="$(command -v "$tool" || true)" + [ -n "$native" ] && ln -sf "$native" "/usr/bin/riscv64-linux-gnu-$tool" + done + + cd cpp + # A multi-hour build's default curses progress output is large enough that + # GitHub drops the job log, taking the failure with it. + bazel build gcs:libstreamergcs.so --config=riscv64 --curses=no --show_progress_rate_limit=60 + + # Stage the licence of every external repository the library is built from. + licenses=/work/runai-model-streamer/py/runai_model_streamer_gcs + external="$(bazel info output_base)/external" + bazel cquery 'deps(gcs:libstreamergcs.so)' --config=riscv64 --output=label > /tmp/deps.txt + sed -n 's|^@@\{0,1\}\([^/]*\)//.*|\1|p' /tmp/deps.txt | sort -u \ + | grep -Ev '^(bazel_tools|bazel_skylib|platforms|toolchain|local_config_.*|local_jdk|remotejdk.*|remote_.*|rules_.*|build_bazel_.*|io_bazel_.*)$' \ + | while read -r repo; do + for f in "$external/$repo"/LICENSE* "$external/$repo"/COPYING* "$external/$repo"/NOTICE*; do + [ -f "$f" ] || continue + case "$(basename "$f")" in + NOTICE*) dest="LICENSE.$repo.NOTICE" ;; + *) dest="LICENSE.$repo"; [ -e "$licenses/$dest" ] && dest="LICENSE.$repo.$(basename "$f")" ;; + esac + cp "$f" "$licenses/$dest" + done + done + ls "$licenses" + + cd ../py/runai_model_streamer_gcs + /opt/python/cp312-cp312/bin/pip install -q setuptools wheel + /opt/python/cp312-cp312/bin/python setup.py bdist_wheel --plat-name manylinux_2_39_riscv64 + cp dist/*.whl /work/dist/ + SCRIPT + + - name: Verify the wheel ships a riscv64 shared object and its licences + run: | + python3 - dist/*.whl <<'EOF' + import sys, zipfile + with zipfile.ZipFile(sys.argv[1]) as zf: + names = zf.namelist() + so = next(n for n in names if n.endswith("libstreamer/lib/libstreamergcs.so")) + header = zf.read(so)[:20] + assert header[:4] == b"\x7fELF", header + assert header[18] == 0xF3, header # EM_RISCV + licenses = {n.rsplit("/", 1)[-1] for n in names if ".dist-info/licenses/" in n} + print(sorted(licenses)) + for dep in ("google_cloud_cpp", "com_google_absl", "com_google_protobuf", "com_github_grpc_grpc", + "boringssl", "com_github_curl_curl", "zlib", "com_github_cares_cares", + "com_googlesource_code_re2", "com_github_google_crc32c", "com_github_nlohmann_json"): + assert f"LICENSE.{dep}" in licenses, dep + EOF + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: runai-model-streamer-gcs-${{ matrix.version }}-py3-none-manylinux_riscv64 + path: dist/*.whl + if-no-files-found: error + + test_wheel: + name: Test runai-model-streamer-gcs ${{ matrix.version }} on Python ${{ matrix.python-version }} + needs: [setup, build_wheel, fake_gcs_server] + if: needs.setup.outputs.versions != '[]' + runs-on: ubuntu-24.04-riscv + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + python-version: ['3.12', '3.13', '3.14'] + + env: + RUNAI_MODEL_STREAMER_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout runai-model-streamer ${{ env.RUNAI_MODEL_STREAMER_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: run-ai/runai-model-streamer + ref: ${{ env.RUNAI_MODEL_STREAMER_VERSION }} + path: runai-model-streamer + persist-credentials: false + + - name: Download wheel + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: runai-model-streamer-gcs-${{ env.RUNAI_MODEL_STREAMER_VERSION }}-py3-none-manylinux_riscv64 + path: wheelhouse + + - name: Download fake-gcs-server + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: fake-gcs-server-riscv64 + path: fake-gcs-server + + - name: Install Python + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + python-version: ${{ matrix.python-version }} + activate-environment: true + enable-cache: false + + - name: Install the wheel and upstream's test dependencies + env: + UV_EXTRA_INDEX_URL: https://pypi.riseproject.dev/simple/ + UV_INDEX_STRATEGY: unsafe-best-match + # requirements.dev's numpy==1.24.4 has no riscv64 wheel and no longer builds on Python >=3.12. + run: | + echo "numpy>=1.26,<3" > "$RUNNER_TEMP/numpy-override.txt" + uv pip install --override "$RUNNER_TEMP/numpy-override.txt" \ + -r runai-model-streamer/py/runai_model_streamer/requirements.dev \ + pytest \ + "runai-model-streamer[gcs]==${RUNAI_MODEL_STREAMER_VERSION}" \ + wheelhouse/*.whl + + - name: Run the GCS package's unit tests against the installed wheel + run: | + site="$(python -c 'import runai_model_streamer_gcs, os; print(os.path.dirname(os.path.dirname(runai_model_streamer_gcs.__file__)))')" + cd "$site" + python -m unittest discover -b runai_model_streamer_gcs + + - name: Run upstream's GCS tests against fake-gcs-server + working-directory: runai-model-streamer/tests + env: + GOOGLE_CLOUD_PROJECT: fake-project + GCS_BUCKET: test-bucket + API_ENDPOINT_OVERRIDE: http://localhost:8000 + CLOUD_STORAGE_EMULATOR_ENDPOINT: http://localhost:8000 + RUNAI_STREAMER_GCS_USE_ANONYMOUS_CREDENTIALS: 'True' + run: | + chmod +x "$GITHUB_WORKSPACE/fake-gcs-server/fake-gcs-server" + root="$RUNNER_TEMP/fake-gcs-server" + mkdir -p "$root/data/test-bucket" + "$GITHUB_WORKSPACE/fake-gcs-server/fake-gcs-server" -scheme http -port 8000 -data "$root/data" -filesystem-root "$root" & + trap 'kill $!' EXIT + pytest -v gcs/ + + gpl_sources: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + name: Collect GPL sources for runai-model-streamer-gcs ${{ matrix.version }} + runs-on: ubuntu-24.04-riscv + + env: + RUNAI_MODEL_STREAMER_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./actions/collect-gpl-sources + with: + image: ${{ env.MANYLINUX_RISCV64_IMAGE }} + packages: gcc + output: gpl-sources.tar + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: runai-model-streamer-gcs-${{ env.RUNAI_MODEL_STREAMER_VERSION }}-gpl-sources + path: gpl-sources.tar + if-no-files-found: error + + publish: + name: Publish runai-model-streamer-gcs ${{ matrix.version }} + needs: [setup, build_wheel, test_wheel, gpl_sources] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + permissions: + contents: write + pull-requests: write + uses: $/.github/workflows/_publish-wheel.yml + secrets: + app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + with: + artifact-pattern: runai-model-streamer-gcs-${{ matrix.version }}-py3-none-manylinux_riscv64 + gpl-sources-artifact: runai-model-streamer-gcs-${{ matrix.version }}-gpl-sources + gpl-sources-description: gcc diff --git a/docs/packages/runai-model-streamer-gcs.yaml b/docs/packages/runai-model-streamer-gcs.yaml new file mode 100644 index 00000000000..f5f0284e626 --- /dev/null +++ b/docs/packages/runai-model-streamer-gcs.yaml @@ -0,0 +1,5 @@ +package-name: runai-model-streamer-gcs +source-code: https://github.com/run-ai/runai-model-streamer +license: Apache-2.0 AND BSD-3-Clause AND MIT AND OpenSSL AND ISC AND curl AND Zlib +versions: +- version: 0.16.1 diff --git a/patches/runai-model-streamer-gcs/0.16.1/0001-toolchain-add-riscv64-as-a-supported-architecture.patch b/patches/runai-model-streamer-gcs/0.16.1/0001-toolchain-add-riscv64-as-a-supported-architecture.patch new file mode 100644 index 00000000000..f255b20b391 --- /dev/null +++ b/patches/runai-model-streamer-gcs/0.16.1/0001-toolchain-add-riscv64-as-a-supported-architecture.patch @@ -0,0 +1,51 @@ +From 820707245ffc7f9f363fc5de3f5375daacb0a8b9 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Thu, 24 Sep 2026 03:32:40 +0000 +Subject: [PATCH] toolchain: add riscv64 as a supported architecture + +configure_toolchain() and its per-arch .bazelrc block are already fully +generic over the arch string (native.platform constraint is +@platforms//cpu:, gcc tools are resolved as +/usr/bin/-linux-gnu-) -- riscv64 was simply never added to +either list. + +No -march flag is added for riscv64: aarch64's exists for crc32c's ARM64 +CRC32C path inside google-cloud-cpp, and on riscv64 crc32c takes its +portable path instead (see the next-but-one patch), so a native riscv64 +build uses the compiler's own default target. + +Upstream-Status: To upstream [this port does not open issues/PRs against third-party repos; riscv64 toolchain support is exactly the kind of change upstream would want once a downstream build proves it works] +--- + cpp/.bazelrc | 3 +++ + cpp/toolchain/configure.bzl | 2 +- + 2 files changed, 4 insertions(+), 1 deletion(-) + +diff --git a/cpp/.bazelrc b/cpp/.bazelrc +index 830e568..4a6ccd2 100644 +--- a/cpp/.bazelrc ++++ b/cpp/.bazelrc +@@ -19,5 +19,8 @@ build:aarch64 --copt="-march=armv8-a+crc+crypto" + build:x86_64 --platform_suffix=x86_64 + build:x86_64 --platforms=@toolchain//:x86_64 + ++build:riscv64 --platform_suffix=riscv64 ++build:riscv64 --platforms=@toolchain//:riscv64 ++ + # Azurite testing support (enables AZURE_STORAGE_ACCOUNT_KEY for local testing) + build:azurite --define azurite_testing=true +diff --git a/cpp/toolchain/configure.bzl b/cpp/toolchain/configure.bzl +index 6b89f54..9a04125 100644 +--- a/cpp/toolchain/configure.bzl ++++ b/cpp/toolchain/configure.bzl +@@ -4,7 +4,7 @@ load("//toolchain:rules.bzl", "get_target_triplet", "runai_crosstool_tools") + + # This should map to the devcontainer Dockerfile + # We install gcc-x86-64-linux-gnu and gcc-aarch64-linux-gnu toolchains +-ARCHITECTURES = ["x86_64", "aarch64"] ++ARCHITECTURES = ["x86_64", "aarch64", "riscv64"] + OS = "linux-gnu" + + def _get_gcc_version(repository_ctx, arch): +-- +2.43.0 + diff --git a/patches/runai-model-streamer-gcs/0.16.1/0002-toolchain-use-gcc-s-own-dumpmachine-triplet-for-builtin-include-dirs.patch b/patches/runai-model-streamer-gcs/0.16.1/0002-toolchain-use-gcc-s-own-dumpmachine-triplet-for-builtin-include-dirs.patch new file mode 100644 index 00000000000..367c1fc785b --- /dev/null +++ b/patches/runai-model-streamer-gcs/0.16.1/0002-toolchain-use-gcc-s-own-dumpmachine-triplet-for-builtin-include-dirs.patch @@ -0,0 +1,134 @@ +From 75c45793987328b8011252f0f24950e94456b336 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Thu, 24 Sep 2026 08:01:06 +0000 +Subject: [PATCH] toolchain: use gcc's own -dumpmachine triplet for builtin + include dirs + +configure_toolchain()'s cxx_builtin_include_directories are built from the +same os+arch naming-convention triplet (-linux-gnu) used to resolve +tool paths (/usr/bin/-linux-gnu-). That equivalence holds for a +genuine Debian cross-toolchain package (a real x86_64-linux-gnu-gcc reports +exactly that as its own -dumpmachine), but not when the tool path is a +same-arch *native* compiler reachable only via a symlink under the expected +name -- its own compiled-in builtin search path (/usr/lib/gcc///include, etc.) still uses its real vendor triplet regardless +of what name it is invoked under. + +This is exactly riscv64's manylinux_riscv64 image: gcc is Red Hat's native +riscv64-redhat-linux build, symlinked to /usr/bin/riscv64-linux-gnu-gcc so +runai_crosstool_tools() finds it. Bazel's own absolute-path sandboxing then +rejects gcc's builtin headers (e.g. stddef.h) because +/usr/lib/gcc/riscv64-redhat-linux//include is not in the declared +riscv64-linux-gnu-based list. + +Thread the compiler's actual -dumpmachine output down as gcc_machine and use +it (falling back to the naming-convention triplet when unset, so x86_64 and +aarch64 are unaffected) for cxx_builtin_include_directories specifically, +while tool_paths keeps using the naming-convention triplet the workflow's +symlinks provide. + +Upstream-Status: To upstream [this port does not open issues/PRs against third-party repos; making cxx_builtin_include_directories match the compiler's own reported triplet is a general robustness fix upstream would likely want] +--- + cpp/toolchain/configure.bzl | 7 ++++++- + cpp/toolchain/template/toolchain.bzl | 23 ++++++++++++++++++++--- + 2 files changed, 26 insertions(+), 4 deletions(-) + +diff --git a/cpp/toolchain/configure.bzl b/cpp/toolchain/configure.bzl +index 9a04125..510ef90 100644 +--- a/cpp/toolchain/configure.bzl ++++ b/cpp/toolchain/configure.bzl +@@ -11,6 +11,10 @@ def _get_gcc_version(repository_ctx, arch): + gcc_tool = runai_crosstool_tools(get_target_triplet(OS, arch))["gcc"] + return repository_ctx.execute([gcc_tool, "-dumpversion"]).stdout.strip() + ++def _get_gcc_machine(repository_ctx, arch): ++ gcc_tool = runai_crosstool_tools(get_target_triplet(OS, arch))["gcc"] ++ return repository_ctx.execute([gcc_tool, "-dumpmachine"]).stdout.strip() ++ + def _get_host_arch(repository_ctx): + return repository_ctx.execute(["/usr/bin/uname", "-m"]).stdout.strip() + +@@ -18,9 +22,10 @@ def _cc_autoconf_toolchain_impl(repository_ctx): + define_statements = [] + for arch in ARCHITECTURES: + gcc_version = _get_gcc_version(repository_ctx, arch) ++ gcc_machine = _get_gcc_machine(repository_ctx, arch) + host_arch = _get_host_arch(repository_ctx) + toolchain_name = arch +- define_statements.append('define_toolchain(name = "%s", os = "%s", host_arch = "%s", arch = "%s", gcc_version = "%s")' % (toolchain_name, OS, host_arch, arch, gcc_version)) ++ define_statements.append('define_toolchain(name = "%s", os = "%s", host_arch = "%s", arch = "%s", gcc_version = "%s", gcc_machine = "%s")' % (toolchain_name, OS, host_arch, arch, gcc_version, gcc_machine)) + + repository_ctx.template( + "BUILD", +diff --git a/cpp/toolchain/template/toolchain.bzl b/cpp/toolchain/template/toolchain.bzl +index 6a4c6cf..36daffc 100644 +--- a/cpp/toolchain/template/toolchain.bzl ++++ b/cpp/toolchain/template/toolchain.bzl +@@ -73,6 +73,15 @@ def _impl(ctx): + target_triplet = get_target_triplet(ctx.attr.os, ctx.attr.arch) + tool_paths = [tool_path(name = k, path = v) for k, v in runai_crosstool_tools(target_triplet).items()] + ++ # gcc_machine is the triplet gcc itself reports (`gcc -dumpmachine`), which is ++ # what its own builtin include search path is actually built from. It is ++ # normally identical to target_triplet (a real x86_64-linux-gnu/aarch64-linux-gnu ++ # cross-gcc reports exactly that), but differs when the "gcc" behind ++ # target_triplet's tool path is a same-arch native compiler under a different ++ # vendor triplet (e.g. Red Hat's riscv64-redhat-linux) made reachable only by a ++ # symlink under the expected name. ++ include_triplet = ctx.attr.gcc_machine if ctx.attr.gcc_machine else target_triplet ++ + # Documented at + # https://docs.bazel.build/versions/main/skylark/lib/cc_common.html#create_cc_toolchain_config_info. + # +@@ -89,7 +98,7 @@ def _impl(ctx): + abi_version = "unknown", + abi_libc_version = "unknown", + tool_paths = tool_paths, +- cxx_builtin_include_directories = _get_include_directories(target_triplet, ctx.attr.gcc_version, ctx.attr.use_cross), ++ cxx_builtin_include_directories = _get_include_directories(include_triplet, ctx.attr.gcc_version, ctx.attr.use_cross), + features = features + ) + +@@ -111,6 +120,11 @@ _toolchain_config = rule( + mandatory = True, + doc = "GCC major version to use (eg: 9)", + ), ++ "gcc_machine": attr.string( ++ mandatory = False, ++ default = "", ++ doc = "The triplet `gcc -dumpmachine` reports, if it differs from os/arch's own naming convention", ++ ), + "use_cross": attr.bool( + mandatory = False, + default = False, +@@ -121,7 +135,7 @@ _toolchain_config = rule( + ) + + +-def define_toolchain(name, os, host_arch, arch, gcc_version): ++def define_toolchain(name, os, host_arch, arch, gcc_version, gcc_machine = ""): + """"define_toolchain creates rules for a cc_toolchain. + + This expects toolchain tools to exist at a canonical path. +@@ -134,6 +148,9 @@ def define_toolchain(name, os, host_arch, arch, gcc_version): + host_arch: The host architecture for this toolchain + arch: The target architecture for this toochain + gcc_version: The GCC version of this toolchain ++ gcc_machine: The triplet `gcc -dumpmachine` reports for this toolchain's ++ compiler, if it differs from the os/arch naming convention used for ++ the tool paths themselves + """ + + native.platform( +@@ -145,7 +162,7 @@ def define_toolchain(name, os, host_arch, arch, gcc_version): + ) + + toolchain_config_name = "%s_toolchain_config" % name +- _toolchain_config(name = toolchain_config_name, os = os, arch = arch, gcc_version = gcc_version, use_cross = host_arch != arch) ++ _toolchain_config(name = toolchain_config_name, os = os, arch = arch, gcc_version = gcc_version, gcc_machine = gcc_machine, use_cross = host_arch != arch) + + empty_target_name = "%s_empty" % name + empty_target_label = ":%s" % empty_target_name +-- +2.43.0 + diff --git a/patches/runai-model-streamer-gcs/0.16.1/0003-third_party-give-google-cloud-cpp-s-crc32c-config-a-portable-default.patch b/patches/runai-model-streamer-gcs/0.16.1/0003-third_party-give-google-cloud-cpp-s-crc32c-config-a-portable-default.patch new file mode 100644 index 00000000000..c103c1d0433 --- /dev/null +++ b/patches/runai-model-streamer-gcs/0.16.1/0003-third_party-give-google-cloud-cpp-s-crc32c-config-a-portable-default.patch @@ -0,0 +1,63 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Mon, 28 Sep 2026 04:00:00 +0000 +Subject: [PATCH] third_party: give google-cloud-cpp's crc32c config a + portable default + +google-cloud-cpp 2.37.0's bazel/crc32c.BUILD generates crc32c_config.h +by first rewriting every `#cmakedefine01 X` to `#define X` and then +appending the value per substitution. HAVE_SSE42 and HAVE_ARM64_CRC32C +only get a value on @platforms//cpu:x86_64 and :arm64; on any other CPU +the "//conditions:default" branch is empty, so both end up as bare +`#define HAVE_SSE42`, and crc32c's own `#if HAVE_SSE42 && ...` / +`#if HAVE_ARM64_CRC32C` fail to preprocess ("operator '&&' has no left +operand", "#if with no expression"). google-cloud-cpp main still has +the empty default. + +Patch the fetched google_cloud_cpp archive so the default branch sets +both to 0, i.e. crc32c's portable implementation. x86_64 and aarch64 +builds are unaffected. + +Upstream-Status: To upstream [this port does not open issues/PRs against third-party repos; the empty default is a google-cloud-cpp bug on every CPU other than x86_64/arm64, and runai-model-streamer can carry the patch until it is fixed there] +--- + cpp/third_party/gcp_repo.bzl | 2 ++ + cpp/third_party/google_cloud_cpp_crc32c_default.patch | 17 +++++++++++++++++ + 2 files changed, 19 insertions(+) + +diff --git a/cpp/third_party/gcp_repo.bzl b/cpp/third_party/gcp_repo.bzl +index e91b830..17803dd 100644 +--- a/cpp/third_party/gcp_repo.bzl ++++ b/cpp/third_party/gcp_repo.bzl +@@ -8,4 +8,6 @@ def load_gcp_repo(): + sha256 = "10867580483cb338e7d50920c2383698f3572cc6b4c7d072e38d5f43755cbd80", + strip_prefix = "google-cloud-cpp-2.37.0", + url = "https://github.com/googleapis/google-cloud-cpp/archive/v2.37.0.tar.gz", ++ patches = [Label("//third_party:google_cloud_cpp_crc32c_default.patch")], ++ patch_args = ["-p1"], + ) +diff --git a/cpp/third_party/google_cloud_cpp_crc32c_default.patch b/cpp/third_party/google_cloud_cpp_crc32c_default.patch +new file mode 100644 +index 0000000..d357c31 +--- /dev/null ++++ b/cpp/third_party/google_cloud_cpp_crc32c_default.patch +@@ -0,0 +1,17 @@ ++diff --git a/bazel/crc32c.BUILD b/bazel/crc32c.BUILD ++index c8d8b3c..78b083f 100644 ++--- a/bazel/crc32c.BUILD +++++ b/bazel/crc32c.BUILD ++@@ -115,7 +115,11 @@ configure_template( ++ " HAVE_SSE42": " HAVE_SSE42 0", ++ " HAVE_ARM64_CRC32C": " HAVE_ARM64_CRC32C 1", ++ }, ++- "//conditions:default": {}, +++ # Any other CPU (e.g. riscv64) uses crc32c's portable implementation. +++ "//conditions:default": { +++ " HAVE_SSE42": " HAVE_SSE42 0", +++ " HAVE_ARM64_CRC32C": " HAVE_ARM64_CRC32C 0", +++ }, ++ }), ++ template = "src/crc32c_config.h.in", ++ ) +-- +2.43.0 + diff --git a/patches/runai-model-streamer-gcs/0.16.1/0004-gcs-ship-the-licences-of-the-statically-linked-dependencies.patch b/patches/runai-model-streamer-gcs/0.16.1/0004-gcs-ship-the-licences-of-the-statically-linked-dependencies.patch new file mode 100644 index 00000000000..6626621e166 --- /dev/null +++ b/patches/runai-model-streamer-gcs/0.16.1/0004-gcs-ship-the-licences-of-the-statically-linked-dependencies.patch @@ -0,0 +1,34 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Mon, 28 Sep 2026 04:00:00 +0000 +Subject: [PATCH] gcs: ship the licences of the statically linked dependencies + +libstreamergcs.so statically links google-cloud-cpp and everything it +pulls in through bazel (abseil, protobuf, gRPC, BoringSSL, libcurl, +zlib, c-ares, re2, crc32c, nlohmann/json, ...), but the explicit +license_files=("LICENSE",) only ships the project's own licence. The +BSD/MIT/curl/OpenSSL licences and the Apache-2.0 NOTICE files require +their notices to travel with the binary. Also match LICENSE.* next to +setup.py, where the build stages each dependency's licence text. + +Upstream-Status: To upstream [this port does not open issues/PRs against third-party repos; the upstream wheels have the same gap] +--- + py/runai_model_streamer_gcs/setup.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/py/runai_model_streamer_gcs/setup.py b/py/runai_model_streamer_gcs/setup.py +index 1379a4f..03f2ff3 100644 +--- a/py/runai_model_streamer_gcs/setup.py ++++ b/py/runai_model_streamer_gcs/setup.py +@@ -22,7 +22,7 @@ assert_lib_exists() + setup( + name="runai-model-streamer-gcs", + version=VERSION, +- license_files=("LICENSE",), ++ license_files=("LICENSE", "LICENSE.*"), + packages=find_packages(), + install_requires=["google-cloud-storage", "google-auth"], + data_files=[("/runai_model_streamer/libstreamer/lib/", [LIB])], +-- +2.43.0 +