runai-model-streamer-gcs: Add version 0.16.1

.github/workflows/build-runai-model-streamer-gcs.yml

@@ -0,0 +1,386 @@
+# SPDX-FileCopyrightText: 2026 The RISE Project
+# SPDX-License-Identifier: MIT
+#
+# This workflow is based on: https://github.com/run-ai/runai-model-streamer/blob/master/Makefile
+# and https://github.com/run-ai/runai-model-streamer/blob/master/tests/Makefile
+---
+name: Build runai-model-streamer-gcs wheels (riscv64)
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version glob to (re)build; empty builds every version of docs/packages/runai-model-streamer-gcs.yaml not released yet'
+        required: false
+        default: ''
+  pull_request:
+    branches: [main]
+    paths:
+      - '.github/workflows/build-runai-model-streamer-gcs.yml'
+      - 'docs/packages/runai-model-streamer-gcs.yaml'
+      - 'patches/runai-model-streamer-gcs/**'
+  push:
+    branches: [main]
+    paths:
+      - '.github/workflows/build-runai-model-streamer-gcs.yml'
+      - 'docs/packages/runai-model-streamer-gcs.yaml'
+      - 'patches/runai-model-streamer-gcs/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+env:
+  MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64
+  BAZEL_VERSION: '7.6.1'
+  RULES_PYTHON_VERSION: '0.33.2'
+  RULES_JAVA_VERSION: '7.6.5'
+  # Pinned by upstream's .devcontainer/Dockerfile, which downloads the amd64/arm64 release binary.
+  FAKE_GCS_SERVER_VERSION: 'v1.52.2'
+
+jobs:
+  setup:
+    uses: $/.github/workflows/_setup.yml
+    with:
+      package: runai-model-streamer-gcs
+      version: ${{ inputs.version }}
+
+  bazel:
+    needs: [setup]
+    if: needs.setup.outputs.versions != '[]'
+    name: Bootstrap bazel (riscv64)
+    runs-on: ubuntu-24.04-riscv
+    timeout-minutes: 120
+
+    steps:
+      - name: Restore bazel binary
+        id: cache
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
+        with:
+          path: bazel-bin
+          key: bazel-${{ env.BAZEL_VERSION }}-manylinux_riscv64
+
+      - name: Bootstrap bazel ${{ env.BAZEL_VERSION }}
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          mkdir -p bazel-bin
+          docker run --rm -i --network=host \
+            -v "${GITHUB_WORKSPACE}:/work" \
+            -w /work \
+            -e BAZEL_VERSION="${BAZEL_VERSION}" \
+            -e RULES_PYTHON_VERSION="${RULES_PYTHON_VERSION}" \
+            -e RULES_JAVA_VERSION="${RULES_JAVA_VERSION}" \
+            "${MANYLINUX_RISCV64_IMAGE}" \
+            bash <<'SCRIPT'
+          set -eux
+
+          dnf install -y --setopt=install_weak_deps=False java-21-openjdk-devel zip unzip
+          JAVA_HOME="$(dirname "$(dirname "$(readlink -f "$(command -v javac)")")")"
+          export JAVA_HOME
+
+          # rules_python 0.33.2 has no riscv64 PLATFORMS entry (bazelbuild/bazel#23018).
+          mkdir -p /tmp/rules_python
+          curl -fsSLo /tmp/rules_python.tar.gz "https://github.com/bazel-contrib/rules_python/releases/download/${RULES_PYTHON_VERSION}/rules_python-${RULES_PYTHON_VERSION}.tar.gz"
+          tar -xzf /tmp/rules_python.tar.gz -C /tmp/rules_python --strip-components=1
+          sed -i 's|fail("No platform declared for host OS {} on arch {}".format(os_name, arch))|return "x86_64-unknown-linux-gnu"|' \
+            /tmp/rules_python/python/private/toolchains_repo.bzl
+
+          # rules_java 7.x maps riscv64 to a stray-colon include path for jni_md.h.
+          mkdir -p /tmp/rules_java
+          curl -fsSLo /tmp/rules_java.tar.gz "https://github.com/bazelbuild/rules_java/releases/download/${RULES_JAVA_VERSION}/rules_java-${RULES_JAVA_VERSION}.tar.gz"
+          tar -xzf /tmp/rules_java.tar.gz -C /tmp/rules_java
+          sed -i 's|\[":include/linux"\]|["include/linux"]|g' /tmp/rules_java/toolchains/BUILD
+
+          mkdir -p /tmp/bazel-src
+          cd /tmp/bazel-src
+          curl -fsSLo dist.zip "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-dist.zip"
+          unzip -q dist.zip
+
+          EXTRA_BAZEL_ARGS="--tool_java_runtime_version=local_jdk \
+            --override_module=rules_python=/tmp/rules_python \
+            --override_module=rules_java=/tmp/rules_java" \
+            bash ./compile.sh
+          install -m 0755 output/bazel /work/bazel-bin/bazel
+          SCRIPT
+
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: bazel-${{ env.BAZEL_VERSION }}-riscv64
+          path: bazel-bin/bazel
+          if-no-files-found: error
+
+  fake_gcs_server:
+    needs: [setup]
+    if: needs.setup.outputs.versions != '[]'
+    name: Build fake-gcs-server (riscv64)
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    steps:
+      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e  # v7.0.0
+        with:
+          go-version: 'stable'
+          cache: false
+
+      - name: Build fake-gcs-server ${{ env.FAKE_GCS_SERVER_VERSION }}
+        env:
+          CGO_ENABLED: '0'
+          GOOS: linux
+          GOARCH: riscv64
+        run: |
+          go install "github.com/fsouza/fake-gcs-server@${FAKE_GCS_SERVER_VERSION}"
+          mkdir -p fake-gcs-server
+          cp "$(go env GOPATH)/bin/linux_riscv64/fake-gcs-server" fake-gcs-server/fake-gcs-server
+
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: fake-gcs-server-riscv64
+          path: fake-gcs-server/fake-gcs-server
+          if-no-files-found: error
+
+  build_wheel:
+    name: Build runai-model-streamer-gcs ${{ matrix.version }} py3-none-manylinux_riscv64
+    needs: [setup, bazel]
+    if: needs.setup.outputs.versions != '[]'
+    runs-on: ubuntu-24.04-riscv
+    timeout-minutes: 720
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ${{ fromJSON(needs.setup.outputs.versions) }}
+
+    env:
+      RUNAI_MODEL_STREAMER_VERSION: ${{ matrix.version }}
+
+    steps:
+      - name: Checkout runai-model-streamer ${{ env.RUNAI_MODEL_STREAMER_VERSION }}
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          repository: run-ai/runai-model-streamer
+          ref: ${{ env.RUNAI_MODEL_STREAMER_VERSION }}
+          path: runai-model-streamer
+          persist-credentials: false
+
+      - name: Checkout python-wheels
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          path: python-wheels
+          persist-credentials: false
+
+      - name: Patch runai-model-streamer source
+        working-directory: runai-model-streamer
+        run: git apply -v ../python-wheels/patches/runai-model-streamer-gcs/${{ env.RUNAI_MODEL_STREAMER_VERSION }}/*.patch
+
+      - name: Download bazel
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
+        with:
+          name: bazel-${{ env.BAZEL_VERSION }}-riscv64
+          path: bazel-bin
+
+      - name: Build the GCS streamer library and package the wheel
+        run: |
+          mkdir -p dist
+          docker run --rm -i --network=host \
+            -v "${GITHUB_WORKSPACE}:/work" \
+            -w /work/runai-model-streamer \
+            -e PACKAGE_VERSION="${RUNAI_MODEL_STREAMER_VERSION}" \
+            "${MANYLINUX_RISCV64_IMAGE}" \
+            bash <<'SCRIPT'
+          set -eux
+
+          # cpp/toolchain links -l:libstdc++.a, which on this image is its own CRB package.
+          dnf install -y --setopt=install_weak_deps=False java-21-openjdk-devel libstdc++-static
+          JAVA_HOME="$(dirname "$(dirname "$(readlink -f "$(command -v javac)")")")"
+          export JAVA_HOME
+          install -m 0755 /work/bazel-bin/bazel /usr/local/bin/bazel
+          git config --global --add safe.directory '*'
+
+          # cpp/toolchain/rules.bzl expects the Debian cross-toolchain names; this image's gcc is native.
+          for tool in gcc g++ ar ld cpp gcov nm objdump strip; do
+            native="$(command -v "$tool" || true)"
+            [ -n "$native" ] && ln -sf "$native" "/usr/bin/riscv64-linux-gnu-$tool"
+          done
+
+          cd cpp
+          # A multi-hour build's default curses progress output is large enough that
+          # GitHub drops the job log, taking the failure with it.
+          bazel build gcs:libstreamergcs.so --config=riscv64 --curses=no --show_progress_rate_limit=60
+
+          # Stage the licence of every external repository the library is built from.
+          licenses=/work/runai-model-streamer/py/runai_model_streamer_gcs
+          external="$(bazel info output_base)/external"
+          bazel cquery 'deps(gcs:libstreamergcs.so)' --config=riscv64 --output=label > /tmp/deps.txt
+          sed -n 's|^@@\{0,1\}\([^/]*\)//.*|\1|p' /tmp/deps.txt | sort -u \
+            | grep -Ev '^(bazel_tools|bazel_skylib|platforms|toolchain|local_config_.*|local_jdk|remotejdk.*|remote_.*|rules_.*|build_bazel_.*|io_bazel_.*)$' \
+            | while read -r repo; do
+                for f in "$external/$repo"/LICENSE* "$external/$repo"/COPYING* "$external/$repo"/NOTICE*; do
+                  [ -f "$f" ] || continue
+                  case "$(basename "$f")" in
+                    NOTICE*) dest="LICENSE.$repo.NOTICE" ;;
+                    *) dest="LICENSE.$repo"; [ -e "$licenses/$dest" ] && dest="LICENSE.$repo.$(basename "$f")" ;;
+                  esac
+                  cp "$f" "$licenses/$dest"
+                done
+              done
+          ls "$licenses"
+
+          cd ../py/runai_model_streamer_gcs
+          /opt/python/cp312-cp312/bin/pip install -q setuptools wheel
+          /opt/python/cp312-cp312/bin/python setup.py bdist_wheel --plat-name manylinux_2_39_riscv64
+          cp dist/*.whl /work/dist/
+          SCRIPT
+
+      - name: Verify the wheel ships a riscv64 shared object and its licences
+        run: |
+          python3 - dist/*.whl <<'EOF'
+          import sys, zipfile
+          with zipfile.ZipFile(sys.argv[1]) as zf:
+              names = zf.namelist()
+              so = next(n for n in names if n.endswith("libstreamer/lib/libstreamergcs.so"))
+              header = zf.read(so)[:20]
+              assert header[:4] == b"\x7fELF", header
+              assert header[18] == 0xF3, header  # EM_RISCV
+              licenses = {n.rsplit("/", 1)[-1] for n in names if ".dist-info/licenses/" in n}
+              print(sorted(licenses))
+              for dep in ("google_cloud_cpp", "com_google_absl", "com_google_protobuf", "com_github_grpc_grpc",
+                          "boringssl", "com_github_curl_curl", "zlib", "com_github_cares_cares",
+                          "com_googlesource_code_re2", "com_github_google_crc32c", "com_github_nlohmann_json"):
+                  assert f"LICENSE.{dep}" in licenses, dep
+          EOF
+
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: runai-model-streamer-gcs-${{ matrix.version }}-py3-none-manylinux_riscv64
+          path: dist/*.whl
+          if-no-files-found: error
+
+  test_wheel:
+    name: Test runai-model-streamer-gcs ${{ matrix.version }} on Python ${{ matrix.python-version }}
+    needs: [setup, build_wheel, fake_gcs_server]
+    if: needs.setup.outputs.versions != '[]'
+    runs-on: ubuntu-24.04-riscv
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ${{ fromJSON(needs.setup.outputs.versions) }}
+        python-version: ['3.12', '3.13', '3.14']
+
+    env:
+      RUNAI_MODEL_STREAMER_VERSION: ${{ matrix.version }}
+
+    steps:
+      - name: Checkout runai-model-streamer ${{ env.RUNAI_MODEL_STREAMER_VERSION }}
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          repository: run-ai/runai-model-streamer
+          ref: ${{ env.RUNAI_MODEL_STREAMER_VERSION }}
+          path: runai-model-streamer
+          persist-credentials: false
+
+      - name: Download wheel
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
+        with:
+          name: runai-model-streamer-gcs-${{ env.RUNAI_MODEL_STREAMER_VERSION }}-py3-none-manylinux_riscv64
+          path: wheelhouse
+
+      - name: Download fake-gcs-server
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
+        with:
+          name: fake-gcs-server-riscv64
+          path: fake-gcs-server
+
+      - name: Install Python
+        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d  # v10.0.1
+        with:
+          python-version: ${{ matrix.python-version }}
+          activate-environment: true
+          enable-cache: false
+
+      - name: Install the wheel and upstream's test dependencies
+        env:
+          UV_EXTRA_INDEX_URL: https://pypi.riseproject.dev/simple/
+          UV_INDEX_STRATEGY: unsafe-best-match
+        # requirements.dev's numpy==1.24.4 has no riscv64 wheel and no longer builds on Python >=3.12.
+        run: |
+          echo "numpy>=1.26,<3" > "$RUNNER_TEMP/numpy-override.txt"
+          uv pip install --override "$RUNNER_TEMP/numpy-override.txt" \
+            -r runai-model-streamer/py/runai_model_streamer/requirements.dev \
+            pytest \
+            "runai-model-streamer[gcs]==${RUNAI_MODEL_STREAMER_VERSION}" \
+            wheelhouse/*.whl
+
+      - name: Run the GCS package's unit tests against the installed wheel
+        run: |
+          site="$(python -c 'import runai_model_streamer_gcs, os; print(os.path.dirname(os.path.dirname(runai_model_streamer_gcs.__file__)))')"
+          cd "$site"
+          python -m unittest discover -b runai_model_streamer_gcs
+
+      - name: Run upstream's GCS tests against fake-gcs-server
+        working-directory: runai-model-streamer/tests
+        env:
+          GOOGLE_CLOUD_PROJECT: fake-project
+          GCS_BUCKET: test-bucket
+          API_ENDPOINT_OVERRIDE: http://localhost:8000
+          CLOUD_STORAGE_EMULATOR_ENDPOINT: http://localhost:8000
+          RUNAI_STREAMER_GCS_USE_ANONYMOUS_CREDENTIALS: 'True'
+        run: |
+          chmod +x "$GITHUB_WORKSPACE/fake-gcs-server/fake-gcs-server"
+          root="$RUNNER_TEMP/fake-gcs-server"
+          mkdir -p "$root/data/test-bucket"
+          "$GITHUB_WORKSPACE/fake-gcs-server/fake-gcs-server" -scheme http -port 8000 -data "$root/data" -filesystem-root "$root" &
+          trap 'kill $!' EXIT
+          pytest -v gcs/
+
+  gpl_sources:
+    needs: [setup]
+    if: needs.setup.outputs.versions != '[]'
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ${{ fromJSON(needs.setup.outputs.versions) }}
+    name: Collect GPL sources for runai-model-streamer-gcs ${{ matrix.version }}
+    runs-on: ubuntu-24.04-riscv
+
+    env:
+      RUNAI_MODEL_STREAMER_VERSION: ${{ matrix.version }}
+
+    steps:
+      - name: Checkout python-wheels
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+
+      - uses: ./actions/collect-gpl-sources
+        with:
+          image: ${{ env.MANYLINUX_RISCV64_IMAGE }}
+          packages: gcc
+          output: gpl-sources.tar
+
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: runai-model-streamer-gcs-${{ env.RUNAI_MODEL_STREAMER_VERSION }}-gpl-sources
+          path: gpl-sources.tar
+          if-no-files-found: error
+
+  publish:
+    name: Publish runai-model-streamer-gcs ${{ matrix.version }}
+    needs: [setup, build_wheel, test_wheel, gpl_sources]
+    if: needs.setup.outputs.versions != '[]'
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ${{ fromJSON(needs.setup.outputs.versions) }}
+    permissions:
+      contents: write
+      pull-requests: write
+    uses: $/.github/workflows/_publish-wheel.yml
+    secrets:
+      app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }}
+    with:
+      artifact-pattern: runai-model-streamer-gcs-${{ matrix.version }}-py3-none-manylinux_riscv64
+      gpl-sources-artifact: runai-model-streamer-gcs-${{ matrix.version }}-gpl-sources
+      gpl-sources-description: gcc

docs/packages/runai-model-streamer-gcs.yaml

@@ -0,0 +1,5 @@
+package-name: runai-model-streamer-gcs
+source-code: https://github.com/run-ai/runai-model-streamer
+license: Apache-2.0 AND BSD-3-Clause AND MIT AND OpenSSL AND ISC AND curl AND Zlib
+versions:
+- version: 0.16.1