diff --git a/.github/workflows/build-grain.yml b/.github/workflows/build-grain.yml new file mode 100644 index 00000000000..d7c84081c18 --- /dev/null +++ b/.github/workflows/build-grain.yml @@ -0,0 +1,381 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +# +# This workflow is based on: +# https://github.com/google/grain/blob/main/.github/workflows/build_and_publish_template.yml +# and https://github.com/google/grain/blob/main/grain/oss/build_whl.sh +--- +name: Build grain wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'Version glob to (re)build; empty builds every version of docs/packages/grain.yaml not released yet' + required: false + default: '' + pull_request: + branches: [main] + paths: + - '.github/workflows/build-grain.yml' + - 'docs/packages/grain.yaml' + push: + branches: [main] + paths: + - '.github/workflows/build-grain.yml' + - 'docs/packages/grain.yaml' + +run-name: build-grain ${{ inputs.version && format('- {0}', inputs.version) || '' }} + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +env: + MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 + # Upstream pins 7.2.1, fetched through setup-bazel, which publishes no riscv64 + # binary; 7.5.0 is the version this repo bootstraps from source. + BAZEL_VERSION: '7.5.0' + # versions bazel 7.5.0's MODULE.bazel pins; both need a riscv64 fix below + RULES_PYTHON_VERSION: '0.33.2' + RULES_JAVA_VERSION: '7.6.5' + +jobs: + setup: + uses: $/.github/workflows/_setup.yml + with: + package: grain + version: ${{ inputs.version }} + + bazel: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + name: Bootstrap bazel (riscv64) + runs-on: ubuntu-24.04-riscv + timeout-minutes: 720 + + env: + GRAIN_VERSION: ${{ matrix.version }} + + steps: + - name: Restore bazel binary + id: cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: bazel-bin + key: bazel-${{ env.BAZEL_VERSION }}-manylinux_riscv64 + + - name: Bootstrap bazel ${{ env.BAZEL_VERSION }} + if: steps.cache.outputs.cache-hit != 'true' + run: | + mkdir -p bazel-bin + docker run --rm -i --network=host \ + -v "${GITHUB_WORKSPACE}:/work" \ + -w /work \ + -e BAZEL_VERSION="${BAZEL_VERSION}" \ + -e RULES_PYTHON_VERSION="${RULES_PYTHON_VERSION}" \ + -e RULES_JAVA_VERSION="${RULES_JAVA_VERSION}" \ + "${MANYLINUX_RISCV64_IMAGE}" \ + bash <<'SCRIPT' + set -eux + + dnf install -y --setopt=install_weak_deps=False java-21-openjdk-devel zip unzip + JAVA_HOME="$(dirname "$(dirname "$(readlink -f "$(command -v javac)")")")" + export JAVA_HOME + + # rules_python 0.33.2's PLATFORMS has no riscv64 entry, aborting the bootstrap + # (bazelbuild/bazel#23018). Any linux entry is a safe stand-in: the toolchain it names + # is never selected on a riscv64 host. Fixed in bazel 8.2.0; the 7.x backport is open. + mkdir -p /tmp/rules_python + curl -fsSLo /tmp/rules_python.tar.gz "https://github.com/bazel-contrib/rules_python/releases/download/${RULES_PYTHON_VERSION}/rules_python-${RULES_PYTHON_VERSION}.tar.gz" + tar -xzf /tmp/rules_python.tar.gz -C /tmp/rules_python --strip-components=1 + sed -i 's|fail("No platform declared for host OS {} on arch {}".format(os_name, arch))|return "x86_64-unknown-linux-gnu"|' \ + /tmp/rules_python/python/private/toolchains_repo.bzl + + # rules_java 7.x maps riscv64 to a stray-colon include path, so a JNI library + # can't find jni_md.h. Fixed in rules_java 8.x, never backported. + mkdir -p /tmp/rules_java + curl -fsSLo /tmp/rules_java.tar.gz "https://github.com/bazelbuild/rules_java/releases/download/${RULES_JAVA_VERSION}/rules_java-${RULES_JAVA_VERSION}.tar.gz" + tar -xzf /tmp/rules_java.tar.gz -C /tmp/rules_java + sed -i 's|\[":include/linux"\]|["include/linux"]|g' /tmp/rules_java/toolchains/BUILD + + mkdir -p /tmp/bazel-src + cd /tmp/bazel-src + curl -fsSLo dist.zip "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-dist.zip" + unzip -q dist.zip + + EXTRA_BAZEL_ARGS="--tool_java_runtime_version=local_jdk \ + --override_module=rules_python=/tmp/rules_python \ + --override_module=rules_java=/tmp/rules_java" \ + bash ./compile.sh + install -m 0755 output/bazel /work/bazel-bin/bazel + SCRIPT + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: bazel-${{ env.BAZEL_VERSION }}-riscv64 + path: bazel-bin/bazel + if-no-files-found: error + + build_wheels: + name: Build grain ${{ matrix.version }} ${{ matrix.tag }}-manylinux_riscv64 + runs-on: ubuntu-24.04-riscv + timeout-minutes: 720 + needs: [setup, bazel] + if: needs.setup.outputs.versions != '[]' + + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + # Upstream supports 3.11-3.14 and publishes no free-threaded wheel; 3.11 is + # dropped because array-record ships no cp311 riscv64 wheel. + # tag is a real dimension: legs introduced only through include collapse + # into a single job, keeping the last one (gotcha 402). + tag: [cp312, cp313, cp314] + include: + - {tag: cp312, python: '3.12'} + - {tag: cp313, python: '3.13'} + - {tag: cp314, python: '3.14'} + + env: + GRAIN_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout grain ${{ matrix.version }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: google/grain + ref: v${{ env.GRAIN_VERSION }} + path: grain + fetch-depth: 1 + persist-credentials: false + + - name: Download bazel + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: bazel-${{ env.BAZEL_VERSION }}-riscv64 + path: bazel-bin + + - name: Build wheel + env: + PYTHON_VERSION: ${{ matrix.python }} + PROJECT_RULES_PYTHON_VERSION: '1.6.0' + run: | + mkdir -p wheelhouse + set -o pipefail + docker run --rm -i --network=host \ + -v "${GITHUB_WORKSPACE}:/work" \ + -w /work \ + -e PYTHON_VERSION \ + -e PROJECT_RULES_PYTHON_VERSION \ + "${MANYLINUX_RISCV64_IMAGE}" \ + bash <<'SCRIPT' 2>&1 | tee build.log + set -eux + + dnf install -y --setopt=install_weak_deps=False java-21-openjdk-devel zip unzip rsync + JAVA_HOME="$(dirname "$(dirname "$(readlink -f "$(command -v javac)")")")" + export JAVA_HOME + install -m 0755 /work/bazel-bin/bazel /usr/local/bin/bazel + + PYTAG="cp${PYTHON_VERSION/./}" + export PYTHON_BIN="/opt/python/${PYTAG}-${PYTAG}/bin/python" + export PATH="/opt/python/${PYTAG}-${PYTAG}/bin:${PATH}" + "${PYTHON_BIN}" -m pip install -q -U setuptools wheel auditwheel + + # protobuf declares a pip hub for CPython 3.8, which python-build-standalone + # does not publish for riscv64, and rules_python aborts on the missing + # interpreter even though nothing uses that hub. + mkdir -p /tmp/rules_python + curl -fsSLo /tmp/rules_python.tar.gz \ + "https://github.com/bazel-contrib/rules_python/releases/download/${PROJECT_RULES_PYTHON_VERSION}/rules_python-${PROJECT_RULES_PYTHON_VERSION}.tar.gz" + tar -xzf /tmp/rules_python.tar.gz -C /tmp/rules_python --strip-components=1 + python3 - <<'PATCH' + path = "/tmp/rules_python/python/private/pypi/extension.bzl" + source = open(path).read() + old = " if python_name not in available_interpreters:\n fail((" + new = ( + " if python_name not in available_interpreters:\n" + " python_name = sorted(available_interpreters)[0]\n" + " if False:\n" + " fail((" + ) + assert old in source + open(path, "w").write(source.replace(old, new, 1)) + PATCH + + cd /work/grain + + rm -f .bazelrc + cat > .bazelrc <<'RC' + build --incompatible_default_to_explicit_init_py + build --enable_platform_specific_config + build --cxxopt=-Wno-deprecated-declarations --host_cxxopt=-Wno-deprecated-declarations + build --cxxopt=-Wno-parentheses --host_cxxopt=-Wno-parentheses + build --cxxopt=-Wno-sign-compare --host_cxxopt=-Wno-sign-compare + common --check_direct_dependencies=error + RC + { + echo "build --@rules_python//python/config_settings:python_version=${PYTHON_VERSION}" + echo "build --override_module=rules_python=/tmp/rules_python" + # A long build's default curses progress output is large enough that + # GitHub drops the job log, taking the failure with it. + echo "common --curses=no --show_progress_rate_limit=60" + } >> .bazelrc + + unset PYTHONPATH + unset PYTHONHOME + + # Upstream builds `...`, which needs the @pypi hub; its lock pins jaxlib and + # scipy wheels that exist for no riscv64 and ship no sdist. These are the + # only targets whose output reaches the wheel. + bazel build \ + //grain/_src/python/experimental/index_shuffle/python:index_shuffle_module.so \ + //grain/_src/python/experimental/index_shuffle:index_shuffle \ + //grain/proto:execution_summary_py_pb2 \ + --action_env PYTHON_BIN_PATH="${PYTHON_BIN}" + + STAGE="$(mktemp -d)" + cp README.md setup.py pyproject.toml LICENSE "${STAGE}" + rsync -avm -L --exclude="__pycache__/*" grain "${STAGE}" + rsync -avm -L --include="*.so" --include="*_pb2.py" \ + --exclude="*.runfiles" --exclude="*_obj" --include="*/" --exclude="*" \ + bazel-bin/grain "${STAGE}" + + # The extension statically links these; upstream ships neither licence, so + # collect them from the sources bazel built. + OUTPUT_BASE="$(bazel info output_base)" + for repo in abseil-cpp pybind11; do + dir="$(find "${OUTPUT_BASE}/external" -mindepth 1 -maxdepth 1 -type d \ + \( -name "${repo}" -o -name "${repo}~" -o -name "*~${repo}" \) | head -1)" + file="$(find "${dir}" -maxdepth 1 -type f -iname 'LICENSE*' | head -1)" + cp "${file}" "${STAGE}/LICENSE.${repo}" + done + + cd "${STAGE}" + "${PYTHON_BIN}" setup.py bdist_wheel --python-tag "py3${PYTHON_VERSION#*.}" + auditwheel repair --plat manylinux_2_39_riscv64 -w /work/wheelhouse dist/*.whl + SCRIPT + + - name: Upload build log + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: grain-${{ env.GRAIN_VERSION }}-${{ matrix.tag }}-build-log + path: build.log + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: grain-${{ env.GRAIN_VERSION }}-${{ matrix.tag }}-manylinux_riscv64 + path: wheelhouse/*.whl + if-no-files-found: error + + - name: Test wheel + env: + PYTHON_VERSION: ${{ matrix.python }} + PIP_EXTRA_INDEX_URL: https://pypi.riseproject.dev/simple/ + run: | + docker run --rm -i --network=host \ + -v "${GITHUB_WORKSPACE}:/work" \ + -w /work \ + -e PYTHON_VERSION \ + -e PIP_EXTRA_INDEX_URL \ + "${MANYLINUX_RISCV64_IMAGE}" \ + bash <<'SCRIPT' + set -eux + + PYTAG="cp${PYTHON_VERSION/./}" + PYTHON_BIN="/opt/python/${PYTAG}-${PYTAG}/bin/python" + "${PYTHON_BIN}" -m pip install /work/wheelhouse/*.whl + + # jax is the one test requirement with no riscv64 build: it resolves jaxlib, + # which ships no riscv64 wheel and no sdist. + grep -v '^jax$' /work/grain/test_requirements.in > /tmp/test_requirements.in + "${PYTHON_BIN}" -m pip install -r /tmp/test_requirements.in --only-binary pyarrow + + SITE="$("${PYTHON_BIN}" -c 'import grain, pathlib; print(pathlib.Path(grain.__file__).parent)')" + + "${PYTHON_BIN}" - <<'PY' + import importlib.metadata + + from grain._src.python.experimental.index_shuffle.python import index_shuffle_module + + assert index_shuffle_module.__file__.endswith(".so"), index_shuffle_module.__file__ + assert index_shuffle_module.index_shuffle(3, 10, 7, 4) in range(10) + + licenses = { + str(f).rsplit("/", 1)[1] + for f in importlib.metadata.files("grain") + if ".dist-info/licenses/" in str(f) + } + assert licenses == {"LICENSE", "LICENSE.abseil-cpp", "LICENSE.pybind11"}, licenses + PY + + # what upstream's profiler_test_no_framework target sets + export EXPECTED_FRAMEWORK=NO_FRAMEWORK + + # One process per file, as upstream's py_test targets run: a single pytest + # over the package leaks state between files. + # /work holds the checkout, whose root would shadow the installed wheel. + cd /tmp + : > /tmp/failed + find "${SITE}" -name '*_test.py' | sort | while read -r test; do + case "$(basename "${test}")" in + # need jax or tensorflow, neither of which builds for riscv64 + base_test.py | batch_test.py | device_put_test.py | \ + jax_import_smoke_test.py | packing_test.py | \ + shared_memory_array_test.py | tf_import_smoke_test.py | \ + tree_lib_jax_test.py) continue ;; + # upstream declares no test target for it and its __main__ calls an + # undefined name + multiprocessing_test.py) continue ;; + esac + # by module, not by path: bazel puts the runfiles root on sys.path, and + # ipc/queue.py shadows the stdlib queue if its own directory goes first + module="grain.$(printf '%s' "${test#"${SITE}/"}" | sed 's/\.py$//; s|/|.|g')" + case "$(basename "${test}")" in + # the only three targets upstream passes this to; the files that do + # not use absltest reject the flag + data_loader_test.py | data_sources_test.py | tfrecord_dataset_test.py) + "${PYTHON_BIN}" -m "${module}" --test_srcdir="${SITE}/_src/python" \ + || echo "${module}" >> /tmp/failed ;; + # its two execution-summary tests wait on a summary-logging thread that + # only reports under upstream's own runner; pytest is what makes the + # RUN_IN_PYTEST expectedFailure grain ships for one of them apply + dataset_test.py) + "${PYTHON_BIN}" -m pytest "${test}" -q -k 'not test_execution_summary' \ + || echo "${module}" >> /tmp/failed ;; + *) + "${PYTHON_BIN}" -m "${module}" || echo "${module}" >> /tmp/failed ;; + esac + done + if [ -s /tmp/failed ]; then + echo 'failed:' + cat /tmp/failed + exit 1 + fi + SCRIPT + + publish: + name: Publish grain ${{ matrix.version }} + needs: [setup, build_wheels] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + permissions: + contents: write + pull-requests: write + uses: $/.github/workflows/_publish-wheel.yml + secrets: + app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + with: + artifact-pattern: grain-${{ matrix.version }}-*-manylinux_riscv64 diff --git a/docs/packages/grain.yaml b/docs/packages/grain.yaml new file mode 100644 index 00000000000..86fffb99203 --- /dev/null +++ b/docs/packages/grain.yaml @@ -0,0 +1,5 @@ +package-name: grain +source-code: https://github.com/google/grain +license: Apache-2.0 +versions: +- version: 0.2.18