docs: Update projects #16
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-FileCopyrightText: 2026 The RISE Project | |
| # SPDX-License-Identifier: MIT | |
| --- | |
| # This workflow is based on: | |
| # https://github.com/meta-pytorch/tokenizers/blob/v1.4.1/.github/workflows/build-wheels-linux.yml | |
| name: Build pytorch-tokenizers wheels (riscv64) | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| version: | |
| description: 'Version glob to (re)build; empty builds every version of docs/packages/pytorch-tokenizers.yaml not released yet' | |
| required: false | |
| default: '' | |
| pull_request: | |
| branches: [main] | |
| paths: | |
| - '.github/workflows/build-pytorch-tokenizers.yml' | |
| - 'docs/packages/pytorch-tokenizers.yaml' | |
| push: | |
| branches: [main] | |
| paths: | |
| - '.github/workflows/build-pytorch-tokenizers.yml' | |
| - 'docs/packages/pytorch-tokenizers.yaml' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| env: | |
| MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 | |
| # The tag CMakeLists.txt's FetchContent_Declare(pybind11) pins. | |
| PYBIND11_REF: v2.13.6 | |
| jobs: | |
| setup: | |
| uses: $/.github/workflows/_setup.yml | |
| with: | |
| package: pytorch-tokenizers | |
| version: ${{ inputs.version }} | |
| build_wheels: | |
| needs: [setup] | |
| if: needs.setup.outputs.versions != '[]' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: ${{ fromJSON(needs.setup.outputs.versions) }} | |
| python: ["cp312", "cp313", "cp314"] | |
| name: Build pytorch-tokenizers ${{ matrix.version }} ${{ matrix.python }}-manylinux_riscv64 | |
| runs-on: ubuntu-24.04-riscv | |
| timeout-minutes: 360 | |
| env: | |
| PYTORCH_TOKENIZERS_VERSION: ${{ matrix.version }} | |
| steps: | |
| - name: Checkout tokenizers v${{ env.PYTORCH_TOKENIZERS_VERSION }} | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| repository: meta-pytorch/tokenizers | |
| ref: v${{ env.PYTORCH_TOKENIZERS_VERSION }} | |
| submodules: recursive | |
| persist-credentials: false | |
| # Every tree below is compiled into the extension, and setuptools' default licence | |
| # glob only reaches the project root. | |
| - name: Stage the vendored licences | |
| run: | | |
| set -euo pipefail | |
| cp third-party/abseil-cpp/LICENSE LICENSE.abseil-cpp | |
| cp third-party/re2/LICENSE LICENSE.re2 | |
| cp third-party/sentencepiece/LICENSE LICENSE.sentencepiece | |
| cp third-party/sentencepiece/third_party/darts_clone/LICENSE LICENSE.darts-clone | |
| cp third-party/sentencepiece/third_party/esaxx/LICENSE LICENSE.esaxx | |
| cp third-party/sentencepiece/third_party/protobuf-lite/LICENSE LICENSE.protobuf-lite | |
| cp third-party/pcre2/LICENCE.md LICENSE.pcre2 | |
| cp third-party/json/LICENSE.MIT LICENSE.nlohmann-json | |
| sed -n '/^MIT License$/,/^SOFTWARE\.$/p' third-party/llama.cpp-unicode/src/unicode.cpp > LICENSE.llama.cpp-unicode | |
| curl -fsSL --retry 5 "https://raw.githubusercontent.com/pybind/pybind11/${PYBIND11_REF}/LICENSE" -o LICENSE.pybind11 | |
| - uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0 | |
| with: | |
| only: ${{ matrix.python }}-manylinux_riscv64 | |
| env: | |
| CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }} | |
| CIBW_ENVIRONMENT: >- | |
| CMAKE_BUILD_PARALLEL_LEVEL=8 | |
| PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/ | |
| # Upstream's smoke test (.ci/scripts/wheel/test_wheel.py) copies test/ into a | |
| # temp directory and runs pytest there, which is what test-sources reproduces. | |
| CIBW_TEST_SOURCES: test | |
| CIBW_TEST_REQUIRES: pytest transformers | |
| CIBW_TEST_COMMAND: python -m pytest -vv test | |
| - name: Check the extension and the vendored licences made it into the wheel | |
| run: | | |
| python3 - wheelhouse/*.whl <<'EOF' | |
| import sys, zipfile | |
| expected = {"LICENSE", "LICENSE.abseil-cpp", "LICENSE.darts-clone", "LICENSE.esaxx", | |
| "LICENSE.llama.cpp-unicode", "LICENSE.nlohmann-json", "LICENSE.pcre2", | |
| "LICENSE.protobuf-lite", "LICENSE.pybind11", "LICENSE.re2", | |
| "LICENSE.sentencepiece"} | |
| for whl in sys.argv[1:]: | |
| names = zipfile.ZipFile(whl).namelist() | |
| assert [n for n in names if n.startswith("pytorch_tokenizers/pytorch_tokenizers_cpp.") | |
| and n.endswith(".so")], whl | |
| licenses = {n.split(".dist-info/licenses/", 1)[1] for n in names | |
| if ".dist-info/licenses/" in n and not n.endswith("/")} | |
| assert licenses == expected, f"{whl}: {sorted(licenses)}" | |
| print(whl, "ok") | |
| EOF | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: pytorch-tokenizers-${{ env.PYTORCH_TOKENIZERS_VERSION }}-${{ matrix.python }}-manylinux_riscv64 | |
| path: wheelhouse/*.whl | |
| if-no-files-found: error | |
| publish: | |
| name: Publish pytorch-tokenizers ${{ matrix.version }} | |
| needs: [setup, build_wheels] | |
| if: needs.setup.outputs.versions != '[]' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: ${{ fromJSON(needs.setup.outputs.versions) }} | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| uses: $/.github/workflows/_publish-wheel.yml | |
| secrets: | |
| app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} | |
| with: | |
| artifact-pattern: pytorch-tokenizers-${{ matrix.version }}-*-manylinux_riscv64 |