diff --git a/.github/workflows/kernels_data_python_release.yaml b/.github/workflows/kernels_data_python_release.yaml deleted file mode 100644 index 706a47133..000000000 --- a/.github/workflows/kernels_data_python_release.yaml +++ /dev/null @@ -1,237 +0,0 @@ -# This file is autogenerated by maturin v1.13.1 -# To update, run -# -# maturin generate-ci github -m kernels-data/bindings/python/Cargo.toml -o .github/workflows/kernels_data_python_release.yaml -# -# Post-generation edits: -# - manifest paths made relative -# - publish step switched from `uv publish` + API token to -# `PyO3/maturin-action` + PyPI trusted publishing (see plan.md ยง3a, ยง3b) -# - pull_request trigger gained `paths-ignore` for docs-only PRs -# - concurrency block added (cancel-in-progress only for PRs, never on tag/main) -# - free-threaded 3.13t build steps removed: not supported by maturin -# anymore. -name: CI - -on: - push: - branches: - - main - - master - tags: - - "*" - pull_request: - paths-ignore: - - "docs/**" - - "*.md" - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -permissions: - contents: read - -jobs: - linux: - runs-on: ${{ matrix.platform.runner }} - strategy: - matrix: - platform: - - runner: ubuntu-22.04 - target: x86_64 - - runner: ubuntu-22.04 - target: x86 - - runner: ubuntu-22.04 - target: aarch64 - - runner: ubuntu-22.04 - target: armv7 - - runner: ubuntu-22.04 - target: s390x - - runner: ubuntu-22.04 - target: ppc64le - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: 3.8 - - name: Build wheels - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path kernels-data/bindings/python/Cargo.toml - sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} - manylinux: auto - - name: Build free-threaded wheels (3.14t) - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path kernels-data/bindings/python/Cargo.toml -i python3.14t - sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} - manylinux: auto - - name: Upload wheels - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: wheels-linux-${{ matrix.platform.target }} - path: dist - - musllinux: - runs-on: ${{ matrix.platform.runner }} - strategy: - matrix: - platform: - - runner: ubuntu-22.04 - target: x86_64 - - runner: ubuntu-22.04 - target: x86 - - runner: ubuntu-22.04 - target: aarch64 - - runner: ubuntu-22.04 - target: armv7 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: 3.8 - - name: Build wheels - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path kernels-data/bindings/python/Cargo.toml - sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} - manylinux: musllinux_1_2 - - name: Build free-threaded wheels (3.14t) - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path kernels-data/bindings/python/Cargo.toml -i python3.14t - sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} - manylinux: musllinux_1_2 - - name: Upload wheels - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: wheels-musllinux-${{ matrix.platform.target }} - path: dist - - windows: - runs-on: ${{ matrix.platform.runner }} - strategy: - matrix: - platform: - - runner: windows-latest - target: x64 - python_arch: x64 - - runner: windows-latest - target: x86 - python_arch: x86 - - runner: windows-11-arm - target: aarch64 - python_arch: arm64 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: 3.13 - architecture: ${{ matrix.platform.python_arch }} - - name: Build wheels - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path kernels-data/bindings/python/Cargo.toml - sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - if: matrix.platform.target != 'aarch64' - with: - python-version: 3.14t - architecture: ${{ matrix.platform.python_arch }} - - name: Build free-threaded wheels (3.14t) - if: matrix.platform.target != 'aarch64' - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path kernels-data/bindings/python/Cargo.toml -i python3.14t - sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} - - name: Upload wheels - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: wheels-windows-${{ matrix.platform.target }} - path: dist - - macos: - runs-on: ${{ matrix.platform.runner }} - strategy: - matrix: - platform: - - runner: macos-15-intel - target: x86_64 - - runner: macos-latest - target: aarch64 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: 3.8 - - name: Build wheels - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path kernels-data/bindings/python/Cargo.toml - sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: 3.14t - - name: Build free-threaded wheels (3.14t) - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path kernels-data/bindings/python/Cargo.toml -i python3.14t - sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} - - name: Upload wheels - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: wheels-macos-${{ matrix.platform.target }} - path: dist - - sdist: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Build sdist - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 - with: - command: sdist - args: --out dist --manifest-path kernels-data/bindings/python/Cargo.toml - - name: Upload sdist - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: wheels-sdist - path: dist - - release: - name: Release - runs-on: ubuntu-latest - if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} - needs: [linux, musllinux, windows, macos, sdist] - environment: - name: pypi - url: https://pypi.org/p/kernels-data - permissions: - # Use to sign the release artifacts - id-token: write - # Used to upload release artifacts - contents: write - # Used to generate artifact attestation - attestations: write - steps: - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - - name: Generate artifact attestation - uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 - with: - subject-path: "wheels-*/*" - - name: Publish to PyPI - if: ${{ startsWith(github.ref, 'refs/tags/') }} - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 - with: - command: upload - args: --non-interactive --skip-existing wheels-*/* diff --git a/.github/workflows/publish_kernels.yml b/.github/workflows/publish_kernels.yml index ddaae9ad7..0c42f7f7f 100644 --- a/.github/workflows/publish_kernels.yml +++ b/.github/workflows/publish_kernels.yml @@ -1,40 +1,238 @@ +# This file is autogenerated by maturin v1.13.1 +# To update, run +# +# maturin generate-ci github -m kernels/Cargo.toml -o .github/workflows/publish_kernels.yaml +# +# Post-generation edits: +# - manifest paths made relative +# - publish step switched from `uv publish` + API token to +# `PyO3/maturin-action` + PyPI trusted publishing (see plan.md ยง3a, ยง3b) +# - pull_request trigger gained `paths-ignore` for docs-only PRs +# - concurrency block added (cancel-in-progress only for PRs, never on tag/main) +# - free-threaded 3.13t build steps removed: not supported by maturin +# anymore. +# - publish-to-testpypi gated on `github.event_name != 'pull_request'`, so +# PR runs (same-repo or forked) only build and never publish. +# - persist-credentials: false on all build jobs' checkout steps, since they run +# on pull_request (including fork PRs) and build untrusted code via +# build.rs/proc-macros. name: Publish Python ๐Ÿ distribution ๐Ÿ“ฆ to PyPI and TestPyPI -on: push +on: + push: + branches: + - main + - master + tags: + - "*" + pull_request: + paths-ignore: + - "docs/**" + - "*.md" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read jobs: - build: - name: Build distribution ๐Ÿ“ฆ - runs-on: ubuntu-latest + linux: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: ubuntu-22.04 + target: x86_64 + - runner: ubuntu-22.04 + target: x86 + - runner: ubuntu-22.04 + target: aarch64 + - runner: ubuntu-22.04 + target: armv7 + - runner: ubuntu-22.04 + target: s390x + - runner: ubuntu-22.04 + target: ppc64le + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: 3.8 + - name: Build wheels + uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --manifest-path kernels/Cargo.toml + sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} + manylinux: auto + - name: Build free-threaded wheels (3.14t) + uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --manifest-path kernels/Cargo.toml -i python3.14t + sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} + manylinux: auto + - name: Upload wheels + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: wheels-linux-${{ matrix.platform.target }} + path: dist + musllinux: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: ubuntu-22.04 + target: x86_64 + - runner: ubuntu-22.04 + target: x86 + - runner: ubuntu-22.04 + target: aarch64 + - runner: ubuntu-22.04 + target: armv7 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: - python-version: "3.9" - - name: Install pypa/build - run: >- - python3 -m - pip install - build - --user - - name: Build a binary wheel and a source tarball - run: ( cd kernels && python3 -m build ) - - name: Store the distribution packages - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: python-package-distributions - path: kernels/dist/ + python-version: 3.8 + - name: Build wheels + uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --manifest-path kernels/Cargo.toml + sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} + manylinux: musllinux_1_2 + - name: Build free-threaded wheels (3.14t) + uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --manifest-path kernels/Cargo.toml -i python3.14t + sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} + manylinux: musllinux_1_2 + - name: Upload wheels + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: wheels-musllinux-${{ matrix.platform.target }} + path: dist + + windows: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: windows-latest + target: x64 + python_arch: x64 + - runner: windows-latest + target: x86 + python_arch: x86 + - runner: windows-11-arm + target: aarch64 + python_arch: arm64 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: 3.13 + architecture: ${{ matrix.platform.python_arch }} + - name: Build wheels + uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --manifest-path kernels/Cargo.toml + sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + if: matrix.platform.target != 'aarch64' + with: + python-version: 3.14t + architecture: ${{ matrix.platform.python_arch }} + - name: Build free-threaded wheels (3.14t) + if: matrix.platform.target != 'aarch64' + uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --manifest-path kernels/Cargo.toml -i python3.14t + sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} + - name: Upload wheels + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: wheels-windows-${{ matrix.platform.target }} + path: dist + + macos: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: macos-15-intel + target: x86_64 + - runner: macos-latest + target: aarch64 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: 3.8 + - name: Build wheels + uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --manifest-path kernels/Cargo.toml + sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: 3.14t + - name: Build free-threaded wheels (3.14t) + uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --manifest-path kernels/Cargo.toml -i python3.14t + sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} + - name: Upload wheels + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: wheels-macos-${{ matrix.platform.target }} + path: dist + + sdist: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Build sdist + uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 + with: + command: sdist + args: --out dist --manifest-path kernels/Cargo.toml + - name: Upload sdist + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: wheels-sdist + path: dist publish-to-pypi: name: >- Publish Python ๐Ÿ distribution ๐Ÿ“ฆ to PyPI if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes needs: - - build + - linux + - musllinux + - windows + - macos + - sdist runs-on: ubuntu-latest environment: name: pypi @@ -44,12 +242,13 @@ jobs: steps: - name: Download all the dists - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: python-package-distributions - path: dist/ + pattern: wheels-* + path: dist + merge-multiple: true - name: Publish distribution ๐Ÿ“ฆ to PyPI - uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247 # release/v1 + uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247 # release/v1 github-release: name: >- @@ -65,12 +264,13 @@ jobs: steps: - name: Download all the dists - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: python-package-distributions - path: dist/ + pattern: wheels-* + path: dist + merge-multiple: true - name: Sign the dists with Sigstore - uses: sigstore/gh-action-sigstore-python@5b79a39c381910c090341a2c9b0bf022c8b387e1 # v3.4.0 + uses: sigstore/gh-action-sigstore-python@5b79a39c381910c090341a2c9b0bf022c8b387e1 # v3.4.0 with: inputs: >- ./dist/*.tar.gz @@ -96,8 +296,13 @@ jobs: publish-to-testpypi: name: Publish Python ๐Ÿ distribution ๐Ÿ“ฆ to TestPyPI + if: github.event_name != 'pull_request' needs: - - build + - linux + - musllinux + - windows + - macos + - sdist runs-on: ubuntu-latest environment: @@ -109,12 +314,13 @@ jobs: steps: - name: Download all the dists - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: python-package-distributions - path: dist/ + pattern: wheels-* + path: dist + merge-multiple: true - name: Publish distribution ๐Ÿ“ฆ to TestPyPI - uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247 # release/v1 + uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247 # release/v1 with: repository-url: https://test.pypi.org/legacy/ skip-existing: true # Only upload when the version is unique. diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml index 2fddfdeb7..ba9326fed 100644 --- a/.github/workflows/rust.yaml +++ b/.github/workflows/rust.yaml @@ -26,7 +26,7 @@ jobs: - name: Cargo fmt (kernels-data) run: | ( cd kernels-data && cargo fmt --all -- --check ) - ( cd kernels-data/bindings/python && cargo fmt --all -- --check ) + ( cd kernels && cargo fmt --all -- --check ) - name: Cargo fmt (kernel-builder) run: ( cd kernel-builder && cargo fmt --all -- --check ) @@ -53,7 +53,7 @@ jobs: - name: Clippy (kernels-data) run: | ( cd kernels-data && cargo clippy -- -D warnings ) - ( cd kernels-data/bindings/python && cargo clippy -- -D warnings ) + ( cd kernels && cargo clippy -- -D warnings ) - name: Clippy (kernel-builder) run: ( cd kernel-builder && cargo clippy -- -D warnings ) diff --git a/.github/workflows/test_e2e.yaml b/.github/workflows/test_e2e.yaml index 3bd307591..b7f61aed1 100644 --- a/.github/workflows/test_e2e.yaml +++ b/.github/workflows/test_e2e.yaml @@ -8,6 +8,7 @@ on: paths: - "kernel-builder/**" - "kernels/src/**" + - "kernels/rust/**" - "nix-builder/**" - "kernels-data/**" - ".github/workflows/test_e2e.yaml" @@ -123,6 +124,9 @@ jobs: with: python-version: "3.12" + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 + - name: Install Python deps working-directory: ./kernels run: | @@ -130,8 +134,6 @@ jobs: CUDA_TAG=$(echo "$VARIANT" | grep -oP 'cu\d+') echo "Installing torch matching variant $VARIANT (CUDA tag: $CUDA_TAG)" uv sync --all-extras --dev - # E2E validates this checkout, so use the local kernels-data binding instead of the released wheel from uv.lock. - uv pip install --reinstall ../kernels-data/bindings/python uv pip install --upgrade torch --index-url https://download.pytorch.org/whl/$CUDA_TAG uv run --no-sync python -c "import torch; print(f'torch={torch.__version__}, cuda={torch.version.cuda}, cxx11_abi={torch.compiled_with_cxx11_abi()}')" diff --git a/.github/workflows/test_kernels.yaml b/.github/workflows/test_kernels.yaml index e63b014e1..cdc71bd98 100644 --- a/.github/workflows/test_kernels.yaml +++ b/.github/workflows/test_kernels.yaml @@ -44,9 +44,8 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Override kernels-data source to local bindings - working-directory: ./kernels - run: uv add ../kernels-data/bindings/python --no-sync + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 - name: Override the Torch version working-directory: ./kernels @@ -64,6 +63,10 @@ jobs: working-directory: ./kernels run: uv run mypy src/kernels + - name: Check type stubs match runtime + working-directory: ./kernels + run: uv run python -m mypy.stubtest kernels._data + - name: Run tests working-directory: ./kernels env: diff --git a/.github/workflows/test_python.yaml b/.github/workflows/test_python.yaml deleted file mode 100644 index 103caf430..000000000 --- a/.github/workflows/test_python.yaml +++ /dev/null @@ -1,45 +0,0 @@ -name: Test Python bindings - -on: - push: - branches: [main] - pull_request: - branches: [main] - types: [opened, synchronize, reopened] # trigger on PRs - paths-ignore: - - "docs/**" - - "*.md" - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - kernels-data: - name: kernels-data bindings - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.12"] - - env: - UV_PYTHON_PREFERENCE: only-managed - - steps: - - name: Checkout code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Install uv and set the python version - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 - with: - python-version: ${{ matrix.python-version }} - - - name: Install the project - run: ( cd kernels-data/bindings/python && uv sync --all-extras --dev) - - - name: Run tests - run: ( cd kernels-data/bindings/python && uv run pytest tests ) - - - name: Check type stubs match runtime - run: ( cd kernels-data/bindings/python && uv run python -m mypy.stubtest kernels_data --allowlist stubtest-allowlist.txt ) diff --git a/Cargo.lock b/Cargo.lock index aa2188816..487875ebb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1056,7 +1056,7 @@ dependencies = [ [[package]] name = "hf-kernel-builder" -version = "0.17.0-dev0" +version = "0.17.0-dev1" dependencies = [ "base32", "built", @@ -1537,7 +1537,7 @@ dependencies = [ [[package]] name = "kernel-abi-check" -version = "0.17.0-dev0" +version = "0.17.0-dev1" dependencies = [ "cpp_demangle", "eyre", @@ -1551,7 +1551,7 @@ dependencies = [ [[package]] name = "kernels-data" -version = "0.17.0-dev0" +version = "0.17.0-dev1" dependencies = [ "base64", "digest", @@ -1571,8 +1571,8 @@ dependencies = [ ] [[package]] -name = "kernels-data-python" -version = "0.17.0-dev0" +name = "kernels-python" +version = "0.17.0-dev1" dependencies = [ "kernels-data", "pyo3", diff --git a/Cargo.toml b/Cargo.toml index 3a1f79264..780fd294e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ members = [ "kernel-abi-check", "kernel-builder", + "kernels", "kernels-data", - "kernels-data/bindings/python", ] resolver = "2" diff --git a/Makefile b/Makefile index b19055d87..0c5f4bb86 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: style kernel-builder-cli-docs quality bump-dev bump-dev-dry-run bump-dev-major bump-dev-major-dry-run bump-release bump-release-dry-run bump-major bump-major-dry-run pin-actions -export check_dirs := kernels/src kernels/tests kernels-data/bindings/python +export check_dirs := kernels/src kernels/tests all: kernels/src/kernels/python_depends.json diff --git a/flake.nix b/flake.nix index b42ce72e7..37ea433d7 100644 --- a/flake.nix +++ b/flake.nix @@ -177,8 +177,8 @@ huggingface-hub jax jax-tvm-ffi - kernels-data matplotlib + maturin mktestdocs openssl.dev pytest @@ -201,7 +201,7 @@ postVenvCreation = '' unset SOURCE_DATE_EPOCH - ( python -m pip install --no-build-isolation --no-dependencies -e kernels ) + ( cd kernels && maturin develop ) pre-commit install ''; diff --git a/kernel-abi-check/Cargo.toml b/kernel-abi-check/Cargo.toml index dd20c75ea..6cc8a80ad 100644 --- a/kernel-abi-check/Cargo.toml +++ b/kernel-abi-check/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "kernel-abi-check" -version = "0.17.0-dev0" +version = "0.17.0-dev1" edition = "2021" description = "Check the ABI of Hub Kernels" homepage = "https://github.com/huggingface/kernel-builder" diff --git a/kernel-builder/Cargo.toml b/kernel-builder/Cargo.toml index ada512ff2..7b354047e 100644 --- a/kernel-builder/Cargo.toml +++ b/kernel-builder/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hf-kernel-builder" -version = "0.17.0-dev0" +version = "0.17.0-dev1" edition = "2021" description = "Build Hugging Face Hub kernels" homepage = "https://github.com/huggingface/kernels" @@ -10,8 +10,8 @@ repository = "https://github.com/huggingface/kernels" [dependencies] base32 = "0.5" -kernel-abi-check = { path = "../kernel-abi-check", version = "0.17.0-dev0" } -kernels-data = { path = "../kernels-data", version = "0.17.0-dev0" } +kernel-abi-check = { path = "../kernel-abi-check", version = "0.17.0-dev1" } +kernels-data = { path = "../kernels-data", version = "0.17.0-dev1" } object = "0.36.7" clap = { version = "4", features = ["derive"] } clap-markdown = "0.1.5" diff --git a/kernels-data/Cargo.toml b/kernels-data/Cargo.toml index 0205c5b89..51d5aacba 100644 --- a/kernels-data/Cargo.toml +++ b/kernels-data/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "kernels-data" -version = "0.17.0-dev0" +version = "0.17.0-dev1" edition = "2024" description = "Kernels data structures" homepage = "https://github.com/huggingface/kernels" diff --git a/kernels-data/bindings/python/MANIFEST.in b/kernels-data/bindings/python/MANIFEST.in deleted file mode 100644 index 5444e43d7..000000000 --- a/kernels-data/bindings/python/MANIFEST.in +++ /dev/null @@ -1,4 +0,0 @@ -include Cargo.toml -include pyproject.toml -include ../../../LICENSE -recursive-include src * diff --git a/kernels-data/bindings/python/pyproject.toml b/kernels-data/bindings/python/pyproject.toml deleted file mode 100644 index 6d16671f7..000000000 --- a/kernels-data/bindings/python/pyproject.toml +++ /dev/null @@ -1,27 +0,0 @@ -[project] -name = "kernels-data" -requires-python = '>=3.8' -classifiers = [ - "Programming Language :: Rust", - "Programming Language :: Python :: Implementation :: CPython", -] -dynamic = [ - 'description', - 'readme', - 'version', -] - -[build-system] -requires = ["maturin>=1,<2"] -build-backend = "maturin" - -[dependency-groups] -dev = [ - "pytest>=8", - "mypy>=1.11", -] - -[tool.maturin] -module-name = "kernels_data" -bindings = 'pyo3' -features = ["pyo3/extension-module"] diff --git a/kernels-data/bindings/python/stubtest-allowlist.txt b/kernels-data/bindings/python/stubtest-allowlist.txt deleted file mode 100644 index 0bd36ed3a..000000000 --- a/kernels-data/bindings/python/stubtest-allowlist.txt +++ /dev/null @@ -1,14 +0,0 @@ -# The `.so` submodule `kernels_data.kernels_data` is a pyo3/maturin -# implementation detail; end users import from the `kernels_data` package. -kernels_data\.kernels_data - -# pyo3 complex enums compile to a Rust-backed (disjoint) base class. The -# `@disjoint_base` marker only lives in `typing_extensions`, which this -# dependency-free extension package does not pull in, so the stub omits it. -kernels_data\.DigestViolation -kernels_data\.KernelVersion - -# `DigestValidationError.violations` is attached as a per-instance attribute by -# the `Digest.validate` binding (via `setattr`), so it is not visible on the -# class object that stubtest inspects, even though instances always have it. -kernels_data\.DigestValidationError\.violations diff --git a/kernels-data/bindings/python/Cargo.toml b/kernels/Cargo.toml similarity index 73% rename from kernels-data/bindings/python/Cargo.toml rename to kernels/Cargo.toml index 9bb2feae2..6f8bcf8de 100644 --- a/kernels-data/bindings/python/Cargo.toml +++ b/kernels/Cargo.toml @@ -1,6 +1,6 @@ [package] -name = "kernels-data-python" -version = "0.17.0-dev0" +name = "kernels-python" +version = "0.17.0-dev1" edition = "2024" description = "Kernels data structures (Python bindings)" homepage = "https://github.com/huggingface/kernels" @@ -8,7 +8,8 @@ license = "Apache-2.0" repository = "https://github.com/huggingface/kernels" [lib] -name = "kernels_data" +name = "_data" +path = "rust/lib.rs" crate-type = ["cdylib"] [dependencies] @@ -16,5 +17,5 @@ pyo3 = { version = "0.26", features = ["abi3", "abi3-py38"] } serde_json = "1" [dependencies.kernels-data] -path = "../.." -version = "0.17.0-dev0" +path = "../kernels-data" +version = "0.17.0-dev1" diff --git a/kernels/pyproject.toml b/kernels/pyproject.toml index a31b5d938..617698bf6 100644 --- a/kernels/pyproject.toml +++ b/kernels/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "kernels" -version = "0.17.0.dev0" +version = "0.17.0.dev1" description = "Download compute kernels" authors = [ { name = "Daniel de Kok", email = "daniel@huggingface.co" }, @@ -11,7 +11,6 @@ readme = "README.md" requires-python = ">= 3.10" dependencies = [ "huggingface-hub>=1.10.0", - "kernels-data>=0.14.0.dev1", "packaging>=20.0", "pyyaml>=6", "sigstore>=4,<5", @@ -21,8 +20,8 @@ dependencies = [ ] [build-system] -requires = ["setuptools"] -build-backend = "setuptools.build_meta" +requires = ["maturin>=1,<2"] +build-backend = "maturin" [dependency-groups] dev = [ @@ -66,8 +65,11 @@ kernels = "kernels.cli:main" [project.entry-points."egg_info.writers"] "kernels.lock" = "kernels.lockfile:write_egg_lockfile" -[tool.setuptools.package-data] -kernels = ["python_depends.json"] +[tool.maturin] +module-name = "kernels._data" +python-source = "src" +bindings = 'pyo3' +features = ["pyo3/extension-module"] [tool.ruff] # If the version is changed, apply the change in the Nix overlay diff --git a/kernels-data/bindings/python/src/config.rs b/kernels/rust/config.rs similarity index 100% rename from kernels-data/bindings/python/src/config.rs rename to kernels/rust/config.rs diff --git a/kernels-data/bindings/python/src/lib.rs b/kernels/rust/lib.rs similarity index 99% rename from kernels-data/bindings/python/src/lib.rs rename to kernels/rust/lib.rs index 315c71332..821cd247d 100644 --- a/kernels-data/bindings/python/src/lib.rs +++ b/kernels/rust/lib.rs @@ -658,7 +658,7 @@ impl PyDigestViolation { } pyo3::create_exception!( - kernels_data, + _data, DigestValidationError, PyException, "Raised by `Digest.validate` when the actual digest does not match the \ @@ -750,8 +750,8 @@ impl PyDigest { } } -#[pyo3::pymodule(name = "kernels_data")] -fn kernels_data_py(m: &PyBound<'_, PyModule>) -> PyResult<()> { +#[pyo3::pymodule(name = "_data")] +fn data_py(m: &PyBound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/kernels-data/bindings/python/src/lock.rs b/kernels/rust/lock.rs similarity index 100% rename from kernels-data/bindings/python/src/lock.rs rename to kernels/rust/lock.rs diff --git a/kernels-data/bindings/python/src/version.rs b/kernels/rust/version.rs similarity index 100% rename from kernels-data/bindings/python/src/version.rs rename to kernels/rust/version.rs diff --git a/kernels/src/kernels/__init__.py b/kernels/src/kernels/__init__.py index cc6c31123..f05706411 100644 --- a/kernels/src/kernels/__init__.py +++ b/kernels/src/kernels/__init__.py @@ -2,8 +2,7 @@ __version__ = importlib.metadata.version("kernels") -from kernels_data import Metadata - +from kernels._data import Metadata from kernels._windows import _add_additional_dll_paths from kernels.benchmark import Benchmark from kernels.deps import get_kernel_dep diff --git a/kernels-data/bindings/python/kernels_data.pyi b/kernels/src/kernels/_data.pyi similarity index 95% rename from kernels-data/bindings/python/kernels_data.pyi rename to kernels/src/kernels/_data.pyi index 6acab2dfa..3c5263a34 100644 --- a/kernels-data/bindings/python/kernels_data.pyi +++ b/kernels/src/kernels/_data.pyi @@ -1,4 +1,4 @@ -"""Type stubs for kernels_data module.""" +"""Type stubs for the `kernels._data` module.""" import os from collections.abc import Iterator @@ -193,9 +193,7 @@ class Digest: """Source digest for a kernel build variant.""" @staticmethod - def hash_variant( - algorithm: DigestAlgorithm, variant_path: os.PathLike[str] | str - ) -> "Digest": + def hash_variant(algorithm: DigestAlgorithm, variant_path: os.PathLike[str] | str) -> "Digest": """Hash the files in `variant_path` using `algorithm`. Args: @@ -262,9 +260,7 @@ class DigestViolation: expected: str got: str __match_args__ = ("path", "expected", "got") - def __new__( - cls, path: str, expected: str, got: str - ) -> "DigestViolation.HashMismatch": ... + def __new__(cls, path: str, expected: str, got: str) -> "DigestViolation.HashMismatch": ... @final class AlgorithmMismatch(DigestViolation): @@ -277,9 +273,7 @@ class DigestViolation: expected: DigestAlgorithm got: DigestAlgorithm __match_args__ = ("expected", "got") - def __new__( - cls, expected: DigestAlgorithm, got: DigestAlgorithm - ) -> "DigestViolation.AlgorithmMismatch": ... + def __new__(cls, expected: DigestAlgorithm, got: DigestAlgorithm) -> "DigestViolation.AlgorithmMismatch": ... def __str__(self) -> str: ... @@ -390,9 +384,7 @@ class KernelLocks: def __contains__(self, dependency: object, /) -> bool: ... def __iter__(self) -> Iterator[KernelDependency]: ... - def get( - self, dependency: KernelDependency, default: Optional[KernelLock] = None - ) -> Optional[KernelLock]: + def get(self, dependency: KernelDependency, default: Optional[KernelLock] = None) -> Optional[KernelLock]: """Get the lock for `dependency`, or `default` if it is not locked.""" ... @@ -482,9 +474,7 @@ class NixKernelLocks: Behaves as a read-only mapping from `KernelDependency` to `NixKernelLock`. """ - def __new__( - cls, locks: dict[KernelDependency, NixKernelLock] - ) -> "NixKernelLocks": ... + def __new__(cls, locks: dict[KernelDependency, NixKernelLock]) -> "NixKernelLocks": ... def __len__(self) -> int: ... def __getitem__(self, dependency: KernelDependency, /) -> NixKernelLock: """Get the lock for `dependency`. @@ -496,9 +486,7 @@ class NixKernelLocks: def __contains__(self, dependency: object, /) -> bool: ... def __iter__(self) -> Iterator[KernelDependency]: ... - def get( - self, dependency: KernelDependency, default: Optional[NixKernelLock] = None - ) -> Optional[NixKernelLock]: + def get(self, dependency: KernelDependency, default: Optional[NixKernelLock] = None) -> Optional[NixKernelLock]: """Get the lock for `dependency`, or `default` if it is not locked.""" ... @@ -542,9 +530,7 @@ class KernelPaths: Behaves as a read-only mapping from `KernelDependency` to `pathlib.Path`. """ - def __new__( - cls, paths: dict[KernelDependency, os.PathLike[str] | str] - ) -> "KernelPaths": ... + def __new__(cls, paths: dict[KernelDependency, os.PathLike[str] | str]) -> "KernelPaths": ... def __len__(self) -> int: ... def __getitem__(self, dependency: KernelDependency, /) -> Path: """Get the path for `dependency`. diff --git a/kernels/src/kernels/_versions.py b/kernels/src/kernels/_versions.py index 9b12c5a57..67333b276 100644 --- a/kernels/src/kernels/_versions.py +++ b/kernels/src/kernels/_versions.py @@ -5,7 +5,8 @@ from huggingface_hub import constants from huggingface_hub.file_download import repo_folder_name from huggingface_hub.hf_api import GitRefInfo -from kernels_data import KernelDependency, KernelVersion + +from kernels._data import KernelDependency, KernelVersion logger = logging.getLogger(__name__) diff --git a/kernels/src/kernels/archs.py b/kernels/src/kernels/archs.py index a9346f588..30d5a2ce1 100644 --- a/kernels/src/kernels/archs.py +++ b/kernels/src/kernels/archs.py @@ -1,5 +1,4 @@ -from kernels_data import Backend, Metadata - +from kernels._data import Backend, Metadata from kernels.compat import has_torch diff --git a/kernels/src/kernels/cli/download.py b/kernels/src/kernels/cli/download.py index 7860a6eee..7dcde2acc 100644 --- a/kernels/src/kernels/cli/download.py +++ b/kernels/src/kernels/cli/download.py @@ -1,7 +1,6 @@ import sys -from kernels_data import KernelLocks - +from kernels._data import KernelLocks from kernels.hf_hub import CACHE_DIR, _get_hf_api from kernels.resolver import _BYTECODE_IGNORE_PATTERNS, resolve_hub_kernel diff --git a/kernels/src/kernels/cli/info.py b/kernels/src/kernels/cli/info.py index cfa020025..f5c3b177a 100644 --- a/kernels/src/kernels/cli/info.py +++ b/kernels/src/kernels/cli/info.py @@ -4,8 +4,8 @@ from typing import Any from huggingface_hub import constants -from kernels_data import Metadata +from kernels._data import Metadata from kernels._versions import _get_available_versions, resolve_version_spec_as_ref from kernels.hf_hub import CACHE_DIR, _get_hf_api from kernels.variants import ( diff --git a/kernels/src/kernels/cli/lock.py b/kernels/src/kernels/cli/lock.py index f7b19ff3e..9f456d34b 100644 --- a/kernels/src/kernels/cli/lock.py +++ b/kernels/src/kernels/cli/lock.py @@ -1,5 +1,4 @@ -from kernels_data import KernelDependency, KernelVersion - +from kernels._data import KernelDependency, KernelVersion from kernels.compat import tomllib from kernels.hf_hub import _get_hf_api from kernels.locking import extract_dependency_locks diff --git a/kernels/src/kernels/deps.py b/kernels/src/kernels/deps.py index 5fd480638..7b26df83f 100644 --- a/kernels/src/kernels/deps.py +++ b/kernels/src/kernels/deps.py @@ -4,8 +4,8 @@ from typing import TYPE_CHECKING, Generic, TypeVar from huggingface_hub.hf_api import HfApi -from kernels_data import KernelDependency, KernelVersion +from kernels._data import KernelDependency, KernelVersion from kernels.hf_hub import RepoInfo from kernels.importer import _import_from_path from kernels.resolver import ( diff --git a/kernels/src/kernels/importer.py b/kernels/src/kernels/importer.py index ccafea8ae..802607415 100644 --- a/kernels/src/kernels/importer.py +++ b/kernels/src/kernels/importer.py @@ -4,8 +4,7 @@ from pathlib import Path from types import ModuleType -from kernels_data import Metadata - +from kernels._data import Metadata from kernels.hf_hub import RepoInfo diff --git a/kernels/src/kernels/install.py b/kernels/src/kernels/install.py index 3db327ea8..88fd8dfcf 100644 --- a/kernels/src/kernels/install.py +++ b/kernels/src/kernels/install.py @@ -1,7 +1,6 @@ from pathlib import Path -from kernels_data import KernelDependency - +from kernels._data import KernelDependency from kernels._versions import revision_or_version from kernels.deps import resolve_kernel_tree from kernels.hf_hub import CACHE_DIR, _get_hf_api diff --git a/kernels/src/kernels/layer/func.py b/kernels/src/kernels/layer/func.py index 77040ee28..c8471cae0 100644 --- a/kernels/src/kernels/layer/func.py +++ b/kernels/src/kernels/layer/func.py @@ -5,8 +5,8 @@ from typing import TYPE_CHECKING, Protocol, Type from huggingface_hub import constants -from kernels_data import KernelDependency, KernelLocks +from kernels._data import KernelDependency, KernelLocks from kernels.resolver import LockedHubCacheResolver, LockedHubResolver from .._versions import select_revision_or_version diff --git a/kernels/src/kernels/layer/layer.py b/kernels/src/kernels/layer/layer.py index 10a4409c7..2f008a50b 100644 --- a/kernels/src/kernels/layer/layer.py +++ b/kernels/src/kernels/layer/layer.py @@ -9,8 +9,8 @@ from typing import TYPE_CHECKING, Callable, Protocol, Type from huggingface_hub import constants -from kernels_data import KernelDependency, KernelLocks +from kernels._data import KernelDependency, KernelLocks from kernels.resolver import LockedHubCacheResolver, LockedHubResolver from .._versions import select_revision_or_version diff --git a/kernels/src/kernels/load.py b/kernels/src/kernels/load.py index 93354fe83..d96b986a1 100644 --- a/kernels/src/kernels/load.py +++ b/kernels/src/kernels/load.py @@ -4,8 +4,8 @@ from types import ModuleType from huggingface_hub import HfApi, constants -from kernels_data import KernelDependency, KernelVersion +from kernels._data import KernelDependency, KernelVersion from kernels._versions import revision_or_version from kernels.deps import resolve_kernel_tree from kernels.hf_hub import _get_hf_api diff --git a/kernels/src/kernels/locking.py b/kernels/src/kernels/locking.py index 0a393c5b9..e821fb992 100644 --- a/kernels/src/kernels/locking.py +++ b/kernels/src/kernels/locking.py @@ -5,13 +5,13 @@ from types import ModuleType from huggingface_hub.hf_api import HfApi -from kernels_data import ( + +from kernels._data import ( KernelDependency, KernelLock, KernelLocks, Metadata, ) - from kernels._versions import resolve_kernel_version from kernels.compat import tomllib from kernels.hf_hub import CACHE_DIR, _check_trust_remote_code diff --git a/kernels/src/kernels/python_deps.py b/kernels/src/kernels/python_deps.py index 4723f418d..182830cb8 100644 --- a/kernels/src/kernels/python_deps.py +++ b/kernels/src/kernels/python_deps.py @@ -4,8 +4,8 @@ from pathlib import Path from huggingface_hub.dataclasses import strict -from kernels_data import Metadata +from kernels._data import Metadata from kernels.backends import Backend, _backend diff --git a/kernels/src/kernels/resolver.py b/kernels/src/kernels/resolver.py index 8221f79cd..0c9e6f24d 100644 --- a/kernels/src/kernels/resolver.py +++ b/kernels/src/kernels/resolver.py @@ -4,8 +4,8 @@ from huggingface_hub.errors import LocalEntryNotFoundError from huggingface_hub.hf_api import HfApi -from kernels_data import KernelDependency, KernelLocks, KernelPaths, Metadata +from kernels._data import KernelDependency, KernelLocks, KernelPaths, Metadata from kernels._versions import _get_available_versions, resolve_kernel_version from kernels.hf_hub import CACHE_DIR, _check_trust_remote_code from kernels.variants import ( diff --git a/kernels/src/kernels/validate.py b/kernels/src/kernels/validate.py index 02306e088..4fa453dbe 100644 --- a/kernels/src/kernels/validate.py +++ b/kernels/src/kernels/validate.py @@ -2,9 +2,9 @@ from dataclasses import dataclass from typing import Protocol -from kernels_data import Metadata, Version from packaging.version import InvalidVersion, parse +from kernels._data import Metadata, Version from kernels.archs import _check_arch_incompatibility from kernels.backends import _backend from kernels.python_deps import validate_dependencies diff --git a/kernels/src/kernels/verify.py b/kernels/src/kernels/verify.py index 54ebd7ade..12ed5b211 100644 --- a/kernels/src/kernels/verify.py +++ b/kernels/src/kernels/verify.py @@ -3,12 +3,13 @@ from typing import TypeAlias, final from cryptography.x509 import Certificate -from kernels_data import Digest, DigestValidationError, DigestViolation, Metadata from sigstore.errors import VerificationError from sigstore.models import Bundle, InvalidBundle from sigstore.verify import Verifier, policy from sigstore.verify.policy import VerificationPolicy +from kernels._data import Digest, DigestValidationError, DigestViolation, Metadata + class GitHubWorkflowPolicy(VerificationPolicy): """ diff --git a/kernels/tests/conftest.py b/kernels/tests/conftest.py index 36807676d..f72023de7 100644 --- a/kernels/tests/conftest.py +++ b/kernels/tests/conftest.py @@ -3,8 +3,8 @@ import sys import pytest -from kernels_data import Metadata +from kernels._data import Metadata from kernels.backends import _get_torch_privateuse_backend_name try: diff --git a/kernels/tests/test_archs.py b/kernels/tests/test_archs.py index 2c7cde65f..7580d8a0b 100644 --- a/kernels/tests/test_archs.py +++ b/kernels/tests/test_archs.py @@ -1,8 +1,8 @@ import pytest import torch -from kernels_data import Metadata from kernels import get_kernel, has_kernel, install_kernel +from kernels._data import Metadata from kernels.archs import _supports_cuda_capability diff --git a/kernels-data/bindings/python/tests/test_config.py b/kernels/tests/test_data_config.py similarity index 76% rename from kernels-data/bindings/python/tests/test_config.py rename to kernels/tests/test_data_config.py index 5d072557f..e8701e1f6 100644 --- a/kernels-data/bindings/python/tests/test_config.py +++ b/kernels/tests/test_data_config.py @@ -1,6 +1,6 @@ import pytest -from kernels_data import Backend, Build, KernelDependency, KernelVersion +from kernels._data import Backend, Build, KernelDependency, KernelVersion def _write_build_toml(path, backends): @@ -42,17 +42,11 @@ def test_build_all_kernel_depends(tmp_path): build = Build.open(tmp_path) assert build.all_kernel_depends(Backend.CUDA) == [ - KernelDependency( - repo_id="kernels-community/activation", version=KernelVersion.Version(1) - ), - KernelDependency( - repo_id="kernels-community/cuda-helper", version=KernelVersion.Version(2) - ), + KernelDependency(repo_id="kernels-community/activation", version=KernelVersion.Version(1)), + KernelDependency(repo_id="kernels-community/cuda-helper", version=KernelVersion.Version(2)), ] assert build.all_kernel_depends(Backend.CPU) == [ - KernelDependency( - repo_id="kernels-community/activation", version=KernelVersion.Version(1) - ) + KernelDependency(repo_id="kernels-community/activation", version=KernelVersion.Version(1)) ] diff --git a/kernels-data/bindings/python/tests/test_digest.py b/kernels/tests/test_data_digest.py similarity index 96% rename from kernels-data/bindings/python/tests/test_digest.py rename to kernels/tests/test_data_digest.py index e3a2a7de6..b5991ae6d 100644 --- a/kernels-data/bindings/python/tests/test_digest.py +++ b/kernels/tests/test_data_digest.py @@ -1,6 +1,6 @@ import pytest -from kernels_data import Digest, DigestAlgorithm, DigestValidationError, DigestViolation +from kernels._data import Digest, DigestAlgorithm, DigestValidationError, DigestViolation def test_hash_variant_hashes_file_with_sha256(tmp_path): @@ -9,9 +9,7 @@ def test_hash_variant_hashes_file_with_sha256(tmp_path): digest = Digest.hash_variant(DigestAlgorithm.SHA256, tmp_path) assert len(digest.files) == 1 - assert ( - digest.files["_extension.so"] == "uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=" - ) + assert digest.files["_extension.so"] == "uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=" def test_hash_variant_hashes_file_with_sha512(tmp_path): diff --git a/kernels-data/bindings/python/tests/test_kernel_lock.py b/kernels/tests/test_data_kernel_lock.py similarity index 99% rename from kernels-data/bindings/python/tests/test_kernel_lock.py rename to kernels/tests/test_data_kernel_lock.py index 0a60b53e3..3989d47c6 100644 --- a/kernels-data/bindings/python/tests/test_kernel_lock.py +++ b/kernels/tests/test_data_kernel_lock.py @@ -2,7 +2,7 @@ import pytest -from kernels_data import ( +from kernels._data import ( KernelDependency, KernelLock, KernelLocks, diff --git a/kernels-data/bindings/python/tests/test_kernels_data.py b/kernels/tests/test_data_metadata.py similarity index 98% rename from kernels-data/bindings/python/tests/test_kernels_data.py rename to kernels/tests/test_data_metadata.py index b80d9f182..0d7916f75 100644 --- a/kernels-data/bindings/python/tests/test_kernels_data.py +++ b/kernels/tests/test_data_metadata.py @@ -2,7 +2,7 @@ import pytest -from kernels_data import ( +from kernels._data import ( Backend, KernelDependency, KernelName, @@ -196,9 +196,7 @@ def test_kernel_version_eq_and_hash(): assert KernelVersion.Version(1) != KernelVersion.Revision("1") assert hash(KernelVersion.Version(1)) == hash(KernelVersion.Version(1)) assert hash(KernelVersion.Revision("abc")) == hash(KernelVersion.Revision("abc")) - assert {KernelVersion.Version(1), KernelVersion.Version(1)} == { - KernelVersion.Version(1) - } + assert {KernelVersion.Version(1), KernelVersion.Version(1)} == {KernelVersion.Version(1)} def test_kernel_dependency_eq_and_hash(): diff --git a/kernels-data/bindings/python/tests/test_version.py b/kernels/tests/test_data_version.py similarity index 96% rename from kernels-data/bindings/python/tests/test_version.py rename to kernels/tests/test_data_version.py index 1f153402f..280a234a6 100644 --- a/kernels-data/bindings/python/tests/test_version.py +++ b/kernels/tests/test_data_version.py @@ -1,6 +1,6 @@ import pytest -from kernels_data import Version +from kernels._data import Version def test_version_parse(): diff --git a/kernels/tests/test_deps.py b/kernels/tests/test_deps.py index 837e5b2a0..23b95c2b1 100644 --- a/kernels/tests/test_deps.py +++ b/kernels/tests/test_deps.py @@ -4,8 +4,8 @@ from types import ModuleType import pytest -from kernels_data import KernelDependency, KernelVersion, Metadata +from kernels._data import KernelDependency, KernelVersion, Metadata from kernels.deps import get_kernel_dep, resolve_kernel_tree, use_kernel_deps from kernels.resolver import LocalKernel diff --git a/kernels/tests/test_resolver.py b/kernels/tests/test_resolver.py index 9cadc4e99..b700ee8ca 100644 --- a/kernels/tests/test_resolver.py +++ b/kernels/tests/test_resolver.py @@ -5,7 +5,9 @@ import pytest from huggingface_hub.hf_api import GitRefInfo -from kernels_data import ( + +import kernels.resolver as resolver_module +from kernels._data import ( KernelDependency, KernelLock, KernelLocks, @@ -14,8 +16,6 @@ KernelVersion, Metadata, ) - -import kernels.resolver as resolver_module from kernels._versions import resolve_version_spec_as_ref from kernels.hf_hub import _get_hf_api from kernels.install import install_kernel diff --git a/kernels/tests/test_validate.py b/kernels/tests/test_validate.py index 5d9c74189..2e23f06a7 100644 --- a/kernels/tests/test_validate.py +++ b/kernels/tests/test_validate.py @@ -5,10 +5,10 @@ import pytest import torch -from kernels_data import Metadata, Version import kernels import kernels.validate as validate_module +from kernels._data import Metadata, Version from kernels.deps import DepTreeNode from kernels.resolver import LocalKernel from kernels.validate import ( diff --git a/kernels/tests/test_verify.py b/kernels/tests/test_verify.py index b6493fc4f..d6e7f03d3 100644 --- a/kernels/tests/test_verify.py +++ b/kernels/tests/test_verify.py @@ -1,9 +1,9 @@ from pathlib import Path -from kernels_data import DigestViolation from sigstore.verify import policy from kernels import install_kernel +from kernels._data import DigestViolation from kernels._versions import select_revision_or_version from kernels.hf_hub import CACHE_DIR, _get_hf_api from kernels.resolver import _BYTECODE_IGNORE_PATTERNS diff --git a/kernels/uv.lock b/kernels/uv.lock index 306a3d205..b40f52eb6 100644 --- a/kernels/uv.lock +++ b/kernels/uv.lock @@ -8,7 +8,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-06-19T07:02:11.14082404Z" +exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. exclude-newer-span = "P7D" [[package]] @@ -374,7 +374,7 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/54/eb9bfc647b19f2009dd5c7f5ec51c4e6ca831725f1aea7a993034f483147/contourpy-1.3.2.tar.gz", hash = "sha256:b6945942715a034c671b7fc54f9588126b0b8bf23db2696e3ca8328f3ff0ab54", size = 13466130, upload-time = "2025-04-15T17:47:53.79Z" } wheels = [ @@ -445,7 +445,7 @@ resolution-markers = [ "python_full_version == '3.11.*'", ] dependencies = [ - { name = "numpy", version = "2.4.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" } wheels = [ @@ -834,7 +834,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1158,11 +1158,10 @@ wheels = [ [[package]] name = "kernels" -version = "0.17.0.dev0" +version = "0.17.0.dev1" source = { editable = "." } dependencies = [ { name = "huggingface-hub" }, - { name = "kernels-data" }, { name = "packaging" }, { name = "pyyaml" }, { name = "sigstore" }, @@ -1217,7 +1216,6 @@ requires-dist = [ { name = "einops", marker = "extra == 'curated-xpu'" }, { name = "hf-doc-builder", marker = "extra == 'docs'" }, { name = "huggingface-hub", specifier = ">=1.10.0" }, - { name = "kernels-data", specifier = ">=0.14.0.dev1" }, { name = "matplotlib", marker = "extra == 'benchmark'", specifier = ">=3.7.0" }, { name = "numpy", marker = "extra == 'benchmark'", specifier = ">=2.0.2" }, { name = "nvidia-cutlass-dsl", marker = "extra == 'curated'" }, @@ -1247,59 +1245,6 @@ dev = [ { name = "types-tabulate" }, ] -[[package]] -name = "kernels-data" -version = "0.14.0.dev1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/63/f3/2c32dfce91fdc5115b16abaf2fb8f41c29b5327be458f430d8a7318e30a3/kernels_data-0.14.0.dev1.tar.gz", hash = "sha256:ddabe0cf0a9476e2aa631838e0ee0f748420156478b7e4b7103b687f04aaf823", size = 40310, upload-time = "2026-04-22T14:42:16.939Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ba/3f/20dd68c6d3b6cc662678c7fb81945221b0433f80d0c890b7975322dab74d/kernels_data-0.14.0.dev1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:d09f608e0cb1b05fc26fc5895d027e38e0e504fa314a9d8797b3122f0582f4c2", size = 1078614, upload-time = "2026-04-22T14:41:56.262Z" }, - { url = "https://files.pythonhosted.org/packages/04/87/708bc67d9152c53aefc7181478ef5beac955f622e34b553e75bb67a844e6/kernels_data-0.14.0.dev1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8c4b79e7e9b5b534890c0bb52bed9bcbe9c1c4ccb527a7595ee39d11d40e7481", size = 1030171, upload-time = "2026-04-22T14:41:52.707Z" }, - { url = "https://files.pythonhosted.org/packages/dc/e3/bfcc1e9c7815fd2e5d0523c829b04173f4b56c663d84866ccdeecfcb144b/kernels_data-0.14.0.dev1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3fd4106930aeca7e8070599148701ce90d309b5f1a50b4ba9ba12d2b595afe1", size = 1165979, upload-time = "2026-04-22T14:41:31.388Z" }, - { url = "https://files.pythonhosted.org/packages/e4/ae/1e33029c24f35d3fea7e81f096d6701aae54c6d88ddfb7678524184b7e6f/kernels_data-0.14.0.dev1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9fe3ff0af1aaa8ae1016eafe4dc7a56a23f09b624fc9017ba9494a826b9791c8", size = 1111545, upload-time = "2026-04-22T14:41:35.178Z" }, - { url = "https://files.pythonhosted.org/packages/0c/a6/ce6be40293d57ae22db25dbbe839b8c826df54ed2ee3ba450e779ad461c2/kernels_data-0.14.0.dev1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:25c4d3afbfa844c5683fbe0bf7ae32cc674a86e8fa5c8e4b5c2745ebd522d7b5", size = 1411110, upload-time = "2026-04-22T14:41:38.658Z" }, - { url = "https://files.pythonhosted.org/packages/05/70/abc2cba1e169f57f88a17fcf407b1d0d3e5ba54da44b252db237f840e529/kernels_data-0.14.0.dev1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:637dc3b9e638fc1c4e8ef179c1e8944e40557c7e46bd713211efaa796b652ff6", size = 1231365, upload-time = "2026-04-22T14:41:42.431Z" }, - { url = "https://files.pythonhosted.org/packages/32/7b/1b9d448ff032a4e02711a47b0853244a2d452fc34618b52607f3c31cb518/kernels_data-0.14.0.dev1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b4294a0ad03b1ecb3ed8b058999f8135ac5e066aabfc09276f257184eb31725", size = 1183961, upload-time = "2026-04-22T14:41:49.328Z" }, - { url = "https://files.pythonhosted.org/packages/89/09/e3c549eb301e0cbf5d03c5ec2afffcdd5859cec0eaef4f4c6fec6128177e/kernels_data-0.14.0.dev1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3bfda8f476e9c9a414259287a31ab363b39955a7209b53597107271837a7a160", size = 1208543, upload-time = "2026-04-22T14:41:45.872Z" }, - { url = "https://files.pythonhosted.org/packages/33/da/e422c1dc642f93be6ccc7428d17c448c1df422cef572b7c68e9af52ae13f/kernels_data-0.14.0.dev1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a6ce5477758cec4d7df2a31db7a97017e4434e398e33828719b261ab8ecf9cae", size = 1343472, upload-time = "2026-04-22T14:42:00.842Z" }, - { url = "https://files.pythonhosted.org/packages/31/7a/6807081aa96d8a30bddbf76845dc94c11fc29e8095e0ecbb8647b7114e95/kernels_data-0.14.0.dev1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:72c4536c8a4dac4cfb5607982fed453fa0a6ff001efd860766aa2d8def4cb5a4", size = 1387779, upload-time = "2026-04-22T14:42:04.678Z" }, - { url = "https://files.pythonhosted.org/packages/1a/9a/ca394ca3aa8b7fa735f1f851a432c500d3f2a4a625c57e68f4adceded66e/kernels_data-0.14.0.dev1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:378722b27b3629aa5723ed3c8cd93e09af00b9f65c59128743ca5d2eeaacc545", size = 1391135, upload-time = "2026-04-22T14:42:08.658Z" }, - { url = "https://files.pythonhosted.org/packages/f9/a8/36fc40954f2d766113e1b43bbeecdc07b79e6f8e3f94c86252d8f582e12b/kernels_data-0.14.0.dev1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:799a828f0630a1f95e21c44c0177c3551d6f008b86fc43cb0d1c3f76f3fc182b", size = 1420797, upload-time = "2026-04-22T14:42:12.638Z" }, - { url = "https://files.pythonhosted.org/packages/fe/ab/0cd226879baefa2ff8144f0b37c274fa97f5b3e06cf26558722cef988324/kernels_data-0.14.0.dev1-cp313-cp313t-win32.whl", hash = "sha256:02b23c2f2d919719a68791a7324ec9789bc7e5551c9e6c4724cc1d2f97a9371f", size = 845031, upload-time = "2026-04-22T14:42:24.906Z" }, - { url = "https://files.pythonhosted.org/packages/3f/d3/c9274f4aca11821c74ee0034ff333a602c15d53e0e6b807683e8b829795b/kernels_data-0.14.0.dev1-cp313-cp313t-win_amd64.whl", hash = "sha256:da6441a7cd3ea211e4d4ba01a89fc5244638ee6b879a578d90a3ec60abd78fc1", size = 926990, upload-time = "2026-04-22T14:42:21.654Z" }, - { url = "https://files.pythonhosted.org/packages/8f/ef/b9654fc399c927b08d19029e4fd1ad00ccd141e85769fb472dcd9b864af5/kernels_data-0.14.0.dev1-cp313-cp313t-win_arm64.whl", hash = "sha256:556bf0aa870bf5f28ad56b62f1c614e69ded2a3ac6dc5bb61ec0d73b0ad36bc2", size = 872405, upload-time = "2026-04-22T14:42:17.759Z" }, - { url = "https://files.pythonhosted.org/packages/18/cd/773017c5dd50493917874f38ad8207904525f242d5cc6d98c4976f782d4a/kernels_data-0.14.0.dev1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:adbf6c062ec3b0f9a17f2e64b0836b9a61d7478fc7edc02b5b15e7ec1a63b5b4", size = 1077450, upload-time = "2026-04-22T14:41:57.954Z" }, - { url = "https://files.pythonhosted.org/packages/6e/52/daa34fa639bf029cdfdb5b9db9c37de15ba694fc3282f729779d7e59fea1/kernels_data-0.14.0.dev1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0f02ee31a2b26fc72b14df5e9ea0be42dae5895a644989ce2395c368118c3eae", size = 1029565, upload-time = "2026-04-22T14:41:53.833Z" }, - { url = "https://files.pythonhosted.org/packages/2d/1e/6e5ca2ff12b41f0f2d74148f9c8fa162f03e8ee485084c8f2d8ae2b5ea6a/kernels_data-0.14.0.dev1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e04d30af73b06a1baca1d2ce448e359bef4a60e30bf1240bd00a310fd25316a5", size = 1165234, upload-time = "2026-04-22T14:41:32.826Z" }, - { url = "https://files.pythonhosted.org/packages/fd/34/3f53007476b9e292f1d7336c20d2a5d17c767c530d6a0a056b63aa35fcb7/kernels_data-0.14.0.dev1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c4937f6c2138771fc0991a49850666911fa8e14c6384fc88797d30cf3095cdc0", size = 1109200, upload-time = "2026-04-22T14:41:36.181Z" }, - { url = "https://files.pythonhosted.org/packages/0e/39/c35950b74d34d07ef54ea7e4f4dd161b402c47f1c004897eeb37a1c24711/kernels_data-0.14.0.dev1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:47fe0f6dc497b0c35f2a3c62f6c2fef60364162b1e5019399755c5fff4f014b4", size = 1410660, upload-time = "2026-04-22T14:41:39.914Z" }, - { url = "https://files.pythonhosted.org/packages/a4/61/ed6d656a3cf8ef9a70283d428790cb305acf073f812c9ceddcfd319c1fe1/kernels_data-0.14.0.dev1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6090469f4f4b4ea8ea8c1517bb5d1d55a3bb430e84f0627b5ab4225d502fddac", size = 1231016, upload-time = "2026-04-22T14:41:43.68Z" }, - { url = "https://files.pythonhosted.org/packages/99/1d/557976470f09a967cc93509df8454f0c54d869f7b8717cc108d36b58a428/kernels_data-0.14.0.dev1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d93de6691fcb26d5e27917a708e24394df98b419466f593fc22c3cb1be064f0d", size = 1182062, upload-time = "2026-04-22T14:41:50.449Z" }, - { url = "https://files.pythonhosted.org/packages/ba/9a/03954655274c284c6b694de0163b0f2cf5e679083ea7c1bf292211f835d9/kernels_data-0.14.0.dev1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bc5a0439db41eca4790f86637a4a42a35fd89eef11ee9050c93c88a999e8c337", size = 1208105, upload-time = "2026-04-22T14:41:47.081Z" }, - { url = "https://files.pythonhosted.org/packages/34/5c/bbe95aaa50ac1b7c4d5fc5c6958340e004fb814f79fa03a80d36f190b42b/kernels_data-0.14.0.dev1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3abb7a5032b0aff58288b5854757601f76e3abb01e9b6d275bccb9ea38193a42", size = 1342258, upload-time = "2026-04-22T14:42:02.037Z" }, - { url = "https://files.pythonhosted.org/packages/08/db/6306c04d49c975dbef5d35fe17a9497a5c2dea65ac83e3801f14b68cfef9/kernels_data-0.14.0.dev1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:7ecf1cd8fe9b6bf8d6ef93a8a2ef299f99737360921f5606b9f4c1a1de33924e", size = 1386877, upload-time = "2026-04-22T14:42:05.973Z" }, - { url = "https://files.pythonhosted.org/packages/b9/5a/d552ae088d4ec49e4ed3606fbd3e3e032e8920fbee6ee6ae8da4d9d6809c/kernels_data-0.14.0.dev1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:de6d37c5cb56c64bce4edec68c1cb291513d8c2bd51c4c67779086ecef26d097", size = 1390631, upload-time = "2026-04-22T14:42:09.956Z" }, - { url = "https://files.pythonhosted.org/packages/64/8e/8c284a4edcbda449ece568a8c66803c5c9b2c0e03f99d47de4148251c55a/kernels_data-0.14.0.dev1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:72aa92be0343d0588c002fc3f3a6932e55a255a2068dcec209689b8ff672e9a2", size = 1419837, upload-time = "2026-04-22T14:42:13.837Z" }, - { url = "https://files.pythonhosted.org/packages/fa/40/55f4d1505fd28e3b393ee5e24d04bd9bb77825ae5a80a355495702ec652c/kernels_data-0.14.0.dev1-cp314-cp314t-win32.whl", hash = "sha256:3f1a69afe7436a1ac0211ab37d0c5a7e2e9fa4f646102dc0dcc98848022ba8be", size = 843884, upload-time = "2026-04-22T14:42:26.304Z" }, - { url = "https://files.pythonhosted.org/packages/71/68/c82022547e81d2f96fb1bb0fe08fd8219b9c5841be8b31d3f468fc8f7254/kernels_data-0.14.0.dev1-cp314-cp314t-win_amd64.whl", hash = "sha256:0462d518623a573474359ec27fc117bebf0f632db9fa12f5679314be29a68680", size = 926187, upload-time = "2026-04-22T14:42:22.637Z" }, - { url = "https://files.pythonhosted.org/packages/c5/13/da09e658826dfd2a7258666c56d128b32782e64a1f9b753b438072017ed3/kernels_data-0.14.0.dev1-cp314-cp314t-win_arm64.whl", hash = "sha256:cf005b191d65aadd3f97ecca5c48ba3aab9d83bdfd939b5b410f0e6c46c16206", size = 871161, upload-time = "2026-04-22T14:42:18.984Z" }, - { url = "https://files.pythonhosted.org/packages/f8/39/3d4dcb761dab3f049bafe6cb3573378342083ec21749e1189c8ec8b3a795/kernels_data-0.14.0.dev1-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c834c1edbb666cafa76202c2ad7d14c26870ef5198392d166d458a73ecbc4a77", size = 1085611, upload-time = "2026-04-22T14:41:59.304Z" }, - { url = "https://files.pythonhosted.org/packages/d3/0f/b21d53901585cf712c799b26b4e15dfedf8df0bafb6381910527659b0e7d/kernels_data-0.14.0.dev1-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:896d9d69602ba247dbc9ae3fa1ccda20b4cad1ebf9acd57643babd943c45cf6d", size = 1038121, upload-time = "2026-04-22T14:41:55.166Z" }, - { url = "https://files.pythonhosted.org/packages/60/db/cc67f8833b4f649f5c5ca95595d28d4010ef1ea0e92c65f94e9862ef3908/kernels_data-0.14.0.dev1-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fab1a2576ce69c4f3f3bc28933f55df545a2013dd2f2a2a6d0ad4452a785a000", size = 1171424, upload-time = "2026-04-22T14:41:33.985Z" }, - { url = "https://files.pythonhosted.org/packages/f2/b9/763f7c7bea1eff65e8621c6135a7fcee7b08895731f219b8426632fde4a1/kernels_data-0.14.0.dev1-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d12631c8edc176103dbebc2a96ab95c6ca15bf260e0f51ac023f920b2afc7929", size = 1120537, upload-time = "2026-04-22T14:41:37.441Z" }, - { url = "https://files.pythonhosted.org/packages/a9/0f/6d3c3792c08d36e6699834adb91663122f382ed19b286c1075e75c7eec93/kernels_data-0.14.0.dev1-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e163e5a2a8f39ebc62b37f7619dbfa9aa0c375eee72116b668fedb4cde52ecf5", size = 1414585, upload-time = "2026-04-22T14:41:41.119Z" }, - { url = "https://files.pythonhosted.org/packages/e4/0c/c39769627d65461f85b4f265af4964558f7c917571434bb828365b80f836/kernels_data-0.14.0.dev1-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f5d5b9112fa07281ec0444bc496cc63693577c3ee89baff5940ad74d261fa26a", size = 1236688, upload-time = "2026-04-22T14:41:44.737Z" }, - { url = "https://files.pythonhosted.org/packages/76/97/3c3658bc386b73a7e4bfbb323b19e38caca29fe746481d668765323c58e5/kernels_data-0.14.0.dev1-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f10e1aaa6460800a046d0790d64e27d3a6a6d75c45e5d86f01ee63f4c9022a7c", size = 1189649, upload-time = "2026-04-22T14:41:51.534Z" }, - { url = "https://files.pythonhosted.org/packages/9c/1b/7344ed92a3ace61896be2762fd87f96582e25a4774e8bd9be9a8a8dc8722/kernels_data-0.14.0.dev1-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b95470763f8b9b5ef652751de296f378797e506362ca5557d59a5ffdf0b1c798", size = 1215890, upload-time = "2026-04-22T14:41:48.201Z" }, - { url = "https://files.pythonhosted.org/packages/4d/1c/1bcaafa69c5d5ae53c93ccccc5d9f707de8b558fdedabe4798829ebf80f8/kernels_data-0.14.0.dev1-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:797de9e0d4f2f3173c81733db8b0a6e2cb025aa114a6c86eb6b8f4a13a3adde7", size = 1346846, upload-time = "2026-04-22T14:42:03.239Z" }, - { url = "https://files.pythonhosted.org/packages/36/79/b4d316f377e66d608664988550d4e51f64d3604c9cedd35c24d3a9a8999a/kernels_data-0.14.0.dev1-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:acd8e1ae3f5c2935da5995909ee96460d5d47692880ef5a7678396c1ff454585", size = 1392736, upload-time = "2026-04-22T14:42:07.18Z" }, - { url = "https://files.pythonhosted.org/packages/89/a5/48ee3a6ab97ce7349097b545cb1a6dbdfb5b739d4129f0b077319824c4c7/kernels_data-0.14.0.dev1-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:2a305e1ad93350cfbceeb2caa8a65c2ac7a2dab0998aaa33d9366a1236e3247c", size = 1397338, upload-time = "2026-04-22T14:42:11.46Z" }, - { url = "https://files.pythonhosted.org/packages/0d/94/4a1abaa151fa758c7f19fd7337400d5177e538c13f6c4da544ad0a74712b/kernels_data-0.14.0.dev1-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e28cdcb3a765587083c7a6630154311bd72bc29ab9b7be9ef3260a4b7b1a319a", size = 1426094, upload-time = "2026-04-22T14:42:15.252Z" }, - { url = "https://files.pythonhosted.org/packages/66/82/5d7390113d2131086511dfb4d91c5267e08472938ea2d2d79e16b7669eae/kernels_data-0.14.0.dev1-cp38-abi3-win32.whl", hash = "sha256:3ff69776f44daaa0f051252d20d7c4a938918d20fcc69d56aed536e8c836df69", size = 847904, upload-time = "2026-04-22T14:42:27.878Z" }, - { url = "https://files.pythonhosted.org/packages/d9/40/3e1dd3d026bdde08d8e613ae7bde8007cd4c169f5b756e854d4c6cef2302/kernels_data-0.14.0.dev1-cp38-abi3-win_amd64.whl", hash = "sha256:2704b93c1babd2a2e3bf2a45435eec21bd989701324c6b92faa15e2ae6d12068", size = 931593, upload-time = "2026-04-22T14:42:23.69Z" }, - { url = "https://files.pythonhosted.org/packages/a9/f1/3751e7451ddc3b1deb5563a0fb03524837a2a1f51691f8d538b457a3df2d/kernels_data-0.14.0.dev1-cp38-abi3-win_arm64.whl", hash = "sha256:6645692cfaa9540f16ee8b2808bd5897cbfe0cc3f0a9ec97166f8ed6c5b1679e", size = 875232, upload-time = "2026-04-22T14:42:20.106Z" }, -] - [[package]] name = "kiwisolver" version = "1.5.0" diff --git a/nix-builder/overlay.nix b/nix-builder/overlay.nix index 4d114aaea..e7e2f7c2d 100644 --- a/nix-builder/overlay.nix +++ b/nix-builder/overlay.nix @@ -194,8 +194,6 @@ final: prev: kernels = callPackage ./pkgs/python-modules/kernels { }; - kernels-data = callPackage ./pkgs/python-modules/kernels-data { }; - pyclibrary = python-self.callPackage ./pkgs/python-modules/pyclibrary { }; mkTorch = callPackage ./pkgs/python-modules/torch/binary { }; diff --git a/nix-builder/pkgs/crate-dirs.nix b/nix-builder/pkgs/crate-dirs.nix index 22697a7f7..3a24005b3 100644 --- a/nix-builder/pkgs/crate-dirs.nix +++ b/nix-builder/pkgs/crate-dirs.nix @@ -8,6 +8,7 @@ toSource { ../../Cargo.toml (fileFilter sourceFiles ../../kernel-abi-check) (fileFilter sourceFiles ../../kernel-builder) + (fileFilter sourceFiles ../../kernels) (fileFilter sourceFiles ../../kernels-data) ]; } diff --git a/nix-builder/pkgs/get-kernel-check/get-kernel-check-hook.py b/nix-builder/pkgs/get-kernel-check/get-kernel-check-hook.py index 772465442..7cd35150a 100755 --- a/nix-builder/pkgs/get-kernel-check/get-kernel-check-hook.py +++ b/nix-builder/pkgs/get-kernel-check/get-kernel-check-hook.py @@ -5,7 +5,7 @@ from kernels.load import get_kernel_with_resolver from kernels.resolver import KernelPathsResolver, RepoPathsResolver, SequentialResolver from kernels.validate import AllValidator, default_metadata_validators -from kernels_data import KernelDependency, KernelPaths, KernelVersion +from kernels._data import KernelDependency, KernelPaths, KernelVersion out = os.getenv("out") if not out: diff --git a/nix-builder/pkgs/lock-kernel-deps/lock_kernel_deps.py b/nix-builder/pkgs/lock-kernel-deps/lock_kernel_deps.py index 79d1068e0..97b20f9ed 100755 --- a/nix-builder/pkgs/lock-kernel-deps/lock_kernel_deps.py +++ b/nix-builder/pkgs/lock-kernel-deps/lock_kernel_deps.py @@ -15,7 +15,7 @@ from huggingface_hub.hf_api import HfApi from kernels.locking import extract_dependency_locks -from kernels_data import Build, KernelLocks, NixKernelLock, NixKernelLocks +from kernels._data import Build, KernelLocks, NixKernelLock, NixKernelLocks def nix_hash(repo_id: str, revision: str) -> str: diff --git a/nix-builder/pkgs/python-modules/kernels-data/default.nix b/nix-builder/pkgs/python-modules/kernels-data/default.nix deleted file mode 100644 index 3bf183ab4..000000000 --- a/nix-builder/pkgs/python-modules/kernels-data/default.nix +++ /dev/null @@ -1,52 +0,0 @@ -{ - lib, - buildPythonPackage, - rustPlatform, -}: - -let - version = - (builtins.fromTOML (builtins.readFile ../../../../kernels-data/Cargo.toml)).package.version; - cargoFlags = [ - "-m" - "kernels-data/bindings/python/Cargo.toml" - ]; -in -buildPythonPackage { - pname = "kernels-data"; - inherit version; - format = "pyproject"; - - src = - let - sourceFiles = - file: - file.name == "Cargo.toml" - || file.name == "Cargo.lock" - || file.hasExt "pyi" - || file.name == "pyproject.toml" - || file.name == "python_dependencies.json" - || file.hasExt "rs"; - in - import ../../crate-dirs.nix { - inherit lib sourceFiles; - }; - - cargoDeps = rustPlatform.importCargoLock { - lockFile = ../../../../Cargo.lock; - outputHashes = { - "hf-hub-1.1.0" = "sha256-wClUTCmphrO4QM+IYwYrNxyvDp8qBGAPdP+Wca8TgRA="; - }; - }; - - maturinBuildFlags = cargoFlags; - - build-system = [ - rustPlatform.cargoSetupHook - rustPlatform.maturinBuildHook - ]; - - meta = with lib; { - description = "Kernels data structures"; - }; -} diff --git a/nix-builder/pkgs/python-modules/kernels/default.nix b/nix-builder/pkgs/python-modules/kernels/default.nix index 8188d21f0..f503d320c 100644 --- a/nix-builder/pkgs/python-modules/kernels/default.nix +++ b/nix-builder/pkgs/python-modules/kernels/default.nix @@ -1,10 +1,9 @@ { lib, buildPythonPackage, - setuptools, + rustPlatform, huggingface-hub, - kernels-data, pyyaml, sigstore, tomlkit, @@ -14,27 +13,50 @@ let version = (builtins.fromTOML (builtins.readFile ../../../../kernels/pyproject.toml)).project.version; + cargoFlags = [ + "-m" + "kernels/Cargo.toml" + ]; in buildPythonPackage { pname = "kernels"; inherit version; - pyproject = true; + format = "pyproject"; src = let sourceFiles = - file: file.hasExt "lock" || file.hasExt "json" || file.hasExt "toml" || file.hasExt "py"; + file: + file.name == "README.md" + || file.name == "Cargo.toml" + || file.name == "Cargo.lock" + || file.hasExt "rs" + || file.hasExt "pyi" + || file.hasExt "lock" + || file.hasExt "json" + || file.hasExt "toml" + || file.hasExt "py"; in - lib.fileset.toSource { - root = ../../../../kernels; - fileset = lib.fileset.fileFilter sourceFiles ../../../../kernels; + import ../../crate-dirs.nix { + inherit lib sourceFiles; + }; + + cargoDeps = rustPlatform.importCargoLock { + lockFile = ../../../../Cargo.lock; + outputHashes = { + "hf-hub-1.1.0" = "sha256-wClUTCmphrO4QM+IYwYrNxyvDp8qBGAPdP+Wca8TgRA="; }; + }; + + maturinBuildFlags = cargoFlags; - build-system = [ setuptools ]; + build-system = [ + rustPlatform.cargoSetupHook + rustPlatform.maturinBuildHook + ]; dependencies = [ huggingface-hub - kernels-data pyyaml sigstore tomlkit diff --git a/nix-builder/tests/Dockerfile.test-kernel b/nix-builder/tests/Dockerfile.test-kernel index 3462e3803..3194e4a78 100644 --- a/nix-builder/tests/Dockerfile.test-kernel +++ b/nix-builder/tests/Dockerfile.test-kernel @@ -14,7 +14,8 @@ ENV DEBIAN_FRONTEND=noninteractive \ NVIDIA_VISIBLE_DEVICES=all \ NVIDIA_DRIVER_CAPABILITIES=compute,utility -# Install uv and Rust toolchain (Rust is needed to build kernels-data from source). +# Install uv and Rust toolchain (Rust is needed to build kernels from source, which +# bundles a maturin/pyo3 extension module). RUN dnf install -y --allowerasing curl && \ curl -LsSf https://astral.sh/uv/install.sh | sh && \ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \ @@ -62,11 +63,10 @@ RUN CUDA_MAJOR_MINOR=$(echo ${CUDA_VERSION} | cut -d'.' -f1,2) && \ RUN uv add "apache-tvm-ffi~=0.1.9" numpy pytest # Copy kernels-data and kernels source to install from source. -COPY kernels-data ./kernels-data-src +COPY kernels-data ./kernels-data COPY kernels ./kernels-src -# Install kernels-data (Rust/maturin package) and kernels from source. -RUN uv add ./kernels-data-src/bindings/python +# Install kernels (Rust/maturin package) from source. RUN uv add ./kernels-src # Copy kernels and tests diff --git a/scripts/_version_common.py b/scripts/_version_common.py index d680aefcb..0f50900aa 100644 --- a/scripts/_version_common.py +++ b/scripts/_version_common.py @@ -19,7 +19,7 @@ CARGO_FILES = [ REPO_ROOT / "kernels-data" / "Cargo.toml", - REPO_ROOT / "kernels-data" / "bindings" / "python" / "Cargo.toml", + REPO_ROOT / "kernels" / "Cargo.toml", REPO_ROOT / "kernel-builder" / "Cargo.toml", REPO_ROOT / "kernel-abi-check" / "Cargo.toml", ]