diff --git a/.github/workflows/build-kuzu.yml b/.github/workflows/build-kuzu.yml new file mode 100644 index 0000000000..0acadf9e00 --- /dev/null +++ b/.github/workflows/build-kuzu.yml @@ -0,0 +1,191 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +--- +# This workflow is based on: https://github.com/kuzudb/kuzu/blob/v0.11.3/.github/workflows/build-and-deploy.yml +# (package-python-sdist) and https://github.com/kuzudb/kuzu/blob/v0.11.3/.github/workflows/linux-wheel-workflow.yml +name: Build kuzu wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'Version glob to (re)build; empty builds every version of docs/packages/kuzu.yaml not released yet' + required: false + default: '' + pull_request: + branches: [main] + paths: + - '.github/workflows/build-kuzu.yml' + - 'docs/packages/kuzu.yaml' + - 'patches/kuzu/**' + push: + branches: [main] + paths: + - '.github/workflows/build-kuzu.yml' + - 'docs/packages/kuzu.yaml' + - 'patches/kuzu/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +env: + MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 + +jobs: + setup: + uses: $/.github/workflows/_setup.yml + with: + package: kuzu + version: ${{ inputs.version }} + + package_python_sdist: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + name: Package kuzu ${{ matrix.version }} sdist + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + + env: + KUZU_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout kuzu v${{ env.KUZU_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: kuzudb/kuzu + ref: v${{ env.KUZU_VERSION }} + persist-credentials: false + + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + python-version: '3.12' + activate-environment: true + enable-cache: false + + - name: Package Python sdist + working-directory: scripts/pip-package + run: | + uv pip install setuptools + python package_tar.py + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: kuzu-${{ env.KUZU_VERSION }}-sdist + path: scripts/pip-package/*.tar.gz + if-no-files-found: error + + build_wheels: + needs: [setup, package_python_sdist] + if: needs.setup.outputs.versions != '[]' + name: Build kuzu ${{ matrix.version }} ${{ matrix.python }}-manylinux_riscv64 + runs-on: ubuntu-24.04-riscv + timeout-minutes: 1440 + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + python: ["cp312", "cp313", "cp314", "cp314t"] + + env: + KUZU_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout kuzu v${{ env.KUZU_VERSION }} tests + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: kuzudb/kuzu + ref: v${{ env.KUZU_VERSION }} + sparse-checkout: | + dataset + tools/python_api/test + persist-credentials: false + + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: python-wheels + persist-credentials: false + + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: kuzu-${{ env.KUZU_VERSION }}-sdist + path: dist + + # CIBW_TEST_SOURCES copies the tests from this checkout, not the sdist, so test patches apply here too. + - name: Extract and patch the sdist and tests + run: | + tar xzf dist/kuzu-${{ env.KUZU_VERSION }}.tar.gz + git apply --directory=sdist/kuzu-source python-wheels/patches/kuzu/${{ env.KUZU_VERSION }}/*.patch + git apply --include='tools/python_api/test/*' python-wheels/patches/kuzu/${{ env.KUZU_VERSION }}/*.patch + + # The wheel statically links the vendored third_party libraries but upstream ships only kuzu's own LICENSE. + - name: Add the vendored libraries' licences + run: | + for f in sdist/kuzu-source/third_party/*/LICENSE* sdist/kuzu-source/extension/*/third_party/*/LICENSE*; do + cp "$f" "sdist/LICENSE.$(basename "$(dirname "$f")")" + done + ls sdist/LICENSE* + + - name: Build wheels + uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0 + with: + package-dir: sdist + output-dir: wheelhouse/ + only: ${{ matrix.python }}-manylinux_riscv64 + env: + CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }} + CIBW_ENVIRONMENT: PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/ + # Not PIP_ONLY_BINARY: torch-geometric 2.3 is sdist-only (pure Python). + CIBW_TEST_ENVIRONMENT: PIP_PREFER_BINARY=1 + CIBW_TEST_SOURCES: dataset tools/python_api/test + # Upstream's requirements_dev.txt test deps; pyarrow 25 is the only riscv64 build, and cp314t has no pandas/polars. + CIBW_TEST_REQUIRES: >- + pytest pytest-asyncio~=1.0 networkx~=3.0 numpy~=2.0 pyarrow torch torch-geometric~=2.3.0 + ${{ matrix.python != 'cp314t' && 'pandas~=2.2 polars~=1.30' || '' }} + # test_extension downloads prebuilt extensions from upstream's server, which has none for riscv64. + CIBW_TEST_COMMAND: >- + python -m pytest -vv tools/python_api/test + --ignore=tools/python_api/test/test_extension.py + ${{ matrix.python == 'cp314t' && '--ignore=tools/python_api/test/test_arrow.py --ignore=tools/python_api/test/test_datatype.py --ignore=tools/python_api/test/test_df.py --ignore=tools/python_api/test/test_networkx.py --ignore=tools/python_api/test/test_scan_pandas.py --ignore=tools/python_api/test/test_scan_pandas_pyarrow.py --ignore=tools/python_api/test/test_scan_polars.py --ignore=tools/python_api/test/test_udf.py' || '' }} + + - name: Check the extension module and licences made it into the wheel + run: | + python3 - wheelhouse/*.whl <<'EOF' + import sys, zipfile + for whl in sys.argv[1:]: + names = zipfile.ZipFile(whl).namelist() + assert any(n.startswith("kuzu/_kuzu.") and n.endswith(".so") for n in names), f"no compiled extension in {whl}" + licences = {n.rsplit("/", 1)[1] for n in names if ".dist-info/licenses/" in n} - {""} + assert {"LICENSE", "LICENSE.zstd", "LICENSE.re2", "LICENSE.pybind11"} <= licences, licences + print(whl, sorted(licences)) + EOF + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: kuzu-${{ env.KUZU_VERSION }}-${{ matrix.python }}-manylinux_riscv64 + path: wheelhouse/*.whl + if-no-files-found: error + + publish: + name: Publish kuzu ${{ matrix.version }} + needs: [setup, build_wheels] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + permissions: + contents: write + pull-requests: write + uses: $/.github/workflows/_publish-wheel.yml + secrets: + app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + with: + artifact-pattern: kuzu-${{ matrix.version }}-*-manylinux_riscv64 diff --git a/docs/packages/kuzu.yaml b/docs/packages/kuzu.yaml new file mode 100644 index 0000000000..0599a512b5 --- /dev/null +++ b/docs/packages/kuzu.yaml @@ -0,0 +1,10 @@ +package-name: kuzu +source-code: https://github.com/kuzudb/kuzu +license: MIT +versions: +- version: 0.11.3 + patched: true + comment: >- + When the default 8TB buffer-manager reservation does not fit in the process address space + (riscv64 Sv39 gives 256GB), the database falls back to the largest power-of-two region that + does, which caps its maximum size accordingly. diff --git a/patches/kuzu/0.11.3/0001-storage-shrink-the-VMRegion-reservation-until-it-fits.patch b/patches/kuzu/0.11.3/0001-storage-shrink-the-VMRegion-reservation-until-it-fits.patch new file mode 100644 index 0000000000..47efcb2b2e --- /dev/null +++ b/patches/kuzu/0.11.3/0001-storage-shrink-the-VMRegion-reservation-until-it-fits.patch @@ -0,0 +1,48 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Fri, 25 Sep 2026 00:00:00 +0000 +Subject: [PATCH] storage: shrink the VMRegion reservation until it fits + +Upstream-Status: To upstream [kuzudb/kuzu is archived and accepts no pull requests] + +Every Database reserves its buffer-manager region with one mmap of +max_db_size bytes, and the Python API defaults that to 1 << 43 (8TB). +riscv64 Sv39 gives a process 256GB of user address space (the T-Head +C910/C920 cores the riscv64 runners use only implement Sv39), so the +reservation fails with ENOMEM and every Database() created with the +default settings throws "Mmap for size 8796093022208 failed." The same +happens under a 39-bit VA arm64 kernel, or on x86-64 under +`ulimit -v 268435456`. + +On ENOMEM, halve the reservation until it fits. Where the full region +fits nothing changes; elsewhere the database is capped at the largest +power-of-two region the address space can hold, the limit +max_db_size already expresses. +--- +diff --git a/src/storage/buffer_manager/vm_region.cpp b/src/storage/buffer_manager/vm_region.cpp +index bf987b8..a0a81ae 100644 +--- a/src/storage/buffer_manager/vm_region.cpp ++++ b/src/storage/buffer_manager/vm_region.cpp +@@ -10,6 +10,8 @@ + #include + #else + #include ++ ++#include + #endif + + #include "common/exception/buffer_manager.h" +@@ -38,6 +40,13 @@ VMRegion::VMRegion(PageSizeClass pageSizeClass, uint64_t maxRegionSize) : numFra + // backed by any file, and its content are initialized to zero. + region = static_cast(mmap(NULL, getMaxRegionSize(), PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1 /* fd */, 0 /* offset */)); ++ // The default 8TB region does not fit in a smaller user address space (256GB under riscv64 ++ // Sv39, 512GB under a 39-bit VA arm64 kernel), so halve the reservation until it does. ++ while (region == MAP_FAILED && errno == ENOMEM && maxNumFrameGroups > 1) { ++ maxNumFrameGroups /= 2; ++ region = static_cast(mmap(NULL, getMaxRegionSize(), PROT_READ | PROT_WRITE, ++ MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1 /* fd */, 0 /* offset */)); ++ } + if (region == MAP_FAILED) { + throw BufferManagerException( + "Mmap for size " + std::to_string(getMaxRegionSize()) + " failed."); diff --git a/patches/kuzu/0.11.3/0002-test-repeat-the-interrupt-until-the-query-stops.patch b/patches/kuzu/0.11.3/0002-test-repeat-the-interrupt-until-the-query-stops.patch new file mode 100644 index 0000000000..13bdb7aaf8 --- /dev/null +++ b/patches/kuzu/0.11.3/0002-test-repeat-the-interrupt-until-the-query-stops.patch @@ -0,0 +1,40 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Fri, 25 Sep 2026 00:00:00 +0000 +Subject: [PATCH] test: repeat the interrupt until the query stops + +Upstream-Status: To upstream [kuzudb/kuzu is archived and accepts no pull requests] + +test_connection_interrupt starts a long query on a thread, sleeps 5s, +calls conn.interrupt() once and expects the thread to end within 100s. +Binding constant-folds each RANGE(1, 1000000) into a million-element +list literal (and names it by its toString()), and executeNoLock() +calls resetActiveQuery(), which clears the interrupted flag, only once +compilation is done. On the riscv64 runners compiling that query takes +longer than 5s, so the interrupt lands during compilation, is wiped, +and the query runs to completion (~2 hours) instead of raising +"Interrupted": the assertion fails after 100s and the interpreter then +blocks at exit on the non-daemon thread. + +Re-issue the interrupt every second until the thread ends, within the +same 100s budget. On a fast machine the first interrupt still sticks +and the test behaves as before. +--- +diff --git a/tools/python_api/test/test_connection.py b/tools/python_api/test/test_connection.py +index 4206516..0632591 100644 +--- a/tools/python_api/test/test_connection.py ++++ b/tools/python_api/test/test_connection.py +@@ -44,6 +44,11 @@ def test_connection_interrupt(conn_db_readwrite: ConnDB) -> None: + execute_thread = threading.Thread(target=run_long_query, args=(conn,)) + execute_thread.start() + time.sleep(5) +- conn.interrupt() +- execute_thread.join(timeout=100) ++ # Binding folds each RANGE(1, 1000000) into a million-element list literal, which can take ++ # longer than the sleep on a slow machine, and the interrupt flag is cleared when execution ++ # starts, so an interrupt that lands during compilation is lost; repeat it until one sticks. ++ deadline = time.monotonic() + 100 ++ while execute_thread.is_alive() and time.monotonic() < deadline: ++ conn.interrupt() ++ execute_thread.join(timeout=1) + assert not execute_thread.is_alive()