From 00aabfcc1ce379af790bdfd990ddd35a6ed6f264 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Fri, 25 Sep 2026 15:56:06 +0000 Subject: [PATCH 1/2] kuzu: add build-kuzu.yml for riscv64 wheels Mirrors upstream's package-python-sdist job (scripts/pip-package/ package_tar.py on ubuntu-latest) and the manylinux wheel build it feeds, narrowed to manylinux_riscv64 and cp312/cp313/cp314/cp314t. The wheel is built from that sdist; the tag's dataset/ and tools/python_api/test are checked out beside it for upstream's pytest suite. - 0001 halves the buffer manager's mmap reservation on ENOMEM. The Python API defaults max_db_size to 8TB, which does not fit in the 256GB user address space of riscv64 Sv39, so every default Database() throws "Mmap for size 8796093022208 failed." (reproduced on x86-64 with the PyPI wheel under `ulimit -v 268435456`). kuzudb/kuzu is archived, so it cannot be sent upstream. - The wheel statically links ~20 vendored third_party libraries but upstream ships only kuzu's own LICENSE; their licence files are copied to the sdist root as LICENSE. and asserted in the wheel. - Tests: upstream's requirements_dev.txt deps, except pyarrow (25 is the only riscv64 build) and pandas/polars on cp314t (no wheels); test_extension.py is skipped as it downloads prebuilt extensions from upstream's server, which has none for riscv64. Rehearsed on x86-64 with the PyPI 0.11.3 wheel and exactly this test layout and dependency set: 212 passed (107 with the cp314t set). --- .github/workflows/build-kuzu.yml | 189 ++++++++++++++++++ docs/packages/kuzu.yaml | 10 + ...e-VMRegion-reservation-until-it-fits.patch | 48 +++++ 3 files changed, 247 insertions(+) create mode 100644 .github/workflows/build-kuzu.yml create mode 100644 docs/packages/kuzu.yaml create mode 100644 patches/kuzu/0.11.3/0001-storage-shrink-the-VMRegion-reservation-until-it-fits.patch diff --git a/.github/workflows/build-kuzu.yml b/.github/workflows/build-kuzu.yml new file mode 100644 index 0000000000..6ca102692f --- /dev/null +++ b/.github/workflows/build-kuzu.yml @@ -0,0 +1,189 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +--- +# This workflow is based on: https://github.com/kuzudb/kuzu/blob/v0.11.3/.github/workflows/build-and-deploy.yml +# (package-python-sdist) and https://github.com/kuzudb/kuzu/blob/v0.11.3/.github/workflows/linux-wheel-workflow.yml +name: Build kuzu wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'Version glob to (re)build; empty builds every version of docs/packages/kuzu.yaml not released yet' + required: false + default: '' + pull_request: + branches: [main] + paths: + - '.github/workflows/build-kuzu.yml' + - 'docs/packages/kuzu.yaml' + - 'patches/kuzu/**' + push: + branches: [main] + paths: + - '.github/workflows/build-kuzu.yml' + - 'docs/packages/kuzu.yaml' + - 'patches/kuzu/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +env: + MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 + +jobs: + setup: + uses: $/.github/workflows/_setup.yml + with: + package: kuzu + version: ${{ inputs.version }} + + package_python_sdist: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + name: Package kuzu ${{ matrix.version }} sdist + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + + env: + KUZU_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout kuzu v${{ env.KUZU_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: kuzudb/kuzu + ref: v${{ env.KUZU_VERSION }} + persist-credentials: false + + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + python-version: '3.12' + activate-environment: true + enable-cache: false + + - name: Package Python sdist + working-directory: scripts/pip-package + run: | + uv pip install setuptools + python package_tar.py + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: kuzu-${{ env.KUZU_VERSION }}-sdist + path: scripts/pip-package/*.tar.gz + if-no-files-found: error + + build_wheels: + needs: [setup, package_python_sdist] + if: needs.setup.outputs.versions != '[]' + name: Build kuzu ${{ matrix.version }} ${{ matrix.python }}-manylinux_riscv64 + runs-on: ubuntu-24.04-riscv + timeout-minutes: 1440 + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + python: ["cp312", "cp313", "cp314", "cp314t"] + + env: + KUZU_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout kuzu v${{ env.KUZU_VERSION }} tests + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: kuzudb/kuzu + ref: v${{ env.KUZU_VERSION }} + sparse-checkout: | + dataset + tools/python_api/test + persist-credentials: false + + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: python-wheels + persist-credentials: false + + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: kuzu-${{ env.KUZU_VERSION }}-sdist + path: dist + + - name: Extract and patch the sdist + run: | + tar xzf dist/kuzu-${{ env.KUZU_VERSION }}.tar.gz + git apply --directory=sdist/kuzu-source python-wheels/patches/kuzu/${{ env.KUZU_VERSION }}/*.patch + + # The wheel statically links the vendored third_party libraries but upstream ships only kuzu's own LICENSE. + - name: Add the vendored libraries' licences + run: | + for f in sdist/kuzu-source/third_party/*/LICENSE* sdist/kuzu-source/extension/*/third_party/*/LICENSE*; do + cp "$f" "sdist/LICENSE.$(basename "$(dirname "$f")")" + done + ls sdist/LICENSE* + + - name: Build wheels + uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0 + with: + package-dir: sdist + output-dir: wheelhouse/ + only: ${{ matrix.python }}-manylinux_riscv64 + env: + CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }} + CIBW_ENVIRONMENT: PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/ + # Not PIP_ONLY_BINARY: torch-geometric 2.3 is sdist-only (pure Python). + CIBW_TEST_ENVIRONMENT: PIP_PREFER_BINARY=1 + CIBW_TEST_SOURCES: dataset tools/python_api/test + # Upstream's requirements_dev.txt test deps; pyarrow 25 is the only riscv64 build, and cp314t has no pandas/polars. + CIBW_TEST_REQUIRES: >- + pytest pytest-asyncio~=1.0 networkx~=3.0 numpy~=2.0 pyarrow torch torch-geometric~=2.3.0 + ${{ matrix.python != 'cp314t' && 'pandas~=2.2 polars~=1.30' || '' }} + # test_extension downloads prebuilt extensions from upstream's server, which has none for riscv64. + CIBW_TEST_COMMAND: >- + python -m pytest -vv tools/python_api/test + --ignore=tools/python_api/test/test_extension.py + ${{ matrix.python == 'cp314t' && '--ignore=tools/python_api/test/test_arrow.py --ignore=tools/python_api/test/test_datatype.py --ignore=tools/python_api/test/test_df.py --ignore=tools/python_api/test/test_networkx.py --ignore=tools/python_api/test/test_scan_pandas.py --ignore=tools/python_api/test/test_scan_pandas_pyarrow.py --ignore=tools/python_api/test/test_scan_polars.py --ignore=tools/python_api/test/test_udf.py' || '' }} + + - name: Check the extension module and licences made it into the wheel + run: | + python3 - wheelhouse/*.whl <<'EOF' + import sys, zipfile + for whl in sys.argv[1:]: + names = zipfile.ZipFile(whl).namelist() + assert any(n.startswith("kuzu/_kuzu.") and n.endswith(".so") for n in names), f"no compiled extension in {whl}" + licences = {n.rsplit("/", 1)[1] for n in names if ".dist-info/licenses/" in n} - {""} + assert {"LICENSE", "LICENSE.zstd", "LICENSE.re2", "LICENSE.pybind11"} <= licences, licences + print(whl, sorted(licences)) + EOF + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: kuzu-${{ env.KUZU_VERSION }}-${{ matrix.python }}-manylinux_riscv64 + path: wheelhouse/*.whl + if-no-files-found: error + + publish: + name: Publish kuzu ${{ matrix.version }} + needs: [setup, build_wheels] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + permissions: + contents: write + pull-requests: write + uses: $/.github/workflows/_publish-wheel.yml + secrets: + app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + with: + artifact-pattern: kuzu-${{ matrix.version }}-*-manylinux_riscv64 diff --git a/docs/packages/kuzu.yaml b/docs/packages/kuzu.yaml new file mode 100644 index 0000000000..0599a512b5 --- /dev/null +++ b/docs/packages/kuzu.yaml @@ -0,0 +1,10 @@ +package-name: kuzu +source-code: https://github.com/kuzudb/kuzu +license: MIT +versions: +- version: 0.11.3 + patched: true + comment: >- + When the default 8TB buffer-manager reservation does not fit in the process address space + (riscv64 Sv39 gives 256GB), the database falls back to the largest power-of-two region that + does, which caps its maximum size accordingly. diff --git a/patches/kuzu/0.11.3/0001-storage-shrink-the-VMRegion-reservation-until-it-fits.patch b/patches/kuzu/0.11.3/0001-storage-shrink-the-VMRegion-reservation-until-it-fits.patch new file mode 100644 index 0000000000..47efcb2b2e --- /dev/null +++ b/patches/kuzu/0.11.3/0001-storage-shrink-the-VMRegion-reservation-until-it-fits.patch @@ -0,0 +1,48 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Fri, 25 Sep 2026 00:00:00 +0000 +Subject: [PATCH] storage: shrink the VMRegion reservation until it fits + +Upstream-Status: To upstream [kuzudb/kuzu is archived and accepts no pull requests] + +Every Database reserves its buffer-manager region with one mmap of +max_db_size bytes, and the Python API defaults that to 1 << 43 (8TB). +riscv64 Sv39 gives a process 256GB of user address space (the T-Head +C910/C920 cores the riscv64 runners use only implement Sv39), so the +reservation fails with ENOMEM and every Database() created with the +default settings throws "Mmap for size 8796093022208 failed." The same +happens under a 39-bit VA arm64 kernel, or on x86-64 under +`ulimit -v 268435456`. + +On ENOMEM, halve the reservation until it fits. Where the full region +fits nothing changes; elsewhere the database is capped at the largest +power-of-two region the address space can hold, the limit +max_db_size already expresses. +--- +diff --git a/src/storage/buffer_manager/vm_region.cpp b/src/storage/buffer_manager/vm_region.cpp +index bf987b8..a0a81ae 100644 +--- a/src/storage/buffer_manager/vm_region.cpp ++++ b/src/storage/buffer_manager/vm_region.cpp +@@ -10,6 +10,8 @@ + #include + #else + #include ++ ++#include + #endif + + #include "common/exception/buffer_manager.h" +@@ -38,6 +40,13 @@ VMRegion::VMRegion(PageSizeClass pageSizeClass, uint64_t maxRegionSize) : numFra + // backed by any file, and its content are initialized to zero. + region = static_cast(mmap(NULL, getMaxRegionSize(), PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1 /* fd */, 0 /* offset */)); ++ // The default 8TB region does not fit in a smaller user address space (256GB under riscv64 ++ // Sv39, 512GB under a 39-bit VA arm64 kernel), so halve the reservation until it does. ++ while (region == MAP_FAILED && errno == ENOMEM && maxNumFrameGroups > 1) { ++ maxNumFrameGroups /= 2; ++ region = static_cast(mmap(NULL, getMaxRegionSize(), PROT_READ | PROT_WRITE, ++ MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1 /* fd */, 0 /* offset */)); ++ } + if (region == MAP_FAILED) { + throw BufferManagerException( + "Mmap for size " + std::to_string(getMaxRegionSize()) + " failed."); From 360d9b039da4565b26540739acd0fb8c00ecc42f Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Fri, 25 Sep 2026 20:42:21 +0000 Subject: [PATCH 2/2] kuzu: repeat the interrupt in test_connection_interrupt cp312 and cp313 both failed only test_connection_interrupt (211 other tests passed, so the VMRegion patch from gotcha 572 works; this is not another address-space reservation). The test interrupts a RANGE(1,1000000) x RANGE(1,1000000) query once, 5s after starting it. Binding constant-folds each RANGE into a million-element list literal, and on the riscv64 runners compiling the query takes longer than 5s. executeNoLock() then calls resetActiveQuery(), which clears the interrupted flag, so the interrupt is lost and the query runs to completion: the assertion fails after 100s, and the interpreter waits ~1h55m at exit for the non-daemon query thread. That wait is why the jobs ran 4+ hours. This is gotcha 38 (a slow runner turns a latent test race into a hard failure). Patch the test to repeat the interrupt every second within the same 100s budget, and apply test patches to the checkout CIBW_TEST_SOURCES copies the tests from as well as to the sdist. --- .github/workflows/build-kuzu.yml | 4 +- ...-the-interrupt-until-the-query-stops.patch | 40 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 patches/kuzu/0.11.3/0002-test-repeat-the-interrupt-until-the-query-stops.patch diff --git a/.github/workflows/build-kuzu.yml b/.github/workflows/build-kuzu.yml index 6ca102692f..0acadf9e00 100644 --- a/.github/workflows/build-kuzu.yml +++ b/.github/workflows/build-kuzu.yml @@ -118,10 +118,12 @@ jobs: name: kuzu-${{ env.KUZU_VERSION }}-sdist path: dist - - name: Extract and patch the sdist + # CIBW_TEST_SOURCES copies the tests from this checkout, not the sdist, so test patches apply here too. + - name: Extract and patch the sdist and tests run: | tar xzf dist/kuzu-${{ env.KUZU_VERSION }}.tar.gz git apply --directory=sdist/kuzu-source python-wheels/patches/kuzu/${{ env.KUZU_VERSION }}/*.patch + git apply --include='tools/python_api/test/*' python-wheels/patches/kuzu/${{ env.KUZU_VERSION }}/*.patch # The wheel statically links the vendored third_party libraries but upstream ships only kuzu's own LICENSE. - name: Add the vendored libraries' licences diff --git a/patches/kuzu/0.11.3/0002-test-repeat-the-interrupt-until-the-query-stops.patch b/patches/kuzu/0.11.3/0002-test-repeat-the-interrupt-until-the-query-stops.patch new file mode 100644 index 0000000000..13bdb7aaf8 --- /dev/null +++ b/patches/kuzu/0.11.3/0002-test-repeat-the-interrupt-until-the-query-stops.patch @@ -0,0 +1,40 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Fri, 25 Sep 2026 00:00:00 +0000 +Subject: [PATCH] test: repeat the interrupt until the query stops + +Upstream-Status: To upstream [kuzudb/kuzu is archived and accepts no pull requests] + +test_connection_interrupt starts a long query on a thread, sleeps 5s, +calls conn.interrupt() once and expects the thread to end within 100s. +Binding constant-folds each RANGE(1, 1000000) into a million-element +list literal (and names it by its toString()), and executeNoLock() +calls resetActiveQuery(), which clears the interrupted flag, only once +compilation is done. On the riscv64 runners compiling that query takes +longer than 5s, so the interrupt lands during compilation, is wiped, +and the query runs to completion (~2 hours) instead of raising +"Interrupted": the assertion fails after 100s and the interpreter then +blocks at exit on the non-daemon thread. + +Re-issue the interrupt every second until the thread ends, within the +same 100s budget. On a fast machine the first interrupt still sticks +and the test behaves as before. +--- +diff --git a/tools/python_api/test/test_connection.py b/tools/python_api/test/test_connection.py +index 4206516..0632591 100644 +--- a/tools/python_api/test/test_connection.py ++++ b/tools/python_api/test/test_connection.py +@@ -44,6 +44,11 @@ def test_connection_interrupt(conn_db_readwrite: ConnDB) -> None: + execute_thread = threading.Thread(target=run_long_query, args=(conn,)) + execute_thread.start() + time.sleep(5) +- conn.interrupt() +- execute_thread.join(timeout=100) ++ # Binding folds each RANGE(1, 1000000) into a million-element list literal, which can take ++ # longer than the sleep on a slow machine, and the interrupt flag is cleared when execution ++ # starts, so an interrupt that lands during compilation is lost; repeat it until one sticks. ++ deadline = time.monotonic() + 100 ++ while execute_thread.is_alive() and time.monotonic() < deadline: ++ conn.interrupt() ++ execute_thread.join(timeout=1) + assert not execute_thread.is_alive()