Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
191 changes: 191 additions & 0 deletions .github/workflows/build-kuzu.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,191 @@
# SPDX-FileCopyrightText: 2026 The RISE Project
# SPDX-License-Identifier: MIT
---
# This workflow is based on: https://github.com/kuzudb/kuzu/blob/v0.11.3/.github/workflows/build-and-deploy.yml
# (package-python-sdist) and https://github.com/kuzudb/kuzu/blob/v0.11.3/.github/workflows/linux-wheel-workflow.yml
name: Build kuzu wheels (riscv64)

on:
workflow_dispatch:
inputs:
version:
description: 'Version glob to (re)build; empty builds every version of docs/packages/kuzu.yaml not released yet'
required: false
default: ''
pull_request:
branches: [main]
paths:
- '.github/workflows/build-kuzu.yml'
- 'docs/packages/kuzu.yaml'
- 'patches/kuzu/**'
push:
branches: [main]
paths:
- '.github/workflows/build-kuzu.yml'
- 'docs/packages/kuzu.yaml'
- 'patches/kuzu/**'

concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true

permissions:
contents: read # to fetch code (actions/checkout)

env:
MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64

jobs:
setup:
uses: $/.github/workflows/_setup.yml
with:
package: kuzu
version: ${{ inputs.version }}

package_python_sdist:
needs: [setup]
if: needs.setup.outputs.versions != '[]'
name: Package kuzu ${{ matrix.version }} sdist
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
version: ${{ fromJSON(needs.setup.outputs.versions) }}

env:
KUZU_VERSION: ${{ matrix.version }}

steps:
- name: Checkout kuzu v${{ env.KUZU_VERSION }}
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: kuzudb/kuzu
ref: v${{ env.KUZU_VERSION }}
persist-credentials: false

- uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
with:
python-version: '3.12'
activate-environment: true
enable-cache: false

- name: Package Python sdist
working-directory: scripts/pip-package
run: |
uv pip install setuptools
python package_tar.py

- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: kuzu-${{ env.KUZU_VERSION }}-sdist
path: scripts/pip-package/*.tar.gz
if-no-files-found: error

build_wheels:
needs: [setup, package_python_sdist]
if: needs.setup.outputs.versions != '[]'
name: Build kuzu ${{ matrix.version }} ${{ matrix.python }}-manylinux_riscv64
runs-on: ubuntu-24.04-riscv
timeout-minutes: 1440
strategy:
fail-fast: false
matrix:
version: ${{ fromJSON(needs.setup.outputs.versions) }}
python: ["cp312", "cp313", "cp314", "cp314t"]

env:
KUZU_VERSION: ${{ matrix.version }}

steps:
- name: Checkout kuzu v${{ env.KUZU_VERSION }} tests
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: kuzudb/kuzu
ref: v${{ env.KUZU_VERSION }}
sparse-checkout: |
dataset
tools/python_api/test
persist-credentials: false

- name: Checkout python-wheels
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
path: python-wheels
persist-credentials: false

- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: kuzu-${{ env.KUZU_VERSION }}-sdist
path: dist

# CIBW_TEST_SOURCES copies the tests from this checkout, not the sdist, so test patches apply here too.
- name: Extract and patch the sdist and tests
run: |
tar xzf dist/kuzu-${{ env.KUZU_VERSION }}.tar.gz
git apply --directory=sdist/kuzu-source python-wheels/patches/kuzu/${{ env.KUZU_VERSION }}/*.patch
git apply --include='tools/python_api/test/*' python-wheels/patches/kuzu/${{ env.KUZU_VERSION }}/*.patch

# The wheel statically links the vendored third_party libraries but upstream ships only kuzu's own LICENSE.
- name: Add the vendored libraries' licences
run: |
for f in sdist/kuzu-source/third_party/*/LICENSE* sdist/kuzu-source/extension/*/third_party/*/LICENSE*; do
cp "$f" "sdist/LICENSE.$(basename "$(dirname "$f")")"
done
ls sdist/LICENSE*

- name: Build wheels
uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0
with:
package-dir: sdist
output-dir: wheelhouse/
only: ${{ matrix.python }}-manylinux_riscv64
env:
CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }}
CIBW_ENVIRONMENT: PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/
# Not PIP_ONLY_BINARY: torch-geometric 2.3 is sdist-only (pure Python).
CIBW_TEST_ENVIRONMENT: PIP_PREFER_BINARY=1
CIBW_TEST_SOURCES: dataset tools/python_api/test
# Upstream's requirements_dev.txt test deps; pyarrow 25 is the only riscv64 build, and cp314t has no pandas/polars.
CIBW_TEST_REQUIRES: >-
pytest pytest-asyncio~=1.0 networkx~=3.0 numpy~=2.0 pyarrow torch torch-geometric~=2.3.0
${{ matrix.python != 'cp314t' && 'pandas~=2.2 polars~=1.30' || '' }}
# test_extension downloads prebuilt extensions from upstream's server, which has none for riscv64.
CIBW_TEST_COMMAND: >-
python -m pytest -vv tools/python_api/test
--ignore=tools/python_api/test/test_extension.py
${{ matrix.python == 'cp314t' && '--ignore=tools/python_api/test/test_arrow.py --ignore=tools/python_api/test/test_datatype.py --ignore=tools/python_api/test/test_df.py --ignore=tools/python_api/test/test_networkx.py --ignore=tools/python_api/test/test_scan_pandas.py --ignore=tools/python_api/test/test_scan_pandas_pyarrow.py --ignore=tools/python_api/test/test_scan_polars.py --ignore=tools/python_api/test/test_udf.py' || '' }}

- name: Check the extension module and licences made it into the wheel
run: |
python3 - wheelhouse/*.whl <<'EOF'
import sys, zipfile
for whl in sys.argv[1:]:
names = zipfile.ZipFile(whl).namelist()
assert any(n.startswith("kuzu/_kuzu.") and n.endswith(".so") for n in names), f"no compiled extension in {whl}"
licences = {n.rsplit("/", 1)[1] for n in names if ".dist-info/licenses/" in n} - {""}
assert {"LICENSE", "LICENSE.zstd", "LICENSE.re2", "LICENSE.pybind11"} <= licences, licences
print(whl, sorted(licences))
EOF

- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: kuzu-${{ env.KUZU_VERSION }}-${{ matrix.python }}-manylinux_riscv64
path: wheelhouse/*.whl
if-no-files-found: error

publish:
name: Publish kuzu ${{ matrix.version }}
needs: [setup, build_wheels]
if: needs.setup.outputs.versions != '[]'
strategy:
fail-fast: false
matrix:
version: ${{ fromJSON(needs.setup.outputs.versions) }}
permissions:
contents: write
pull-requests: write
uses: $/.github/workflows/_publish-wheel.yml
secrets:
app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }}
with:
artifact-pattern: kuzu-${{ matrix.version }}-*-manylinux_riscv64
10 changes: 10 additions & 0 deletions docs/packages/kuzu.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package-name: kuzu
source-code: https://github.com/kuzudb/kuzu
license: MIT
versions:
- version: 0.11.3
patched: true
comment: >-
When the default 8TB buffer-manager reservation does not fit in the process address space
(riscv64 Sv39 gives 256GB), the database falls back to the largest power-of-two region that
does, which caps its maximum size accordingly.
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Ludovic Henry <git@ludovic.dev>
Date: Fri, 25 Sep 2026 00:00:00 +0000
Subject: [PATCH] storage: shrink the VMRegion reservation until it fits

Upstream-Status: To upstream [kuzudb/kuzu is archived and accepts no pull requests]

Every Database reserves its buffer-manager region with one mmap of
max_db_size bytes, and the Python API defaults that to 1 << 43 (8TB).
riscv64 Sv39 gives a process 256GB of user address space (the T-Head
C910/C920 cores the riscv64 runners use only implement Sv39), so the
reservation fails with ENOMEM and every Database() created with the
default settings throws "Mmap for size 8796093022208 failed." The same
happens under a 39-bit VA arm64 kernel, or on x86-64 under
`ulimit -v 268435456`.

On ENOMEM, halve the reservation until it fits. Where the full region
fits nothing changes; elsewhere the database is capped at the largest
power-of-two region the address space can hold, the limit
max_db_size already expresses.
---
diff --git a/src/storage/buffer_manager/vm_region.cpp b/src/storage/buffer_manager/vm_region.cpp
index bf987b8..a0a81ae 100644
--- a/src/storage/buffer_manager/vm_region.cpp
+++ b/src/storage/buffer_manager/vm_region.cpp
@@ -10,6 +10,8 @@
#include <memoryapi.h>
#else
#include <sys/mman.h>
+
+#include <cerrno>
#endif

#include "common/exception/buffer_manager.h"
@@ -38,6 +40,13 @@ VMRegion::VMRegion(PageSizeClass pageSizeClass, uint64_t maxRegionSize) : numFra
// backed by any file, and its content are initialized to zero.
region = static_cast<uint8_t*>(mmap(NULL, getMaxRegionSize(), PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1 /* fd */, 0 /* offset */));
+ // The default 8TB region does not fit in a smaller user address space (256GB under riscv64
+ // Sv39, 512GB under a 39-bit VA arm64 kernel), so halve the reservation until it does.
+ while (region == MAP_FAILED && errno == ENOMEM && maxNumFrameGroups > 1) {
+ maxNumFrameGroups /= 2;
+ region = static_cast<uint8_t*>(mmap(NULL, getMaxRegionSize(), PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1 /* fd */, 0 /* offset */));
+ }
if (region == MAP_FAILED) {
throw BufferManagerException(
"Mmap for size " + std::to_string(getMaxRegionSize()) + " failed.");
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Ludovic Henry <git@ludovic.dev>
Date: Fri, 25 Sep 2026 00:00:00 +0000
Subject: [PATCH] test: repeat the interrupt until the query stops

Upstream-Status: To upstream [kuzudb/kuzu is archived and accepts no pull requests]

test_connection_interrupt starts a long query on a thread, sleeps 5s,
calls conn.interrupt() once and expects the thread to end within 100s.
Binding constant-folds each RANGE(1, 1000000) into a million-element
list literal (and names it by its toString()), and executeNoLock()
calls resetActiveQuery(), which clears the interrupted flag, only once
compilation is done. On the riscv64 runners compiling that query takes
longer than 5s, so the interrupt lands during compilation, is wiped,
and the query runs to completion (~2 hours) instead of raising
"Interrupted": the assertion fails after 100s and the interpreter then
blocks at exit on the non-daemon thread.

Re-issue the interrupt every second until the thread ends, within the
same 100s budget. On a fast machine the first interrupt still sticks
and the test behaves as before.
---
diff --git a/tools/python_api/test/test_connection.py b/tools/python_api/test/test_connection.py
index 4206516..0632591 100644
--- a/tools/python_api/test/test_connection.py
+++ b/tools/python_api/test/test_connection.py
@@ -44,6 +44,11 @@ def test_connection_interrupt(conn_db_readwrite: ConnDB) -> None:
execute_thread = threading.Thread(target=run_long_query, args=(conn,))
execute_thread.start()
time.sleep(5)
- conn.interrupt()
- execute_thread.join(timeout=100)
+ # Binding folds each RANGE(1, 1000000) into a million-element list literal, which can take
+ # longer than the sleep on a slow machine, and the interrupt flag is cleared when execution
+ # starts, so an interrupt that lands during compilation is lost; repeat it until one sticks.
+ deadline = time.monotonic() + 100
+ while execute_thread.is_alive() and time.monotonic() < deadline:
+ conn.interrupt()
+ execute_thread.join(timeout=1)
assert not execute_thread.is_alive()
Loading