From 826e5a471b645e45e37d1d81dd725cdc05bb20ae Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Fri, 25 Sep 2026 16:44:12 +0000 Subject: [PATCH] ladybug: add build-ladybug.yml for riscv64 wheels LadybugDB (formerly Kuzu) is an embedded graph database; the wheel's ladybug/_lbug is a pybind11 module over the whole C++ engine. Mirrors upstream's python-wheel-workflow.yml: the sdist comes from scripts/pip-package/package_tar.py on ubuntu-latest, the static liblbug.a is built once like the Linux compat leg of precompiled-bin-workflow.yml, and cibuildwheel links each interpreter's extension against it. Narrowed to manylinux_riscv64 and cp312/cp313/cp314/cp314t. - liblbug.a is built in manylinux_2_39_riscv64 with its default GCC 14 and openssl-devel (OpenSSL 3), where upstream's manylinux_2_28 image needs gcc-toolset-13 and the openssl3 packages. - 0001 halves the buffer manager's mmap reservation on ENOMEM. The default max_db_size is 8TB, which does not fit in riscv64 Sv39's 256GB user address space, so every default Database() throws "Mmap for size 8796093022208 failed." (reproduced on x86-64 with the PyPI wheel under `ulimit -v 268435456`; still present in v0.20.4). - The wheel statically links the vendored third_party libraries but upstream ships only ladybug's own LICENSE; their licence files are copied to the sdist root as LICENSE. and asserted in the wheel. - Tests mirror ladybug-python's python-ci-pybind job (pytest over the tag's tools/python_api/test with dataset/, pybind backend forced). pyarrow 25 is the only riscv64 build; cp314t has no pandas/polars, so the modules needing them are skipped there. test_fsm.py is skipped: it fails identically with upstream's own x86_64 0.19.1 wheel. Rehearsed on x86-64 with the PyPI 0.19.1 wheel, this test layout and dependency set: 272 passed (7 test_fsm failures); 168 passed with the cp314t set. --- .github/workflows/build-ladybug.yml | 246 ++++++++++++++++++ docs/packages/ladybug.yaml | 10 + ...e-VMRegion-reservation-until-it-fits.patch | 48 ++++ 3 files changed, 304 insertions(+) create mode 100644 .github/workflows/build-ladybug.yml create mode 100644 docs/packages/ladybug.yaml create mode 100644 patches/ladybug/0.19.1/0001-storage-shrink-the-VMRegion-reservation-until-it-fits.patch diff --git a/.github/workflows/build-ladybug.yml b/.github/workflows/build-ladybug.yml new file mode 100644 index 00000000000..17b4624aeff --- /dev/null +++ b/.github/workflows/build-ladybug.yml @@ -0,0 +1,246 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +--- +# This workflow is based on: https://github.com/LadybugDB/ladybug/blob/v0.19.1/.github/workflows/python-wheel-workflow.yml +# and the Linux compat leg of https://github.com/LadybugDB/ladybug/blob/v0.19.1/.github/workflows/precompiled-bin-workflow.yml +name: Build ladybug wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'Version glob to (re)build; empty builds every version of docs/packages/ladybug.yaml not released yet' + required: false + default: '' + pull_request: + branches: [main] + paths: + - '.github/workflows/build-ladybug.yml' + - 'docs/packages/ladybug.yaml' + - 'patches/ladybug/**' + push: + branches: [main] + paths: + - '.github/workflows/build-ladybug.yml' + - 'docs/packages/ladybug.yaml' + - 'patches/ladybug/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +env: + MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 + +jobs: + setup: + uses: $/.github/workflows/_setup.yml + with: + package: ladybug + version: ${{ inputs.version }} + + package_python_sdist: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + name: Package ladybug ${{ matrix.version }} sdist + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + + env: + LADYBUG_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout ladybug v${{ env.LADYBUG_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: LadybugDB/ladybug + ref: v${{ env.LADYBUG_VERSION }} + persist-credentials: false + + - name: Update submodules + run: git submodule update --init --recursive tools/python_api + + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + python-version: '3.12' + activate-environment: true + enable-cache: false + + - name: Package Python sdist + working-directory: scripts/pip-package + run: | + uv pip install setuptools + python package_tar.py + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ladybug-${{ env.LADYBUG_VERSION }}-sdist + path: scripts/pip-package/*.tar.gz + if-no-files-found: error + + build_precompiled_bin: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + name: Build liblbug ${{ matrix.version }} manylinux_riscv64 + runs-on: ubuntu-24.04-riscv + timeout-minutes: 1440 + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + + env: + LADYBUG_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout ladybug v${{ env.LADYBUG_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: LadybugDB/ladybug + ref: v${{ env.LADYBUG_VERSION }} + persist-credentials: false + + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: python-wheels + persist-credentials: false + + - name: Apply patches + run: git apply python-wheels/patches/ladybug/${{ env.LADYBUG_VERSION }}/*.patch + + # Rocky 10's openssl-devel is OpenSSL 3 and its default GCC 14 replaces gcc-toolset-13. + - name: Build + run: | + docker run --rm -v "$(pwd):/work" -w /work "${MANYLINUX_RISCV64_IMAGE}" bash -c ' + set -euxo pipefail + dnf install -y ninja-build openssl-devel + make GEN=Ninja + make install + mkdir precompiled-liblbug + cp install/include/lbug.h install/include/lbug.hpp precompiled-liblbug/ + cp -L install/lib*/liblbug.a precompiled-liblbug/ + ' + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ladybug-${{ env.LADYBUG_VERSION }}-liblbug-static + path: precompiled-liblbug/ + if-no-files-found: error + + build_wheels: + needs: [setup, package_python_sdist, build_precompiled_bin] + if: needs.setup.outputs.versions != '[]' + name: Build ladybug ${{ matrix.version }} ${{ matrix.python }}-manylinux_riscv64 + runs-on: ubuntu-24.04-riscv + timeout-minutes: 720 + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + python: ["cp312", "cp313", "cp314", "cp314t"] + + env: + LADYBUG_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout ladybug v${{ env.LADYBUG_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: LadybugDB/ladybug + ref: v${{ env.LADYBUG_VERSION }} + persist-credentials: false + + - name: Update submodules + run: git submodule update --init --depth 1 dataset tools/python_api + + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ladybug-${{ env.LADYBUG_VERSION }}-sdist + path: scripts/pip-package/ + + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ladybug-${{ env.LADYBUG_VERSION }}-liblbug-static + path: precompiled-liblbug/ + + - name: Prepare wheel source tree + run: | + mkdir -p scripts/pip-package/cibw-source + tar -xzf scripts/pip-package/ladybug-${{ env.LADYBUG_VERSION }}.tar.gz -C scripts/pip-package/cibw-source + + # The wheel statically links the vendored third_party libraries but upstream ships only ladybug's own LICENSE. + - name: Add the vendored libraries' licences + working-directory: scripts/pip-package/cibw-source/sdist + run: | + for f in ladybug-source/third_party/*/LICENSE*; do + cp "$f" "LICENSE.$(basename "$(dirname "$f")")" + done + ls LICENSE* + + - name: Build wheels + uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0 + with: + package-dir: scripts/pip-package/cibw-source/sdist + output-dir: wheelhouse/ + only: ${{ matrix.python }}-manylinux_riscv64 + env: + CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }} + CIBW_BEFORE_ALL_LINUX: dnf install -y openssl-devel pkg-config + CIBW_ENVIRONMENT_LINUX: >- + LBUG_API_PRECOMPILED_LIB_PATH=/project/precompiled-liblbug/liblbug.a + PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/ + CIBW_REPAIR_WHEEL_COMMAND_LINUX: auditwheel repair --exclude libssl.so.3 --exclude libcrypto.so.3 -w {dest_dir} {wheel} + # Not PIP_ONLY_BINARY: torch-geometric is pure Python. + CIBW_TEST_ENVIRONMENT: LBUG_PYTHON_BACKEND=pybind PIP_PREFER_BINARY=1 + CIBW_TEST_SOURCES: dataset tools/python_api/test + # Upstream's [dev] test deps; pyarrow 25 is the only riscv64 build, and cp314t has no pandas/polars. + CIBW_TEST_REQUIRES: >- + pytest pytest-asyncio~=1.0 networkx~=3.0 numpy~=2.0 pyarrow torch torch-geometric>=2.5.0 + ${{ matrix.python != 'cp314t' && 'pandas~=2.2 polars~=1.30' || '' }} + # test_fsm.py fails the same way with upstream's own x86_64 0.19.1 wheel. + CIBW_TEST_COMMAND: >- + python -m pytest -vv tools/python_api/test + --ignore=tools/python_api/test/test_fsm.py + ${{ matrix.python == 'cp314t' && '--ignore=tools/python_api/test/test_arrow.py --ignore=tools/python_api/test/test_arrow_memory_backed_table.py --ignore=tools/python_api/test/test_df.py --ignore=tools/python_api/test/test_networkx.py --ignore=tools/python_api/test/test_scan_pandas.py --ignore=tools/python_api/test/test_scan_pandas_pyarrow.py --ignore=tools/python_api/test/test_scan_polars.py --ignore=tools/python_api/test/test_udf.py -k "not test_get_as_df_json"' || '' }} + + - name: Check the extension module and licences made it into the wheel + run: | + python3 - wheelhouse/*.whl <<'EOF' + import sys, zipfile + for whl in sys.argv[1:]: + names = zipfile.ZipFile(whl).namelist() + assert any(n.startswith("ladybug/_lbug.") and n.endswith(".so") for n in names), f"no compiled extension in {whl}" + licences = {n.rsplit("/", 1)[1] for n in names if ".dist-info/licenses/" in n} - {""} + assert {"LICENSE", "LICENSE.zstd", "LICENSE.re2", "LICENSE.pybind11"} <= licences, licences + print(whl, sorted(licences)) + EOF + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ladybug-${{ env.LADYBUG_VERSION }}-${{ matrix.python }}-manylinux_riscv64 + path: wheelhouse/*.whl + if-no-files-found: error + + publish: + name: Publish ladybug ${{ matrix.version }} + needs: [setup, build_wheels] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + permissions: + contents: write + pull-requests: write + uses: $/.github/workflows/_publish-wheel.yml + secrets: + app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + with: + artifact-pattern: ladybug-${{ matrix.version }}-*-manylinux_riscv64 diff --git a/docs/packages/ladybug.yaml b/docs/packages/ladybug.yaml new file mode 100644 index 00000000000..fa0912f99fb --- /dev/null +++ b/docs/packages/ladybug.yaml @@ -0,0 +1,10 @@ +package-name: ladybug +source-code: https://github.com/LadybugDB/ladybug +license: MIT +versions: +- version: 0.19.1 + patched: true + comment: >- + When the default 8TB buffer-manager reservation does not fit in the process address space + (riscv64 Sv39 gives 256GB), the database falls back to the largest power-of-two region that + does, which caps its maximum size accordingly. diff --git a/patches/ladybug/0.19.1/0001-storage-shrink-the-VMRegion-reservation-until-it-fits.patch b/patches/ladybug/0.19.1/0001-storage-shrink-the-VMRegion-reservation-until-it-fits.patch new file mode 100644 index 00000000000..9c58b6d45f1 --- /dev/null +++ b/patches/ladybug/0.19.1/0001-storage-shrink-the-VMRegion-reservation-until-it-fits.patch @@ -0,0 +1,48 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Fri, 25 Sep 2026 00:00:00 +0000 +Subject: [PATCH] storage: shrink the VMRegion reservation until it fits + +Upstream-Status: To upstream [not yet submitted; python-wheels does not open issues/PRs on third-party repos] + +Every Database reserves its buffer-manager region with one mmap of +max_db_size bytes, and max_db_size defaults to 1 << 43 (8TB). riscv64 +Sv39 gives a process 256GB of user address space (the T-Head C910/C920 +cores the riscv64 runners use only implement Sv39), so the reservation +fails with ENOMEM and every Database() created with the default +settings throws "Mmap for size 8796093022208 failed." The same happens +under a 39-bit VA arm64 kernel, or on x86-64 under +`ulimit -v 268435456`. Still present in v0.20.4. + +On ENOMEM, halve the reservation until it fits. Where the full region +fits nothing changes; elsewhere the database is capped at the largest +power-of-two region the address space can hold, the limit +max_db_size already expresses. +--- +diff --git a/src/storage/buffer_manager/vm_region.cpp b/src/storage/buffer_manager/vm_region.cpp +index 429bc61..d102eef 100644 +--- a/src/storage/buffer_manager/vm_region.cpp ++++ b/src/storage/buffer_manager/vm_region.cpp +@@ -13,6 +13,8 @@ + #else + #include + #include ++ ++#include + #endif + + #include "common/assert.h" +@@ -61,6 +63,13 @@ VMRegion::VMRegion(PageSizeClass pageSizeClass, uint64_t maxRegionSize) : numFra + // backed by any file, and its content are initialized to zero. + region = static_cast(mmap(NULL, getMaxRegionSize(), PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1 /* fd */, 0 /* offset */)); ++ // The default 8TB region does not fit in a smaller user address space (256GB under riscv64 ++ // Sv39, 512GB under a 39-bit VA arm64 kernel), so halve the reservation until it does. ++ while (region == MAP_FAILED && errno == ENOMEM && maxNumFrameGroups > 1) { ++ maxNumFrameGroups /= 2; ++ region = static_cast(mmap(NULL, getMaxRegionSize(), PROT_READ | PROT_WRITE, ++ MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1 /* fd */, 0 /* offset */)); ++ } + if (region == MAP_FAILED) { + throw BufferManagerException( + "Mmap for size " + std::to_string(getMaxRegionSize()) + " failed.");