diff --git a/.github/workflows/build-openviking.yml b/.github/workflows/build-openviking.yml new file mode 100644 index 00000000000..2c76c77002f --- /dev/null +++ b/.github/workflows/build-openviking.yml @@ -0,0 +1,232 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +--- +# This workflow is based on: https://github.com/volcengine/OpenViking/blob/v0.4.16/.github/workflows/_build.yml +name: Build openviking wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'Version glob to (re)build; empty builds every version of docs/packages/openviking.yaml not released yet' + required: false + default: '' + pull_request: + branches: [main] + paths: + - '.github/workflows/build-openviking.yml' + - 'docs/packages/openviking.yaml' + - 'patches/openviking/**' + push: + branches: [main] + paths: + - '.github/workflows/build-openviking.yml' + - 'docs/packages/openviking.yaml' + - 'patches/openviking/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +jobs: + setup: + uses: $/.github/workflows/_setup.yml + with: + package: openviking + version: ${{ inputs.version }} + + web_studio: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + name: Build openviking ${{ matrix.version }} Web Studio + runs-on: ubuntu-latest + env: + OPENVIKING_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout openviking v${{ env.OPENVIKING_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: volcengine/OpenViking + ref: v${{ env.OPENVIKING_VERSION }} + persist-credentials: false + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22' + + - name: Build Web Studio + working-directory: web-studio + run: | + npm ci + npm run build -- --base=/studio/ + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: openviking-${{ env.OPENVIKING_VERSION }}-web-studio + path: web-studio/dist + if-no-files-found: error + + build_wheels: + needs: [setup, web_studio] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + name: Build openviking ${{ matrix.version }} cp310-abi3-manylinux_riscv64 + runs-on: ubuntu-24.04-riscv + timeout-minutes: 360 + env: + OPENVIKING_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout openviking v${{ env.OPENVIKING_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: volcengine/OpenViking + ref: v${{ env.OPENVIKING_VERSION }} + persist-credentials: false + + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: python-wheels + persist-credentials: false + + - name: Apply patches + run: git apply -v python-wheels/patches/openviking/${{ env.OPENVIKING_VERSION }}/*.patch + + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: openviking-${{ env.OPENVIKING_VERSION }}-web-studio + path: openviking/web_studio/dist + + # Upstream ships no notices for the C++ libraries statically linked into the engine. + - name: Add vendored licences + run: | + cp third_party/leveldb-1.23/LICENSE LICENSE.leveldb + cp third_party/spdlog-1.14.1/LICENSE LICENSE.spdlog + cp third_party/croaring/LICENSE LICENSE.croaring + cp third_party/rapidjson/LICENSE LICENSE.rapidjson + + - name: Write smoke test + run: | + cat > smoke_test.py <<'PY' + import importlib + import importlib.util + from importlib.resources import files + + from openviking.pyagfs import get_binding_client + import openviking.storage.vectordb.engine as engine + + binding_client, _ = get_binding_client() + if binding_client is None: + raise SystemExit("ragfs binding client was not installed") + + print(f"Loaded RAGFS binding client {binding_client.__name__}") + print(f"Loaded runtime engine variant {engine.ENGINE_VARIANT}") + print(f"Available engine variants {engine.AVAILABLE_ENGINE_VARIANTS}") + + module_name = f"openviking.storage.vectordb.engine._{engine.ENGINE_VARIANT}" + backend_spec = importlib.util.find_spec(module_name) + if backend_spec is None or backend_spec.origin is None: + raise SystemExit(f"backend module {module_name} was not installed") + + backend = importlib.import_module(module_name) + required_filter_symbols = ( + "_index_engine_set_filter_layout", + "_index_engine_evaluate_filter", + "_index_engine_evaluate_filter_cached", + "_index_engine_search_with_filter_token", + ) + missing_filter_symbols = [ + symbol for symbol in required_filter_symbols if not hasattr(backend, symbol) + ] + if missing_filter_symbols: + raise SystemExit( + f"backend module {module_name} is missing filter ABI symbols: " + f"{', '.join(missing_filter_symbols)}" + ) + + required_filter_methods = ( + "set_filter_layout", + "evaluate_filter", + "search_with_filter_token", + ) + missing_filter_methods = [ + method for method in required_filter_methods if not hasattr(engine.IndexEngine, method) + ] + if missing_filter_methods: + raise SystemExit( + "IndexEngine is missing filter methods: " + f"{', '.join(missing_filter_methods)}" + ) + + studio_index = files("openviking").joinpath("web_studio/dist/index.html") + if not studio_index.is_file(): + raise SystemExit("Web Studio index.html was not installed") + + print(f"Imported backend module {module_name}") + print(f"Backend module origin {backend_spec.origin}") + print(f"Validated filter ABI symbols {required_filter_symbols}") + print(f"Bundled Web Studio index {studio_index}") + PY + + - name: Build and test wheel + uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0 + env: + CIBW_ARCHS: riscv64 + CIBW_BUILD: "cp310-* cp311-* cp312-* cp313-* cp314-*" + CIBW_SKIP: "*-musllinux_*" + CIBW_BEFORE_ALL_LINUX: >- + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal + CIBW_ENVIRONMENT: >- + PATH="$PATH:$HOME/.cargo/bin" + PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/ + OV_REQUIRE_RAGFS_BUILD=1 + SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OPENVIKING=${{ env.OPENVIKING_VERSION }} + CIBW_TEST_ENVIRONMENT: >- + PIP_ONLY_BINARY=:all: + # tiktoken, grpcio and cryptography have no cp310/cp311 riscv64 wheel. + CIBW_TEST_SKIP: "cp310-* cp311-*" + CIBW_TEST_REQUIRES: pytest pytest-asyncio numpy + CIBW_TEST_SOURCES: tests pyproject.toml smoke_test.py + CIBW_TEST_COMMAND: >- + python smoke_test.py && + python -c "import importlib.metadata as m; l = set(p.name for p in m.files('openviking') if '.dist-info/licenses/' in str(p)); assert set(['LICENSE', 'LICENSE.leveldb', 'LICENSE.spdlog', 'LICENSE.croaring', 'LICENSE.rapidjson']) <= l, l" && + python -m pytest -q -o addopts='' + tests/vectordb/test_cuvs_config.py + tests/vectordb/test_cuvs_index.py + tests/vectordb/test_cuvs_collection.py + tests/vectordb/test_str_to_uint64.py + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: openviking-${{ env.OPENVIKING_VERSION }}-cp310-abi3-manylinux_riscv64 + path: ./wheelhouse/*.whl + if-no-files-found: error + + publish: + name: Publish openviking ${{ matrix.version }} + needs: [setup, web_studio, build_wheels] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + permissions: + contents: write + pull-requests: write + uses: $/.github/workflows/_publish-wheel.yml + secrets: + app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + with: + artifact-pattern: openviking-${{ matrix.version }}-*-manylinux_riscv64 diff --git a/docs/packages/openviking.yaml b/docs/packages/openviking.yaml new file mode 100644 index 00000000000..3682abd0844 --- /dev/null +++ b/docs/packages/openviking.yaml @@ -0,0 +1,6 @@ +package-name: openviking +source-code: https://github.com/volcengine/OpenViking +license: AGPL-3.0-only +versions: +- version: 0.4.16 + patched: true diff --git a/patches/openviking/0.4.16/0001-build-only-require-Development.Module-from-FindPytho.patch b/patches/openviking/0.4.16/0001-build-only-require-Development.Module-from-FindPytho.patch new file mode 100644 index 00000000000..2f595cd2d20 --- /dev/null +++ b/patches/openviking/0.4.16/0001-build-only-require-Development.Module-from-FindPytho.patch @@ -0,0 +1,36 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Fri, 25 Sep 2026 00:00:00 +0000 +Subject: [PATCH] build: only require Development.Module from FindPython3 + +The vector engine is a Python extension module, so it needs the Python +headers but never links libpython. Asking FindPython3 for the umbrella +`Development` component also pulls in `Development.Embed`, which requires +libpython to exist. The manylinux interpreters used to build wheels ship +no libpython at all, so configuring fails on every manylinux image before +anything is compiled: + + Could NOT find Python3 (missing: Python3_LIBRARIES Development + Development.Embed) + +Upstream's own Linux wheels build against a CPython compiled with +--enable-shared, which is why their CI does not hit it. Reproduces on +manylinux_2_28_x86_64 as well as manylinux_2_39_riscv64. + +Upstream-Status: To upstream [not yet submitted; still present on volcengine/OpenViking main] + +Signed-off-by: Ludovic Henry +--- +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index 1a79b8a4..da2a9495 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -43,7 +43,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error -Wno-deprecated-declarations + set(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE} -lpthread") + set(Python3_ARCH_INCLUDE_DIR "/usr/include/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/") + +-find_package(Python3 COMPONENTS Interpreter Development REQUIRED) ++find_package(Python3 COMPONENTS Interpreter Development.Module REQUIRED) + if(WIN32 AND CMAKE_VERSION VERSION_GREATER_EQUAL "3.26") + find_package(Python3 COMPONENTS Development.SABIModule QUIET) + endif()