rerun-sdk: Add versions 0.37.2, 0.38.0, 0.38.1 (#2037) #31
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-FileCopyrightText: 2026 The RISE Project | |
| # SPDX-License-Identifier: MIT | |
| --- | |
| # Based on the `linux-x64` leg of upstream's own wheel pipeline: | |
| # https://github.com/rerun-io/rerun/blob/0.37.1/.github/workflows/reusable_build_and_upload_wheels.yml | |
| # (MODE=pr: `--no-default-features --features perf_telemetry,extension-module`, | |
| # i.e. the crate's own default features -- no --features flags needed). | |
| # | |
| # Scope: this builds only the `rerun_bindings` Python SDK extension (rerun_py/), | |
| # not the `rerun-cli` native viewer. Upstream's published wheel also bundles a | |
| # prebuilt `rerun-cli` binary (a full wgpu/egui desktop+web viewer, built by a | |
| # separate ~16-core job that itself needs a wasm/JS toolchain for the bundled | |
| # web viewer) at rerun_sdk/rerun_cli/rerun; building that from source is out of | |
| # scope here. rerun_py's own build.rs fails without it unless | |
| # RERUN_ALLOW_MISSING_BIN is set (see below), and upstream's own | |
| # `[tool.maturin] include` comment already documents that a missing binary is | |
| # "not a packaging failure" -- `import rerun` and all logging/recording/gRPC | |
| # APIs work; only `rerun.spawn()`/`serve()` and the `rerun` console script | |
| # (which shell out to the bundled binary) raise a clear error instead of | |
| # launching a viewer. | |
| name: Build rerun-sdk wheels (riscv64) | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| version: | |
| description: 'Version glob to (re)build; empty builds every version of docs/packages/rerun-sdk.yaml not released yet' | |
| required: false | |
| default: '' | |
| pull_request: | |
| branches: [main] | |
| paths: | |
| - '.github/workflows/build-rerun-sdk.yml' | |
| - 'docs/packages/rerun-sdk.yaml' | |
| - 'patches/rerun-sdk/**' | |
| push: | |
| branches: [main] | |
| paths: | |
| - '.github/workflows/build-rerun-sdk.yml' | |
| - 'docs/packages/rerun-sdk.yaml' | |
| - 'patches/rerun-sdk/**' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read # to fetch code (actions/checkout) | |
| env: | |
| MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 | |
| jobs: | |
| setup: | |
| uses: $/.github/workflows/_setup.yml | |
| with: | |
| package: rerun-sdk | |
| version: ${{ inputs.version }} | |
| build_wheels: | |
| needs: [setup] | |
| if: needs.setup.outputs.versions != '[]' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: ${{ fromJSON(needs.setup.outputs.versions) }} | |
| # pyo3 carries `abi3-py310` unconditionally in rerun_py/Cargo.toml, so | |
| # upstream ships one cp310-abi3 wheel. Our registry has no riscv64 pyarrow | |
| # (a hard runtime dependency) for cp310/cp311, so the build floor here is | |
| # cp312 instead -- the wheel is still tagged cp310-abi3 by the pyo3 | |
| # feature regardless of which interpreter compiles it (CLAUDE.md gotcha | |
| # 96), it is simply never tested below cp312. | |
| name: Build rerun-sdk ${{ matrix.version }} cp310-abi3-manylinux_riscv64 | |
| runs-on: ubuntu-24.04-riscv | |
| timeout-minutes: 1440 | |
| env: | |
| RERUN_SDK_VERSION: ${{ matrix.version }} | |
| steps: | |
| # rerun_py is a member of the rerun Cargo workspace (~760 crates on | |
| # riscv64: datafusion, lance, tonic/tokio, hdf5-pure, ...) and builds | |
| # against its sibling crates, so the checkout root has to be the | |
| # workspace root, not rerun_py itself. | |
| # lfs: true is required: the test suite's binary fixtures (HDF5, MP4, | |
| # ...) are Git LFS pointers without it, and get fed straight into their | |
| # native parsers -- "HDF5 signature not found" / "MP4 ... box with a | |
| # larger size than it" is what that looks like from the Python side. | |
| - name: Checkout rerun ${{ env.RERUN_SDK_VERSION }} | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| repository: rerun-io/rerun | |
| ref: ${{ env.RERUN_SDK_VERSION }} | |
| lfs: true | |
| persist-credentials: false | |
| - name: Checkout python-wheels | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| path: python-wheels | |
| persist-credentials: false | |
| # lance-core and lance-linalg (build-dependencies of re_datafusion via | |
| # lance) have per-arch SIMD structs/closures with cfg arms for | |
| # aarch64/x86_64/loongarch64 and no catch-all, so they don't compile on | |
| # riscv64: lance-core's SIMD-tier-detection closure evaluates to `()` | |
| # instead of `SimdSupport` (E0308), and lance-linalg's f32x8/f32x16/ | |
| # f64x4/f64x8/i32x8 types don't exist at all there (E0425) (CLAUDE.md | |
| # gotchas 287/300). There is no compatible bugfix release to `cargo | |
| # update` to, so riscv64-fixed copies are vendored here (the crates.io | |
| # tarballs, already free of the workspace-inherited fields the git | |
| # checkouts carry) and wired in via `[patch.crates-io]`. The pinned | |
| # lance version moves between rerun releases (9.0.0 through 0.37.x, | |
| # 10.0.0 from 0.38.0), so it is read out of the just-checked-out | |
| # Cargo.lock rather than hardcoded. | |
| - name: Vendor riscv64-fixed lance-core and lance-linalg | |
| run: | | |
| LANCE_VERSION=$(awk '$0 == "name = \"lance-core\""{getline; print; exit}' Cargo.lock | sed -E 's/^version = "(.*)"$/\1/') | |
| curl -fsSL -A "python-wheels (https://github.com/riseproject-dev/python-wheels)" -o lance-core.tar.gz "https://crates.io/api/v1/crates/lance-core/${LANCE_VERSION}/download" | |
| mkdir "lance-core-${LANCE_VERSION}-riscv64" | |
| tar xzf lance-core.tar.gz -C "lance-core-${LANCE_VERSION}-riscv64" --strip-components=1 | |
| rm lance-core.tar.gz | |
| patch -p1 -d "lance-core-${LANCE_VERSION}-riscv64" < python-wheels/patches/rerun-sdk/${{ env.RERUN_SDK_VERSION }}/0001-lance-core-riscv64-simd-fallback.patch | |
| curl -fsSL -A "python-wheels (https://github.com/riseproject-dev/python-wheels)" -o lance-linalg.tar.gz "https://crates.io/api/v1/crates/lance-linalg/${LANCE_VERSION}/download" | |
| mkdir "lance-linalg-${LANCE_VERSION}-riscv64" | |
| tar xzf lance-linalg.tar.gz -C "lance-linalg-${LANCE_VERSION}-riscv64" --strip-components=1 | |
| rm lance-linalg.tar.gz | |
| patch -p1 -d "lance-linalg-${LANCE_VERSION}-riscv64" < python-wheels/patches/rerun-sdk/${{ env.RERUN_SDK_VERSION }}/0003-lance-linalg-riscv64-simd-fallback.patch | |
| git apply python-wheels/patches/rerun-sdk/${{ env.RERUN_SDK_VERSION }}/0002-cargo-patch-lance-crates-for-riscv64.patch | |
| # re_importer moved from crates/store/re_importer (0.37.x) to | |
| # crates/data_flow/re_importer (0.38.0+) upstream; CIBW_TEST_SOURCES | |
| # below needs its importer_mcap test-assets path to track whichever one | |
| # the checked-out tag actually has. | |
| - name: Resolve re_importer crate path | |
| run: | | |
| for dir in crates/data_flow/re_importer crates/store/re_importer; do | |
| if [ -d "$dir" ]; then | |
| echo "RE_IMPORTER_DIR=$dir" >> "$GITHUB_ENV" | |
| break | |
| fi | |
| done | |
| # This workspace links ~760 crates (datafusion, lance, tonic, hdf5-pure, | |
| # ...) concurrently; same OOM guard build-deltalake.yml/ | |
| # build-polars-runtime.yml need for a comparably sized Rust build. | |
| - name: Set swap space | |
| uses: pierotofy/set-swap-space@fc79b3f67fa8a838184ce84a674ca12238d2c761 # master | |
| with: | |
| swap-size-gb: 10 | |
| - name: Build wheel | |
| uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0 | |
| with: | |
| package-dir: rerun_py | |
| output-dir: wheelhouse/ | |
| # musllinux is dropped: rustup.rs ships no riscv64 musl toolchain. | |
| # cp310/cp311 are dropped: our registry has no riscv64 pyarrow | |
| # there (see the floor note above); cibuildwheel builds once on | |
| # cp312 and re-tests the same abi3 wheel on cp313/cp314. | |
| # Upstream ships no free-threaded wheel, so cp314t is not added. | |
| env: | |
| CIBW_BUILD: cp312-manylinux_riscv64 cp313-manylinux_riscv64 cp314-manylinux_riscv64 | |
| CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }} | |
| # rerun_py ships no [tool.cibuildwheel]; the Rust toolchain its | |
| # maturin backend needs is installed in-container here. lance (a | |
| # build-dependency of re_datafusion) needs protoc at build time to | |
| # compile its manifest .proto files -- upstream's own before-build | |
| # pulls a wheel-packaged protoc, which has no riscv64 build; Rocky | |
| # 10's CRB repo (enabled in the image) has one (CLAUDE.md gotcha | |
| # 100). | |
| CIBW_BEFORE_ALL_LINUX: >- | |
| curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && | |
| yum install -y protobuf-compiler protobuf-devel | |
| # The repo's own .cargo/config.toml unconditionally points | |
| # PYO3_CONFIG_FILE at rerun_py/pyo3-build.cfg for every cargo | |
| # invocation, expecting a pixi activation hook to have generated it | |
| # first; outside pixi it must be written by hand (upstream's own | |
| # fallback: `python scripts/generate_pyo3_config.py`). abi3 pins the | |
| # `version` field to 3.10 regardless of the interpreter that writes | |
| # it, so which one runs this does not matter. | |
| CIBW_BEFORE_BUILD_LINUX: | | |
| python -c " | |
| import sys; sys.path.insert(0, 'rerun_pixi_env/src') | |
| from pathlib import Path | |
| from rerun_pixi_env.pyo3_config import generate_config_file | |
| generate_config_file(Path('rerun_py/pyo3-build.cfg'))" | |
| # rerun's workspace Cargo.toml bakes `lto = "thin"` + | |
| # `codegen-units = 1` straight into the standard `release` profile | |
| # (not a custom-named one), so it cannot be swapped out the way | |
| # deltalake's `python-release` is; override it with | |
| # `CARGO_PROFILE_RELEASE_*` instead (CLAUDE.md gotcha 141). 0.38.x's | |
| # larger workspace (re_datafusion, re_hdf5, re_mcap, ... added since | |
| # 0.37.2) pushed the single-codegen-unit LTO link of rerun_bindings | |
| # past what CARGO_BUILD_JOBS=2 + 10 GB swap alone can hold, and | |
| # rustc aborted with a heap-corruption crash instead of a clean OOM. | |
| CIBW_ENVIRONMENT_LINUX: >- | |
| PATH="$PATH:$HOME/.cargo/bin" | |
| PROTOC_INCLUDE=/usr/include | |
| RERUN_ALLOW_MISSING_BIN=1 | |
| CARGO_BUILD_JOBS=2 | |
| CARGO_PROFILE_RELEASE_LTO=off | |
| CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16 | |
| PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/ | |
| # Without this pip prefers PyPI's newer numpy/pillow/pandas/torch, | |
| # none of which have riscv64 wheels, and falls back to slow source | |
| # builds in the container (CLAUDE.md gotcha 12). | |
| CIBW_TEST_ENVIRONMENT_LINUX: PIP_ONLY_BINARY=numpy,pillow,pandas,torch,pyarrow | |
| # Subset of upstream's `tests` extra (pyproject.toml | |
| # [project.optional-dependencies]) that has riscv64 wheels | |
| # somewhere: av, torchvision, datafusion (the PyPI package) and | |
| # polars publish none, so the handful of test files that need them | |
| # are ignored below instead. opencv-python is on our registry as a | |
| # cp37-abi3 wheel, forward-compatible with cp312+. | |
| CIBW_TEST_REQUIRES: >- | |
| pytest opencv-python>4.6 torch>=2.5 pandas>=2 | |
| syrupy==5.0.0 inline-snapshot==0.31.1 semver>=3.0,<3.1 tomli==2.0.1 | |
| # test-sources resolves against the checkout root, not package-dir. | |
| # tests/assets is the workspace-level fixture dir test_asset3d.py | |
| # reads via `Path(__file__).parents[3] / "tests" / "assets"`; the | |
| # importer_mcap assets (path resolved above -- see the | |
| # "Resolve re_importer crate path" step) and the animated_urdf | |
| # example data are the same shape for test_mcap_reader.py / | |
| # test_urdf_tree.py. Cargo.toml is test_version.py's own fixture | |
| # (it parses the workspace version straight out of it). The | |
| # `tests/assets/lerobot` fixtures (new in 0.38.x, present but unused | |
| # by any test in 0.37.2) are a sibling of importer_mcap under | |
| # `${RE_IMPORTER_DIR}/tests`, not under `src/importer_mcap/tests`; | |
| # test_lerobot_reader.py resolves them via | |
| # `Path(__file__).parents[3] / "crates/data_flow/re_importer/tests/ | |
| # assets/lerobot"` and without this line the whole file fails with | |
| # `FileNotFoundError: LeRobot dataset not found` -- not a datafusion | |
| # issue, just a missing test-sources entry from the 0.38.x bump. | |
| CIBW_TEST_SOURCES: >- | |
| rerun_py/tests rerun_py/pyproject.toml tests/assets | |
| ${{ env.RE_IMPORTER_DIR }}/src/importer_mcap/tests/assets | |
| ${{ env.RE_IMPORTER_DIR }}/tests/assets/lerobot | |
| examples/rust/animated_urdf/data Cargo.toml | |
| # e2e_redap_tests needs the "server" Cargo feature (not built, see | |
| # the scope note above) to start its local catalog server. | |
| # api_sandbox mixes in drafts of a future, unreleased API. Both are | |
| # upstream-labelled sandboxes/integration suites, not the SDK's | |
| # regression tests. test_headless_viewer.py spawns the (unbuilt) | |
| # rerun-cli binary, same as test_rrd_reader_multi_store.py's | |
| # session fixture (`rerun rrd merge`). test_server.py needs the | |
| # `datafusion` Python package (no riscv64 wheel anywhere, same as | |
| # test_datafusion_utils.py and, new in 0.38.x, test_chunk_index.py, | |
| # which imports `from datafusion import col, lit` at module scope); | |
| # the remaining ignored files all reach | |
| # rerun.experimental.dataloader, whose decoders import torchvision | |
| # or datafusion transitively. `-o markers=local_only` registers the | |
| # mark e2e_redap_tests/conftest.py normally defines -- pytest only | |
| # loads conftest.py files under the paths it actually collects, so | |
| # excluding that directory drops the registration too and | |
| # `filterwarnings = error` turns the resulting "unknown mark" | |
| # warning into a collection failure. The deselected tests are | |
| # single failures in otherwise-passing files, for the same three | |
| # reasons (rerun-cli, av, datafusion) plus | |
| # test_multiprocessing_gc, whose own comment says | |
| # torch.multiprocessing (torch is on CIBW_TEST_REQUIRES) "causes | |
| # more issues" than stdlib multiprocessing for this fork+gc race; | |
| # it hits the test's 5s join() timeout consistently on this | |
| # runner but not the reason any other test here was dropped -- | |
| # deselect, don't chase (CLAUDE.md gotcha 14). | |
| # test_optimized_stream.py (new in 0.38.x) is the same "single | |
| # failures in an otherwise-passing file" shape as | |
| # test_multi_stream.py::test_isolated_streams and | |
| # test_send_dataframe.py::test_send_dataframe_roundtrip above: only | |
| # the 4 tests that call `ChunkStore.reader()` (which lazily does | |
| # `import datafusion` in rerun_sdk/rerun/chunk/_chunk_store.py, same | |
| # missing riscv64 wheel as test_datafusion_utils.py/test_server.py) | |
| # fail; its other 4 tests (test_merge_quality, test_layout_snapshot, | |
| # test_laziness, test_own_chunk_rules) don't touch `.reader()` and | |
| # pass, so they're deselected individually rather than the whole | |
| # file ignored. | |
| CIBW_TEST_COMMAND: >- | |
| cd rerun_py && | |
| python -c "import rerun_bindings.rerun_bindings as m; assert m.__file__.endswith('.so'), m.__file__" && | |
| python -m pytest tests -v -o markers=local_only | |
| --ignore=tests/e2e_redap_tests | |
| --ignore=tests/api_sandbox | |
| --ignore=tests/integration/test_headless_viewer.py | |
| --ignore=tests/integration/test_dataloader_video.py | |
| --ignore=tests/integration/test_dataloader_video_codecs.py | |
| --ignore=tests/integration/test_chunk_store_reader.py | |
| --ignore=tests/integration/test_chunk_index.py | |
| --ignore=tests/integration/test_rrd_reader_multi_store.py | |
| --ignore=tests/unit/test_dataloader_decoder_helpers.py | |
| --ignore=tests/unit/test_dataloader_manifest.py | |
| --ignore=tests/unit/test_dataloader_query_indices.py | |
| --ignore=tests/unit/test_dataloader_skip_incomplete.py | |
| --ignore=tests/unit/test_sample_index.py | |
| --ignore=tests/unit/test_datafusion_utils.py | |
| --ignore=tests/unit/test_viewer_client.py | |
| --ignore=tests/unit/test_server.py | |
| --deselect=tests/integration/test_chunk_store.py::test_collect_optimize_video_stream_summary | |
| --deselect=tests/integration/test_lazy_chunk_stream.py::test_identity_roundtrip | |
| --deselect=tests/unit/test_binary_stream.py::test_binary_stream | |
| --deselect=tests/unit/test_multi_stream.py::test_isolated_streams | |
| --deselect=tests/unit/test_send_dataframe.py::test_send_dataframe_roundtrip | |
| --deselect=tests/unit/test_multiprocessing_gc.py::test_multiprocessing_gc | |
| --deselect=tests/integration/test_optimized_stream.py::test_data_equality | |
| --deselect=tests/integration/test_optimized_stream.py::test_chunk_store_path | |
| --deselect=tests/integration/test_optimized_stream.py::test_sweep_order_observable | |
| --deselect=tests/integration/test_optimized_stream.py::test_own_chunk_default | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: rerun-sdk-${{ env.RERUN_SDK_VERSION }}-cp310-abi3-manylinux_riscv64 | |
| path: wheelhouse/*.whl | |
| if-no-files-found: error | |
| publish: | |
| name: Publish rerun-sdk ${{ matrix.version }} | |
| needs: [setup, build_wheels] | |
| if: needs.setup.outputs.versions != '[]' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: ${{ fromJSON(needs.setup.outputs.versions) }} | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| uses: $/.github/workflows/_publish-wheel.yml | |
| secrets: | |
| app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} | |
| with: | |
| artifact-pattern: rerun-sdk-${{ matrix.version }}-*-manylinux_riscv64 |