rerun-sdk: Add versions 0.37.2, 0.38.0, 0.38.1

.github/workflows/build-rerun-sdk.yml

@@ -100,32 +100,50 @@ jobs:
           path: python-wheels
           persist-credentials: false
 
-      # lance-core and lance-linalg 9.0.0 (build-dependencies of re_datafusion
-      # via lance) have per-arch SIMD structs/closures with cfg arms for
+      # lance-core and lance-linalg (build-dependencies of re_datafusion via
+      # lance) have per-arch SIMD structs/closures with cfg arms for
       # aarch64/x86_64/loongarch64 and no catch-all, so they don't compile on
       # riscv64: lance-core's SIMD-tier-detection closure evaluates to `()`
       # instead of `SimdSupport` (E0308), and lance-linalg's f32x8/f32x16/
       # f64x4/f64x8/i32x8 types don't exist at all there (E0425) (CLAUDE.md
       # gotchas 287/300). There is no compatible bugfix release to `cargo
       # update` to, so riscv64-fixed copies are vendored here (the crates.io
       # tarballs, already free of the workspace-inherited fields the git
-      # checkouts carry) and wired in via `[patch.crates-io]`.
+      # checkouts carry) and wired in via `[patch.crates-io]`. The pinned
+      # lance version moves between rerun releases (9.0.0 through 0.37.x,
+      # 10.0.0 from 0.38.0), so it is read out of the just-checked-out
+      # Cargo.lock rather than hardcoded.
       - name: Vendor riscv64-fixed lance-core and lance-linalg
         run: |
-          curl -fsSL -A "python-wheels (https://github.com/riseproject-dev/python-wheels)" -o lance-core.tar.gz https://crates.io/api/v1/crates/lance-core/9.0.0/download
-          mkdir lance-core-9.0.0-riscv64
-          tar xzf lance-core.tar.gz -C lance-core-9.0.0-riscv64 --strip-components=1
+          LANCE_VERSION=$(awk '$0 == "name = \"lance-core\""{getline; print; exit}' Cargo.lock | sed -E 's/^version = "(.*)"$/\1/')
+
+          curl -fsSL -A "python-wheels (https://github.com/riseproject-dev/python-wheels)" -o lance-core.tar.gz "https://crates.io/api/v1/crates/lance-core/${LANCE_VERSION}/download"
+          mkdir "lance-core-${LANCE_VERSION}-riscv64"
+          tar xzf lance-core.tar.gz -C "lance-core-${LANCE_VERSION}-riscv64" --strip-components=1
           rm lance-core.tar.gz
-          patch -p1 -d lance-core-9.0.0-riscv64 < python-wheels/patches/rerun-sdk/${{ env.RERUN_SDK_VERSION }}/0001-lance-core-riscv64-simd-fallback.patch
+          patch -p1 -d "lance-core-${LANCE_VERSION}-riscv64" < python-wheels/patches/rerun-sdk/${{ env.RERUN_SDK_VERSION }}/0001-lance-core-riscv64-simd-fallback.patch
 
-          curl -fsSL -A "python-wheels (https://github.com/riseproject-dev/python-wheels)" -o lance-linalg.tar.gz https://crates.io/api/v1/crates/lance-linalg/9.0.0/download
-          mkdir lance-linalg-9.0.0-riscv64
-          tar xzf lance-linalg.tar.gz -C lance-linalg-9.0.0-riscv64 --strip-components=1
+          curl -fsSL -A "python-wheels (https://github.com/riseproject-dev/python-wheels)" -o lance-linalg.tar.gz "https://crates.io/api/v1/crates/lance-linalg/${LANCE_VERSION}/download"
+          mkdir "lance-linalg-${LANCE_VERSION}-riscv64"
+          tar xzf lance-linalg.tar.gz -C "lance-linalg-${LANCE_VERSION}-riscv64" --strip-components=1
           rm lance-linalg.tar.gz
-          patch -p1 -d lance-linalg-9.0.0-riscv64 < python-wheels/patches/rerun-sdk/${{ env.RERUN_SDK_VERSION }}/0003-lance-linalg-riscv64-simd-fallback.patch
+          patch -p1 -d "lance-linalg-${LANCE_VERSION}-riscv64" < python-wheels/patches/rerun-sdk/${{ env.RERUN_SDK_VERSION }}/0003-lance-linalg-riscv64-simd-fallback.patch
 
           git apply python-wheels/patches/rerun-sdk/${{ env.RERUN_SDK_VERSION }}/0002-cargo-patch-lance-crates-for-riscv64.patch
 
+      # re_importer moved from crates/store/re_importer (0.37.x) to
+      # crates/data_flow/re_importer (0.38.0+) upstream; CIBW_TEST_SOURCES
+      # below needs its importer_mcap test-assets path to track whichever one
+      # the checked-out tag actually has.
+      - name: Resolve re_importer crate path
+        run: |
+          for dir in crates/data_flow/re_importer crates/store/re_importer; do
+            if [ -d "$dir" ]; then
+              echo "RE_IMPORTER_DIR=$dir" >> "$GITHUB_ENV"
+              break
+            fi
+          done
+
       # This workspace links ~760 crates (datafusion, lance, tonic, hdf5-pure,
       # ...) concurrently; same OOM guard build-deltalake.yml/
       # build-polars-runtime.yml need for a comparably sized Rust build.
@@ -170,11 +188,22 @@ jobs:
             from pathlib import Path
             from rerun_pixi_env.pyo3_config import generate_config_file
             generate_config_file(Path('rerun_py/pyo3-build.cfg'))"
+          # rerun's workspace Cargo.toml bakes `lto = "thin"` +
+          # `codegen-units = 1` straight into the standard `release` profile
+          # (not a custom-named one), so it cannot be swapped out the way
+          # deltalake's `python-release` is; override it with
+          # `CARGO_PROFILE_RELEASE_*` instead (CLAUDE.md gotcha 141). 0.38.x's
+          # larger workspace (re_datafusion, re_hdf5, re_mcap, ... added since
+          # 0.37.2) pushed the single-codegen-unit LTO link of rerun_bindings
+          # past what CARGO_BUILD_JOBS=2 + 10 GB swap alone can hold, and
+          # rustc aborted with a heap-corruption crash instead of a clean OOM.
           CIBW_ENVIRONMENT_LINUX: >-
             PATH="$PATH:$HOME/.cargo/bin"
             PROTOC_INCLUDE=/usr/include
             RERUN_ALLOW_MISSING_BIN=1
             CARGO_BUILD_JOBS=2
+            CARGO_PROFILE_RELEASE_LTO=off
+            CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
             PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/
           # Without this pip prefers PyPI's newer numpy/pillow/pandas/torch,
           # none of which have riscv64 wheels, and falls back to slow source
@@ -192,13 +221,23 @@ jobs:
           # test-sources resolves against the checkout root, not package-dir.
           # tests/assets is the workspace-level fixture dir test_asset3d.py
           # reads via `Path(__file__).parents[3] / "tests" / "assets"`; the
-          # importer_mcap assets and the animated_urdf example data are the
-          # same shape for test_mcap_reader.py / test_urdf_tree.py. Cargo.toml
-          # is test_version.py's own fixture (it parses the workspace version
-          # straight out of it).
+          # importer_mcap assets (path resolved above -- see the
+          # "Resolve re_importer crate path" step) and the animated_urdf
+          # example data are the same shape for test_mcap_reader.py /
+          # test_urdf_tree.py. Cargo.toml is test_version.py's own fixture
+          # (it parses the workspace version straight out of it). The
+          # `tests/assets/lerobot` fixtures (new in 0.38.x, present but unused
+          # by any test in 0.37.2) are a sibling of importer_mcap under
+          # `${RE_IMPORTER_DIR}/tests`, not under `src/importer_mcap/tests`;
+          # test_lerobot_reader.py resolves them via
+          # `Path(__file__).parents[3] / "crates/data_flow/re_importer/tests/
+          # assets/lerobot"` and without this line the whole file fails with
+          # `FileNotFoundError: LeRobot dataset not found` -- not a datafusion
+          # issue, just a missing test-sources entry from the 0.38.x bump.
           CIBW_TEST_SOURCES: >-
             rerun_py/tests rerun_py/pyproject.toml tests/assets
-            crates/store/re_importer/src/importer_mcap/tests/assets
+            ${{ env.RE_IMPORTER_DIR }}/src/importer_mcap/tests/assets
+            ${{ env.RE_IMPORTER_DIR }}/tests/assets/lerobot
             examples/rust/animated_urdf/data Cargo.toml
           # e2e_redap_tests needs the "server" Cargo feature (not built, see
           # the scope note above) to start its local catalog server.
@@ -208,7 +247,9 @@ jobs:
           # rerun-cli binary, same as test_rrd_reader_multi_store.py's
           # session fixture (`rerun rrd merge`). test_server.py needs the
           # `datafusion` Python package (no riscv64 wheel anywhere, same as
-          # test_datafusion_utils.py); the remaining ignored files all reach
+          # test_datafusion_utils.py and, new in 0.38.x, test_chunk_index.py,
+          # which imports `from datafusion import col, lit` at module scope);
+          # the remaining ignored files all reach
           # rerun.experimental.dataloader, whose decoders import torchvision
           # or datafusion transitively. `-o markers=local_only` registers the
           # mark e2e_redap_tests/conftest.py normally defines -- pytest only
@@ -224,6 +265,17 @@ jobs:
           # it hits the test's 5s join() timeout consistently on this
           # runner but not the reason any other test here was dropped --
           # deselect, don't chase (CLAUDE.md gotcha 14).
+          # test_optimized_stream.py (new in 0.38.x) is the same "single
+          # failures in an otherwise-passing file" shape as
+          # test_multi_stream.py::test_isolated_streams and
+          # test_send_dataframe.py::test_send_dataframe_roundtrip above: only
+          # the 4 tests that call `ChunkStore.reader()` (which lazily does
+          # `import datafusion` in rerun_sdk/rerun/chunk/_chunk_store.py, same
+          # missing riscv64 wheel as test_datafusion_utils.py/test_server.py)
+          # fail; its other 4 tests (test_merge_quality, test_layout_snapshot,
+          # test_laziness, test_own_chunk_rules) don't touch `.reader()` and
+          # pass, so they're deselected individually rather than the whole
+          # file ignored.
           CIBW_TEST_COMMAND: >-
             cd rerun_py &&
             python -c "import rerun_bindings.rerun_bindings as m; assert m.__file__.endswith('.so'), m.__file__" &&
@@ -234,6 +286,7 @@ jobs:
             --ignore=tests/integration/test_dataloader_video.py
             --ignore=tests/integration/test_dataloader_video_codecs.py
             --ignore=tests/integration/test_chunk_store_reader.py
+            --ignore=tests/integration/test_chunk_index.py
             --ignore=tests/integration/test_rrd_reader_multi_store.py
             --ignore=tests/unit/test_dataloader_decoder_helpers.py
             --ignore=tests/unit/test_dataloader_manifest.py
@@ -249,6 +302,10 @@ jobs:
             --deselect=tests/unit/test_multi_stream.py::test_isolated_streams
             --deselect=tests/unit/test_send_dataframe.py::test_send_dataframe_roundtrip
             --deselect=tests/unit/test_multiprocessing_gc.py::test_multiprocessing_gc
+            --deselect=tests/integration/test_optimized_stream.py::test_data_equality
+            --deselect=tests/integration/test_optimized_stream.py::test_chunk_store_path
+            --deselect=tests/integration/test_optimized_stream.py::test_sweep_order_observable
+            --deselect=tests/integration/test_optimized_stream.py::test_own_chunk_default
 
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
         with:

docs/packages/rerun-sdk.yaml

@@ -9,3 +9,6 @@ versions:
   - filename: rerun_sdk-0.37.1-cp310-abi3-manylinux_2_39_riscv64.whl
     sha256: c6306ffeb0f552fd3f2c499223a1121065cec447cc3fef0d17c6a00e6fdbe198
     requires-python: '>=3.10'
+- version: 0.37.2
+- version: 0.38.0
+- version: 0.38.1

patches/rerun-sdk/0.37.2/0001-lance-core-riscv64-simd-fallback.patch

@@ -0,0 +1,22 @@
+Upstream-Status: To upstream [lance's own SIMD-tier cfg arms have no catch-all for architectures without a dedicated tier (riscv64 included); not filed upstream per this session's policy against opening external issues/PRs]
+
+--- a/src/utils/cpu.rs
++++ b/src/utils/cpu.rs
+@@ -210,6 +210,17 @@
+             SimdSupport::None
+         }
+     }
++    // riscv64 (and any other architecture without a dedicated tier above) has
++    // no SIMD kernels in lance yet; upstream's cfg arms have no catch-all, so
++    // this closure otherwise evaluates to `()` here instead of `SimdSupport`.
++    #[cfg(not(any(
++        target_arch = "aarch64",
++        target_arch = "x86_64",
++        target_arch = "loongarch64"
++    )))]
++    {
++        SimdSupport::None
++    }
+ });
+
+ #[cfg(target_arch = "x86_64")]

patches/rerun-sdk/0.37.2/0002-cargo-patch-lance-crates-for-riscv64.patch

@@ -0,0 +1,37 @@
+Upstream-Status: Inappropriate [points at a directory this port's workflow materializes at runtime; not a fix rerun itself would carry]
+
+diff --git a/Cargo.lock b/Cargo.lock
+index 52b6e5b..4e5ec02 100644
+--- a/Cargo.lock
++++ b/Cargo.lock
+@@ -5622,8 +5622,6 @@ dependencies = [
+ [[package]]
+ name = "lance-core"
+ version = "9.0.0"
+-source = "registry+https://github.com/rust-lang/crates.io-index"
+-checksum = "238c8a58308e7718d6bd96b53494eb7953fa299778bc4911cc571c3576e9446d"
+ dependencies = [
+  "arrow-array",
+  "arrow-buffer",
+@@ -5923,8 +5921,6 @@ dependencies = [
+ [[package]]
+ name = "lance-linalg"
+ version = "9.0.0"
+-source = "registry+https://github.com/rust-lang/crates.io-index"
+-checksum = "0e5a9b99bd1f49bc2fe5afb81323141abc506c818f589de95dbab4f6143d9a88"
+ dependencies = [
+  "arrow-array",
+  "arrow-buffer",
+diff --git a/Cargo.toml b/Cargo.toml
+index ac0093e..b2093a8 100644
+--- a/Cargo.toml
++++ b/Cargo.toml
+@@ -845,6 +845,8 @@ self_named_module_files = "allow" # Disabled waiting on https://github.com/rust-
+ significant_drop_tightening = "allow" # An update of parking_lot made this trigger in a lot of places. TODO(emilk): fix those places
+
+ [patch.crates-io]
++lance-core = { path = "lance-core-9.0.0-riscv64" }
++lance-linalg = { path = "lance-linalg-9.0.0-riscv64" }
+ # Try to avoid patching crates! It prevents us from publishing the crates on crates.io.
+ # If you do patch always prefer to patch to the trunk branch of the upstream repo (i.e. `main`, `master`, …).
+ # If that is not possible, patch to a branch that has a PR open on the upstream repo.