From b4997f8384a9fa9130bcb129e32d23b2fad8d6a1 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Mon, 21 Sep 2026 20:52:52 +0000 Subject: [PATCH] rawpy: Add version 0.27.0 Builds LibRaw from the vendored git submodule via cibuildwheel, the same CMake-driven recipe rawpy's own setup.py uses on every platform. Rocky 10 riscv64 ships lcms2/jasper directly; libjpeg-turbo is built from source with -DWITH_JPEG8=ON to get the DNG lossy codec's ABI the same way upstream's own build-linux.sh does, since riscv64 has no libjpeg-turbo SIMD kernels either way (-DWITH_SIMD=0). No architectural blocker: this LibRaw release carries no x86 SIMD intrinsics at all, and every optional codec dependency (LCMS2, JPEG, Jasper) degrades gracefully through CMake's own find_package when absent. One patch moves 16 OpenMP loops in the vendored LibRaw copy (raw2image.cpp, ahd_demosaic.cpp, dht_demosaic.cpp) from schedule(dynamic)/schedule(guided) to schedule(static), working around the riscv64 runners' libgomp defect (gotcha 166) that segfaults on those schedules; raw2image.cpp's loops run on every single postprocess() call. --- .github/workflows/build-rawpy.yml | 151 +++++++++++++++ docs/packages/rawpy.yaml | 6 + ...-the-dynamic-and-guided-OpenMP-loops.patch | 181 ++++++++++++++++++ 3 files changed, 338 insertions(+) create mode 100644 .github/workflows/build-rawpy.yml create mode 100644 docs/packages/rawpy.yaml create mode 100644 patches/rawpy/0.27.0/0001-use-schedule-static-for-the-dynamic-and-guided-OpenMP-loops.patch diff --git a/.github/workflows/build-rawpy.yml b/.github/workflows/build-rawpy.yml new file mode 100644 index 0000000000..993422000a --- /dev/null +++ b/.github/workflows/build-rawpy.yml @@ -0,0 +1,151 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +--- +name: Build rawpy wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'Version glob to (re)build; empty builds every version of docs/packages/rawpy.yaml not released yet' + required: false + default: '' + pull_request: + branches: [main] + paths: + - '.github/workflows/build-rawpy.yml' + - 'docs/packages/rawpy.yaml' + - 'patches/rawpy/**' + push: + branches: [main] + paths: + - '.github/workflows/build-rawpy.yml' + - 'docs/packages/rawpy.yaml' + - 'patches/rawpy/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +env: + MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 + +jobs: + setup: + uses: $/.github/workflows/_setup.yml + with: + package: rawpy + version: ${{ inputs.version }} + + build_wheels: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + name: Build rawpy ${{ matrix.version }} ${{ matrix.python }}-manylinux_riscv64 + runs-on: ubuntu-24.04-riscv + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + python: ["cp312", "cp313", "cp314", "cp314t"] + + env: + RAWPY_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout rawpy v${{ env.RAWPY_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: letmaik/rawpy + ref: v${{ env.RAWPY_VERSION }} + submodules: true + persist-credentials: false + + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: python-wheels + persist-credentials: false + + - name: Patch rawpy source + run: git apply python-wheels/patches/rawpy/${{ env.RAWPY_VERSION }}/*.patch + + - name: Build wheels + uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0 + with: + output-dir: wheelhouse/ + only: ${{ matrix.python }}-manylinux_riscv64 + env: + CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }} + # Rocky 10 riscv64 ships lcms2/jasper directly; its libjpeg-turbo lacks the + # JPEG8 ABI (DNG lossy codec) upstream's own build-linux.sh gets by building + # from source with -DWITH_JPEG8=ON, so only that one is built the same way + # upstream does (riscv64 has no libjpeg-turbo SIMD kernels, hence -DWITH_SIMD=0). + CIBW_BEFORE_BUILD: > + dnf install -y zlib-ng-compat-devel lcms2-devel jasper-devel && + curl -fsSL -o /tmp/libjpeg-turbo.tar.gz + https://github.com/libjpeg-turbo/libjpeg-turbo/releases/download/3.1.3/libjpeg-turbo-3.1.3.tar.gz && + echo "075920b826834ac4ddf97661cc73491047855859affd671d52079c6867c1c6c0 /tmp/libjpeg-turbo.tar.gz" | sha256sum -c - && + tar xzf /tmp/libjpeg-turbo.tar.gz -C /tmp && + cmake -S /tmp/libjpeg-turbo-3.1.3 -B /tmp/libjpeg-turbo-3.1.3/build + -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release + -DENABLE_SHARED=ON -DENABLE_STATIC=OFF -DWITH_JPEG8=ON -DWITH_SIMD=0 && + cmake --build /tmp/libjpeg-turbo-3.1.3/build -j"$(nproc)" --target install && + rm -f /usr/include/jconfig-32.h /usr/include/jconfig-64.h && + curl -fsSL -o {project}/LICENSE.lcms2 https://raw.githubusercontent.com/mm2/Little-CMS/lcms2.16/LICENSE && + curl -fsSL -o {project}/LICENSE.libjpeg-turbo https://raw.githubusercontent.com/libjpeg-turbo/libjpeg-turbo/3.1.3/LICENSE.md && + curl -fsSL -o {project}/LICENSE.jasper https://raw.githubusercontent.com/jasper-software/jasper/version-4.1.0/LICENSE.txt + CIBW_ENVIRONMENT: PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/ + CIBW_TEST_ENVIRONMENT: CI=true + CIBW_TEST_EXTRAS: test + CIBW_TEST_SOURCES: test examples + CIBW_TEST_COMMAND: >- + python -c "import importlib.metadata as m; names={p.name for p in m.files('rawpy') if '.dist-info/licenses/' in str(p)}; assert names == {'LICENSE','LICENSE.LibRaw','LICENSE.lcms2','LICENSE.libjpeg-turbo','LICENSE.jasper'}, names" && + python -m pytest -v test + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: rawpy-${{ env.RAWPY_VERSION }}-${{ matrix.python }}-manylinux_riscv64 + path: wheelhouse/*.whl + if-no-files-found: error + + gpl_sources: + name: Collect GPL sources (gcc) for rawpy ${{ matrix.version }} + needs: [setup] + if: needs.setup.outputs.versions != '[]' + runs-on: ubuntu-24.04-riscv + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + steps: + - uses: riseproject-dev/python-wheels/actions/collect-gpl-sources@main + with: + image: ${{ env.MANYLINUX_RISCV64_IMAGE }} + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: rawpy-${{ matrix.version }}-gpl-sources + path: gpl-sources.tar + if-no-files-found: error + + publish: + name: Publish rawpy ${{ matrix.version }} + needs: [setup, build_wheels, gpl_sources] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + permissions: + contents: write + pull-requests: write + uses: $/.github/workflows/_publish-wheel.yml + secrets: + app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + with: + artifact-pattern: rawpy-${{ matrix.version }}-*-manylinux_riscv64 + gpl-sources-artifact: rawpy-${{ matrix.version }}-gpl-sources + gpl-sources-description: gcc diff --git a/docs/packages/rawpy.yaml b/docs/packages/rawpy.yaml new file mode 100644 index 0000000000..a58afa7675 --- /dev/null +++ b/docs/packages/rawpy.yaml @@ -0,0 +1,6 @@ +package-name: rawpy +source-code: https://github.com/letmaik/rawpy +license: MIT +versions: +- version: 0.27.0 + patched: true diff --git a/patches/rawpy/0.27.0/0001-use-schedule-static-for-the-dynamic-and-guided-OpenMP-loops.patch b/patches/rawpy/0.27.0/0001-use-schedule-static-for-the-dynamic-and-guided-OpenMP-loops.patch new file mode 100644 index 0000000000..65527505f7 --- /dev/null +++ b/patches/rawpy/0.27.0/0001-use-schedule-static-for-the-dynamic-and-guided-OpenMP-loops.patch @@ -0,0 +1,181 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Mon, 21 Sep 2026 20:00:00 +0000 +Subject: [PATCH] use schedule(static) for the dynamic and guided OpenMP loops + +Upstream-Status: Inappropriate [works around a libgomp defect on the riscv64 runners, riseproject-dev/python-wheels#617] + +libgomp's dynamic and guided work-share schedules fault on the riscv64 +machines these wheels are built and tested on (see CLAUDE.md gotcha 166). +copy_fuji_uncropped/copy_bayer in raw2image.cpp run schedule(dynamic) on +every single postprocess() call regardless of camera or demosaic +algorithm, and ahd_demosaic.cpp (the default demosaic algorithm) and +dht_demosaic.cpp (the optional DHT algorithm) run schedule(dynamic)/ +schedule(guided) too, so this is reachable by ordinary decoding, not an +edge case. + +Moving all 16 affected loops in the vendored LibRaw copy to +schedule(static) costs only load balancing on rows/columns whose +iterations differ in cost, and buys a wheel that does not crash. Revert +once the runners' toolchain is fixed. +--- + external/LibRaw/src/demosaic/ahd_demosaic.cpp | 2 +- + external/LibRaw/src/demosaic/dht_demosaic.cpp | 26 +++++++++++++------------- + external/LibRaw/src/preprocessing/raw2image.cpp | 4 ++-- + 3 files changed, 16 insertions(+), 16 deletions(-) + +diff --git a/external/LibRaw/src/demosaic/ahd_demosaic.cpp b/external/LibRaw/src/demosaic/ahd_demosaic.cpp +index ae87b28..ae6ad6d 100644 +--- a/external/LibRaw/src/demosaic/ahd_demosaic.cpp ++++ b/external/LibRaw/src/demosaic/ahd_demosaic.cpp +@@ -306,7 +306,7 @@ void LibRaw::ahd_interpolate() + char** buffers = malloc_omp_buffers(buffer_count, buffer_size); + + #ifdef LIBRAW_USE_OPENMP +-#pragma omp parallel for schedule(dynamic) default(none) shared(terminate_flag) firstprivate(buffers) ++#pragma omp parallel for schedule(static) default(none) shared(terminate_flag) firstprivate(buffers) + #endif + for (int top = 2; top < height - 5; top += LIBRAW_AHD_TILE - 6) + { +diff --git a/external/LibRaw/src/demosaic/dht_demosaic.cpp b/external/LibRaw/src/demosaic/dht_demosaic.cpp +index 9fa603f..b23dc61 100644 +--- a/external/LibRaw/src/demosaic/dht_demosaic.cpp ++++ b/external/LibRaw/src/demosaic/dht_demosaic.cpp +@@ -259,7 +259,7 @@ void DHT::hide_hots() + { + int iwidth = libraw.imgdata.sizes.iwidth; + #if defined(LIBRAW_USE_OPENMP) +-#pragma omp parallel for schedule(guided) firstprivate(iwidth) ++#pragma omp parallel for schedule(static) firstprivate(iwidth) + #endif + for (int i = 0; i < libraw.imgdata.sizes.iheight; ++i) + { +@@ -401,7 +401,7 @@ void DHT::restore_hots() + #ifdef _MSC_VER + #pragma omp parallel for firstprivate(iwidth) + #else +-#pragma omp parallel for schedule(guided) firstprivate(iwidth) collapse(2) ++#pragma omp parallel for schedule(static) firstprivate(iwidth) collapse(2) + #endif + #endif + for (int i = 0; i < libraw.imgdata.sizes.iheight; ++i) +@@ -423,7 +423,7 @@ void DHT::restore_hots() + void DHT::make_diag_dirs() + { + #if defined(LIBRAW_USE_OPENMP) +-#pragma omp parallel for schedule(guided) ++#pragma omp parallel for schedule(static) + #endif + for (int i = 0; i < libraw.imgdata.sizes.iheight; ++i) + { +@@ -442,7 +442,7 @@ void DHT::make_diag_dirs() + // refine_diag_dirs(i, (i & 1) ^ 1); + // } + #if defined(LIBRAW_USE_OPENMP) +-#pragma omp parallel for schedule(guided) ++#pragma omp parallel for schedule(static) + #endif + for (int i = 0; i < libraw.imgdata.sizes.iheight; ++i) + { +@@ -453,28 +453,28 @@ void DHT::make_diag_dirs() + void DHT::make_hv_dirs() + { + #if defined(LIBRAW_USE_OPENMP) +-#pragma omp parallel for schedule(guided) ++#pragma omp parallel for schedule(static) + #endif + for (int i = 0; i < libraw.imgdata.sizes.iheight; ++i) + { + make_hv_dline(i); + } + #if defined(LIBRAW_USE_OPENMP) +-#pragma omp parallel for schedule(guided) ++#pragma omp parallel for schedule(static) + #endif + for (int i = 0; i < libraw.imgdata.sizes.iheight; ++i) + { + refine_hv_dirs(i, i & 1); + } + #if defined(LIBRAW_USE_OPENMP) +-#pragma omp parallel for schedule(guided) ++#pragma omp parallel for schedule(static) + #endif + for (int i = 0; i < libraw.imgdata.sizes.iheight; ++i) + { + refine_hv_dirs(i, (i & 1) ^ 1); + } + #if defined(LIBRAW_USE_OPENMP) +-#pragma omp parallel for schedule(guided) ++#pragma omp parallel for schedule(static) + #endif + for (int i = 0; i < libraw.imgdata.sizes.iheight; ++i) + { +@@ -689,7 +689,7 @@ void DHT::refine_idiag_dirs(int i) + void DHT::make_greens() + { + #if defined(LIBRAW_USE_OPENMP) +-#pragma omp parallel for schedule(guided) ++#pragma omp parallel for schedule(static) + #endif + for (int i = 0; i < libraw.imgdata.sizes.iheight; ++i) + { +@@ -765,7 +765,7 @@ void DHT::make_gline(int i) + void DHT::illustrate_dirs() + { + #if defined(LIBRAW_USE_OPENMP) +-#pragma omp parallel for schedule(guided) ++#pragma omp parallel for schedule(static) + #endif + for (int i = 0; i < libraw.imgdata.sizes.iheight; ++i) + { +@@ -956,7 +956,7 @@ void DHT::make_rb() + { + #if defined(LIBRAW_USE_OPENMP) + #pragma omp barrier +-#pragma omp parallel for schedule(guided) ++#pragma omp parallel for schedule(static) + #endif + for (int i = 0; i < libraw.imgdata.sizes.iheight; ++i) + { +@@ -964,7 +964,7 @@ void DHT::make_rb() + } + #if defined(LIBRAW_USE_OPENMP) + #pragma omp barrier +-#pragma omp parallel for schedule(guided) ++#pragma omp parallel for schedule(static) + #endif + for (int i = 0; i < libraw.imgdata.sizes.iheight; ++i) + { +@@ -982,7 +982,7 @@ void DHT::copy_to_image() + #ifdef _MSC_VER + #pragma omp parallel for + #else +-#pragma omp parallel for schedule(guided) collapse(2) ++#pragma omp parallel for schedule(static) collapse(2) + #endif + #endif + for (int i = 0; i < libraw.imgdata.sizes.iheight; ++i) +diff --git a/external/LibRaw/src/preprocessing/raw2image.cpp b/external/LibRaw/src/preprocessing/raw2image.cpp +index e343762..9fd2f1e 100644 +--- a/external/LibRaw/src/preprocessing/raw2image.cpp ++++ b/external/LibRaw/src/preprocessing/raw2image.cpp +@@ -221,7 +221,7 @@ void LibRaw::copy_fuji_uncropped(unsigned short cblack[4], + unsigned short *dmaxp) + { + #if defined(LIBRAW_USE_OPENMP) +-#pragma omp parallel for schedule(dynamic) default(none) firstprivate(cblack) shared(dmaxp) ++#pragma omp parallel for schedule(static) default(none) firstprivate(cblack) shared(dmaxp) + #endif + for (int row = 0; row < int(S.raw_height) - int(S.top_margin) * 2; row++) + { +@@ -276,7 +276,7 @@ void LibRaw::copy_bayer(unsigned short cblack[4], unsigned short *dmaxp) + // Both cropped and uncropped + int maxHeight = MIN(int(S.height),int(S.raw_height)-int(S.top_margin)); + #if defined(LIBRAW_USE_OPENMP) +-#pragma omp parallel for schedule(dynamic) default(none) shared(dmaxp) firstprivate(cblack, maxHeight) ++#pragma omp parallel for schedule(static) default(none) shared(dmaxp) firstprivate(cblack, maxHeight) + #endif + for (int row = 0; row < maxHeight ; row++) + { +-- +2.43.0