From 1045894b8ed9a1b6ff5c6e74669ce6f3e5734d6d Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Thu, 24 Sep 2026 05:02:28 +0000 Subject: [PATCH 1/7] xprof: add build-xprof.yml for riscv64 wheels XProf ships a compiled profiler_plugin_c_api.so and libpywrap_events_db_c_api.so behind a py3-none wheel (the "abi: py3" in the queue note is upstream's own retag, not a pure-Python build): they are ctypes/C-API loaded, not linked against a specific CPython, so one build serves every interpreter. Confirmed this is not the jaxlib (PR #526, parked) XLA-codegen wall: xprof's native targets (xprof/pywrap, xprof/convert/events_db/python) depend only on xla/pjrt, xla/tsl/* and xla/hlo/ir - none of it reachable from xla/codegen/intrinsic/cpp, which is the genrule that has no RISC-V LLVM backend wired in and is used exclusively by xla/backends/cpu/codegen, xla/service/cpu and xla/service/gpu, none of which xprof touches. The embedded/llo_analysis path that would pull in more of XLA sits behind enable_embedded_features, which defaults off in the OSS build. The frontend (Angular/rollup, trace_viewer_v2 wasm) has no riscv64-specific code and node.js publishes no riscv64 release, so it is built once on its own job with upstream's hermetic toolchain (confirmed byte-identical across upstream's macOS/Linux/Windows wheels) and only the two native extensions are built on the riscv64 runner, following build-ray.yml's precedent for splitting an arch-independent JS asset out of a Bazel build. Bazel bootstrap follows build-ai-edge-litert.yml/build-ray.yml (rules_python 0.33.2 and rules_java 7.6.5 both need a riscv64 stand-in; xprof's own .bazelversion, 7.4.1, resolves bazel_features 1.11.0 and does not need that fix). --config=clang_local is xprof's own off-switch for XLA's hermetic C++ toolchain, already used upstream for non-x86_64/aarch64 hosts. --- .github/workflows/build-xprof.yml | 326 ++++++++++++++++++++++++++++++ docs/packages/xprof.yaml | 5 + 2 files changed, 331 insertions(+) create mode 100644 .github/workflows/build-xprof.yml create mode 100644 docs/packages/xprof.yaml diff --git a/.github/workflows/build-xprof.yml b/.github/workflows/build-xprof.yml new file mode 100644 index 00000000000..fbd91dcbe79 --- /dev/null +++ b/.github/workflows/build-xprof.yml @@ -0,0 +1,326 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +# +# This workflow is based on: +# https://github.com/openxla/xprof/blob/main/README.md#build-from-source (bazel run +# plugin:build_pip_package) and plugin/build_pip_package.sh. +--- +name: Build xprof wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'Version glob to (re)build; empty builds every version of docs/packages/xprof.yaml not released yet' + required: false + default: '' + pull_request: + branches: [main] + paths: + - '.github/workflows/build-xprof.yml' + - 'docs/packages/xprof.yaml' + push: + branches: [main] + paths: + - '.github/workflows/build-xprof.yml' + - 'docs/packages/xprof.yaml' + +run-name: build-xprof ${{ inputs.version && format('- {0}', inputs.version) || '' }} + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +env: + MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 + # xprof's .bazelversion (7.4.1) predates the bazel_features/apple_support chain that + # forces bootstrapping a newer release here (see build-ai-edge-litert.yml), so it + # bootstraps directly. + BAZEL_VERSION: '7.4.1' + # versions bazel 7.4.1's MODULE.bazel pins; both need a riscv64 fix below + RULES_PYTHON_VERSION: '0.33.2' + RULES_JAVA_VERSION: '7.6.5' + HERMETIC_PYTHON_VERSION: '3.12' + +jobs: + setup: + uses: $/.github/workflows/_setup.yml + with: + package: xprof + version: ${{ inputs.version }} + + # xprof's frontend (trace_viewer_v2.wasm, bundle.js, ...) is an Angular/rollup build + # with no riscv64-specific code; node.js publishes no riscv64 release, so it is built + # once here with upstream's own hermetic toolchain, on the host arch node supports. + # Byte-identical output was confirmed across the upstream macOS/Linux/Windows wheels. + reference_build: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + name: Build xprof ${{ matrix.version }} reference package (frontend + python) + runs-on: ubuntu-24.04 + timeout-minutes: 360 + + env: + XPROF_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout xprof xprof-v${{ env.XPROF_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: openxla/xprof + ref: xprof-v${{ env.XPROF_VERSION }} + fetch-depth: 1 + persist-credentials: false + + - name: Install bazel ${{ env.BAZEL_VERSION }} + run: | + sudo curl -fsSLo /usr/local/bin/bazel \ + "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-linux-x86_64" + sudo chmod +x /usr/local/bin/bazel + + - name: Build reference package + run: | + bazel run //plugin:build_pip_package -- --output "${GITHUB_WORKSPACE}/dist" + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: xprof-${{ env.XPROF_VERSION }}-reference + path: dist + if-no-files-found: error + + bazel: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + name: Bootstrap bazel (riscv64) + runs-on: ubuntu-24.04-riscv + timeout-minutes: 720 + + steps: + - name: Restore bazel binary + id: cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: bazel-bin + key: bazel-${{ env.BAZEL_VERSION }}-manylinux_riscv64 + + - name: Bootstrap bazel ${{ env.BAZEL_VERSION }} + if: steps.cache.outputs.cache-hit != 'true' + run: | + mkdir -p bazel-bin + docker run --rm -i --network=host \ + -v "${GITHUB_WORKSPACE}:/work" \ + -w /work \ + -e BAZEL_VERSION \ + -e RULES_PYTHON_VERSION \ + -e RULES_JAVA_VERSION \ + "${MANYLINUX_RISCV64_IMAGE}" \ + bash <<'SCRIPT' + set -eux + + dnf install -y --setopt=install_weak_deps=False java-21-openjdk-devel zip unzip + JAVA_HOME="$(dirname "$(dirname "$(readlink -f "$(command -v javac)")")")" + export JAVA_HOME + + # rules_python 0.33.2's PLATFORMS has no riscv64 entry, aborting the bootstrap + # (bazelbuild/bazel#23018). Any linux entry is a safe stand-in: the toolchain it names + # is never selected on a riscv64 host. Fixed in bazel 8.2.0; the 7.x backport is open. + mkdir -p /tmp/rules_python + curl -fsSLo /tmp/rules_python.tar.gz "https://github.com/bazel-contrib/rules_python/releases/download/${RULES_PYTHON_VERSION}/rules_python-${RULES_PYTHON_VERSION}.tar.gz" + tar -xzf /tmp/rules_python.tar.gz -C /tmp/rules_python --strip-components=1 + sed -i 's|fail("No platform declared for host OS {} on arch {}".format(os_name, arch))|return "x86_64-unknown-linux-gnu"|' \ + /tmp/rules_python/python/private/toolchains_repo.bzl + + # rules_java 7.x maps riscv64 to a stray-colon include path, so a JNI library + # can't find jni_md.h. Fixed in rules_java 8.x, never backported. + mkdir -p /tmp/rules_java + curl -fsSLo /tmp/rules_java.tar.gz "https://github.com/bazelbuild/rules_java/releases/download/${RULES_JAVA_VERSION}/rules_java-${RULES_JAVA_VERSION}.tar.gz" + tar -xzf /tmp/rules_java.tar.gz -C /tmp/rules_java + sed -i 's|\[":include/linux"\]|["include/linux"]|g' /tmp/rules_java/toolchains/BUILD + + mkdir -p /tmp/bazel-src + cd /tmp/bazel-src + curl -fsSLo dist.zip "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-dist.zip" + unzip -q dist.zip + + EXTRA_BAZEL_ARGS="--tool_java_runtime_version=local_jdk \ + --override_module=rules_python=/tmp/rules_python \ + --override_module=rules_java=/tmp/rules_java" \ + bash ./compile.sh + install -m 0755 output/bazel /work/bazel-bin/bazel + SCRIPT + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: bazel-${{ env.BAZEL_VERSION }}-riscv64 + path: bazel-bin/bazel + if-no-files-found: error + + build_wheels: + name: Build xprof ${{ matrix.version }} py3-none-manylinux_riscv64 + runs-on: ubuntu-24.04-riscv + timeout-minutes: 1440 # 24h + needs: [setup, bazel, reference_build] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + + env: + XPROF_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout xprof xprof-v${{ env.XPROF_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: openxla/xprof + ref: xprof-v${{ env.XPROF_VERSION }} + path: xprof + fetch-depth: 1 + persist-credentials: false + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: python-wheels + fetch-depth: 1 + persist-credentials: false + + - name: Download bazel + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: bazel-${{ env.BAZEL_VERSION }}-riscv64 + path: bazel-bin + + - name: Download reference package + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: xprof-${{ env.XPROF_VERSION }}-reference + path: dist-reference + + - name: Build wheel + env: + HERMETIC_PYTHON_VERSION: ${{ env.HERMETIC_PYTHON_VERSION }} + run: | + mkdir -p wheelhouse + set -o pipefail + docker run --rm -i --network=host \ + -v "${GITHUB_WORKSPACE}:/work" \ + -w /work \ + -e HERMETIC_PYTHON_VERSION \ + "${MANYLINUX_RISCV64_IMAGE}" \ + bash <<'SCRIPT' 2>&1 | tee build.log + set -eux + + # clang, because xprof (via @xla) ships no hermetic riscv64 C++ toolchain and + # --config=clang_local (already used upstream for non-x86_64/aarch64 hosts, see + # xla/tsl:ci_linux_aarch64) is the documented off-switch. + dnf install -y --setopt=install_weak_deps=False java-21-openjdk-devel zip unzip clang + JAVA_HOME="$(dirname "$(dirname "$(readlink -f "$(command -v javac)")")")" + export JAVA_HOME + install -m 0755 /work/bazel-bin/bazel /usr/local/bin/bazel + git config --global --add safe.directory '*' + + cd /work/xprof + + # Only the two native extensions: the frontend and every pure-Python file come + # from the reference package built in the reference_build job, which is + # architecture-independent (confirmed byte-identical across upstream's own + # macOS/Linux/Windows wheels). + bazel build -c opt --config=clang_local \ + --repo_env=HERMETIC_PYTHON_VERSION="${HERMETIC_PYTHON_VERSION}" \ + --repo_env=PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/ \ + --curses=no --show_progress_rate_limit=60 \ + //xprof/pywrap:profiler_plugin_c_api.so \ + //xprof/convert/events_db/python:libpywrap_events_db_c_api.so + + cp -a /work/dist-reference /work/dist + cp bazel-bin/xprof/pywrap/profiler_plugin_c_api.so \ + /work/dist/xprof/convert/profiler_plugin_c_api.so + cp bazel-bin/xprof/convert/events_db/python/libpywrap_events_db_c_api.so \ + /work/dist/xprof/convert/events_db/python/libpywrap_events_db_c_api.so + + PYTHON_BIN="/opt/python/cp312-cp312/bin/python" + "${PYTHON_BIN}" -m pip install -q -U pip setuptools wheel auditwheel + + cd /work/dist + "${PYTHON_BIN}" -m pip wheel . --no-deps --no-build-isolation -w /work/wheelhouse-raw + + auditwheel repair --plat manylinux_2_39_riscv64 -w /work/wheelhouse \ + /work/wheelhouse-raw/xprof-*.whl + + "${PYTHON_BIN}" - <<'PY' + import glob, zipfile + + whl = sorted(glob.glob("/work/wheelhouse/xprof-*.whl"))[0] + names = zipfile.ZipFile(whl).namelist() + for want in ( + "xprof/convert/profiler_plugin_c_api.so", + "xprof/convert/events_db/python/libpywrap_events_db_c_api.so", + "xprof/static/bundle.js", + "xprof/static/trace_viewer_v2.wasm", + ): + if not any(want in n for n in names): + raise SystemExit(f"error: {want} missing from {whl}") + print(whl) + PY + SCRIPT + + - name: Upload build log + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: xprof-${{ env.XPROF_VERSION }}-build-log + path: build.log + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: xprof-${{ env.XPROF_VERSION }}-py3-none-manylinux_riscv64 + path: wheelhouse/*.whl + if-no-files-found: error + + - name: Test wheel + env: + PIP_EXTRA_INDEX_URL: https://pypi.riseproject.dev/simple/ + run: | + docker run --rm -i --network=host \ + -v "${GITHUB_WORKSPACE}:/work" \ + -w /tmp \ + -e PIP_EXTRA_INDEX_URL \ + "${MANYLINUX_RISCV64_IMAGE}" \ + bash <<'SCRIPT' + set -eux + + PYTHON_BIN="/opt/python/cp312-cp312/bin/python" + "${PYTHON_BIN}" -m pip install --only-binary=:all: /work/wheelhouse/*.whl + + "${PYTHON_BIN}" - <<'PY' + from xprof.convert import _pywrap_profiler_plugin # noqa: F401 + from xprof import version + + print("xprof", version.__version__) + PY + SCRIPT + + publish: + name: Publish xprof ${{ matrix.version }} + needs: [setup, build_wheels] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + permissions: + contents: write + pull-requests: write + uses: $/.github/workflows/_publish-wheel.yml + secrets: + app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + with: + artifact-pattern: xprof-${{ matrix.version }}-*-manylinux_riscv64 diff --git a/docs/packages/xprof.yaml b/docs/packages/xprof.yaml new file mode 100644 index 00000000000..06d1f2216f8 --- /dev/null +++ b/docs/packages/xprof.yaml @@ -0,0 +1,5 @@ +package-name: xprof +source-code: https://github.com/openxla/xprof +license: Apache-2.0 +versions: +- version: 2.23.1 From 5e0a49d90838bea187a7bc92207c92be2bef310c Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Thu, 24 Sep 2026 05:04:43 +0000 Subject: [PATCH 2/7] build-xprof: free disk space before the x86_64 reference bazel build The reference_build job compiles the same Arrow/gRPC/protobuf/abseil/XLA-TSL tree the riscv64 job does, on a GitHub-hosted runner with a much smaller disk than the self-hosted riscv64 boxes; build-daft.yml/build-lancedb.yml hit the same ceiling on a comparably sized dependency tree. --- .github/workflows/build-xprof.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-xprof.yml b/.github/workflows/build-xprof.yml index fbd91dcbe79..5c8440c5007 100644 --- a/.github/workflows/build-xprof.yml +++ b/.github/workflows/build-xprof.yml @@ -79,6 +79,9 @@ jobs: fetch-depth: 1 persist-credentials: false + - name: Free disk space + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + - name: Install bazel ${{ env.BAZEL_VERSION }} run: | sudo curl -fsSLo /usr/local/bin/bazel \ From 212e296710486c4f4de1775bfaf0e6166057be13 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Thu, 24 Sep 2026 07:14:15 +0000 Subject: [PATCH 3/7] build-xprof: force standalone javac to dodge a worker-JVM glibc crash The riscv64 bazel bootstrap died mid-compile: Fatal glibc error: pthread_mutex_lock.c:94 (___pthread_mutex_lock): assertion failed: mutex->__data.__owner == 0 inside a Javac multiplex-worker process building bazel's own libtop_level_status_events.jar, which took the whole self-build down (ERROR: Could not build Bazel). --strategy=Javac=standalone forks a fresh javac per compile action instead of routing them through one persistent worker JVM, which is what races on this image's glibc. Applied to both the from-source bazel bootstrap and the xprof build itself, since either could hit the same worker path. --- .github/workflows/build-xprof.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-xprof.yml b/.github/workflows/build-xprof.yml index 5c8440c5007..c3c168147a4 100644 --- a/.github/workflows/build-xprof.yml +++ b/.github/workflows/build-xprof.yml @@ -152,9 +152,14 @@ jobs: curl -fsSLo dist.zip "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-dist.zip" unzip -q dist.zip + # java-21-openjdk's persistent Javac worker crashes under this image's glibc + # ("Fatal glibc error: pthread_mutex_lock.c: assertion failed: + # mutex->__data.__owner == 0") once several multiplex-worker threads race; + # standalone forks a fresh javac per action instead of reusing one worker JVM. EXTRA_BAZEL_ARGS="--tool_java_runtime_version=local_jdk \ --override_module=rules_python=/tmp/rules_python \ - --override_module=rules_java=/tmp/rules_java" \ + --override_module=rules_java=/tmp/rules_java \ + --strategy=Javac=standalone" \ bash ./compile.sh install -m 0755 output/bazel /work/bazel-bin/bazel SCRIPT @@ -239,6 +244,7 @@ jobs: bazel build -c opt --config=clang_local \ --repo_env=HERMETIC_PYTHON_VERSION="${HERMETIC_PYTHON_VERSION}" \ --repo_env=PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/ \ + --strategy=Javac=standalone \ --curses=no --show_progress_rate_limit=60 \ //xprof/pywrap:profiler_plugin_c_api.so \ //xprof/convert/events_db/python:libpywrap_events_db_c_api.so From 4e4adda3f8c3074da914b7d69e3ea5c5d694b29e Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Thu, 24 Sep 2026 13:05:12 +0000 Subject: [PATCH 4/7] build-xprof: drop libpywrap_events_db_c_api.so, not shipped at 2.23.1 Checked the workflow against the actual xprof-v2.23.1 git tag (rather than main, which I had been reading): xprof/convert/events_db (the Arrow/Parquet events database) does not exist yet at that tag, so //xprof/convert/events_db/python:libpywrap_events_db_c_api.so is not a buildable target and the real released wheel does not carry it either (confirmed by unzipping it - only profiler_plugin_c_api.so is present). CI caught this: 'no such package xprof/convert/events_db/python'. Removes the target from the build, the copy step and the wheel sanity check; xprof/pywrap:profiler_plugin_c_api.so's own dependency chain (checked against the tag too) is unaffected. --- .github/workflows/build-xprof.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-xprof.yml b/.github/workflows/build-xprof.yml index c3c168147a4..f9501cb9e88 100644 --- a/.github/workflows/build-xprof.yml +++ b/.github/workflows/build-xprof.yml @@ -237,23 +237,23 @@ jobs: cd /work/xprof - # Only the two native extensions: the frontend and every pure-Python file come + # Only the native extension: the frontend and every pure-Python file come # from the reference package built in the reference_build job, which is # architecture-independent (confirmed byte-identical across upstream's own - # macOS/Linux/Windows wheels). + # macOS/Linux/Windows wheels). 2.23.1 predates xprof/convert/events_db (the + # Arrow/Parquet events database, added upstream after this release) - the + # real released wheel carries only profiler_plugin_c_api.so, confirmed by + # unzipping it. bazel build -c opt --config=clang_local \ --repo_env=HERMETIC_PYTHON_VERSION="${HERMETIC_PYTHON_VERSION}" \ --repo_env=PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/ \ --strategy=Javac=standalone \ --curses=no --show_progress_rate_limit=60 \ - //xprof/pywrap:profiler_plugin_c_api.so \ - //xprof/convert/events_db/python:libpywrap_events_db_c_api.so + //xprof/pywrap:profiler_plugin_c_api.so cp -a /work/dist-reference /work/dist cp bazel-bin/xprof/pywrap/profiler_plugin_c_api.so \ /work/dist/xprof/convert/profiler_plugin_c_api.so - cp bazel-bin/xprof/convert/events_db/python/libpywrap_events_db_c_api.so \ - /work/dist/xprof/convert/events_db/python/libpywrap_events_db_c_api.so PYTHON_BIN="/opt/python/cp312-cp312/bin/python" "${PYTHON_BIN}" -m pip install -q -U pip setuptools wheel auditwheel @@ -271,7 +271,6 @@ jobs: names = zipfile.ZipFile(whl).namelist() for want in ( "xprof/convert/profiler_plugin_c_api.so", - "xprof/convert/events_db/python/libpywrap_events_db_c_api.so", "xprof/static/bundle.js", "xprof/static/trace_viewer_v2.wasm", ): From fe7d1506c284b0376b9cf256621a89beceb4beb9 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Thu, 24 Sep 2026 17:43:37 +0000 Subject: [PATCH 5/7] build-xprof: export CC/CXX=clang so bazel actually uses it --config=clang_local only disables the hermetic cc toolchain and toolchain resolution; it doesn't select clang. Without CC set, Bazel's local_config_cc autodetection falls back to plain gcc, which silently built most of the tree but failed on com_google_highway's riscv64 copts: they unconditionally add the Clang-only -menable-experimental-extensions flag, which gcc has never implemented. --- .github/workflows/build-xprof.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-xprof.yml b/.github/workflows/build-xprof.yml index f9501cb9e88..ab06e10f476 100644 --- a/.github/workflows/build-xprof.yml +++ b/.github/workflows/build-xprof.yml @@ -228,10 +228,21 @@ jobs: # clang, because xprof (via @xla) ships no hermetic riscv64 C++ toolchain and # --config=clang_local (already used upstream for non-x86_64/aarch64 hosts, see - # xla/tsl:ci_linux_aarch64) is the documented off-switch. + # xla/tsl:ci_linux_aarch64) is the documented off-switch. clang_local only disables + # the hermetic/toolchain-resolution machinery though - it does not itself select + # clang, so CC must be set explicitly or Bazel's local_config_cc autodetection + # defaults to plain `gcc` (bazelbuild/bazel tools/cpp/unix_cc_configure.bzl + # find_cc()). Left unset, that silently built most of the tree with gcc and only + # broke on com_google_highway's riscv64 copts, which unconditionally add the + # Clang-only `-menable-experimental-extensions` flag (google/highway BUILD.bazel) + # that gcc has never implemented (absent from GCC's RISC-V Options docs, any + # version). dnf install -y --setopt=install_weak_deps=False java-21-openjdk-devel zip unzip clang JAVA_HOME="$(dirname "$(dirname "$(readlink -f "$(command -v javac)")")")" export JAVA_HOME + CC=clang + CXX=clang++ + export CC CXX install -m 0755 /work/bazel-bin/bazel /usr/local/bin/bazel git config --global --add safe.directory '*' From dca6b4450901dd06495936abd6a1ec943c0c6fa0 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Fri, 25 Sep 2026 01:06:29 +0000 Subject: [PATCH 6/7] build-xprof: backport LLVM's clang-21 SFINAE crash workaround into @llvm-raw The build died after ~3.5h with clang-21 exiting 132 (SIGILL) while compiling @llvm-project//mlir:IR's lib/IR/BuiltinDialectBytecode.cpp [for tool], instantiating mlir::getChecked (BytecodeImplementation.h:472). This is not stack or memory exhaustion: the backtrace is 87 frames deep and ends in OverloadCandidateSet::CompleteCandidates <- NoteCandidates, a stack overflow would be SIGSEGV, and it reproduces deterministically on x86_64 with the same clang (21.1.8, RESF 21.1.8-1.el10, Rocky 10) from a CMake build of the same llvm-project c9b25a6437fd - also under `ulimit -s unlimited`, with -std=c++20, and without PCH. A 30-line reduction crashes too. Cause: MLIR's is_detected probe of T::getChecked(emitError, params...) fails overload resolution against the inherited StorageUserBase::getChecked( function_ref, MLIRContext *, Args...) template for every type without a no-context getChecked. Clang 21's CWG2369 handling skips the function_ref parameter, rejects the MLIRContext * one, and leaves a conversion slot uninitialized that building the candidate notes then reads (llvm/llvm-project#188249, #198278). Upstream fixed it on the MLIR side in llvm/llvm-project#208359 (9722a2ceddb6, 2026-07-09), after the LLVM that xprof 2.23.1's XLA pins. Carry that change unmodified as patches/xprof/2.23.1/0001 and apply it to Bazel's fetched @llvm-raw between a --nobuild pass and the real build. Verified locally: with it the crashing TU and 853 MLIR IR/bytecode/dialect objects compile with clang 21.1.8 (XLA's own llvm patches applied too); without it the same TU still crashes. The fetch/patch/build sequence was checked with bazel 7.4.1 against an @llvm-raw -> llvm_configure-style symlink overlay. --- .github/workflows/build-xprof.yml | 27 +++++++-- docs/packages/xprof.yaml | 1 + ...ang-21-crash-during-SFINAE-overload-.patch | 59 +++++++++++++++++++ 3 files changed, 81 insertions(+), 6 deletions(-) create mode 100644 patches/xprof/2.23.1/0001-mlir-IR-Avoid-Clang-21-crash-during-SFINAE-overload-.patch diff --git a/.github/workflows/build-xprof.yml b/.github/workflows/build-xprof.yml index ab06e10f476..62dddbc1728 100644 --- a/.github/workflows/build-xprof.yml +++ b/.github/workflows/build-xprof.yml @@ -19,11 +19,13 @@ on: paths: - '.github/workflows/build-xprof.yml' - 'docs/packages/xprof.yaml' + - 'patches/xprof/**' push: branches: [main] paths: - '.github/workflows/build-xprof.yml' - 'docs/packages/xprof.yaml' + - 'patches/xprof/**' run-name: build-xprof ${{ inputs.version && format('- {0}', inputs.version) || '' }} @@ -222,6 +224,7 @@ jobs: -v "${GITHUB_WORKSPACE}:/work" \ -w /work \ -e HERMETIC_PYTHON_VERSION \ + -e XPROF_VERSION \ "${MANYLINUX_RISCV64_IMAGE}" \ bash <<'SCRIPT' 2>&1 | tee build.log set -eux @@ -255,12 +258,24 @@ jobs: # Arrow/Parquet events database, added upstream after this release) - the # real released wheel carries only profiler_plugin_c_api.so, confirmed by # unzipping it. - bazel build -c opt --config=clang_local \ - --repo_env=HERMETIC_PYTHON_VERSION="${HERMETIC_PYTHON_VERSION}" \ - --repo_env=PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/ \ - --strategy=Javac=standalone \ - --curses=no --show_progress_rate_limit=60 \ - //xprof/pywrap:profiler_plugin_c_api.so + BAZEL_FLAGS=( + -c opt --config=clang_local + --repo_env=HERMETIC_PYTHON_VERSION="${HERMETIC_PYTHON_VERSION}" + --repo_env=PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/ + --strategy=Javac=standalone + --curses=no --show_progress_rate_limit=60 + ) + TARGET=//xprof/pywrap:profiler_plugin_c_api.so + + # patches/xprof/ are against @llvm-raw (the LLVM @xla pins), not + # xprof: the image's clang 21.1.8 crashes on MLIR's BuiltinDialectBytecode.cpp + # (llvm/llvm-project#188249). Fetch without building, then patch in place. + bazel build "${BAZEL_FLAGS[@]}" --nobuild "${TARGET}" + LLVM_RAW="$(bazel info output_base)/external/llvm-raw" + for patch in /work/python-wheels/patches/xprof/"${XPROF_VERSION}"/*.patch; do + git -C "${LLVM_RAW}" apply -v "${patch}" + done + bazel build "${BAZEL_FLAGS[@]}" "${TARGET}" cp -a /work/dist-reference /work/dist cp bazel-bin/xprof/pywrap/profiler_plugin_c_api.so \ diff --git a/docs/packages/xprof.yaml b/docs/packages/xprof.yaml index 06d1f2216f8..12c505a0be6 100644 --- a/docs/packages/xprof.yaml +++ b/docs/packages/xprof.yaml @@ -3,3 +3,4 @@ source-code: https://github.com/openxla/xprof license: Apache-2.0 versions: - version: 2.23.1 + patched: true diff --git a/patches/xprof/2.23.1/0001-mlir-IR-Avoid-Clang-21-crash-during-SFINAE-overload-.patch b/patches/xprof/2.23.1/0001-mlir-IR-Avoid-Clang-21-crash-during-SFINAE-overload-.patch new file mode 100644 index 00000000000..40421db03c9 --- /dev/null +++ b/patches/xprof/2.23.1/0001-mlir-IR-Avoid-Clang-21-crash-during-SFINAE-overload-.patch @@ -0,0 +1,59 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Fri, 25 Sep 2026 03:00:00 +0200 +Subject: [PATCH] [mlir][IR] Avoid Clang 21 crash during SFINAE overload + resolution + +Upstream-Status: Backport [https://github.com/llvm/llvm-project/commit/9722a2ceddb6e4fc098f89c9c55c5ad7afd1a64c] + +Applies to @llvm-raw (llvm/llvm-project c9b25a6437fd, the commit pinned by +the openxla/xla c520e3fb3f00 that xprof 2.23.1 builds against), not to the +xprof checkout. + +mlir::getChecked() in mlir/Bytecode/BytecodeImplementation.h probes +T::getChecked(emitError, params...) through llvm::is_detected. For every +attribute/type without a no-context getChecked (AffineMapAttr, ArrayAttr, ...) +that probe fails overload resolution against the inherited +StorageUserBase::getChecked(function_ref, MLIRContext *, Args...) template. +Clang 21's CWG2369 handling skips the function_ref parameter (it has a +converting constructor), finds the MLIRContext * one bad, and returns with +the first conversion slot never initialized; building the candidate notes +then reads it. Clang 21.1.8 (the manylinux_2_39 image's clang, RESF +21.1.8-1.el10) dies there with SIGILL (exit 132) compiling +mlir/lib/IR/BuiltinDialectBytecode.cpp, identically on x86_64, independent +of stack size (ulimit -s unlimited still crashes) - llvm/llvm-project#188249, +#198278. Clang 22.1.0 is not affected. + +This is upstream commit 9722a2ceddb6 by Kazu Hirata (llvm/llvm-project#208359, +2026-07-09), unchanged: the context parameter becomes a constrained Arg1 &&, +so the probe fails template argument deduction instead of hitting the +uninitialized conversion. The resulting header is byte-identical to upstream +main's. +--- + mlir/include/mlir/IR/StorageUniquerSupport.h | 11 +++++++++-- + 1 file changed, 9 insertions(+), 2 deletions(-) + +diff --git a/mlir/include/mlir/IR/StorageUniquerSupport.h b/mlir/include/mlir/IR/StorageUniquerSupport.h +--- a/mlir/include/mlir/IR/StorageUniquerSupport.h ++++ b/mlir/include/mlir/IR/StorageUniquerSupport.h +@@ -194,9 +194,16 @@ + /// Get or create a new ConcreteT instance within the ctx. If the arguments + /// provided are invalid, errors are emitted using the provided `emitError` + /// and a null object is returned. +- template ++ /// ++ /// Workaround: We use Arg1 && instead of MLIRContext * in the parameter list ++ /// to work around a bug in Clang 21.1.8 where Clang segfaults during SFINAE ++ /// overload resolution when attempting to match non-pointer arguments ++ /// against an unconstrained MLIRContext * parameter. ++ template , MLIRContext *>>> + static ConcreteT getChecked(function_ref emitErrorFn, +- MLIRContext *ctx, Args... args) { ++ Arg1 &&ctx, Args... args) { + // If the construction invariants fail then we return a null attribute. + if (failed(ConcreteT::verifyInvariants(emitErrorFn, args...))) + return ConcreteT(); +-- +2.47.0 From 18d0dfc09719a42ca908618d96f1cfb3bb276889 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Fri, 25 Sep 2026 17:51:38 +0000 Subject: [PATCH 7/7] build-xprof: install lld so --icf=all links //xprof/pywrap:profiler_plugin_c_api.so hard-codes -Wl,--icf=all in its Linux linkopts. Upstream always links with its hermetic clang+lld toolchain, but under --config=clang_local the image's clang falls back to GNU ld, which rejects the option and failed the final link after 13h of compilation. Installing lld 21.1.8 (matching clang 21.1.8) lets Bazel's local_config_cc linker probe pick it up and add -fuse-ld=lld. --- .github/workflows/build-xprof.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-xprof.yml b/.github/workflows/build-xprof.yml index 62dddbc1728..ec287ac1513 100644 --- a/.github/workflows/build-xprof.yml +++ b/.github/workflows/build-xprof.yml @@ -240,7 +240,10 @@ jobs: # Clang-only `-menable-experimental-extensions` flag (google/highway BUILD.bazel) # that gcc has never implemented (absent from GCC's RISC-V Options docs, any # version). - dnf install -y --setopt=install_weak_deps=False java-21-openjdk-devel zip unzip clang + # lld too: profiler_plugin_c_api.so links with -Wl,--icf=all (upstream's hermetic + # toolchain is clang+lld), which GNU ld rejects; with lld present, Bazel's + # local_config_cc autodetects it and adds -fuse-ld=lld to every link. + dnf install -y --setopt=install_weak_deps=False java-21-openjdk-devel zip unzip clang lld JAVA_HOME="$(dirname "$(dirname "$(readlink -f "$(command -v javac)")")")" export JAVA_HOME CC=clang