From 2b215b34fd478db181902474a08570c52c50f463 Mon Sep 17 00:00:00 2001 From: snigupta Date: Tue, 21 Jul 2026 15:35:31 -0600 Subject: [PATCH 1/7] Add XDP multi-platform CI build workflow and VE2 Yocto cache Signed-off-by: snigupta --- .github/workflows/xdp-board-test.yml | 30 ++ .github/workflows/xdp-xrt-build.yml | 397 ++++++++++++++++------ ci/board/edge/xdp_edge_board_exec.sh | 17 + ci/board/ve2/tests/01_xrt_smi_version.sh | 5 + ci/board/ve2/tests/02_xrt_smi_examine.sh | 5 + ci/board/ve2/tests/03_xrt_smi_validate.sh | 7 + ci/board/ve2/tests/04_xdp_smoke.sh | 20 ++ ci/board/ve2/xdp_ve2_board_exec.sh | 90 +++++ ci/ve2/ensure_yocto_cache.sh | 58 ++++ 9 files changed, 526 insertions(+), 103 deletions(-) create mode 100644 .github/workflows/xdp-board-test.yml create mode 100755 ci/board/edge/xdp_edge_board_exec.sh create mode 100755 ci/board/ve2/tests/01_xrt_smi_version.sh create mode 100755 ci/board/ve2/tests/02_xrt_smi_examine.sh create mode 100755 ci/board/ve2/tests/03_xrt_smi_validate.sh create mode 100755 ci/board/ve2/tests/04_xdp_smoke.sh create mode 100755 ci/board/ve2/xdp_ve2_board_exec.sh create mode 100755 ci/ve2/ensure_yocto_cache.sh diff --git a/.github/workflows/xdp-board-test.yml b/.github/workflows/xdp-board-test.yml new file mode 100644 index 00000000..686e41c9 --- /dev/null +++ b/.github/workflows/xdp-board-test.yml @@ -0,0 +1,30 @@ +name: XDP Board Test + +# Disabled for now. Re-enable after vek385 B-revB2 board farm access is confirmed. +# Scripts live under ci/board/ve2/ for future zboard + xrt-smi tests. + +on: + workflow_call: + inputs: + build_run_id: + required: true + type: string + workflow_dispatch: + inputs: + build_run_id: + description: XDP Build run ID with VE2 RPM artifacts + required: true + type: string + +jobs: + board-test-ve2: + if: false + runs-on: [self-hosted, Ubuntu-22.04] + steps: + - run: echo "VE2 board test disabled" + + board-test-edge: + if: false + runs-on: [self-hosted, Ubuntu-22.04] + steps: + - run: echo "Edge board test disabled" diff --git a/.github/workflows/xdp-xrt-build.yml b/.github/workflows/xdp-xrt-build.yml index 0f54b722..be6b5ad4 100644 --- a/.github/workflows/xdp-xrt-build.yml +++ b/.github/workflows/xdp-xrt-build.yml @@ -2,122 +2,313 @@ name: XDP Build on: pull_request: - types: [opened, synchronize, reopened, labeled] - branches: [ master ] - + types: [opened, synchronize, reopened, labeled, unlabeled] + branches: [master] + concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + group: xdp-build-${{ github.event.pull_request.number }} cancel-in-progress: true - + jobs: build-xrt-alveo: - if: github.event_name == 'pull_request' + if: | + contains(github.event.pull_request.labels.*.name, 'build-xrt-alveo') || + ( + !contains(github.event.pull_request.labels.*.name, 'build-xrt-alveo') && + !contains(github.event.pull_request.labels.*.name, 'build-xrt-edge') && + !contains(github.event.pull_request.labels.*.name, 'build-xrt-ve2') && + !contains(github.event.pull_request.labels.*.name, 'build-xrt-client') + ) runs-on: ubuntu-latest timeout-minutes: 60 - + concurrency: + group: xdp-alveo-${{ github.event.pull_request.number }} + cancel-in-progress: true + steps: - - name: Checkout XDP PR - uses: actions/checkout@v4 - with: - path: xdp-pr - fetch-depth: 0 - - - name: Clone XRT Repository - run: | - git clone https://github.com/Xilinx/XRT.git XRT - cd XRT - git submodule update --init --recursive - - - name: Apply XDP PR Changes to XRT - run: | - rm -rf XRT/src/runtime_src/xdp - cp -r xdp-pr XRT/src/runtime_src/xdp - - - name: Setup Build Environment Using XRT Scripts - run: | - cd XRT/src/runtime_src/tools/scripts - chmod +x xrtdeps.sh - sudo ./xrtdeps.sh - # Install cross-compilation tools for edge builds - sudo apt-get update - sudo apt-get install -y \ - gcc-aarch64-linux-gnu \ - g++-aarch64-linux-gnu \ - crossbuild-essential-arm64 - - - name: Build XRT for Alveo - run: | - cd XRT/build - chmod +x build.sh - ./build.sh -noinit -noert - - - name: Validate build - run: | - if ls XRT/build/Release/opt/xilinx/xrt/lib/libxrt_core.so* 1> /dev/null 2>&1; then - echo "✓ XRT Build successful" - else - echo "✗ XRT Build failed" - exit 1 - fi - - if ls XRT/build/Release/opt/xilinx/xrt/lib/libxdp_core.so* 1> /dev/null 2>&1; then - echo "✓ XDP Build successful" - else - echo "✗ XDP Build failed" - exit 1 - fi - - - name: Upload Build Artifacts - uses: actions/upload-artifact@v4 - with: - name: xrt-alveo-build-PR-${{ github.event.pull_request.number }} - path: | - XRT/build/Release/opt/xilinx/xrt/lib/libxrt*.so* - XRT/build/Release/opt/xilinx/xrt/lib/libxdp*.so* - XRT/build/Release/opt/xilinx/xrt/lib/xrt/module/*.so* - retention-days: 1 - if-no-files-found: warn - - - name: Generate Build Summary - if: always() - run: | - echo "## XDP Build Summary" >> $GITHUB_STEP_SUMMARY - echo "- Build Date: $(date)" >> $GITHUB_STEP_SUMMARY - echo "- XDP Source: ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY - - if ls XRT/build/Release/opt/xilinx/xrt/lib/libxrt_core.so* 1> /dev/null 2>&1; then - echo "- XRT Build: ✅ Success" >> $GITHUB_STEP_SUMMARY - else - echo "- XRT Build: ❌ Failed" >> $GITHUB_STEP_SUMMARY - fi - - if ls XRT/build/Release/opt/xilinx/xrt/lib/libxdp_core.so* 1> /dev/null 2>&1; then - echo "- XDP Build: ✅ Success" >> $GITHUB_STEP_SUMMARY - else - echo "- XDP Build: ❌ Failed" >> $GITHUB_STEP_SUMMARY - fi - - # Future job for edge builds + - name: Checkout XDP + uses: actions/checkout@v4 + with: + path: xdp-pr + fetch-depth: 0 + + - name: Clone XRT + run: | + git clone https://github.com/Xilinx/XRT.git XRT + git -C XRT submodule update --init --recursive + + - name: Overlay XDP + run: | + rm -rf XRT/src/runtime_src/xdp + cp -r xdp-pr XRT/src/runtime_src/xdp + + - name: Setup build environment + run: | + cd XRT/src/runtime_src/tools/scripts + chmod +x xrtdeps.sh + sudo ./xrtdeps.sh + sudo apt-get update + sudo apt-get install -y \ + gcc-aarch64-linux-gnu \ + g++-aarch64-linux-gnu \ + crossbuild-essential-arm64 + + - name: Build XRT for Alveo + run: | + cd XRT/build + chmod +x build.sh + ./build.sh -noinit -noert + + - name: Validate build outputs + run: | + set -euo pipefail + XRT_ROOT="XRT/build/Release/opt/xilinx/xrt" + + echo "Built shared libraries under ${XRT_ROOT}:" + find "${XRT_ROOT}" -name '*.so*' | sort || true + + XRT_CORE="$(find "${XRT_ROOT}" -name 'libxrt_core.so*' -type f | head -1)" + XDP_CORE="$(find "${XRT_ROOT}" -name 'libxdp_core.so*' -type f | head -1)" + + test -n "${XRT_CORE}" || { echo "libxrt_core not found under ${XRT_ROOT}"; exit 1; } + test -n "${XDP_CORE}" || { echo "libxdp_core not found under ${XRT_ROOT}"; exit 1; } + echo "Validated ${XRT_CORE}" + echo "Validated ${XDP_CORE}" + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: xrt-alveo-build-pr-${{ github.event.pull_request.number }} + path: | + XRT/build/Release/opt/xilinx/xrt/**/libxrt*.so* + XRT/build/Release/opt/xilinx/xrt/**/libxdp*.so* + XRT/build/Release/opt/xilinx/xrt/**/xrt/module/*.so* + retention-days: 1 + + - name: Build summary + if: always() + run: | + XRT_ROOT="XRT/build/Release/opt/xilinx/xrt" + echo "## XDP Alveo Build" >> "$GITHUB_STEP_SUMMARY" + echo "- Commit: ${{ github.sha }}" >> "$GITHUB_STEP_SUMMARY" + echo "- PR: #${{ github.event.pull_request.number }}" >> "$GITHUB_STEP_SUMMARY" + echo "- Platform: Alveo (native)" >> "$GITHUB_STEP_SUMMARY" + if find "${XRT_ROOT}" -name 'libxrt_core.so*' -type f 2>/dev/null | grep -q . && \ + find "${XRT_ROOT}" -name 'libxdp_core.so*' -type f 2>/dev/null | grep -q .; then + echo "- Result: success" >> "$GITHUB_STEP_SUMMARY" + else + echo "- Result: failed" >> "$GITHUB_STEP_SUMMARY" + fi + build-xrt-edge: - if: contains(github.event.pull_request.labels.*.name, 'build-edge') - runs-on: ubuntu-latest + if: | + contains(github.event.pull_request.labels.*.name, 'build-xrt-edge') || + ( + !contains(github.event.pull_request.labels.*.name, 'build-xrt-alveo') && + !contains(github.event.pull_request.labels.*.name, 'build-xrt-edge') && + !contains(github.event.pull_request.labels.*.name, 'build-xrt-ve2') && + !contains(github.event.pull_request.labels.*.name, 'build-xrt-client') + ) + runs-on: [self-hosted, Ubuntu-22.04] + timeout-minutes: 360 + concurrency: + group: xdp-edge-${{ github.event.pull_request.number }} + cancel-in-progress: true + env: + SUDO: /tools/aecg-sudo/sudo/bin/sudo + + steps: + - name: Checkout XDP + uses: actions/checkout@v4 + with: + path: xdp-pr + fetch-depth: 0 + + - name: Clone XRT + run: | + rm -rf XRT + git clone https://github.com/Xilinx/XRT.git XRT + git -C XRT submodule update --init --recursive + + - name: Overlay XDP + run: | + rm -rf XRT/src/runtime_src/xdp + cp -r xdp-pr XRT/src/runtime_src/xdp + + - name: Setup build environment + run: | + cd XRT/src/runtime_src/tools/scripts + chmod +x xrtdeps.sh + $SUDO ./xrtdeps.sh + + - name: Build XRT for Edge + run: | + cd XRT/build + chmod +x build_edge.sh + ./build_edge.sh -aarch versal -noinit + + - name: Validate build outputs + run: | + ls XRT/build/versal/rpms/xrt*.rpm + ls XRT/build/versal/rpms/zocl*.rpm + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: xrt-edge-build-pr-${{ github.event.pull_request.number }} + path: | + XRT/build/versal/rpms/*.rpm + XRT/build/versal/install_xrt.sh + XRT/build/versal/reinstall_xrt.sh + XRT/build/versal/rpm.txt + retention-days: 1 + + - name: Build summary + if: always() + run: | + echo "## XDP Edge Build" >> "$GITHUB_STEP_SUMMARY" + echo "- Commit: ${{ github.sha }}" >> "$GITHUB_STEP_SUMMARY" + echo "- PR: #${{ github.event.pull_request.number }}" >> "$GITHUB_STEP_SUMMARY" + echo "- Platform: Edge (versal aarch)" >> "$GITHUB_STEP_SUMMARY" + if ls XRT/build/versal/rpms/xrt*.rpm 2>/dev/null | grep -q . && \ + ls XRT/build/versal/rpms/zocl*.rpm 2>/dev/null | grep -q .; then + echo "- Result: success" >> "$GITHUB_STEP_SUMMARY" + else + echo "- Result: failed" >> "$GITHUB_STEP_SUMMARY" + fi + + build-xrt-ve2: + if: | + contains(github.event.pull_request.labels.*.name, 'build-xrt-ve2') || + ( + !contains(github.event.pull_request.labels.*.name, 'build-xrt-alveo') && + !contains(github.event.pull_request.labels.*.name, 'build-xrt-edge') && + !contains(github.event.pull_request.labels.*.name, 'build-xrt-ve2') && + !contains(github.event.pull_request.labels.*.name, 'build-xrt-client') + ) + runs-on: [self-hosted, Ubuntu-22.04] + timeout-minutes: 480 + concurrency: + group: xdp-ve2-${{ github.event.pull_request.number }} + cancel-in-progress: true + env: + # Self-hosted runner paths for VE2 Yocto cache (Telluride rev-b / vek385). + VE2_YOCTO_CACHE_ROOT: /scratch/snigupta/ve2_yocto_cache + VE2_YOCTO_CACHE_TTL_DAYS: '7' + VE2_MACHINE: amd-cortexa78-mali-common + VE2_YOCTO_DIR: /scratch/snigupta/ve2_yocto_cache/Vitis-AI-Telluride/versal_2ve/reference_design/vek385/rev-b/sw/yocto + steps: - - name: Placeholder for edge build - run: echo "Edge build would go here" - - # Future job for client builds + - name: Checkout XDP + uses: actions/checkout@v4 + with: + path: xdp-pr + fetch-depth: 0 + + - name: Ensure Yocto workspace cache + run: | + chmod +x xdp-pr/ci/ve2/ensure_yocto_cache.sh + xdp-pr/ci/ve2/ensure_yocto_cache.sh + + - name: Build XRT RPMs with XDP overlay + env: + YOCTO_DIR: ${{ env.VE2_YOCTO_DIR }} + MACHINE: ${{ env.VE2_MACHINE }} + run: | + cd "${YOCTO_DIR}" + source edf-init-build-env + + if [ ! -d build/workspace/sources/xrt ]; then + MACHINE="${MACHINE}" devtool modify xrt + fi + + XDP_TARGET="build/workspace/sources/xrt/xrt/src/runtime_src/xdp" + rm -rf "${XDP_TARGET}" + mkdir -p "$(dirname "${XDP_TARGET}")" + cp -r "${{ github.workspace }}/xdp-pr" "${XDP_TARGET}" + + MACHINE="${MACHINE}" bitbake xrt -c cleanall + MACHINE="${MACHINE}" bitbake xrt -c package_write_rpm -f + + - name: Validate build outputs + env: + YOCTO_DIR: ${{ env.VE2_YOCTO_DIR }} + run: | + RPM_DIR="${YOCTO_DIR}/build/tmp/deploy/rpm" + find "${RPM_DIR}"/cortexa* -name 'xrt*.rpm' | tee /tmp/ve2-rpms.txt + test -s /tmp/ve2-rpms.txt + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: xrt-ve2-build-pr-${{ github.event.pull_request.number }} + path: ${{ env.VE2_YOCTO_DIR }}/build/tmp/deploy/rpm/cortexa*/xrt*.rpm + retention-days: 1 + + - name: Build summary + if: always() + env: + YOCTO_DIR: ${{ env.VE2_YOCTO_DIR }} + run: | + RPM_DIR="${YOCTO_DIR}/build/tmp/deploy/rpm" + echo "## XDP VE2 Build" >> "$GITHUB_STEP_SUMMARY" + echo "- Commit: ${{ github.sha }}" >> "$GITHUB_STEP_SUMMARY" + echo "- PR: #${{ github.event.pull_request.number }}" >> "$GITHUB_STEP_SUMMARY" + echo "- Platform: VE2 (Yocto RPM)" >> "$GITHUB_STEP_SUMMARY" + echo "- Machine: ${{ env.VE2_MACHINE }}" >> "$GITHUB_STEP_SUMMARY" + echo "- Yocto cache: ${{ env.VE2_YOCTO_CACHE_ROOT }}" >> "$GITHUB_STEP_SUMMARY" + if find "${RPM_DIR}"/cortexa* -name 'xrt*.rpm' 2>/dev/null | grep -q .; then + echo "- Result: success" >> "$GITHUB_STEP_SUMMARY" + else + echo "- Result: failed" >> "$GITHUB_STEP_SUMMARY" + fi + build-xrt-client: - if: contains(github.event.pull_request.labels.*.name, 'build-client') + if: contains(github.event.pull_request.labels.*.name, 'build-xrt-client') runs-on: ubuntu-latest steps: - - name: Placeholder for client build - run: echo "Client build would go here" + - run: echo "Client build not implemented" - # Future job for ve2 builds - build-xrt-ve2: - if: contains(github.event.pull_request.labels.*.name, 'build-ve2') + - name: Build summary + if: always() + run: | + echo "## XDP Client Build" >> "$GITHUB_STEP_SUMMARY" + echo "- Commit: ${{ github.sha }}" >> "$GITHUB_STEP_SUMMARY" + echo "- PR: #${{ github.event.pull_request.number }}" >> "$GITHUB_STEP_SUMMARY" + echo "- Platform: Client" >> "$GITHUB_STEP_SUMMARY" + echo "- Result: skipped (not implemented)" >> "$GITHUB_STEP_SUMMARY" + + ci-required: + if: always() + needs: [build-xrt-alveo, build-xrt-edge, build-xrt-ve2, build-xrt-client] runs-on: ubuntu-latest steps: - - name: Placeholder for ve2 build - run: echo "VE2 build would go here" + - name: Require all scheduled builds to pass + run: | + check_result() { + local job="$1" + local result="$2" + if [ "${result}" = "skipped" ]; then + echo "${job}: skipped" + return 0 + fi + if [ "${result}" = "success" ]; then + echo "${job}: success" + return 0 + fi + echo "${job}: ${result}" + return 1 + } + + failed=0 + check_result build-xrt-alveo "${{ needs.build-xrt-alveo.result }}" || failed=1 + check_result build-xrt-edge "${{ needs.build-xrt-edge.result }}" || failed=1 + check_result build-xrt-ve2 "${{ needs.build-xrt-ve2.result }}" || failed=1 + check_result build-xrt-client "${{ needs.build-xrt-client.result }}" || failed=1 + + if [ "${failed}" -ne 0 ]; then + echo "One or more XDP builds failed" + exit 1 + fi + echo "All scheduled XDP builds passed" diff --git a/ci/board/edge/xdp_edge_board_exec.sh b/ci/board/edge/xdp_edge_board_exec.sh new file mode 100755 index 00000000..5e539f47 --- /dev/null +++ b/ci/board/edge/xdp_edge_board_exec.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Placeholder for Edge board testing. +# +# Open questions before enabling board-test-edge: +# - Which runner label / board access method (SSH vs zboard)? +# - RPM path from edge build (XRT/build/versal/rpms/) vs target arch on board +# - Canonical test design path on NFS +# +# Expected flow once infra is confirmed: +# 1. Install xrt/zocl RPMs built by build-xrt-edge +# 2. Run xrt-smi or platform-specific smoke tests +# 3. Run XDP validation design + +set -euo pipefail + +echo "Edge board test not yet configured" +exit 1 diff --git a/ci/board/ve2/tests/01_xrt_smi_version.sh b/ci/board/ve2/tests/01_xrt_smi_version.sh new file mode 100755 index 00000000..05f3e70e --- /dev/null +++ b/ci/board/ve2/tests/01_xrt_smi_version.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -euo pipefail + +echo "Running: xrt-smi --version" +xrt-smi --version diff --git a/ci/board/ve2/tests/02_xrt_smi_examine.sh b/ci/board/ve2/tests/02_xrt_smi_examine.sh new file mode 100755 index 00000000..24b51c65 --- /dev/null +++ b/ci/board/ve2/tests/02_xrt_smi_examine.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -euo pipefail + +echo "Running: xrt-smi examine" +xrt-smi examine diff --git a/ci/board/ve2/tests/03_xrt_smi_validate.sh b/ci/board/ve2/tests/03_xrt_smi_validate.sh new file mode 100755 index 00000000..0f0180fa --- /dev/null +++ b/ci/board/ve2/tests/03_xrt_smi_validate.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -euo pipefail + +DEVICE="${XRT_VALIDATE_DEVICE:-0000:00:00.0}" + +echo "Running: xrt-smi validate --device ${DEVICE}" +xrt-smi validate --device "${DEVICE}" diff --git a/ci/board/ve2/tests/04_xdp_smoke.sh b/ci/board/ve2/tests/04_xdp_smoke.sh new file mode 100755 index 00000000..2deb91be --- /dev/null +++ b/ci/board/ve2/tests/04_xdp_smoke.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +# Placeholder smoke test until a canonical compile directory is configured. +# Verifies XRT packages are installed after RPM upgrade. +echo "Running: XDP smoke test (package check)" + +if rpm -qa | grep -q '^xrt'; then + echo "xrt RPM package found" + rpm -qa | grep '^xrt' || true +else + echo "ERROR: no xrt RPM package found after install" + exit 1 +fi + +if [ -d /opt/xilinx/xrt/lib ] || [ -d /usr/lib/xrt ]; then + echo "XRT library path present on board" +else + echo "WARNING: expected XRT library path not found (may still be valid for this rootfs)" +fi diff --git a/ci/board/ve2/xdp_ve2_board_exec.sh b/ci/board/ve2/xdp_ve2_board_exec.sh new file mode 100755 index 00000000..c73e4b89 --- /dev/null +++ b/ci/board/ve2/xdp_ve2_board_exec.sh @@ -0,0 +1,90 @@ +#!/bin/bash +set -euo pipefail + +WORK_DIR="/mnt/test" +RPM_DIR="${WORK_DIR}/rpms" +TEST_DIR="${WORK_DIR}/tests" +SUMMARY="${WORK_DIR}/github_step_summary.md" + +echo "################################ XDP VE2 Board Test ################################" +rm -f "${SUMMARY}" +touch "${SUMMARY}" +echo "## XDP VE2 Board Test" >> "${SUMMARY}" +echo "| Step | Status |" >> "${SUMMARY}" +echo "|------|--------|" >> "${SUMMARY}" + +log_pass() { echo "| $1 | pass |" >> "${SUMMARY}"; } +log_fail() { echo "| $1 | fail |" >> "${SUMMARY}"; } + +ve2_set_env() { + echo 1 > /sys/module/rcupdate/parameters/rcu_cpu_stall_suppress + export XRT_AIARM=true + export XLNX_ENABLE_CACHE=0 + export XRT_ELF_FLOW=1 +} + +ve2_run_fpgautil() { + local overlay_dir="${WORK_DIR}/overlay" + local pdi_path="${overlay_dir}/vpl_gen_fixed_pld.pdi" + local dtbo_path="${overlay_dir}/pl_aiarm.dtbo" + + if [ ! -d "${overlay_dir}" ]; then + echo "WARN: ${overlay_dir} not found, skipping fpgautil" + return 0 + fi + + if [ ! -f "${pdi_path}" ] || [ ! -f "${dtbo_path}" ]; then + echo "WARN: overlay files not found under ${overlay_dir}, skipping fpgautil" + return 0 + fi + + echo "Loading FPGA overlay" + fpgautil -b "${pdi_path}" -o "${dtbo_path}" || echo "WARN: fpgautil returned non-zero (overlay may already be loaded)" +} + +install_xrt_rpms() { + echo "Installing XRT RPMs from ${RPM_DIR}" + if ! ls "${RPM_DIR}"/xrt*.rpm >/dev/null 2>&1; then + echo "ERROR: no xrt RPMs found in ${RPM_DIR}" + log_fail "Install XRT RPMs" + exit 1 + fi + + if command -v dnf >/dev/null 2>&1; then + dnf install -y "${RPM_DIR}"/xrt*.rpm + else + rpm -Uvh --force "${RPM_DIR}"/xrt*.rpm + fi + log_pass "Install XRT RPMs" +} + +run_unit_tests() { + if [ ! -d "${TEST_DIR}" ]; then + echo "ERROR: test directory not found: ${TEST_DIR}" + log_fail "Unit tests" + exit 1 + fi + + ve2_set_env + ve2_run_fpgautil + + local test_script + for test_script in "${TEST_DIR}"/*.sh; do + [ -f "${test_script}" ] || continue + local name + name="$(basename "${test_script}")" + echo "---- Running ${name} ----" + if bash "${test_script}"; then + log_pass "${name}" + else + log_fail "${name}" + exit 1 + fi + done +} + +install_xrt_rpms +run_unit_tests + +echo "All board tests passed" +cat "${SUMMARY}" diff --git a/ci/ve2/ensure_yocto_cache.sh b/ci/ve2/ensure_yocto_cache.sh new file mode 100755 index 00000000..5dde64a9 --- /dev/null +++ b/ci/ve2/ensure_yocto_cache.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Prepare a persistent Telluride + Yocto workspace on the self-hosted runner. +# Skips clone + create_yocto.sh -i when cache is younger than VE2_YOCTO_CACHE_TTL_DAYS. +set -euo pipefail + +CACHE_ROOT="${VE2_YOCTO_CACHE_ROOT:-/scratch/snigupta/ve2_yocto_cache}" +CACHE_TTL_DAYS="${VE2_YOCTO_CACHE_TTL_DAYS:-7}" +TELLURIDE_REPO="${TELLURIDE_REPO:-IPS-SSW/Vitis-AI-Telluride.git}" +GITENTERPRISE_HOST="${GITENTERPRISE_HOST:-gitenterprise.xilinx.com}" +YOCTO_REL="${VE2_YOCTO_REL:-versal_2ve/reference_design/vek385/rev-b/sw/yocto}" +FORCE_REFRESH="${VE2_FORCE_CACHE_REFRESH:-false}" + +TELLURIDE_DIR="${CACHE_ROOT}/Vitis-AI-Telluride" +YOCTO_DIR="${TELLURIDE_DIR}/${YOCTO_REL}" +MARKER="${CACHE_ROOT}/.initialized" + +cache_is_valid() { + [ "${FORCE_REFRESH}" = "true" ] && return 1 + [ -f "${MARKER}" ] || return 1 + [ -d "${YOCTO_DIR}/build" ] || return 1 + [ -f "${YOCTO_DIR}/edf-init-build-env" ] || return 1 + + local now marker_age_sec age_days + now="$(date +%s)" + marker_age_sec=$((now - $(stat -c %Y "${MARKER}"))) + age_days=$((marker_age_sec / 86400)) + echo "Cache age: ${age_days} day(s) (ttl ${CACHE_TTL_DAYS} day(s))" + [ "${age_days}" -lt "${CACHE_TTL_DAYS}" ] +} + +mkdir -p "${CACHE_ROOT}" + +if cache_is_valid; then + echo "Reusing cached Yocto workspace at ${YOCTO_DIR}" +else + if [ "${FORCE_REFRESH}" = "true" ]; then + echo "Force refresh: rebuilding Yocto cache at ${CACHE_ROOT}" + else + echo "Cache missing or stale: initializing Yocto workspace at ${CACHE_ROOT}" + fi + + rm -rf "${TELLURIDE_DIR}" + git clone --recurse-submodules \ + "https://${GITENTERPRISE_HOST}/${TELLURIDE_REPO}" \ + "${TELLURIDE_DIR}" + + cd "${YOCTO_DIR}" + chmod +x create_yocto.sh + ./create_yocto.sh -i + touch "${MARKER}" + date -u +"%Y-%m-%dT%H:%M:%SZ" > "${CACHE_ROOT}/.initialized_at" +fi + +if [ -n "${GITHUB_ENV:-}" ]; then + echo "YOCTO_DIR=${YOCTO_DIR}" >> "${GITHUB_ENV}" +fi + +echo "${YOCTO_DIR}" From e15cf631c0d1faeebb547bda73b4992bbbb6afb3 Mon Sep 17 00:00:00 2001 From: snigupta Date: Tue, 21 Jul 2026 15:37:46 -0600 Subject: [PATCH 2/7] Remove board test workflow and scripts Signed-off-by: snigupta --- .github/workflows/xdp-board-test.yml | 30 -------- ci/board/edge/xdp_edge_board_exec.sh | 17 ----- ci/board/ve2/tests/01_xrt_smi_version.sh | 5 -- ci/board/ve2/tests/02_xrt_smi_examine.sh | 5 -- ci/board/ve2/tests/03_xrt_smi_validate.sh | 7 -- ci/board/ve2/tests/04_xdp_smoke.sh | 20 ----- ci/board/ve2/xdp_ve2_board_exec.sh | 90 ----------------------- 7 files changed, 174 deletions(-) delete mode 100644 .github/workflows/xdp-board-test.yml delete mode 100755 ci/board/edge/xdp_edge_board_exec.sh delete mode 100755 ci/board/ve2/tests/01_xrt_smi_version.sh delete mode 100755 ci/board/ve2/tests/02_xrt_smi_examine.sh delete mode 100755 ci/board/ve2/tests/03_xrt_smi_validate.sh delete mode 100755 ci/board/ve2/tests/04_xdp_smoke.sh delete mode 100755 ci/board/ve2/xdp_ve2_board_exec.sh diff --git a/.github/workflows/xdp-board-test.yml b/.github/workflows/xdp-board-test.yml deleted file mode 100644 index 686e41c9..00000000 --- a/.github/workflows/xdp-board-test.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: XDP Board Test - -# Disabled for now. Re-enable after vek385 B-revB2 board farm access is confirmed. -# Scripts live under ci/board/ve2/ for future zboard + xrt-smi tests. - -on: - workflow_call: - inputs: - build_run_id: - required: true - type: string - workflow_dispatch: - inputs: - build_run_id: - description: XDP Build run ID with VE2 RPM artifacts - required: true - type: string - -jobs: - board-test-ve2: - if: false - runs-on: [self-hosted, Ubuntu-22.04] - steps: - - run: echo "VE2 board test disabled" - - board-test-edge: - if: false - runs-on: [self-hosted, Ubuntu-22.04] - steps: - - run: echo "Edge board test disabled" diff --git a/ci/board/edge/xdp_edge_board_exec.sh b/ci/board/edge/xdp_edge_board_exec.sh deleted file mode 100755 index 5e539f47..00000000 --- a/ci/board/edge/xdp_edge_board_exec.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# Placeholder for Edge board testing. -# -# Open questions before enabling board-test-edge: -# - Which runner label / board access method (SSH vs zboard)? -# - RPM path from edge build (XRT/build/versal/rpms/) vs target arch on board -# - Canonical test design path on NFS -# -# Expected flow once infra is confirmed: -# 1. Install xrt/zocl RPMs built by build-xrt-edge -# 2. Run xrt-smi or platform-specific smoke tests -# 3. Run XDP validation design - -set -euo pipefail - -echo "Edge board test not yet configured" -exit 1 diff --git a/ci/board/ve2/tests/01_xrt_smi_version.sh b/ci/board/ve2/tests/01_xrt_smi_version.sh deleted file mode 100755 index 05f3e70e..00000000 --- a/ci/board/ve2/tests/01_xrt_smi_version.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -set -euo pipefail - -echo "Running: xrt-smi --version" -xrt-smi --version diff --git a/ci/board/ve2/tests/02_xrt_smi_examine.sh b/ci/board/ve2/tests/02_xrt_smi_examine.sh deleted file mode 100755 index 24b51c65..00000000 --- a/ci/board/ve2/tests/02_xrt_smi_examine.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -set -euo pipefail - -echo "Running: xrt-smi examine" -xrt-smi examine diff --git a/ci/board/ve2/tests/03_xrt_smi_validate.sh b/ci/board/ve2/tests/03_xrt_smi_validate.sh deleted file mode 100755 index 0f0180fa..00000000 --- a/ci/board/ve2/tests/03_xrt_smi_validate.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -euo pipefail - -DEVICE="${XRT_VALIDATE_DEVICE:-0000:00:00.0}" - -echo "Running: xrt-smi validate --device ${DEVICE}" -xrt-smi validate --device "${DEVICE}" diff --git a/ci/board/ve2/tests/04_xdp_smoke.sh b/ci/board/ve2/tests/04_xdp_smoke.sh deleted file mode 100755 index 2deb91be..00000000 --- a/ci/board/ve2/tests/04_xdp_smoke.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# Placeholder smoke test until a canonical compile directory is configured. -# Verifies XRT packages are installed after RPM upgrade. -echo "Running: XDP smoke test (package check)" - -if rpm -qa | grep -q '^xrt'; then - echo "xrt RPM package found" - rpm -qa | grep '^xrt' || true -else - echo "ERROR: no xrt RPM package found after install" - exit 1 -fi - -if [ -d /opt/xilinx/xrt/lib ] || [ -d /usr/lib/xrt ]; then - echo "XRT library path present on board" -else - echo "WARNING: expected XRT library path not found (may still be valid for this rootfs)" -fi diff --git a/ci/board/ve2/xdp_ve2_board_exec.sh b/ci/board/ve2/xdp_ve2_board_exec.sh deleted file mode 100755 index c73e4b89..00000000 --- a/ci/board/ve2/xdp_ve2_board_exec.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash -set -euo pipefail - -WORK_DIR="/mnt/test" -RPM_DIR="${WORK_DIR}/rpms" -TEST_DIR="${WORK_DIR}/tests" -SUMMARY="${WORK_DIR}/github_step_summary.md" - -echo "################################ XDP VE2 Board Test ################################" -rm -f "${SUMMARY}" -touch "${SUMMARY}" -echo "## XDP VE2 Board Test" >> "${SUMMARY}" -echo "| Step | Status |" >> "${SUMMARY}" -echo "|------|--------|" >> "${SUMMARY}" - -log_pass() { echo "| $1 | pass |" >> "${SUMMARY}"; } -log_fail() { echo "| $1 | fail |" >> "${SUMMARY}"; } - -ve2_set_env() { - echo 1 > /sys/module/rcupdate/parameters/rcu_cpu_stall_suppress - export XRT_AIARM=true - export XLNX_ENABLE_CACHE=0 - export XRT_ELF_FLOW=1 -} - -ve2_run_fpgautil() { - local overlay_dir="${WORK_DIR}/overlay" - local pdi_path="${overlay_dir}/vpl_gen_fixed_pld.pdi" - local dtbo_path="${overlay_dir}/pl_aiarm.dtbo" - - if [ ! -d "${overlay_dir}" ]; then - echo "WARN: ${overlay_dir} not found, skipping fpgautil" - return 0 - fi - - if [ ! -f "${pdi_path}" ] || [ ! -f "${dtbo_path}" ]; then - echo "WARN: overlay files not found under ${overlay_dir}, skipping fpgautil" - return 0 - fi - - echo "Loading FPGA overlay" - fpgautil -b "${pdi_path}" -o "${dtbo_path}" || echo "WARN: fpgautil returned non-zero (overlay may already be loaded)" -} - -install_xrt_rpms() { - echo "Installing XRT RPMs from ${RPM_DIR}" - if ! ls "${RPM_DIR}"/xrt*.rpm >/dev/null 2>&1; then - echo "ERROR: no xrt RPMs found in ${RPM_DIR}" - log_fail "Install XRT RPMs" - exit 1 - fi - - if command -v dnf >/dev/null 2>&1; then - dnf install -y "${RPM_DIR}"/xrt*.rpm - else - rpm -Uvh --force "${RPM_DIR}"/xrt*.rpm - fi - log_pass "Install XRT RPMs" -} - -run_unit_tests() { - if [ ! -d "${TEST_DIR}" ]; then - echo "ERROR: test directory not found: ${TEST_DIR}" - log_fail "Unit tests" - exit 1 - fi - - ve2_set_env - ve2_run_fpgautil - - local test_script - for test_script in "${TEST_DIR}"/*.sh; do - [ -f "${test_script}" ] || continue - local name - name="$(basename "${test_script}")" - echo "---- Running ${name} ----" - if bash "${test_script}"; then - log_pass "${name}" - else - log_fail "${name}" - exit 1 - fi - done -} - -install_xrt_rpms -run_unit_tests - -echo "All board tests passed" -cat "${SUMMARY}" From e31ce3b41c057eae96e5ad3c4ad70c5621add10e Mon Sep 17 00:00:00 2001 From: snigupta Date: Thu, 23 Jul 2026 10:00:34 -0600 Subject: [PATCH 3/7] Add VE2 CSR board test; enable push trigger for fork testing Signed-off-by: snigupta --- .github/workflows/xdp-xrt-build.yml | 100 +++++++++++---- ci/csr/config.defaults.sh | 26 ++++ ci/csr/generate_csr_run.sh | 78 ++++++++++++ ci/csr/patch_merged_tql.sh | 87 +++++++++++++ ci/csr/run_ve2_board_test.sh | 86 +++++++++++++ ci/csr/stage_ve2_rpms.sh | 78 ++++++++++++ ci/csr/submit_csr_jenkins.sh | 183 ++++++++++++++++++++++++++++ 7 files changed, 617 insertions(+), 21 deletions(-) create mode 100755 ci/csr/config.defaults.sh create mode 100755 ci/csr/generate_csr_run.sh create mode 100755 ci/csr/patch_merged_tql.sh create mode 100755 ci/csr/run_ve2_board_test.sh create mode 100755 ci/csr/stage_ve2_rpms.sh create mode 100755 ci/csr/submit_csr_jenkins.sh diff --git a/.github/workflows/xdp-xrt-build.yml b/.github/workflows/xdp-xrt-build.yml index be6b5ad4..4d49353d 100644 --- a/.github/workflows/xdp-xrt-build.yml +++ b/.github/workflows/xdp-xrt-build.yml @@ -4,25 +4,31 @@ on: pull_request: types: [opened, synchronize, reopened, labeled, unlabeled] branches: [master] + # Fork testing: push to this branch runs VE2 build + CSR board test. + push: + branches: [ci/xdp-multi-platform-builds] + workflow_dispatch: concurrency: - group: xdp-build-${{ github.event.pull_request.number }} + group: xdp-build-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: build-xrt-alveo: if: | - contains(github.event.pull_request.labels.*.name, 'build-xrt-alveo') || - ( - !contains(github.event.pull_request.labels.*.name, 'build-xrt-alveo') && - !contains(github.event.pull_request.labels.*.name, 'build-xrt-edge') && - !contains(github.event.pull_request.labels.*.name, 'build-xrt-ve2') && - !contains(github.event.pull_request.labels.*.name, 'build-xrt-client') + github.event_name == 'pull_request' && ( + contains(github.event.pull_request.labels.*.name, 'build-xrt-alveo') || + ( + !contains(github.event.pull_request.labels.*.name, 'build-xrt-alveo') && + !contains(github.event.pull_request.labels.*.name, 'build-xrt-edge') && + !contains(github.event.pull_request.labels.*.name, 'build-xrt-ve2') && + !contains(github.event.pull_request.labels.*.name, 'build-xrt-client') + ) ) runs-on: ubuntu-latest timeout-minutes: 60 concurrency: - group: xdp-alveo-${{ github.event.pull_request.number }} + group: xdp-alveo-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: true steps: @@ -78,7 +84,7 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: xrt-alveo-build-pr-${{ github.event.pull_request.number }} + name: xrt-alveo-build-${{ github.event.pull_request.number || github.run_id }} path: | XRT/build/Release/opt/xilinx/xrt/**/libxrt*.so* XRT/build/Release/opt/xilinx/xrt/**/libxdp*.so* @@ -102,17 +108,19 @@ jobs: build-xrt-edge: if: | - contains(github.event.pull_request.labels.*.name, 'build-xrt-edge') || - ( - !contains(github.event.pull_request.labels.*.name, 'build-xrt-alveo') && - !contains(github.event.pull_request.labels.*.name, 'build-xrt-edge') && - !contains(github.event.pull_request.labels.*.name, 'build-xrt-ve2') && - !contains(github.event.pull_request.labels.*.name, 'build-xrt-client') + github.event_name == 'pull_request' && ( + contains(github.event.pull_request.labels.*.name, 'build-xrt-edge') || + ( + !contains(github.event.pull_request.labels.*.name, 'build-xrt-alveo') && + !contains(github.event.pull_request.labels.*.name, 'build-xrt-edge') && + !contains(github.event.pull_request.labels.*.name, 'build-xrt-ve2') && + !contains(github.event.pull_request.labels.*.name, 'build-xrt-client') + ) ) runs-on: [self-hosted, Ubuntu-22.04] timeout-minutes: 360 concurrency: - group: xdp-edge-${{ github.event.pull_request.number }} + group: xdp-edge-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: true env: SUDO: /tools/aecg-sudo/sudo/bin/sudo @@ -155,7 +163,7 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: xrt-edge-build-pr-${{ github.event.pull_request.number }} + name: xrt-edge-build-${{ github.event.pull_request.number || github.run_id }} path: | XRT/build/versal/rpms/*.rpm XRT/build/versal/install_xrt.sh @@ -179,6 +187,8 @@ jobs: build-xrt-ve2: if: | + github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'build-xrt-ve2') || ( !contains(github.event.pull_request.labels.*.name, 'build-xrt-alveo') && @@ -189,7 +199,7 @@ jobs: runs-on: [self-hosted, Ubuntu-22.04] timeout-minutes: 480 concurrency: - group: xdp-ve2-${{ github.event.pull_request.number }} + group: xdp-ve2-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: true env: # Self-hosted runner paths for VE2 Yocto cache (Telluride rev-b / vek385). @@ -241,7 +251,7 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: xrt-ve2-build-pr-${{ github.event.pull_request.number }} + name: xrt-ve2-build-${{ github.event.pull_request.number || github.run_id }} path: ${{ env.VE2_YOCTO_DIR }}/build/tmp/deploy/rpm/cortexa*/xrt*.rpm retention-days: 1 @@ -264,7 +274,9 @@ jobs: fi build-xrt-client: - if: contains(github.event.pull_request.labels.*.name, 'build-xrt-client') + if: | + github.event_name == 'pull_request' && + contains(github.event.pull_request.labels.*.name, 'build-xrt-client') runs-on: ubuntu-latest steps: - run: echo "Client build not implemented" @@ -278,8 +290,54 @@ jobs: echo "- Platform: Client" >> "$GITHUB_STEP_SUMMARY" echo "- Result: skipped (not implemented)" >> "$GITHUB_STEP_SUMMARY" + board-test-ve2: + if: | + github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'board-test-ve2') + needs: build-xrt-ve2 + runs-on: [self-hosted, Ubuntu-22.04] + timeout-minutes: 720 + concurrency: + group: xdp-board-ve2-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + env: + VE2_YOCTO_CACHE_ROOT: /scratch/snigupta/ve2_yocto_cache + VE2_YOCTO_DIR: /scratch/snigupta/ve2_yocto_cache/Vitis-AI-Telluride/versal_2ve/reference_design/vek385/rev-b/sw/yocto + CSR_ROOT: /wrk/csr/snigupta/xdp_ci + JENKINS_USER: ${{ secrets.JENKINS_USER }} + JENKINS_API_TOKEN: ${{ secrets.JENKINS_API_TOKEN }} + + steps: + - name: Checkout XDP + uses: actions/checkout@v4 + + - name: Run VE2 CSR board test + env: + XDP_CSR_DESCRIPTION: >- + XDP ${{ github.event_name }} ${{ github.ref_name }} @ ${{ github.sha }} + run: | + bash ci/csr/run_ve2_board_test.sh \ + --run-id '${{ github.event.pull_request.number || github.run_id }}-${{ github.run_id }}' + + - name: Board test summary + if: always() + run: | + RUN_ID='${{ github.event.pull_request.number || github.run_id }}-${{ github.run_id }}' + echo "## XDP VE2 Board Test (CSR)" >> "$GITHUB_STEP_SUMMARY" + echo "- Commit: ${{ github.sha }}" >> "$GITHUB_STEP_SUMMARY" + echo "- Trigger: ${{ github.event_name }} (${{ github.ref_name }})" >> "$GITHUB_STEP_SUMMARY" + echo "- Suite: VE2_XDP (hw)" >> "$GITHUB_STEP_SUMMARY" + echo "- CSR workspace: ${CSR_ROOT}/pr-${RUN_ID}" >> "$GITHUB_STEP_SUMMARY" + echo "- Jenkins job: csr_submit_job (BOARD_SELECT_RESOURCE=VE2)" >> "$GITHUB_STEP_SUMMARY" + if [ "${{ job.status }}" = "success" ]; then + echo "- Result: success" >> "$GITHUB_STEP_SUMMARY" + else + echo "- Result: failed" >> "$GITHUB_STEP_SUMMARY" + fi + ci-required: - if: always() + if: always() && github.event_name == 'pull_request' needs: [build-xrt-alveo, build-xrt-edge, build-xrt-ve2, build-xrt-client] runs-on: ubuntu-latest steps: diff --git a/ci/csr/config.defaults.sh b/ci/csr/config.defaults.sh new file mode 100755 index 00000000..dfbd60e3 --- /dev/null +++ b/ci/csr/config.defaults.sh @@ -0,0 +1,26 @@ +# Defaults for XDP VE2 CSR board testing on XCO. +# Override any value via environment before calling the ci/csr scripts. + +: "${CSR_ROOT:=/wrk/csr/snigupta/xdp_ci}" +: "${CSR_SPEC:=/proj/testcases/xtc/HEAD/auto/ymls/config/VAI_6_3_GEN2_REGRESSION.yml}" +: "${CSR_DAY:=sunday}" +: "${CSR_SUITE:=VE2_XDP}" +: "${CSR_TESTMODE:=hw}" +: "${CSR_TQL_REPO:=/proj/testcases/xtc/HEAD/auto/tql/vaiml/}" +: "${CSR_TA:=/proj/xbuilds/HEAD_INT_flexml_verified_vitis/installs}" +: "${CSR_CREATE_SPRITE_RUN:=/proj/testcases/xtc/HEAD/auto/scripts/createSpriteRun/createSpriteRun.sh}" + +# VE2_XDP is commented out in VAI_6_2 runs.yml; it is active under eachday in 6.3. +: "${CSR_SUPER_SUITE:=VAI_6_3_GEN2_REGRESSION}" + +: "${VE2_RPM_DEPLOY_SUBDIR:=cortexa72_cortexa53}" +: "${VE2_RPM_GLOB:=xrt*.rpm}" +: "${VE2_DNF_PACKAGES:=xrt-20261*.rpm xrt-dbg*.rpm xrt-src*.rpm}" + +: "${JENKINS_URL:=http://acasops:8080}" +: "${JENKINS_JOB:=csr_submit_job}" +: "${JENKINS_BOARD_SELECT:=VE2}" +: "${JENKINS_COMPILE_JOBS_ONLY:=False}" +: "${JENKINS_LNX_TA:=DEFAULT}" + +: "${CSR_WAIT_TIMEOUT_SEC:=14400}" diff --git a/ci/csr/generate_csr_run.sh b/ci/csr/generate_csr_run.sh new file mode 100755 index 00000000..f02e5d24 --- /dev/null +++ b/ci/csr/generate_csr_run.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Generate CSR sprite files for a VE2_XDP hw run. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=config.defaults.sh +source "${SCRIPT_DIR}/config.defaults.sh" + +usage() { + cat <<'EOF' +Usage: generate_csr_run.sh --sprite-dir DIR + +Creates CSR merged TQL and launch scripts under --sprite-dir. +EOF +} + +SPRITE_DIR="" + +while [ "$#" -gt 0 ]; do + case "$1" in + --sprite-dir) + SPRITE_DIR="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [ -z "${SPRITE_DIR}" ]; then + echo "Missing required --sprite-dir" >&2 + usage >&2 + exit 1 +fi + +mkdir -p "${SPRITE_DIR}/results" +chmod a+w "${SPRITE_DIR}" "${SPRITE_DIR}/results" +chmod a+w "$(dirname "${SPRITE_DIR}")" 2>/dev/null || true + +export ENVROOT="${ENVROOT:-/proj/rdi/env/stable/hierdesign}" +# g_profile can return 126 under bash -e; createSpriteRunLaunch re-sources RDI env. +# shellcheck disable=SC1091 +set +e +source "${ENVROOT}/g_profile" +set -e +umask 022 + +cd "${SPRITE_DIR}" + +"${CSR_CREATE_SPRITE_RUN}" \ + --day "${CSR_DAY}" \ + --spec "${CSR_SPEC}" \ + --results "${SPRITE_DIR}/results" \ + --flexml \ + --tql-repo "${CSR_TQL_REPO}" \ + --xtc-no-lut \ + --testsuites "${CSR_SUITE}" \ + --testmode "${CSR_TESTMODE}" \ + --ta "${CSR_TA}" + +RDI_SCRIPT="${SPRITE_DIR}/${CSR_SUPER_SUITE}_${CSR_DAY}_rdi.sh" +if [ ! -s "${RDI_SCRIPT}" ]; then + echo "CSR generation did not produce a non-empty RDI script: ${RDI_SCRIPT}" >&2 + ls -la "${SPRITE_DIR}" >&2 || true + exit 1 +fi + +chmod -R a+w "${SPRITE_DIR}" 2>/dev/null || true +echo "Generated CSR run in ${SPRITE_DIR}" +ls -1 "${SPRITE_DIR}"/*"${CSR_SUITE}"*.tql +echo "Launch script: ${RDI_SCRIPT}" diff --git a/ci/csr/patch_merged_tql.sh b/ci/csr/patch_merged_tql.sh new file mode 100755 index 00000000..7165b585 --- /dev/null +++ b/ci/csr/patch_merged_tql.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# Patch CSR merged TQL to copy custom XRT RPMs and install them on the board. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=config.defaults.sh +source "${SCRIPT_DIR}/config.defaults.sh" + +usage() { + cat <<'EOF' +Usage: patch_merged_tql.sh --sprite-dir DIR --rpm-dir DIR [--suite SUITE] + +Appends a GLOBAL APPLY block to the merged VE2_XDP TQL in the sprite directory. +EOF +} + +SPRITE_DIR="" +RPM_DIR="" +SUITE="${CSR_SUITE}" + +while [ "$#" -gt 0 ]; do + case "$1" in + --sprite-dir) + SPRITE_DIR="$2" + shift 2 + ;; + --rpm-dir) + RPM_DIR="$2" + shift 2 + ;; + --suite) + SUITE="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [ -z "${SPRITE_DIR}" ] || [ -z "${RPM_DIR}" ]; then + echo "Missing required --sprite-dir and/or --rpm-dir" >&2 + usage >&2 + exit 1 +fi + +if [ ! -d "${RPM_DIR}" ] || [ -z "$(find "${RPM_DIR}" -maxdepth 1 -name '*.rpm' -print -quit)" ]; then + echo "RPM directory is empty or missing: ${RPM_DIR}" >&2 + exit 1 +fi + +shopt -s nullglob +TQL_FILES=( "${SPRITE_DIR}"/*"${SUITE}"*.tql ) +shopt -u nullglob + +if [ "${#TQL_FILES[@]}" -eq 0 ]; then + echo "No merged TQL matching suite ${SUITE} under ${SPRITE_DIR}" >&2 + ls -la "${SPRITE_DIR}" >&2 || true + exit 1 +fi + +TQL_FILE="${TQL_FILES[0]}" +RPM_BASENAME="$(basename "${RPM_DIR}")" +MARKER="# XDP CI: install custom XRT RPMs on Telluride before board run" + +if grep -qF "${MARKER}" "${TQL_FILE}"; then + echo "TQL already patched: ${TQL_FILE}" + exit 0 +fi + +cat >> "${TQL_FILE}" <&2 + usage >&2 + exit 1 + ;; + esac +done + +if [ -z "${RUN_ID}" ]; then + echo "Missing required --run-id" >&2 + usage >&2 + exit 1 +fi + +SPRITE_DIR="${CSR_ROOT}/pr-${RUN_ID}/sprite" +RPM_DIR="${CSR_ROOT}/pr-${RUN_ID}/rpms" +DESCRIPTION="${XDP_CSR_DESCRIPTION:-XDP PR ${RUN_ID} VE2_XDP board test}" + +mkdir -p "${CSR_ROOT}/pr-${RUN_ID}" +chmod a+w "${CSR_ROOT}" "${CSR_ROOT}/pr-${RUN_ID}" 2>/dev/null || true + +STAGE_ARGS=( --dest "${RPM_DIR}" ) +if [ -n "${RPM_SOURCE}" ]; then + STAGE_ARGS+=( --source "${RPM_SOURCE}" ) +fi + +echo "== Stage VE2 XRT RPMs ==" +bash "${SCRIPT_DIR}/stage_ve2_rpms.sh" "${STAGE_ARGS[@]}" + +echo "== Generate CSR sprite run ==" +bash "${SCRIPT_DIR}/generate_csr_run.sh" --sprite-dir "${SPRITE_DIR}" + +echo "== Patch merged TQL for custom XRT install ==" +bash "${SCRIPT_DIR}/patch_merged_tql.sh" \ + --sprite-dir "${SPRITE_DIR}" \ + --rpm-dir "${RPM_DIR}" + +if [ "${XDP_CSR_SKIP_JENKINS:-0}" = "1" ]; then + echo "XDP_CSR_SKIP_JENKINS=1; CSR artifacts ready at ${SPRITE_DIR}" + exit 0 +fi + +echo "== Submit CSR run to Jenkins ==" +bash "${SCRIPT_DIR}/submit_csr_jenkins.sh" \ + --sprite-dir "${SPRITE_DIR}" \ + --description "${DESCRIPTION}" + +echo "VE2 CSR board test completed for run ${RUN_ID}" diff --git a/ci/csr/stage_ve2_rpms.sh b/ci/csr/stage_ve2_rpms.sh new file mode 100755 index 00000000..63c53909 --- /dev/null +++ b/ci/csr/stage_ve2_rpms.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Copy VE2 XRT RPMs into a CSR run directory for board-side dnf install. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=config.defaults.sh +source "${SCRIPT_DIR}/config.defaults.sh" + +usage() { + cat <<'EOF' +Usage: stage_ve2_rpms.sh --dest DIR [--source DIR] + + --dest DIR Destination rpms/ directory under the CSR sprite run + --source DIR Yocto RPM deploy directory (default: ${VE2_YOCTO_DIR}/build/tmp/deploy/rpm/${VE2_RPM_DEPLOY_SUBDIR}) +EOF +} + +DEST="" +SOURCE="" + +while [ "$#" -gt 0 ]; do + case "$1" in + --dest) + DEST="$2" + shift 2 + ;; + --source) + SOURCE="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [ -z "${DEST}" ]; then + echo "Missing required --dest" >&2 + usage >&2 + exit 1 +fi + +if [ -z "${SOURCE}" ]; then + if [ -z "${VE2_YOCTO_DIR:-}" ]; then + echo "Set VE2_YOCTO_DIR or pass --source" >&2 + exit 1 + fi + SOURCE="${VE2_YOCTO_DIR}/build/tmp/deploy/rpm/${VE2_RPM_DEPLOY_SUBDIR}" +fi + +if [ ! -d "${SOURCE}" ]; then + echo "RPM source directory not found: ${SOURCE}" >&2 + exit 1 +fi + +shopt -s nullglob +RPMS=( "${SOURCE}"/${VE2_RPM_GLOB} ) +shopt -u nullglob + +if [ "${#RPMS[@]}" -eq 0 ]; then + echo "No RPMs matching ${VE2_RPM_GLOB} under ${SOURCE}" >&2 + exit 1 +fi + +mkdir -p "${DEST}" +rm -f "${DEST}"/*.rpm +cp -f "${RPMS[@]}" "${DEST}/" +chmod -R a+rX "${DEST}" +chmod a+w "${DEST}" + +echo "Staged ${#RPMS[@]} RPM(s) to ${DEST}:" +ls -1 "${DEST}"/*.rpm diff --git a/ci/csr/submit_csr_jenkins.sh b/ci/csr/submit_csr_jenkins.sh new file mode 100755 index 00000000..bf4b07ca --- /dev/null +++ b/ci/csr/submit_csr_jenkins.sh @@ -0,0 +1,183 @@ +#!/bin/bash +# Submit a CSR sprite directory to Jenkins csr_submit_job and wait for completion. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=config.defaults.sh +source "${SCRIPT_DIR}/config.defaults.sh" + +usage() { + cat <<'EOF' +Usage: submit_csr_jenkins.sh --sprite-dir DIR [--description TEXT] + +Requires: + JENKINS_USER + JENKINS_API_TOKEN + +Optional: + JENKINS_URL (default: http://acasops:8080) + CSR_WAIT_TIMEOUT_SEC (default: 14400) +EOF +} + +SPRITE_DIR="" +DESCRIPTION="" + +while [ "$#" -gt 0 ]; do + case "$1" in + --sprite-dir) + SPRITE_DIR="$2" + shift 2 + ;; + --description) + DESCRIPTION="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [ -z "${SPRITE_DIR}" ]; then + echo "Missing required --sprite-dir" >&2 + usage >&2 + exit 1 +fi + +if [ ! -d "${SPRITE_DIR}" ]; then + echo "Sprite directory not found: ${SPRITE_DIR}" >&2 + exit 1 +fi + +if [ -z "${JENKINS_USER:-}" ] || [ -z "${JENKINS_API_TOKEN:-}" ]; then + echo "Set JENKINS_USER and JENKINS_API_TOKEN to submit ${JENKINS_JOB}" >&2 + exit 1 +fi + +chmod a+w "${SPRITE_DIR}" "$(dirname "${SPRITE_DIR}")" 2>/dev/null || true + +if [ -z "${DESCRIPTION}" ]; then + DESCRIPTION="XDP VE2 CSR: ${SPRITE_DIR}" +fi + +python3 - <<'PY' "${JENKINS_URL}" "${JENKINS_JOB}" "${SPRITE_DIR}" "${DESCRIPTION}" \ + "${JENKINS_LNX_TA}" "${JENKINS_COMPILE_JOBS_ONLY}" "${JENKINS_BOARD_SELECT}" \ + "${JENKINS_USER}" "${JENKINS_API_TOKEN}" "${CSR_WAIT_TIMEOUT_SEC}" +import base64 +import json +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +( + jenkins_url, + job_name, + sprite_dir, + description, + lnx_ta, + compile_jobs_only, + board_select, + user, + token, + timeout_sec, +) = sys.argv[1:11] +timeout_sec = int(timeout_sec) +base = jenkins_url.rstrip("/") +auth = base64.b64encode(f"{user}:{token}".encode()).decode() + + +def request(url, method="GET", data=None, headers=None): + req_headers = {"Authorization": f"Basic {auth}"} + if headers: + req_headers.update(headers) + req = urllib.request.Request(url, data=data, headers=req_headers, method=method) + with urllib.request.urlopen(req, timeout=60) as resp: + body = resp.read().decode() + return resp, body + + +crumb_resp, crumb_body = request(f"{base}/crumbIssuer/api/json") +crumb = json.loads(crumb_body) +crumb_field = crumb["crumbRequestField"] +crumb_value = crumb["crumb"] + +params = { + "SPRITE_DIR": sprite_dir, + "DESCRIPTION": description[:256], + "LNX_TA": lnx_ta, + "WIN_TA": "DEFAULT", + "COMPILE_JOBS_ONLY": compile_jobs_only, + "LINUX_COMPILE": "False", + "BOARD_SELECT_RESOURCE": board_select, + "VE2_DOCKER": "False", + "VE2_BOOT_IMAGE_PATH": "DEFAULT", +} +query = urllib.parse.urlencode(params) +trigger_url = f"{base}/job/{job_name}/buildWithParameters?{query}" + +trigger_resp, _ = request( + trigger_url, + method="POST", + data=b"", + headers={crumb_field: crumb_value}, +) +queue_url = trigger_resp.headers.get("Location") +if not queue_url: + raise SystemExit("Jenkins did not return a queue Location header") + +print(f"Queued Jenkins job: {queue_url.rstrip('/')}/api/json") + +build_url = None +deadline = time.time() + min(timeout_sec, 600) +while time.time() < deadline: + _, body = request(f"{queue_url.rstrip('/')}/api/json") + data = json.loads(body) + if "executable" in data and data["executable"]: + build_url = data["executable"]["url"] + break + if data.get("cancelled"): + raise SystemExit("Jenkins queue item was cancelled") + time.sleep(5) + +if not build_url: + raise SystemExit("Timed out waiting for Jenkins to start the build") + +print(f"Jenkins build started: {build_url}") + +deadline = time.time() + timeout_sec +result = None +while time.time() < deadline: + _, body = request(f"{build_url.rstrip('/')}/api/json") + data = json.loads(body) + if not data.get("building") and data.get("result") is not None: + result = data["result"] + break + time.sleep(60) + +if result is None: + raise SystemExit(f"Timed out waiting for Jenkins build after {timeout_sec}s: {build_url}") + +print(f"Jenkins result: {result}") +print(f"Console: {build_url}console") + +_, console = request(f"{build_url.rstrip('/')}/consoleText") +for line in console.splitlines(): + if "Check XOAH Link for details:" in line: + print(line.strip()) + if "All (" in line and "runs finished at:" in line: + print(line.strip()) + +if result != "SUCCESS": + raise SystemExit(1) +PY + +echo "Jenkins ${JENKINS_JOB} completed successfully" From 8c4b458a5bdcaddc0074fc1d0d43979433e00b84 Mon Sep 17 00:00:00 2001 From: snigupta Date: Thu, 23 Jul 2026 10:11:56 -0600 Subject: [PATCH 4/7] Fix CSR generation in GHA when LD_LIBRARY_PATH is unset Signed-off-by: snigupta --- ci/csr/generate_csr_run.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ci/csr/generate_csr_run.sh b/ci/csr/generate_csr_run.sh index f02e5d24..1eb94588 100755 --- a/ci/csr/generate_csr_run.sh +++ b/ci/csr/generate_csr_run.sh @@ -44,12 +44,8 @@ mkdir -p "${SPRITE_DIR}/results" chmod a+w "${SPRITE_DIR}" "${SPRITE_DIR}/results" chmod a+w "$(dirname "${SPRITE_DIR}")" 2>/dev/null || true -export ENVROOT="${ENVROOT:-/proj/rdi/env/stable/hierdesign}" -# g_profile can return 126 under bash -e; createSpriteRunLaunch re-sources RDI env. -# shellcheck disable=SC1091 -set +e -source "${ENVROOT}/g_profile" -set -e +# createSpriteRunLaunch sets up RDI/XTC. Do not source g_profile here: under bash +# nounset it fails when LD_LIBRARY_PATH is unset (common in GHA runner env). umask 022 cd "${SPRITE_DIR}" From a064485c892ddbeb69050a5dd35475bd93faed52 Mon Sep 17 00:00:00 2001 From: snigupta Date: Thu, 23 Jul 2026 10:29:21 -0600 Subject: [PATCH 5/7] Use local RDI submit when Jenkins build API is unavailable. Default XDP_CSR_SUBMIT_MODE=local for fork CI; keep Jenkins path for when csr_submit_job Job/Build permission is granted. Signed-off-by: snigupta --- .github/workflows/xdp-xrt-build.yml | 3 +- ci/csr/config.defaults.sh | 1 + ci/csr/run_ve2_board_test.sh | 25 ++++-- ci/csr/submit_csr_local.sh | 116 ++++++++++++++++++++++++++++ 4 files changed, 139 insertions(+), 6 deletions(-) create mode 100755 ci/csr/submit_csr_local.sh diff --git a/.github/workflows/xdp-xrt-build.yml b/.github/workflows/xdp-xrt-build.yml index 4d49353d..14a0cad2 100644 --- a/.github/workflows/xdp-xrt-build.yml +++ b/.github/workflows/xdp-xrt-build.yml @@ -305,6 +305,7 @@ jobs: VE2_YOCTO_CACHE_ROOT: /scratch/snigupta/ve2_yocto_cache VE2_YOCTO_DIR: /scratch/snigupta/ve2_yocto_cache/Vitis-AI-Telluride/versal_2ve/reference_design/vek385/rev-b/sw/yocto CSR_ROOT: /wrk/csr/snigupta/xdp_ci + XDP_CSR_SUBMIT_MODE: local JENKINS_USER: ${{ secrets.JENKINS_USER }} JENKINS_API_TOKEN: ${{ secrets.JENKINS_API_TOKEN }} @@ -329,7 +330,7 @@ jobs: echo "- Trigger: ${{ github.event_name }} (${{ github.ref_name }})" >> "$GITHUB_STEP_SUMMARY" echo "- Suite: VE2_XDP (hw)" >> "$GITHUB_STEP_SUMMARY" echo "- CSR workspace: ${CSR_ROOT}/pr-${RUN_ID}" >> "$GITHUB_STEP_SUMMARY" - echo "- Jenkins job: csr_submit_job (BOARD_SELECT_RESOURCE=VE2)" >> "$GITHUB_STEP_SUMMARY" + echo "- Submit mode: local RDI (set XDP_CSR_SUBMIT_MODE=jenkins when Job/Build granted)" >> "$GITHUB_STEP_SUMMARY" if [ "${{ job.status }}" = "success" ]; then echo "- Result: success" >> "$GITHUB_STEP_SUMMARY" else diff --git a/ci/csr/config.defaults.sh b/ci/csr/config.defaults.sh index dfbd60e3..8de1e5c5 100755 --- a/ci/csr/config.defaults.sh +++ b/ci/csr/config.defaults.sh @@ -24,3 +24,4 @@ : "${JENKINS_LNX_TA:=DEFAULT}" : "${CSR_WAIT_TIMEOUT_SEC:=14400}" +: "${XDP_CSR_SUBMIT_MODE:=local}" diff --git a/ci/csr/run_ve2_board_test.sh b/ci/csr/run_ve2_board_test.sh index 127303de..3f2cc668 100755 --- a/ci/csr/run_ve2_board_test.sh +++ b/ci/csr/run_ve2_board_test.sh @@ -15,7 +15,8 @@ Environment: VE2_YOCTO_DIR Yocto workspace used by build-xrt-ve2 JENKINS_USER Required for Jenkins submission JENKINS_API_TOKEN Required for Jenkins submission - XDP_CSR_SKIP_JENKINS=1 Generate CSR only; do not submit to Jenkins + XDP_CSR_SKIP_JENKINS=1 Generate CSR only; do not submit + XDP_CSR_SUBMIT_MODE local (default) or jenkins EOF } @@ -78,9 +79,23 @@ if [ "${XDP_CSR_SKIP_JENKINS:-0}" = "1" ]; then exit 0 fi -echo "== Submit CSR run to Jenkins ==" -bash "${SCRIPT_DIR}/submit_csr_jenkins.sh" \ - --sprite-dir "${SPRITE_DIR}" \ - --description "${DESCRIPTION}" +SUBMIT_MODE="${XDP_CSR_SUBMIT_MODE:-local}" +case "${SUBMIT_MODE}" in + jenkins) + echo "== Submit CSR run to Jenkins ==" + bash "${SCRIPT_DIR}/submit_csr_jenkins.sh" \ + --sprite-dir "${SPRITE_DIR}" \ + --description "${DESCRIPTION}" + ;; + local) + echo "== Submit CSR run locally (RDI) ==" + bash "${SCRIPT_DIR}/submit_csr_local.sh" \ + --sprite-dir "${SPRITE_DIR}" + ;; + *) + echo "Unknown XDP_CSR_SUBMIT_MODE=${SUBMIT_MODE} (use local or jenkins)" >&2 + exit 1 + ;; +esac echo "VE2 CSR board test completed for run ${RUN_ID}" diff --git a/ci/csr/submit_csr_local.sh b/ci/csr/submit_csr_local.sh new file mode 100755 index 00000000..e65e41f7 --- /dev/null +++ b/ci/csr/submit_csr_local.sh @@ -0,0 +1,116 @@ +#!/bin/bash +# Launch CSR RDI directly on XCO and wait for completion (no Jenkins API). +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=config.defaults.sh +source "${SCRIPT_DIR}/config.defaults.sh" + +usage() { + cat <<'EOF' +Usage: submit_csr_local.sh --sprite-dir DIR + +Runs the generated *_rdi.sh in SPRITE_DIR and polls the RDI log until complete. +Requires LSF permission and RDI env on the self-hosted runner host. +EOF +} + +SPRITE_DIR="" +POLL_SEC="${CSR_LOCAL_POLL_SEC:-60}" +TIMEOUT_SEC="${CSR_WAIT_TIMEOUT_SEC:-14400}" + +while [ "$#" -gt 0 ]; do + case "$1" in + --sprite-dir) + SPRITE_DIR="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [ -z "${SPRITE_DIR}" ] || [ ! -d "${SPRITE_DIR}" ]; then + echo "Missing or invalid --sprite-dir" >&2 + exit 1 +fi + +shopt -s nullglob +RDI_SCRIPTS=( "${SPRITE_DIR}"/*_rdi.sh ) +shopt -u nullglob + +if [ "${#RDI_SCRIPTS[@]}" -eq 0 ]; then + echo "No *_rdi.sh found under ${SPRITE_DIR}" >&2 + exit 1 +fi + +RDI_SCRIPT="${RDI_SCRIPTS[0]}" +RESULTS_DIR="${SPRITE_DIR}/results" +mkdir -p "${RESULTS_DIR}" +chmod a+w "${SPRITE_DIR}" "${RESULTS_DIR}" "$(dirname "${SPRITE_DIR}")" 2>/dev/null || true + +if [ -f /proj/xtools/dsv/rdi2/utils/setRDIEnv.csh ]; then + # tcsh env file; source from bash with nounset disabled for optional vars. + set +u + # shellcheck disable=SC1091 + source /proj/xtools/dsv/rdi2/utils/setRDIEnv.csh 2>/dev/null || true + set -u +fi + +echo "Launching CSR RDI locally: ${RDI_SCRIPT}" +cd "${SPRITE_DIR}" +bash "${RDI_SCRIPT}" + +mapfile -t LOG_FILES < <(find "${RESULTS_DIR}" -maxdepth 1 -name 'rdi_*.log' -type f | sort) +if [ "${#LOG_FILES[@]}" -eq 0 ]; then + echo "No rdi_*.log files found under ${RESULTS_DIR}" >&2 + exit 1 +fi + +echo "Monitoring ${#LOG_FILES[@]} RDI log(s):" +printf ' %s\n' "${LOG_FILES[@]}" + +deadline=$(( $(date +%s) + TIMEOUT_SEC )) +pending=("${LOG_FILES[@]}") +failed=0 + +while [ "${#pending[@]}" -gt 0 ]; do + now=$(date +%s) + if [ "${now}" -ge "${deadline}" ]; then + echo "Timed out after ${TIMEOUT_SEC}s waiting for RDI logs" >&2 + exit 1 + fi + + still_pending=() + for log in "${pending[@]}"; do + if grep -q 'Finished at:' "${log}" 2>/dev/null; then + echo "Completed: ${log}" + if grep -Eiq 'ERROR|CRITICAL WARNING|RDI regression failed|exit code [1-9]' "${log}"; then + echo "Failure indicators found in ${log}" >&2 + failed=1 + fi + else + still_pending+=( "${log}" ) + echo "Still running: ${log}" + fi + done + + pending=( "${still_pending[@]}" ) + if [ "${#pending[@]}" -gt 0 ]; then + sleep "${POLL_SEC}" + fi +done + +if [ "${failed}" -ne 0 ]; then + echo "One or more RDI runs reported failures" >&2 + exit 1 +fi + +echo "All RDI runs completed successfully" From 1e172b67264250694efb38dc817a094ca6ea58cb Mon Sep 17 00:00:00 2001 From: snigupta Date: Thu, 23 Jul 2026 11:39:48 -0600 Subject: [PATCH 6/7] Limit VE2 CSR trial runs to a single XDP design. Default to ResNet18_MLTimeline and fix local RDI launch via setRDIEnv.csh. Signed-off-by: snigupta --- .github/workflows/xdp-xrt-build.yml | 3 +- ci/csr/config.defaults.sh | 4 ++ ci/csr/limit_merged_tql.sh | 97 +++++++++++++++++++++++++++++ ci/csr/run_ve2_board_test.sh | 4 ++ ci/csr/submit_csr_local.sh | 29 ++++++--- 5 files changed, 127 insertions(+), 10 deletions(-) create mode 100755 ci/csr/limit_merged_tql.sh diff --git a/.github/workflows/xdp-xrt-build.yml b/.github/workflows/xdp-xrt-build.yml index 14a0cad2..5ab3bbc0 100644 --- a/.github/workflows/xdp-xrt-build.yml +++ b/.github/workflows/xdp-xrt-build.yml @@ -306,6 +306,7 @@ jobs: VE2_YOCTO_DIR: /scratch/snigupta/ve2_yocto_cache/Vitis-AI-Telluride/versal_2ve/reference_design/vek385/rev-b/sw/yocto CSR_ROOT: /wrk/csr/snigupta/xdp_ci XDP_CSR_SUBMIT_MODE: local + XDP_CSR_SINGLE_TEST: ResNet18_MLTimeline JENKINS_USER: ${{ secrets.JENKINS_USER }} JENKINS_API_TOKEN: ${{ secrets.JENKINS_API_TOKEN }} @@ -328,7 +329,7 @@ jobs: echo "## XDP VE2 Board Test (CSR)" >> "$GITHUB_STEP_SUMMARY" echo "- Commit: ${{ github.sha }}" >> "$GITHUB_STEP_SUMMARY" echo "- Trigger: ${{ github.event_name }} (${{ github.ref_name }})" >> "$GITHUB_STEP_SUMMARY" - echo "- Suite: VE2_XDP (hw)" >> "$GITHUB_STEP_SUMMARY" + echo "- Suite: VE2_XDP (hw), single test: ResNet18_MLTimeline" >> "$GITHUB_STEP_SUMMARY" echo "- CSR workspace: ${CSR_ROOT}/pr-${RUN_ID}" >> "$GITHUB_STEP_SUMMARY" echo "- Submit mode: local RDI (set XDP_CSR_SUBMIT_MODE=jenkins when Job/Build granted)" >> "$GITHUB_STEP_SUMMARY" if [ "${{ job.status }}" = "success" ]; then diff --git a/ci/csr/config.defaults.sh b/ci/csr/config.defaults.sh index 8de1e5c5..28ce357a 100755 --- a/ci/csr/config.defaults.sh +++ b/ci/csr/config.defaults.sh @@ -25,3 +25,7 @@ : "${CSR_WAIT_TIMEOUT_SEC:=14400}" : "${XDP_CSR_SUBMIT_MODE:=local}" + +# Trial phase: run one XDP design instead of the full VE2_XDP suite (~15 tests). +# Set empty to run all tests in the merged TQL. +: "${XDP_CSR_SINGLE_TEST:=ResNet18_MLTimeline}" diff --git a/ci/csr/limit_merged_tql.sh b/ci/csr/limit_merged_tql.sh new file mode 100755 index 00000000..3e69c4c5 --- /dev/null +++ b/ci/csr/limit_merged_tql.sh @@ -0,0 +1,97 @@ +#!/bin/bash +# Limit merged CSR TQL to a single testcase directory (trial/smoke runs). +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=config.defaults.sh +source "${SCRIPT_DIR}/config.defaults.sh" + +usage() { + cat <<'EOF' +Usage: limit_merged_tql.sh --sprite-dir DIR [--test NAME] + +Environment: + XDP_CSR_SINGLE_TEST Test subdirectory name under XDP_Suites/test_repo + (default: ResNet18_MLTimeline). Set empty to run all tests. +EOF +} + +SPRITE_DIR="" +SINGLE_TEST="${XDP_CSR_SINGLE_TEST:-ResNet18_MLTimeline}" + +while [ "$#" -gt 0 ]; do + case "$1" in + --sprite-dir) + SPRITE_DIR="$2" + shift 2 + ;; + --test) + SINGLE_TEST="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [ -z "${SPRITE_DIR}" ]; then + echo "Missing required --sprite-dir" >&2 + exit 1 +fi + +if [ -z "${SINGLE_TEST}" ]; then + echo "XDP_CSR_SINGLE_TEST unset; running full suite TQL unchanged" + exit 0 +fi + +shopt -s nullglob +TQL_FILES=( "${SPRITE_DIR}"/*"${CSR_SUITE}"*.tql ) +shopt -u nullglob + +if [ "${#TQL_FILES[@]}" -eq 0 ]; then + echo "No merged TQL for suite ${CSR_SUITE} under ${SPRITE_DIR}" >&2 + exit 1 +fi + +TQL_FILE="${TQL_FILES[0]}" + +python3 - <<'PY' "${TQL_FILE}" "${SINGLE_TEST}" +import re +import sys + +tql_path, single_test = sys.argv[1:3] +text = open(tql_path, encoding="utf-8").read() + +# Split trailing GLOBAL APPLY blocks (e.g. XRT RPM patch) from query blocks. +global_idx = text.find("\nGLOBAL APPLY") +if global_idx == -1: + body, tail = text, "" +else: + body, tail = text[:global_idx], text[global_idx:] + +# Each testcase query starts at a line beginning with FROM. +parts = re.split(r"(?m)^(?=FROM )", body) +header = parts[0] if parts and not parts[0].lstrip().startswith("FROM ") else "" +queries = [p for p in parts if p.lstrip().startswith("FROM ")] + +kept = [q for q in queries if single_test in q.splitlines()[0]] +if not kept: + available = [q.splitlines()[0].strip() for q in queries] + raise SystemExit( + f"No FROM block matches test '{single_test}'.\n" + f"Available FROM lines:\n " + "\n ".join(available) + ) + +new_body = header + "".join(kept) +open(tql_path, "w", encoding="utf-8").write(new_body + tail) +print(f"Limited {tql_path} to 1 test: {single_test}") +print(f" kept FROM: {kept[0].splitlines()[0].strip()}") +print(f" removed {len(queries) - len(kept)} other FROM block(s)") +PY diff --git a/ci/csr/run_ve2_board_test.sh b/ci/csr/run_ve2_board_test.sh index 3f2cc668..c00bb105 100755 --- a/ci/csr/run_ve2_board_test.sh +++ b/ci/csr/run_ve2_board_test.sh @@ -17,6 +17,7 @@ Environment: JENKINS_API_TOKEN Required for Jenkins submission XDP_CSR_SKIP_JENKINS=1 Generate CSR only; do not submit XDP_CSR_SUBMIT_MODE local (default) or jenkins + XDP_CSR_SINGLE_TEST One design under XDP_Suites/test_repo (default: ResNet18_MLTimeline; empty = all) EOF } @@ -69,6 +70,9 @@ bash "${SCRIPT_DIR}/stage_ve2_rpms.sh" "${STAGE_ARGS[@]}" echo "== Generate CSR sprite run ==" bash "${SCRIPT_DIR}/generate_csr_run.sh" --sprite-dir "${SPRITE_DIR}" +echo "== Limit merged TQL to single test (trial) ==" +bash "${SCRIPT_DIR}/limit_merged_tql.sh" --sprite-dir "${SPRITE_DIR}" + echo "== Patch merged TQL for custom XRT install ==" bash "${SCRIPT_DIR}/patch_merged_tql.sh" \ --sprite-dir "${SPRITE_DIR}" \ diff --git a/ci/csr/submit_csr_local.sh b/ci/csr/submit_csr_local.sh index e65e41f7..a3ee7a0c 100755 --- a/ci/csr/submit_csr_local.sh +++ b/ci/csr/submit_csr_local.sh @@ -56,17 +56,14 @@ RESULTS_DIR="${SPRITE_DIR}/results" mkdir -p "${RESULTS_DIR}" chmod a+w "${SPRITE_DIR}" "${RESULTS_DIR}" "$(dirname "${SPRITE_DIR}")" 2>/dev/null || true -if [ -f /proj/xtools/dsv/rdi2/utils/setRDIEnv.csh ]; then - # tcsh env file; source from bash with nounset disabled for optional vars. - set +u - # shellcheck disable=SC1091 - source /proj/xtools/dsv/rdi2/utils/setRDIEnv.csh 2>/dev/null || true - set -u -fi - echo "Launching CSR RDI locally: ${RDI_SCRIPT}" cd "${SPRITE_DIR}" -bash "${RDI_SCRIPT}" + +# setRDIEnv.csh is tcsh; sourcing it from bash does not put rdi on PATH. +tcsh -f </dev/null; then + echo "RDI launch failed in ${log}:" >&2 + cat "${log}" >&2 + exit 1 + fi +done + deadline=$(( $(date +%s) + TIMEOUT_SEC )) pending=("${LOG_FILES[@]}") failed=0 @@ -98,6 +103,12 @@ while [ "${#pending[@]}" -gt 0 ]; do fi else still_pending+=( "${log}" ) + if [ "$(wc -c < "${log}")" -lt 512 ] && \ + find "${log}" -mmin +2 2>/dev/null | grep -q .; then + echo "RDI log stalled (no progress): ${log}" >&2 + cat "${log}" >&2 + exit 1 + fi echo "Still running: ${log}" fi done From 658669ddd2d66643eaeae72b9d39a281dac37648 Mon Sep 17 00:00:00 2001 From: snigupta Date: Thu, 23 Jul 2026 16:07:28 -0600 Subject: [PATCH 7/7] Testing board run Signed-off-by: snigupta --- .github/workflows/xdp-xrt-build.yml | 2 + ci/csr/config.defaults.sh | 4 ++ ci/csr/patch_merged_tql_no_docker.sh | 83 ++++++++++++++++++++++++++++ ci/csr/run_ve2_board_test.sh | 5 ++ ci/csr/submit_csr_jenkins.sh | 62 ++++++++++++++------- ci/csr/submit_csr_local.sh | 49 ++++++++++++++++ 6 files changed, 186 insertions(+), 19 deletions(-) create mode 100755 ci/csr/patch_merged_tql_no_docker.sh diff --git a/.github/workflows/xdp-xrt-build.yml b/.github/workflows/xdp-xrt-build.yml index 5ab3bbc0..788622be 100644 --- a/.github/workflows/xdp-xrt-build.yml +++ b/.github/workflows/xdp-xrt-build.yml @@ -307,6 +307,7 @@ jobs: CSR_ROOT: /wrk/csr/snigupta/xdp_ci XDP_CSR_SUBMIT_MODE: local XDP_CSR_SINGLE_TEST: ResNet18_MLTimeline + XDP_CSR_USE_DOCKER: "0" JENKINS_USER: ${{ secrets.JENKINS_USER }} JENKINS_API_TOKEN: ${{ secrets.JENKINS_API_TOKEN }} @@ -330,6 +331,7 @@ jobs: echo "- Commit: ${{ github.sha }}" >> "$GITHUB_STEP_SUMMARY" echo "- Trigger: ${{ github.event_name }} (${{ github.ref_name }})" >> "$GITHUB_STEP_SUMMARY" echo "- Suite: VE2_XDP (hw), single test: ResNet18_MLTimeline" >> "$GITHUB_STEP_SUMMARY" + echo "- vaiml compile: native FlexML (XDP_CSR_USE_DOCKER=0)" >> "$GITHUB_STEP_SUMMARY" echo "- CSR workspace: ${CSR_ROOT}/pr-${RUN_ID}" >> "$GITHUB_STEP_SUMMARY" echo "- Submit mode: local RDI (set XDP_CSR_SUBMIT_MODE=jenkins when Job/Build granted)" >> "$GITHUB_STEP_SUMMARY" if [ "${{ job.status }}" = "success" ]; then diff --git a/ci/csr/config.defaults.sh b/ci/csr/config.defaults.sh index 28ce357a..78bba9f3 100755 --- a/ci/csr/config.defaults.sh +++ b/ci/csr/config.defaults.sh @@ -29,3 +29,7 @@ # Trial phase: run one XDP design instead of the full VE2_XDP suite (~15 tests). # Set empty to run all tests in the merged TQL. : "${XDP_CSR_SINGLE_TEST:=ResNet18_MLTimeline}" + +# Local RDI trial: native FlexML compile avoids docker.sock on farm hosts. +# Set to 1 for production docker compile (suite default). +: "${XDP_CSR_USE_DOCKER:=0}" diff --git a/ci/csr/patch_merged_tql_no_docker.sh b/ci/csr/patch_merged_tql_no_docker.sh new file mode 100755 index 00000000..46d891f3 --- /dev/null +++ b/ci/csr/patch_merged_tql_no_docker.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# Patch merged TQL to compile vaiml natively (no Docker) for local RDI trial runs. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=config.defaults.sh +source "${SCRIPT_DIR}/config.defaults.sh" + +usage() { + cat <<'EOF' +Usage: patch_merged_tql_no_docker.sh --sprite-dir DIR [--suite SUITE] + +Appends a GLOBAL APPLY block that sets user.use_docker = 0 and drops the +(vai_docker) constraint from tasks.vaiml.select_resource. + +Set XDP_CSR_USE_DOCKER=1 to skip this patch (production docker compile path). +EOF +} + +SPRITE_DIR="" +SUITE="${CSR_SUITE}" + +while [ "$#" -gt 0 ]; do + case "$1" in + --sprite-dir) + SPRITE_DIR="$2" + shift 2 + ;; + --suite) + SUITE="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [ -z "${SPRITE_DIR}" ]; then + echo "Missing required --sprite-dir" >&2 + usage >&2 + exit 1 +fi + +if [ "${XDP_CSR_USE_DOCKER:-0}" = "1" ]; then + echo "XDP_CSR_USE_DOCKER=1; keeping suite default docker compile (no patch)" + exit 0 +fi + +shopt -s nullglob +TQL_FILES=( "${SPRITE_DIR}"/*"${SUITE}"*.tql ) +shopt -u nullglob + +if [ "${#TQL_FILES[@]}" -eq 0 ]; then + echo "No merged TQL matching suite ${SUITE} under ${SPRITE_DIR}" >&2 + exit 1 +fi + +TQL_FILE="${TQL_FILES[0]}" +MARKER="# XDP CI: native vaiml compile (user.use_docker = 0)" + +if grep -qF "${MARKER}" "${TQL_FILE}"; then + echo "TQL already patched for no-docker compile: ${TQL_FILE}" + exit 0 +fi + +cat >> "${TQL_FILE}" <<'EOF' + +# XDP CI: native vaiml compile (user.use_docker = 0) +GLOBAL APPLY * MODIFY + user.use_docker = 0, + tasks.vaiml.select_resource = "(ostype == ubuntu2204 || ostype == ubuntu22044 || ostype == ubuntu24041)"; +EOF + +echo "Patched ${TQL_FILE} for native vaiml compile" +echo " user.use_docker = 0" +echo " tasks.vaiml.select_resource = (ubuntu hosts, no vai_docker tag)" diff --git a/ci/csr/run_ve2_board_test.sh b/ci/csr/run_ve2_board_test.sh index c00bb105..d6d3a181 100755 --- a/ci/csr/run_ve2_board_test.sh +++ b/ci/csr/run_ve2_board_test.sh @@ -18,6 +18,7 @@ Environment: XDP_CSR_SKIP_JENKINS=1 Generate CSR only; do not submit XDP_CSR_SUBMIT_MODE local (default) or jenkins XDP_CSR_SINGLE_TEST One design under XDP_Suites/test_repo (default: ResNet18_MLTimeline; empty = all) + XDP_CSR_USE_DOCKER 0 = native vaiml compile (default trial); 1 = suite docker path EOF } @@ -73,6 +74,10 @@ bash "${SCRIPT_DIR}/generate_csr_run.sh" --sprite-dir "${SPRITE_DIR}" echo "== Limit merged TQL to single test (trial) ==" bash "${SCRIPT_DIR}/limit_merged_tql.sh" --sprite-dir "${SPRITE_DIR}" +echo "== Patch merged TQL for native vaiml compile (no docker) ==" +bash "${SCRIPT_DIR}/patch_merged_tql_no_docker.sh" \ + --sprite-dir "${SPRITE_DIR}" + echo "== Patch merged TQL for custom XRT install ==" bash "${SCRIPT_DIR}/patch_merged_tql.sh" \ --sprite-dir "${SPRITE_DIR}" \ diff --git a/ci/csr/submit_csr_jenkins.sh b/ci/csr/submit_csr_jenkins.sh index bf4b07ca..585c1427 100755 --- a/ci/csr/submit_csr_jenkins.sh +++ b/ci/csr/submit_csr_jenkins.sh @@ -71,6 +71,7 @@ python3 - <<'PY' "${JENKINS_URL}" "${JENKINS_JOB}" "${SPRITE_DIR}" "${DESCRIPTIO "${JENKINS_LNX_TA}" "${JENKINS_COMPILE_JOBS_ONLY}" "${JENKINS_BOARD_SELECT}" \ "${JENKINS_USER}" "${JENKINS_API_TOKEN}" "${CSR_WAIT_TIMEOUT_SEC}" import base64 +import http.cookiejar import json import sys import time @@ -90,23 +91,45 @@ import urllib.request token, timeout_sec, ) = sys.argv[1:11] +user = user.strip() +token = token.strip() timeout_sec = int(timeout_sec) base = jenkins_url.rstrip("/") auth = base64.b64encode(f"{user}:{token}".encode()).decode() +cookie_jar = http.cookiejar.CookieJar() +opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookie_jar)) -def request(url, method="GET", data=None, headers=None): - req_headers = {"Authorization": f"Basic {auth}"} - if headers: - req_headers.update(headers) - req = urllib.request.Request(url, data=data, headers=req_headers, method=method) - with urllib.request.urlopen(req, timeout=60) as resp: - body = resp.read().decode() - return resp, body +def request(url, method="GET", data=None, extra_headers=None): + headers = { + "Authorization": f"Basic {auth}", + "Accept": "application/json", + } + if extra_headers: + headers.update(extra_headers) + req = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with opener.open(req, timeout=120) as resp: + body = resp.read().decode(errors="replace") + return resp, body + except urllib.error.HTTPError as exc: + body = exc.read().decode(errors="replace") + snippet = " ".join(body.split())[:500] + raise SystemExit( + f"Jenkins HTTP {exc.code} for {method} {url}\n{snippet}" + ) from exc -crumb_resp, crumb_body = request(f"{base}/crumbIssuer/api/json") -crumb = json.loads(crumb_body) + +def request_json(url, method="GET", data=None, extra_headers=None): + _, body = request(url, method=method, data=data, extra_headers=extra_headers) + return json.loads(body) + + +whoami = request_json(f"{base}/me/api/json") +print(f"Jenkins user: {whoami.get('fullName', whoami.get('id', user))}") + +crumb = request_json(f"{base}/crumbIssuer/api/json") crumb_field = crumb["crumbRequestField"] crumb_value = crumb["crumb"] @@ -121,14 +144,17 @@ params = { "VE2_DOCKER": "False", "VE2_BOOT_IMAGE_PATH": "DEFAULT", } -query = urllib.parse.urlencode(params) -trigger_url = f"{base}/job/{job_name}/buildWithParameters?{query}" +trigger_url = f"{base}/job/{job_name}/buildWithParameters" +form_body = urllib.parse.urlencode(params).encode() trigger_resp, _ = request( trigger_url, method="POST", - data=b"", - headers={crumb_field: crumb_value}, + data=form_body, + extra_headers={ + crumb_field: crumb_value, + "Content-Type": "application/x-www-form-urlencoded", + }, ) queue_url = trigger_resp.headers.get("Location") if not queue_url: @@ -139,9 +165,8 @@ print(f"Queued Jenkins job: {queue_url.rstrip('/')}/api/json") build_url = None deadline = time.time() + min(timeout_sec, 600) while time.time() < deadline: - _, body = request(f"{queue_url.rstrip('/')}/api/json") - data = json.loads(body) - if "executable" in data and data["executable"]: + data = request_json(f"{queue_url.rstrip('/')}/api/json") + if data.get("executable"): build_url = data["executable"]["url"] break if data.get("cancelled"): @@ -156,8 +181,7 @@ print(f"Jenkins build started: {build_url}") deadline = time.time() + timeout_sec result = None while time.time() < deadline: - _, body = request(f"{build_url.rstrip('/')}/api/json") - data = json.loads(body) + data = request_json(f"{build_url.rstrip('/')}/api/json") if not data.get("building") and data.get("result") is not None: result = data["result"] break diff --git a/ci/csr/submit_csr_local.sh b/ci/csr/submit_csr_local.sh index a3ee7a0c..2dcbc248 100755 --- a/ci/csr/submit_csr_local.sh +++ b/ci/csr/submit_csr_local.sh @@ -124,4 +124,53 @@ if [ "${failed}" -ne 0 ]; then exit 1 fi +# RDI can finish cleanly while individual tests fail; check suite summaries. +check_suite_test_results() { + local results_dir="$1" + local summary fail_count stats line + local -a summaries=() + local saw_summary=0 + local saw_failure=0 + + shopt -s nullglob + summaries=( "${results_dir}"/*/*.summary "${results_dir}"/*.summary ) + shopt -u nullglob + + if [ "${#summaries[@]}" -eq 0 ]; then + echo "No CSR suite .summary files found under ${results_dir}" >&2 + return 1 + fi + + for summary in "${summaries[@]}"; do + [ -f "${summary}" ] || continue + saw_summary=1 + stats="$(sed -n '2p' "${summary}")" + fail_count="$(awk '{print ($3 ~ /^[0-9]+$/) ? $3 : 0}' <<< "${stats}")" + if [ "${fail_count:-0}" -gt 0 ]; then + saw_failure=1 + echo "CSR test failure(s) reported in ${summary}" >&2 + awk '/^ishitag:/,0' "${summary}" >&2 || true + fi + done + + if [ "${saw_summary}" -eq 0 ]; then + echo "No readable CSR suite .summary files under ${results_dir}" >&2 + return 1 + fi + + if [ "${saw_failure}" -ne 0 ]; then + while IFS= read -r line; do + echo "XOAH: ${line}" >&2 + done < <(grep -rh 'xoah_report_url:' "${results_dir}" 2>/dev/null | sed 's/.*xoah_report_url:[[:space:]]*//' | sort -u) + return 1 + fi + + return 0 +} + +if ! check_suite_test_results "${RESULTS_DIR}"; then + echo "CSR suite reported test failure(s); see summary and XOAH links above" >&2 + exit 1 +fi + echo "All RDI runs completed successfully"