diff --git a/.github/workflows/xdp-xrt-build.yml b/.github/workflows/xdp-xrt-build.yml index 0f54b722..788622be 100644 --- a/.github/workflows/xdp-xrt-build.yml +++ b/.github/workflows/xdp-xrt-build.yml @@ -2,122 +2,375 @@ name: XDP Build on: pull_request: - types: [opened, synchronize, reopened, labeled] - branches: [ master ] - + types: [opened, synchronize, reopened, labeled, unlabeled] + branches: [master] + # Fork testing: push to this branch runs VE2 build + CSR board test. + push: + branches: [ci/xdp-multi-platform-builds] + workflow_dispatch: + concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + group: xdp-build-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true - + jobs: build-xrt-alveo: - if: github.event_name == 'pull_request' + if: | + github.event_name == 'pull_request' && ( + contains(github.event.pull_request.labels.*.name, 'build-xrt-alveo') || + ( + !contains(github.event.pull_request.labels.*.name, 'build-xrt-alveo') && + !contains(github.event.pull_request.labels.*.name, 'build-xrt-edge') && + !contains(github.event.pull_request.labels.*.name, 'build-xrt-ve2') && + !contains(github.event.pull_request.labels.*.name, 'build-xrt-client') + ) + ) runs-on: ubuntu-latest timeout-minutes: 60 - + concurrency: + group: xdp-alveo-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + steps: - - name: Checkout XDP PR - uses: actions/checkout@v4 - with: - path: xdp-pr - fetch-depth: 0 - - - name: Clone XRT Repository - run: | - git clone https://github.com/Xilinx/XRT.git XRT - cd XRT - git submodule update --init --recursive - - - name: Apply XDP PR Changes to XRT - run: | - rm -rf XRT/src/runtime_src/xdp - cp -r xdp-pr XRT/src/runtime_src/xdp - - - name: Setup Build Environment Using XRT Scripts - run: | - cd XRT/src/runtime_src/tools/scripts - chmod +x xrtdeps.sh - sudo ./xrtdeps.sh - # Install cross-compilation tools for edge builds - sudo apt-get update - sudo apt-get install -y \ - gcc-aarch64-linux-gnu \ - g++-aarch64-linux-gnu \ - crossbuild-essential-arm64 - - - name: Build XRT for Alveo - run: | - cd XRT/build - chmod +x build.sh - ./build.sh -noinit -noert - - - name: Validate build - run: | - if ls XRT/build/Release/opt/xilinx/xrt/lib/libxrt_core.so* 1> /dev/null 2>&1; then - echo "✓ XRT Build successful" - else - echo "✗ XRT Build failed" - exit 1 - fi - - if ls XRT/build/Release/opt/xilinx/xrt/lib/libxdp_core.so* 1> /dev/null 2>&1; then - echo "✓ XDP Build successful" - else - echo "✗ XDP Build failed" - exit 1 - fi - - - name: Upload Build Artifacts - uses: actions/upload-artifact@v4 - with: - name: xrt-alveo-build-PR-${{ github.event.pull_request.number }} - path: | - XRT/build/Release/opt/xilinx/xrt/lib/libxrt*.so* - XRT/build/Release/opt/xilinx/xrt/lib/libxdp*.so* - XRT/build/Release/opt/xilinx/xrt/lib/xrt/module/*.so* - retention-days: 1 - if-no-files-found: warn - - - name: Generate Build Summary - if: always() - run: | - echo "## XDP Build Summary" >> $GITHUB_STEP_SUMMARY - echo "- Build Date: $(date)" >> $GITHUB_STEP_SUMMARY - echo "- XDP Source: ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY - - if ls XRT/build/Release/opt/xilinx/xrt/lib/libxrt_core.so* 1> /dev/null 2>&1; then - echo "- XRT Build: ✅ Success" >> $GITHUB_STEP_SUMMARY - else - echo "- XRT Build: ❌ Failed" >> $GITHUB_STEP_SUMMARY - fi - - if ls XRT/build/Release/opt/xilinx/xrt/lib/libxdp_core.so* 1> /dev/null 2>&1; then - echo "- XDP Build: ✅ Success" >> $GITHUB_STEP_SUMMARY - else - echo "- XDP Build: ❌ Failed" >> $GITHUB_STEP_SUMMARY - fi - - # Future job for edge builds + - name: Checkout XDP + uses: actions/checkout@v4 + with: + path: xdp-pr + fetch-depth: 0 + + - name: Clone XRT + run: | + git clone https://github.com/Xilinx/XRT.git XRT + git -C XRT submodule update --init --recursive + + - name: Overlay XDP + run: | + rm -rf XRT/src/runtime_src/xdp + cp -r xdp-pr XRT/src/runtime_src/xdp + + - name: Setup build environment + run: | + cd XRT/src/runtime_src/tools/scripts + chmod +x xrtdeps.sh + sudo ./xrtdeps.sh + sudo apt-get update + sudo apt-get install -y \ + gcc-aarch64-linux-gnu \ + g++-aarch64-linux-gnu \ + crossbuild-essential-arm64 + + - name: Build XRT for Alveo + run: | + cd XRT/build + chmod +x build.sh + ./build.sh -noinit -noert + + - name: Validate build outputs + run: | + set -euo pipefail + XRT_ROOT="XRT/build/Release/opt/xilinx/xrt" + + echo "Built shared libraries under ${XRT_ROOT}:" + find "${XRT_ROOT}" -name '*.so*' | sort || true + + XRT_CORE="$(find "${XRT_ROOT}" -name 'libxrt_core.so*' -type f | head -1)" + XDP_CORE="$(find "${XRT_ROOT}" -name 'libxdp_core.so*' -type f | head -1)" + + test -n "${XRT_CORE}" || { echo "libxrt_core not found under ${XRT_ROOT}"; exit 1; } + test -n "${XDP_CORE}" || { echo "libxdp_core not found under ${XRT_ROOT}"; exit 1; } + echo "Validated ${XRT_CORE}" + echo "Validated ${XDP_CORE}" + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: xrt-alveo-build-${{ github.event.pull_request.number || github.run_id }} + path: | + XRT/build/Release/opt/xilinx/xrt/**/libxrt*.so* + XRT/build/Release/opt/xilinx/xrt/**/libxdp*.so* + XRT/build/Release/opt/xilinx/xrt/**/xrt/module/*.so* + retention-days: 1 + + - name: Build summary + if: always() + run: | + XRT_ROOT="XRT/build/Release/opt/xilinx/xrt" + echo "## XDP Alveo Build" >> "$GITHUB_STEP_SUMMARY" + echo "- Commit: ${{ github.sha }}" >> "$GITHUB_STEP_SUMMARY" + echo "- PR: #${{ github.event.pull_request.number }}" >> "$GITHUB_STEP_SUMMARY" + echo "- Platform: Alveo (native)" >> "$GITHUB_STEP_SUMMARY" + if find "${XRT_ROOT}" -name 'libxrt_core.so*' -type f 2>/dev/null | grep -q . && \ + find "${XRT_ROOT}" -name 'libxdp_core.so*' -type f 2>/dev/null | grep -q .; then + echo "- Result: success" >> "$GITHUB_STEP_SUMMARY" + else + echo "- Result: failed" >> "$GITHUB_STEP_SUMMARY" + fi + build-xrt-edge: - if: contains(github.event.pull_request.labels.*.name, 'build-edge') - runs-on: ubuntu-latest + if: | + github.event_name == 'pull_request' && ( + contains(github.event.pull_request.labels.*.name, 'build-xrt-edge') || + ( + !contains(github.event.pull_request.labels.*.name, 'build-xrt-alveo') && + !contains(github.event.pull_request.labels.*.name, 'build-xrt-edge') && + !contains(github.event.pull_request.labels.*.name, 'build-xrt-ve2') && + !contains(github.event.pull_request.labels.*.name, 'build-xrt-client') + ) + ) + runs-on: [self-hosted, Ubuntu-22.04] + timeout-minutes: 360 + concurrency: + group: xdp-edge-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + env: + SUDO: /tools/aecg-sudo/sudo/bin/sudo + steps: - - name: Placeholder for edge build - run: echo "Edge build would go here" - - # Future job for client builds + - name: Checkout XDP + uses: actions/checkout@v4 + with: + path: xdp-pr + fetch-depth: 0 + + - name: Clone XRT + run: | + rm -rf XRT + git clone https://github.com/Xilinx/XRT.git XRT + git -C XRT submodule update --init --recursive + + - name: Overlay XDP + run: | + rm -rf XRT/src/runtime_src/xdp + cp -r xdp-pr XRT/src/runtime_src/xdp + + - name: Setup build environment + run: | + cd XRT/src/runtime_src/tools/scripts + chmod +x xrtdeps.sh + $SUDO ./xrtdeps.sh + + - name: Build XRT for Edge + run: | + cd XRT/build + chmod +x build_edge.sh + ./build_edge.sh -aarch versal -noinit + + - name: Validate build outputs + run: | + ls XRT/build/versal/rpms/xrt*.rpm + ls XRT/build/versal/rpms/zocl*.rpm + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: xrt-edge-build-${{ github.event.pull_request.number || github.run_id }} + path: | + XRT/build/versal/rpms/*.rpm + XRT/build/versal/install_xrt.sh + XRT/build/versal/reinstall_xrt.sh + XRT/build/versal/rpm.txt + retention-days: 1 + + - name: Build summary + if: always() + run: | + echo "## XDP Edge Build" >> "$GITHUB_STEP_SUMMARY" + echo "- Commit: ${{ github.sha }}" >> "$GITHUB_STEP_SUMMARY" + echo "- PR: #${{ github.event.pull_request.number }}" >> "$GITHUB_STEP_SUMMARY" + echo "- Platform: Edge (versal aarch)" >> "$GITHUB_STEP_SUMMARY" + if ls XRT/build/versal/rpms/xrt*.rpm 2>/dev/null | grep -q . && \ + ls XRT/build/versal/rpms/zocl*.rpm 2>/dev/null | grep -q .; then + echo "- Result: success" >> "$GITHUB_STEP_SUMMARY" + else + echo "- Result: failed" >> "$GITHUB_STEP_SUMMARY" + fi + + build-xrt-ve2: + if: | + github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'build-xrt-ve2') || + ( + !contains(github.event.pull_request.labels.*.name, 'build-xrt-alveo') && + !contains(github.event.pull_request.labels.*.name, 'build-xrt-edge') && + !contains(github.event.pull_request.labels.*.name, 'build-xrt-ve2') && + !contains(github.event.pull_request.labels.*.name, 'build-xrt-client') + ) + runs-on: [self-hosted, Ubuntu-22.04] + timeout-minutes: 480 + concurrency: + group: xdp-ve2-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + env: + # Self-hosted runner paths for VE2 Yocto cache (Telluride rev-b / vek385). + VE2_YOCTO_CACHE_ROOT: /scratch/snigupta/ve2_yocto_cache + VE2_YOCTO_CACHE_TTL_DAYS: '7' + VE2_MACHINE: amd-cortexa78-mali-common + VE2_YOCTO_DIR: /scratch/snigupta/ve2_yocto_cache/Vitis-AI-Telluride/versal_2ve/reference_design/vek385/rev-b/sw/yocto + + steps: + - name: Checkout XDP + uses: actions/checkout@v4 + with: + path: xdp-pr + fetch-depth: 0 + + - name: Ensure Yocto workspace cache + run: | + chmod +x xdp-pr/ci/ve2/ensure_yocto_cache.sh + xdp-pr/ci/ve2/ensure_yocto_cache.sh + + - name: Build XRT RPMs with XDP overlay + env: + YOCTO_DIR: ${{ env.VE2_YOCTO_DIR }} + MACHINE: ${{ env.VE2_MACHINE }} + run: | + cd "${YOCTO_DIR}" + source edf-init-build-env + + if [ ! -d build/workspace/sources/xrt ]; then + MACHINE="${MACHINE}" devtool modify xrt + fi + + XDP_TARGET="build/workspace/sources/xrt/xrt/src/runtime_src/xdp" + rm -rf "${XDP_TARGET}" + mkdir -p "$(dirname "${XDP_TARGET}")" + cp -r "${{ github.workspace }}/xdp-pr" "${XDP_TARGET}" + + MACHINE="${MACHINE}" bitbake xrt -c cleanall + MACHINE="${MACHINE}" bitbake xrt -c package_write_rpm -f + + - name: Validate build outputs + env: + YOCTO_DIR: ${{ env.VE2_YOCTO_DIR }} + run: | + RPM_DIR="${YOCTO_DIR}/build/tmp/deploy/rpm" + find "${RPM_DIR}"/cortexa* -name 'xrt*.rpm' | tee /tmp/ve2-rpms.txt + test -s /tmp/ve2-rpms.txt + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: xrt-ve2-build-${{ github.event.pull_request.number || github.run_id }} + path: ${{ env.VE2_YOCTO_DIR }}/build/tmp/deploy/rpm/cortexa*/xrt*.rpm + retention-days: 1 + + - name: Build summary + if: always() + env: + YOCTO_DIR: ${{ env.VE2_YOCTO_DIR }} + run: | + RPM_DIR="${YOCTO_DIR}/build/tmp/deploy/rpm" + echo "## XDP VE2 Build" >> "$GITHUB_STEP_SUMMARY" + echo "- Commit: ${{ github.sha }}" >> "$GITHUB_STEP_SUMMARY" + echo "- PR: #${{ github.event.pull_request.number }}" >> "$GITHUB_STEP_SUMMARY" + echo "- Platform: VE2 (Yocto RPM)" >> "$GITHUB_STEP_SUMMARY" + echo "- Machine: ${{ env.VE2_MACHINE }}" >> "$GITHUB_STEP_SUMMARY" + echo "- Yocto cache: ${{ env.VE2_YOCTO_CACHE_ROOT }}" >> "$GITHUB_STEP_SUMMARY" + if find "${RPM_DIR}"/cortexa* -name 'xrt*.rpm' 2>/dev/null | grep -q .; then + echo "- Result: success" >> "$GITHUB_STEP_SUMMARY" + else + echo "- Result: failed" >> "$GITHUB_STEP_SUMMARY" + fi + build-xrt-client: - if: contains(github.event.pull_request.labels.*.name, 'build-client') + if: | + github.event_name == 'pull_request' && + contains(github.event.pull_request.labels.*.name, 'build-xrt-client') runs-on: ubuntu-latest steps: - - name: Placeholder for client build - run: echo "Client build would go here" + - run: echo "Client build not implemented" - # Future job for ve2 builds - build-xrt-ve2: - if: contains(github.event.pull_request.labels.*.name, 'build-ve2') + - name: Build summary + if: always() + run: | + echo "## XDP Client Build" >> "$GITHUB_STEP_SUMMARY" + echo "- Commit: ${{ github.sha }}" >> "$GITHUB_STEP_SUMMARY" + echo "- PR: #${{ github.event.pull_request.number }}" >> "$GITHUB_STEP_SUMMARY" + echo "- Platform: Client" >> "$GITHUB_STEP_SUMMARY" + echo "- Result: skipped (not implemented)" >> "$GITHUB_STEP_SUMMARY" + + board-test-ve2: + if: | + github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'board-test-ve2') + needs: build-xrt-ve2 + runs-on: [self-hosted, Ubuntu-22.04] + timeout-minutes: 720 + concurrency: + group: xdp-board-ve2-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + env: + VE2_YOCTO_CACHE_ROOT: /scratch/snigupta/ve2_yocto_cache + VE2_YOCTO_DIR: /scratch/snigupta/ve2_yocto_cache/Vitis-AI-Telluride/versal_2ve/reference_design/vek385/rev-b/sw/yocto + CSR_ROOT: /wrk/csr/snigupta/xdp_ci + XDP_CSR_SUBMIT_MODE: local + XDP_CSR_SINGLE_TEST: ResNet18_MLTimeline + XDP_CSR_USE_DOCKER: "0" + JENKINS_USER: ${{ secrets.JENKINS_USER }} + JENKINS_API_TOKEN: ${{ secrets.JENKINS_API_TOKEN }} + + steps: + - name: Checkout XDP + uses: actions/checkout@v4 + + - name: Run VE2 CSR board test + env: + XDP_CSR_DESCRIPTION: >- + XDP ${{ github.event_name }} ${{ github.ref_name }} @ ${{ github.sha }} + run: | + bash ci/csr/run_ve2_board_test.sh \ + --run-id '${{ github.event.pull_request.number || github.run_id }}-${{ github.run_id }}' + + - name: Board test summary + if: always() + run: | + RUN_ID='${{ github.event.pull_request.number || github.run_id }}-${{ github.run_id }}' + echo "## XDP VE2 Board Test (CSR)" >> "$GITHUB_STEP_SUMMARY" + echo "- Commit: ${{ github.sha }}" >> "$GITHUB_STEP_SUMMARY" + echo "- Trigger: ${{ github.event_name }} (${{ github.ref_name }})" >> "$GITHUB_STEP_SUMMARY" + echo "- Suite: VE2_XDP (hw), single test: ResNet18_MLTimeline" >> "$GITHUB_STEP_SUMMARY" + echo "- vaiml compile: native FlexML (XDP_CSR_USE_DOCKER=0)" >> "$GITHUB_STEP_SUMMARY" + echo "- CSR workspace: ${CSR_ROOT}/pr-${RUN_ID}" >> "$GITHUB_STEP_SUMMARY" + echo "- Submit mode: local RDI (set XDP_CSR_SUBMIT_MODE=jenkins when Job/Build granted)" >> "$GITHUB_STEP_SUMMARY" + if [ "${{ job.status }}" = "success" ]; then + echo "- Result: success" >> "$GITHUB_STEP_SUMMARY" + else + echo "- Result: failed" >> "$GITHUB_STEP_SUMMARY" + fi + + ci-required: + if: always() && github.event_name == 'pull_request' + needs: [build-xrt-alveo, build-xrt-edge, build-xrt-ve2, build-xrt-client] runs-on: ubuntu-latest steps: - - name: Placeholder for ve2 build - run: echo "VE2 build would go here" + - name: Require all scheduled builds to pass + run: | + check_result() { + local job="$1" + local result="$2" + if [ "${result}" = "skipped" ]; then + echo "${job}: skipped" + return 0 + fi + if [ "${result}" = "success" ]; then + echo "${job}: success" + return 0 + fi + echo "${job}: ${result}" + return 1 + } + + failed=0 + check_result build-xrt-alveo "${{ needs.build-xrt-alveo.result }}" || failed=1 + check_result build-xrt-edge "${{ needs.build-xrt-edge.result }}" || failed=1 + check_result build-xrt-ve2 "${{ needs.build-xrt-ve2.result }}" || failed=1 + check_result build-xrt-client "${{ needs.build-xrt-client.result }}" || failed=1 + + if [ "${failed}" -ne 0 ]; then + echo "One or more XDP builds failed" + exit 1 + fi + echo "All scheduled XDP builds passed" diff --git a/ci/csr/config.defaults.sh b/ci/csr/config.defaults.sh new file mode 100755 index 00000000..78bba9f3 --- /dev/null +++ b/ci/csr/config.defaults.sh @@ -0,0 +1,35 @@ +# Defaults for XDP VE2 CSR board testing on XCO. +# Override any value via environment before calling the ci/csr scripts. + +: "${CSR_ROOT:=/wrk/csr/snigupta/xdp_ci}" +: "${CSR_SPEC:=/proj/testcases/xtc/HEAD/auto/ymls/config/VAI_6_3_GEN2_REGRESSION.yml}" +: "${CSR_DAY:=sunday}" +: "${CSR_SUITE:=VE2_XDP}" +: "${CSR_TESTMODE:=hw}" +: "${CSR_TQL_REPO:=/proj/testcases/xtc/HEAD/auto/tql/vaiml/}" +: "${CSR_TA:=/proj/xbuilds/HEAD_INT_flexml_verified_vitis/installs}" +: "${CSR_CREATE_SPRITE_RUN:=/proj/testcases/xtc/HEAD/auto/scripts/createSpriteRun/createSpriteRun.sh}" + +# VE2_XDP is commented out in VAI_6_2 runs.yml; it is active under eachday in 6.3. +: "${CSR_SUPER_SUITE:=VAI_6_3_GEN2_REGRESSION}" + +: "${VE2_RPM_DEPLOY_SUBDIR:=cortexa72_cortexa53}" +: "${VE2_RPM_GLOB:=xrt*.rpm}" +: "${VE2_DNF_PACKAGES:=xrt-20261*.rpm xrt-dbg*.rpm xrt-src*.rpm}" + +: "${JENKINS_URL:=http://acasops:8080}" +: "${JENKINS_JOB:=csr_submit_job}" +: "${JENKINS_BOARD_SELECT:=VE2}" +: "${JENKINS_COMPILE_JOBS_ONLY:=False}" +: "${JENKINS_LNX_TA:=DEFAULT}" + +: "${CSR_WAIT_TIMEOUT_SEC:=14400}" +: "${XDP_CSR_SUBMIT_MODE:=local}" + +# Trial phase: run one XDP design instead of the full VE2_XDP suite (~15 tests). +# Set empty to run all tests in the merged TQL. +: "${XDP_CSR_SINGLE_TEST:=ResNet18_MLTimeline}" + +# Local RDI trial: native FlexML compile avoids docker.sock on farm hosts. +# Set to 1 for production docker compile (suite default). +: "${XDP_CSR_USE_DOCKER:=0}" diff --git a/ci/csr/generate_csr_run.sh b/ci/csr/generate_csr_run.sh new file mode 100755 index 00000000..1eb94588 --- /dev/null +++ b/ci/csr/generate_csr_run.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# Generate CSR sprite files for a VE2_XDP hw run. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=config.defaults.sh +source "${SCRIPT_DIR}/config.defaults.sh" + +usage() { + cat <<'EOF' +Usage: generate_csr_run.sh --sprite-dir DIR + +Creates CSR merged TQL and launch scripts under --sprite-dir. +EOF +} + +SPRITE_DIR="" + +while [ "$#" -gt 0 ]; do + case "$1" in + --sprite-dir) + SPRITE_DIR="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [ -z "${SPRITE_DIR}" ]; then + echo "Missing required --sprite-dir" >&2 + usage >&2 + exit 1 +fi + +mkdir -p "${SPRITE_DIR}/results" +chmod a+w "${SPRITE_DIR}" "${SPRITE_DIR}/results" +chmod a+w "$(dirname "${SPRITE_DIR}")" 2>/dev/null || true + +# createSpriteRunLaunch sets up RDI/XTC. Do not source g_profile here: under bash +# nounset it fails when LD_LIBRARY_PATH is unset (common in GHA runner env). +umask 022 + +cd "${SPRITE_DIR}" + +"${CSR_CREATE_SPRITE_RUN}" \ + --day "${CSR_DAY}" \ + --spec "${CSR_SPEC}" \ + --results "${SPRITE_DIR}/results" \ + --flexml \ + --tql-repo "${CSR_TQL_REPO}" \ + --xtc-no-lut \ + --testsuites "${CSR_SUITE}" \ + --testmode "${CSR_TESTMODE}" \ + --ta "${CSR_TA}" + +RDI_SCRIPT="${SPRITE_DIR}/${CSR_SUPER_SUITE}_${CSR_DAY}_rdi.sh" +if [ ! -s "${RDI_SCRIPT}" ]; then + echo "CSR generation did not produce a non-empty RDI script: ${RDI_SCRIPT}" >&2 + ls -la "${SPRITE_DIR}" >&2 || true + exit 1 +fi + +chmod -R a+w "${SPRITE_DIR}" 2>/dev/null || true +echo "Generated CSR run in ${SPRITE_DIR}" +ls -1 "${SPRITE_DIR}"/*"${CSR_SUITE}"*.tql +echo "Launch script: ${RDI_SCRIPT}" diff --git a/ci/csr/limit_merged_tql.sh b/ci/csr/limit_merged_tql.sh new file mode 100755 index 00000000..3e69c4c5 --- /dev/null +++ b/ci/csr/limit_merged_tql.sh @@ -0,0 +1,97 @@ +#!/bin/bash +# Limit merged CSR TQL to a single testcase directory (trial/smoke runs). +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=config.defaults.sh +source "${SCRIPT_DIR}/config.defaults.sh" + +usage() { + cat <<'EOF' +Usage: limit_merged_tql.sh --sprite-dir DIR [--test NAME] + +Environment: + XDP_CSR_SINGLE_TEST Test subdirectory name under XDP_Suites/test_repo + (default: ResNet18_MLTimeline). Set empty to run all tests. +EOF +} + +SPRITE_DIR="" +SINGLE_TEST="${XDP_CSR_SINGLE_TEST:-ResNet18_MLTimeline}" + +while [ "$#" -gt 0 ]; do + case "$1" in + --sprite-dir) + SPRITE_DIR="$2" + shift 2 + ;; + --test) + SINGLE_TEST="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [ -z "${SPRITE_DIR}" ]; then + echo "Missing required --sprite-dir" >&2 + exit 1 +fi + +if [ -z "${SINGLE_TEST}" ]; then + echo "XDP_CSR_SINGLE_TEST unset; running full suite TQL unchanged" + exit 0 +fi + +shopt -s nullglob +TQL_FILES=( "${SPRITE_DIR}"/*"${CSR_SUITE}"*.tql ) +shopt -u nullglob + +if [ "${#TQL_FILES[@]}" -eq 0 ]; then + echo "No merged TQL for suite ${CSR_SUITE} under ${SPRITE_DIR}" >&2 + exit 1 +fi + +TQL_FILE="${TQL_FILES[0]}" + +python3 - <<'PY' "${TQL_FILE}" "${SINGLE_TEST}" +import re +import sys + +tql_path, single_test = sys.argv[1:3] +text = open(tql_path, encoding="utf-8").read() + +# Split trailing GLOBAL APPLY blocks (e.g. XRT RPM patch) from query blocks. +global_idx = text.find("\nGLOBAL APPLY") +if global_idx == -1: + body, tail = text, "" +else: + body, tail = text[:global_idx], text[global_idx:] + +# Each testcase query starts at a line beginning with FROM. +parts = re.split(r"(?m)^(?=FROM )", body) +header = parts[0] if parts and not parts[0].lstrip().startswith("FROM ") else "" +queries = [p for p in parts if p.lstrip().startswith("FROM ")] + +kept = [q for q in queries if single_test in q.splitlines()[0]] +if not kept: + available = [q.splitlines()[0].strip() for q in queries] + raise SystemExit( + f"No FROM block matches test '{single_test}'.\n" + f"Available FROM lines:\n " + "\n ".join(available) + ) + +new_body = header + "".join(kept) +open(tql_path, "w", encoding="utf-8").write(new_body + tail) +print(f"Limited {tql_path} to 1 test: {single_test}") +print(f" kept FROM: {kept[0].splitlines()[0].strip()}") +print(f" removed {len(queries) - len(kept)} other FROM block(s)") +PY diff --git a/ci/csr/patch_merged_tql.sh b/ci/csr/patch_merged_tql.sh new file mode 100755 index 00000000..7165b585 --- /dev/null +++ b/ci/csr/patch_merged_tql.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# Patch CSR merged TQL to copy custom XRT RPMs and install them on the board. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=config.defaults.sh +source "${SCRIPT_DIR}/config.defaults.sh" + +usage() { + cat <<'EOF' +Usage: patch_merged_tql.sh --sprite-dir DIR --rpm-dir DIR [--suite SUITE] + +Appends a GLOBAL APPLY block to the merged VE2_XDP TQL in the sprite directory. +EOF +} + +SPRITE_DIR="" +RPM_DIR="" +SUITE="${CSR_SUITE}" + +while [ "$#" -gt 0 ]; do + case "$1" in + --sprite-dir) + SPRITE_DIR="$2" + shift 2 + ;; + --rpm-dir) + RPM_DIR="$2" + shift 2 + ;; + --suite) + SUITE="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [ -z "${SPRITE_DIR}" ] || [ -z "${RPM_DIR}" ]; then + echo "Missing required --sprite-dir and/or --rpm-dir" >&2 + usage >&2 + exit 1 +fi + +if [ ! -d "${RPM_DIR}" ] || [ -z "$(find "${RPM_DIR}" -maxdepth 1 -name '*.rpm' -print -quit)" ]; then + echo "RPM directory is empty or missing: ${RPM_DIR}" >&2 + exit 1 +fi + +shopt -s nullglob +TQL_FILES=( "${SPRITE_DIR}"/*"${SUITE}"*.tql ) +shopt -u nullglob + +if [ "${#TQL_FILES[@]}" -eq 0 ]; then + echo "No merged TQL matching suite ${SUITE} under ${SPRITE_DIR}" >&2 + ls -la "${SPRITE_DIR}" >&2 || true + exit 1 +fi + +TQL_FILE="${TQL_FILES[0]}" +RPM_BASENAME="$(basename "${RPM_DIR}")" +MARKER="# XDP CI: install custom XRT RPMs on Telluride before board run" + +if grep -qF "${MARKER}" "${TQL_FILE}"; then + echo "TQL already patched: ${TQL_FILE}" + exit 0 +fi + +cat >> "${TQL_FILE}" <&2 + usage >&2 + exit 1 + ;; + esac +done + +if [ -z "${SPRITE_DIR}" ]; then + echo "Missing required --sprite-dir" >&2 + usage >&2 + exit 1 +fi + +if [ "${XDP_CSR_USE_DOCKER:-0}" = "1" ]; then + echo "XDP_CSR_USE_DOCKER=1; keeping suite default docker compile (no patch)" + exit 0 +fi + +shopt -s nullglob +TQL_FILES=( "${SPRITE_DIR}"/*"${SUITE}"*.tql ) +shopt -u nullglob + +if [ "${#TQL_FILES[@]}" -eq 0 ]; then + echo "No merged TQL matching suite ${SUITE} under ${SPRITE_DIR}" >&2 + exit 1 +fi + +TQL_FILE="${TQL_FILES[0]}" +MARKER="# XDP CI: native vaiml compile (user.use_docker = 0)" + +if grep -qF "${MARKER}" "${TQL_FILE}"; then + echo "TQL already patched for no-docker compile: ${TQL_FILE}" + exit 0 +fi + +cat >> "${TQL_FILE}" <<'EOF' + +# XDP CI: native vaiml compile (user.use_docker = 0) +GLOBAL APPLY * MODIFY + user.use_docker = 0, + tasks.vaiml.select_resource = "(ostype == ubuntu2204 || ostype == ubuntu22044 || ostype == ubuntu24041)"; +EOF + +echo "Patched ${TQL_FILE} for native vaiml compile" +echo " user.use_docker = 0" +echo " tasks.vaiml.select_resource = (ubuntu hosts, no vai_docker tag)" diff --git a/ci/csr/run_ve2_board_test.sh b/ci/csr/run_ve2_board_test.sh new file mode 100755 index 00000000..d6d3a181 --- /dev/null +++ b/ci/csr/run_ve2_board_test.sh @@ -0,0 +1,110 @@ +#!/bin/bash +# End-to-end VE2 board test via CSR for one XDP PR build. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=config.defaults.sh +source "${SCRIPT_DIR}/config.defaults.sh" + +usage() { + cat <<'EOF' +Usage: run_ve2_board_test.sh --run-id ID [--rpm-source DIR] + +Environment: + CSR_ROOT Base NFS workspace (default: /wrk/csr/snigupta/xdp_ci) + VE2_YOCTO_DIR Yocto workspace used by build-xrt-ve2 + JENKINS_USER Required for Jenkins submission + JENKINS_API_TOKEN Required for Jenkins submission + XDP_CSR_SKIP_JENKINS=1 Generate CSR only; do not submit + XDP_CSR_SUBMIT_MODE local (default) or jenkins + XDP_CSR_SINGLE_TEST One design under XDP_Suites/test_repo (default: ResNet18_MLTimeline; empty = all) + XDP_CSR_USE_DOCKER 0 = native vaiml compile (default trial); 1 = suite docker path +EOF +} + +RUN_ID="" +RPM_SOURCE="" + +while [ "$#" -gt 0 ]; do + case "$1" in + --run-id) + RUN_ID="$2" + shift 2 + ;; + --rpm-source) + RPM_SOURCE="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [ -z "${RUN_ID}" ]; then + echo "Missing required --run-id" >&2 + usage >&2 + exit 1 +fi + +SPRITE_DIR="${CSR_ROOT}/pr-${RUN_ID}/sprite" +RPM_DIR="${CSR_ROOT}/pr-${RUN_ID}/rpms" +DESCRIPTION="${XDP_CSR_DESCRIPTION:-XDP PR ${RUN_ID} VE2_XDP board test}" + +mkdir -p "${CSR_ROOT}/pr-${RUN_ID}" +chmod a+w "${CSR_ROOT}" "${CSR_ROOT}/pr-${RUN_ID}" 2>/dev/null || true + +STAGE_ARGS=( --dest "${RPM_DIR}" ) +if [ -n "${RPM_SOURCE}" ]; then + STAGE_ARGS+=( --source "${RPM_SOURCE}" ) +fi + +echo "== Stage VE2 XRT RPMs ==" +bash "${SCRIPT_DIR}/stage_ve2_rpms.sh" "${STAGE_ARGS[@]}" + +echo "== Generate CSR sprite run ==" +bash "${SCRIPT_DIR}/generate_csr_run.sh" --sprite-dir "${SPRITE_DIR}" + +echo "== Limit merged TQL to single test (trial) ==" +bash "${SCRIPT_DIR}/limit_merged_tql.sh" --sprite-dir "${SPRITE_DIR}" + +echo "== Patch merged TQL for native vaiml compile (no docker) ==" +bash "${SCRIPT_DIR}/patch_merged_tql_no_docker.sh" \ + --sprite-dir "${SPRITE_DIR}" + +echo "== Patch merged TQL for custom XRT install ==" +bash "${SCRIPT_DIR}/patch_merged_tql.sh" \ + --sprite-dir "${SPRITE_DIR}" \ + --rpm-dir "${RPM_DIR}" + +if [ "${XDP_CSR_SKIP_JENKINS:-0}" = "1" ]; then + echo "XDP_CSR_SKIP_JENKINS=1; CSR artifacts ready at ${SPRITE_DIR}" + exit 0 +fi + +SUBMIT_MODE="${XDP_CSR_SUBMIT_MODE:-local}" +case "${SUBMIT_MODE}" in + jenkins) + echo "== Submit CSR run to Jenkins ==" + bash "${SCRIPT_DIR}/submit_csr_jenkins.sh" \ + --sprite-dir "${SPRITE_DIR}" \ + --description "${DESCRIPTION}" + ;; + local) + echo "== Submit CSR run locally (RDI) ==" + bash "${SCRIPT_DIR}/submit_csr_local.sh" \ + --sprite-dir "${SPRITE_DIR}" + ;; + *) + echo "Unknown XDP_CSR_SUBMIT_MODE=${SUBMIT_MODE} (use local or jenkins)" >&2 + exit 1 + ;; +esac + +echo "VE2 CSR board test completed for run ${RUN_ID}" diff --git a/ci/csr/stage_ve2_rpms.sh b/ci/csr/stage_ve2_rpms.sh new file mode 100755 index 00000000..63c53909 --- /dev/null +++ b/ci/csr/stage_ve2_rpms.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Copy VE2 XRT RPMs into a CSR run directory for board-side dnf install. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=config.defaults.sh +source "${SCRIPT_DIR}/config.defaults.sh" + +usage() { + cat <<'EOF' +Usage: stage_ve2_rpms.sh --dest DIR [--source DIR] + + --dest DIR Destination rpms/ directory under the CSR sprite run + --source DIR Yocto RPM deploy directory (default: ${VE2_YOCTO_DIR}/build/tmp/deploy/rpm/${VE2_RPM_DEPLOY_SUBDIR}) +EOF +} + +DEST="" +SOURCE="" + +while [ "$#" -gt 0 ]; do + case "$1" in + --dest) + DEST="$2" + shift 2 + ;; + --source) + SOURCE="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [ -z "${DEST}" ]; then + echo "Missing required --dest" >&2 + usage >&2 + exit 1 +fi + +if [ -z "${SOURCE}" ]; then + if [ -z "${VE2_YOCTO_DIR:-}" ]; then + echo "Set VE2_YOCTO_DIR or pass --source" >&2 + exit 1 + fi + SOURCE="${VE2_YOCTO_DIR}/build/tmp/deploy/rpm/${VE2_RPM_DEPLOY_SUBDIR}" +fi + +if [ ! -d "${SOURCE}" ]; then + echo "RPM source directory not found: ${SOURCE}" >&2 + exit 1 +fi + +shopt -s nullglob +RPMS=( "${SOURCE}"/${VE2_RPM_GLOB} ) +shopt -u nullglob + +if [ "${#RPMS[@]}" -eq 0 ]; then + echo "No RPMs matching ${VE2_RPM_GLOB} under ${SOURCE}" >&2 + exit 1 +fi + +mkdir -p "${DEST}" +rm -f "${DEST}"/*.rpm +cp -f "${RPMS[@]}" "${DEST}/" +chmod -R a+rX "${DEST}" +chmod a+w "${DEST}" + +echo "Staged ${#RPMS[@]} RPM(s) to ${DEST}:" +ls -1 "${DEST}"/*.rpm diff --git a/ci/csr/submit_csr_jenkins.sh b/ci/csr/submit_csr_jenkins.sh new file mode 100755 index 00000000..585c1427 --- /dev/null +++ b/ci/csr/submit_csr_jenkins.sh @@ -0,0 +1,207 @@ +#!/bin/bash +# Submit a CSR sprite directory to Jenkins csr_submit_job and wait for completion. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=config.defaults.sh +source "${SCRIPT_DIR}/config.defaults.sh" + +usage() { + cat <<'EOF' +Usage: submit_csr_jenkins.sh --sprite-dir DIR [--description TEXT] + +Requires: + JENKINS_USER + JENKINS_API_TOKEN + +Optional: + JENKINS_URL (default: http://acasops:8080) + CSR_WAIT_TIMEOUT_SEC (default: 14400) +EOF +} + +SPRITE_DIR="" +DESCRIPTION="" + +while [ "$#" -gt 0 ]; do + case "$1" in + --sprite-dir) + SPRITE_DIR="$2" + shift 2 + ;; + --description) + DESCRIPTION="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [ -z "${SPRITE_DIR}" ]; then + echo "Missing required --sprite-dir" >&2 + usage >&2 + exit 1 +fi + +if [ ! -d "${SPRITE_DIR}" ]; then + echo "Sprite directory not found: ${SPRITE_DIR}" >&2 + exit 1 +fi + +if [ -z "${JENKINS_USER:-}" ] || [ -z "${JENKINS_API_TOKEN:-}" ]; then + echo "Set JENKINS_USER and JENKINS_API_TOKEN to submit ${JENKINS_JOB}" >&2 + exit 1 +fi + +chmod a+w "${SPRITE_DIR}" "$(dirname "${SPRITE_DIR}")" 2>/dev/null || true + +if [ -z "${DESCRIPTION}" ]; then + DESCRIPTION="XDP VE2 CSR: ${SPRITE_DIR}" +fi + +python3 - <<'PY' "${JENKINS_URL}" "${JENKINS_JOB}" "${SPRITE_DIR}" "${DESCRIPTION}" \ + "${JENKINS_LNX_TA}" "${JENKINS_COMPILE_JOBS_ONLY}" "${JENKINS_BOARD_SELECT}" \ + "${JENKINS_USER}" "${JENKINS_API_TOKEN}" "${CSR_WAIT_TIMEOUT_SEC}" +import base64 +import http.cookiejar +import json +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +( + jenkins_url, + job_name, + sprite_dir, + description, + lnx_ta, + compile_jobs_only, + board_select, + user, + token, + timeout_sec, +) = sys.argv[1:11] +user = user.strip() +token = token.strip() +timeout_sec = int(timeout_sec) +base = jenkins_url.rstrip("/") +auth = base64.b64encode(f"{user}:{token}".encode()).decode() + +cookie_jar = http.cookiejar.CookieJar() +opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookie_jar)) + + +def request(url, method="GET", data=None, extra_headers=None): + headers = { + "Authorization": f"Basic {auth}", + "Accept": "application/json", + } + if extra_headers: + headers.update(extra_headers) + req = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with opener.open(req, timeout=120) as resp: + body = resp.read().decode(errors="replace") + return resp, body + except urllib.error.HTTPError as exc: + body = exc.read().decode(errors="replace") + snippet = " ".join(body.split())[:500] + raise SystemExit( + f"Jenkins HTTP {exc.code} for {method} {url}\n{snippet}" + ) from exc + + +def request_json(url, method="GET", data=None, extra_headers=None): + _, body = request(url, method=method, data=data, extra_headers=extra_headers) + return json.loads(body) + + +whoami = request_json(f"{base}/me/api/json") +print(f"Jenkins user: {whoami.get('fullName', whoami.get('id', user))}") + +crumb = request_json(f"{base}/crumbIssuer/api/json") +crumb_field = crumb["crumbRequestField"] +crumb_value = crumb["crumb"] + +params = { + "SPRITE_DIR": sprite_dir, + "DESCRIPTION": description[:256], + "LNX_TA": lnx_ta, + "WIN_TA": "DEFAULT", + "COMPILE_JOBS_ONLY": compile_jobs_only, + "LINUX_COMPILE": "False", + "BOARD_SELECT_RESOURCE": board_select, + "VE2_DOCKER": "False", + "VE2_BOOT_IMAGE_PATH": "DEFAULT", +} +trigger_url = f"{base}/job/{job_name}/buildWithParameters" +form_body = urllib.parse.urlencode(params).encode() + +trigger_resp, _ = request( + trigger_url, + method="POST", + data=form_body, + extra_headers={ + crumb_field: crumb_value, + "Content-Type": "application/x-www-form-urlencoded", + }, +) +queue_url = trigger_resp.headers.get("Location") +if not queue_url: + raise SystemExit("Jenkins did not return a queue Location header") + +print(f"Queued Jenkins job: {queue_url.rstrip('/')}/api/json") + +build_url = None +deadline = time.time() + min(timeout_sec, 600) +while time.time() < deadline: + data = request_json(f"{queue_url.rstrip('/')}/api/json") + if data.get("executable"): + build_url = data["executable"]["url"] + break + if data.get("cancelled"): + raise SystemExit("Jenkins queue item was cancelled") + time.sleep(5) + +if not build_url: + raise SystemExit("Timed out waiting for Jenkins to start the build") + +print(f"Jenkins build started: {build_url}") + +deadline = time.time() + timeout_sec +result = None +while time.time() < deadline: + data = request_json(f"{build_url.rstrip('/')}/api/json") + if not data.get("building") and data.get("result") is not None: + result = data["result"] + break + time.sleep(60) + +if result is None: + raise SystemExit(f"Timed out waiting for Jenkins build after {timeout_sec}s: {build_url}") + +print(f"Jenkins result: {result}") +print(f"Console: {build_url}console") + +_, console = request(f"{build_url.rstrip('/')}/consoleText") +for line in console.splitlines(): + if "Check XOAH Link for details:" in line: + print(line.strip()) + if "All (" in line and "runs finished at:" in line: + print(line.strip()) + +if result != "SUCCESS": + raise SystemExit(1) +PY + +echo "Jenkins ${JENKINS_JOB} completed successfully" diff --git a/ci/csr/submit_csr_local.sh b/ci/csr/submit_csr_local.sh new file mode 100755 index 00000000..2dcbc248 --- /dev/null +++ b/ci/csr/submit_csr_local.sh @@ -0,0 +1,176 @@ +#!/bin/bash +# Launch CSR RDI directly on XCO and wait for completion (no Jenkins API). +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=config.defaults.sh +source "${SCRIPT_DIR}/config.defaults.sh" + +usage() { + cat <<'EOF' +Usage: submit_csr_local.sh --sprite-dir DIR + +Runs the generated *_rdi.sh in SPRITE_DIR and polls the RDI log until complete. +Requires LSF permission and RDI env on the self-hosted runner host. +EOF +} + +SPRITE_DIR="" +POLL_SEC="${CSR_LOCAL_POLL_SEC:-60}" +TIMEOUT_SEC="${CSR_WAIT_TIMEOUT_SEC:-14400}" + +while [ "$#" -gt 0 ]; do + case "$1" in + --sprite-dir) + SPRITE_DIR="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [ -z "${SPRITE_DIR}" ] || [ ! -d "${SPRITE_DIR}" ]; then + echo "Missing or invalid --sprite-dir" >&2 + exit 1 +fi + +shopt -s nullglob +RDI_SCRIPTS=( "${SPRITE_DIR}"/*_rdi.sh ) +shopt -u nullglob + +if [ "${#RDI_SCRIPTS[@]}" -eq 0 ]; then + echo "No *_rdi.sh found under ${SPRITE_DIR}" >&2 + exit 1 +fi + +RDI_SCRIPT="${RDI_SCRIPTS[0]}" +RESULTS_DIR="${SPRITE_DIR}/results" +mkdir -p "${RESULTS_DIR}" +chmod a+w "${SPRITE_DIR}" "${RESULTS_DIR}" "$(dirname "${SPRITE_DIR}")" 2>/dev/null || true + +echo "Launching CSR RDI locally: ${RDI_SCRIPT}" +cd "${SPRITE_DIR}" + +# setRDIEnv.csh is tcsh; sourcing it from bash does not put rdi on PATH. +tcsh -f <&2 + exit 1 +fi + +echo "Monitoring ${#LOG_FILES[@]} RDI log(s):" +printf ' %s\n' "${LOG_FILES[@]}" + +for log in "${LOG_FILES[@]}"; do + if grep -q 'command not found' "${log}" 2>/dev/null; then + echo "RDI launch failed in ${log}:" >&2 + cat "${log}" >&2 + exit 1 + fi +done + +deadline=$(( $(date +%s) + TIMEOUT_SEC )) +pending=("${LOG_FILES[@]}") +failed=0 + +while [ "${#pending[@]}" -gt 0 ]; do + now=$(date +%s) + if [ "${now}" -ge "${deadline}" ]; then + echo "Timed out after ${TIMEOUT_SEC}s waiting for RDI logs" >&2 + exit 1 + fi + + still_pending=() + for log in "${pending[@]}"; do + if grep -q 'Finished at:' "${log}" 2>/dev/null; then + echo "Completed: ${log}" + if grep -Eiq 'ERROR|CRITICAL WARNING|RDI regression failed|exit code [1-9]' "${log}"; then + echo "Failure indicators found in ${log}" >&2 + failed=1 + fi + else + still_pending+=( "${log}" ) + if [ "$(wc -c < "${log}")" -lt 512 ] && \ + find "${log}" -mmin +2 2>/dev/null | grep -q .; then + echo "RDI log stalled (no progress): ${log}" >&2 + cat "${log}" >&2 + exit 1 + fi + echo "Still running: ${log}" + fi + done + + pending=( "${still_pending[@]}" ) + if [ "${#pending[@]}" -gt 0 ]; then + sleep "${POLL_SEC}" + fi +done + +if [ "${failed}" -ne 0 ]; then + echo "One or more RDI runs reported failures" >&2 + exit 1 +fi + +# RDI can finish cleanly while individual tests fail; check suite summaries. +check_suite_test_results() { + local results_dir="$1" + local summary fail_count stats line + local -a summaries=() + local saw_summary=0 + local saw_failure=0 + + shopt -s nullglob + summaries=( "${results_dir}"/*/*.summary "${results_dir}"/*.summary ) + shopt -u nullglob + + if [ "${#summaries[@]}" -eq 0 ]; then + echo "No CSR suite .summary files found under ${results_dir}" >&2 + return 1 + fi + + for summary in "${summaries[@]}"; do + [ -f "${summary}" ] || continue + saw_summary=1 + stats="$(sed -n '2p' "${summary}")" + fail_count="$(awk '{print ($3 ~ /^[0-9]+$/) ? $3 : 0}' <<< "${stats}")" + if [ "${fail_count:-0}" -gt 0 ]; then + saw_failure=1 + echo "CSR test failure(s) reported in ${summary}" >&2 + awk '/^ishitag:/,0' "${summary}" >&2 || true + fi + done + + if [ "${saw_summary}" -eq 0 ]; then + echo "No readable CSR suite .summary files under ${results_dir}" >&2 + return 1 + fi + + if [ "${saw_failure}" -ne 0 ]; then + while IFS= read -r line; do + echo "XOAH: ${line}" >&2 + done < <(grep -rh 'xoah_report_url:' "${results_dir}" 2>/dev/null | sed 's/.*xoah_report_url:[[:space:]]*//' | sort -u) + return 1 + fi + + return 0 +} + +if ! check_suite_test_results "${RESULTS_DIR}"; then + echo "CSR suite reported test failure(s); see summary and XOAH links above" >&2 + exit 1 +fi + +echo "All RDI runs completed successfully" diff --git a/ci/ve2/ensure_yocto_cache.sh b/ci/ve2/ensure_yocto_cache.sh new file mode 100755 index 00000000..5dde64a9 --- /dev/null +++ b/ci/ve2/ensure_yocto_cache.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Prepare a persistent Telluride + Yocto workspace on the self-hosted runner. +# Skips clone + create_yocto.sh -i when cache is younger than VE2_YOCTO_CACHE_TTL_DAYS. +set -euo pipefail + +CACHE_ROOT="${VE2_YOCTO_CACHE_ROOT:-/scratch/snigupta/ve2_yocto_cache}" +CACHE_TTL_DAYS="${VE2_YOCTO_CACHE_TTL_DAYS:-7}" +TELLURIDE_REPO="${TELLURIDE_REPO:-IPS-SSW/Vitis-AI-Telluride.git}" +GITENTERPRISE_HOST="${GITENTERPRISE_HOST:-gitenterprise.xilinx.com}" +YOCTO_REL="${VE2_YOCTO_REL:-versal_2ve/reference_design/vek385/rev-b/sw/yocto}" +FORCE_REFRESH="${VE2_FORCE_CACHE_REFRESH:-false}" + +TELLURIDE_DIR="${CACHE_ROOT}/Vitis-AI-Telluride" +YOCTO_DIR="${TELLURIDE_DIR}/${YOCTO_REL}" +MARKER="${CACHE_ROOT}/.initialized" + +cache_is_valid() { + [ "${FORCE_REFRESH}" = "true" ] && return 1 + [ -f "${MARKER}" ] || return 1 + [ -d "${YOCTO_DIR}/build" ] || return 1 + [ -f "${YOCTO_DIR}/edf-init-build-env" ] || return 1 + + local now marker_age_sec age_days + now="$(date +%s)" + marker_age_sec=$((now - $(stat -c %Y "${MARKER}"))) + age_days=$((marker_age_sec / 86400)) + echo "Cache age: ${age_days} day(s) (ttl ${CACHE_TTL_DAYS} day(s))" + [ "${age_days}" -lt "${CACHE_TTL_DAYS}" ] +} + +mkdir -p "${CACHE_ROOT}" + +if cache_is_valid; then + echo "Reusing cached Yocto workspace at ${YOCTO_DIR}" +else + if [ "${FORCE_REFRESH}" = "true" ]; then + echo "Force refresh: rebuilding Yocto cache at ${CACHE_ROOT}" + else + echo "Cache missing or stale: initializing Yocto workspace at ${CACHE_ROOT}" + fi + + rm -rf "${TELLURIDE_DIR}" + git clone --recurse-submodules \ + "https://${GITENTERPRISE_HOST}/${TELLURIDE_REPO}" \ + "${TELLURIDE_DIR}" + + cd "${YOCTO_DIR}" + chmod +x create_yocto.sh + ./create_yocto.sh -i + touch "${MARKER}" + date -u +"%Y-%m-%dT%H:%M:%SZ" > "${CACHE_ROOT}/.initialized_at" +fi + +if [ -n "${GITHUB_ENV:-}" ]; then + echo "YOCTO_DIR=${YOCTO_DIR}" >> "${GITHUB_ENV}" +fi + +echo "${YOCTO_DIR}"