From e2be63ec827a095304cbf64372e9baa0afeda4f4 Mon Sep 17 00:00:00 2001 From: "Mark F. Brown" Date: Mon, 15 Dec 2025 10:20:36 -0500 Subject: [PATCH 01/14] xpmem: Remove hard polling requirement Do not force hard polling when XPMEM is enabled. This conflicts with counter based polling when used with OFI transport. Issue #1217 Signed-off-by: Mark F. Brown --- .github/workflows/ci.yml | 8 +++++--- configure.ac | 1 - 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff4795a7e..0061c1a33 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,11 +69,13 @@ jobs: xpmem_version: master sos_config: --with-xpmem=${XPMEM_INSTALL_DIR} --enable-error-checking --enable-remote-virtual-addressing --enable-pmi-simple + --enable-hard-polling libfabric_version: v2.1.x - config_name: XPMEM shared atomics xpmem_version: master sos_config: --with-xpmem=${XPMEM_INSTALL_DIR} --enable-shr-atomics --enable-error-checking --enable-pmi-simple + --enable-hard-polling libfabric_version: v2.1.x - config_name: RVA, thread completion sos_config: --enable-error-checking --enable-remote-virtual-addressing @@ -517,7 +519,7 @@ jobs: sos_config: [--enable-pmi-simple --disable-fortran, --with-cma --enable-error-checking --enable-profiling --enable-pmi-simple --disable-fortran --with-hwloc=no, - --with-xpmem --enable-error-checking --enable-pmi-simple --with-hwloc=no] + --with-xpmem --enable-error-checking --enable-pmi-simple --with-hwloc=no --enable-hard-polling] steps: - name: Checking OS version run: | @@ -620,7 +622,7 @@ jobs: matrix: include: - config_name: XPMEM with Shared Atomics - sos_config: --with-xpmem --enable-shr-atomics --enable-error-checking --enable-pmi-simple + sos_config: --with-xpmem --enable-shr-atomics --enable-error-checking --enable-pmi-simple --enable-hard-polling portals4_version: master xpmem_version: master @@ -736,7 +738,7 @@ jobs: include: - config_name: transport_none xpmem_version: master - sos_config: [--with-xpmem --enable-shr-atomics --enable-error-checking --enable-pmi-simple] + sos_config: [--with-xpmem --enable-shr-atomics --enable-error-checking --enable-pmi-simple --enable-hard-polling] steps: - name: Checking OS version diff --git a/configure.ac b/configure.ac index 0145c53f5..19f854db0 100644 --- a/configure.ac +++ b/configure.ac @@ -537,7 +537,6 @@ AM_CONDITIONAL([USE_CMA], [test "$transport_cma" = "yes"]) AS_IF([test "$transport_xpmem" = "yes" -o "$transport_cma" = "yes"], [AC_DEFINE([USE_ON_NODE_COMMS], [1], [Define if any on-node comm transport is available]) - AC_DEFINE([ENABLE_HARD_POLLING], [1], [Enable hard polling]) ]) if test "$enable_shr_atomics" = "yes"; then From 74958a2b0e862ccd867b07e3dd74845180d63764 Mon Sep 17 00:00:00 2001 From: Bryan Morgan Date: Tue, 2 Jun 2026 17:29:40 -0700 Subject: [PATCH 02/14] Remove CI workflow for test branch --- .github/workflows/ci.yml | 806 --------------------------------------- 1 file changed, 806 deletions(-) delete mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 0061c1a33..000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,806 +0,0 @@ -name: SOS-github-actions - -on: [push, pull_request] - -defaults: - run: - shell: bash - -env: - SOS_INSTALL_DIR: ${{ github.workspace }}/install/sos - LIBFABRIC_INSTALL_DIR: ${{ github.workspace }}/install/libfabric - UCX_INSTALL_DIR: ${{ github.workspace }}/install/ucx - PORTALS4_INSTALL_DIR: ${{ github.workspace }}/install/portals4 - LIBEVENT_INSTALL_DIR: ${{ github.workspace }}/install/libevent - PRRTE_INSTALL_DIR: ${{ github.workspace }}/install/prrte - PMIX_INSTALL_DIR: ${{ github.workspace }}/install/pmix - SOS_BUILD_STATIC_OPTS: --disable-shared --enable-static - XPMEM_INSTALL_DIR: ${{ github.workspace }}/install/xpmem - SOS_PM: mpiexec.hydra - SOS_PM_POST: true - -jobs: - OFI: - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - include: - - config_name: PMI simple - sos_config: --enable-pmi-simple - libfabric_version: v1.7.x - - config_name: PMI simple - sos_config: --enable-pmi-simple - libfabric_version: v1.9.x - - config_name: PMI simple - sos_config: --enable-pmi-simple - libfabric_version: v1.11.x - - config_name: PMI simple - sos_config: --enable-pmi-simple - libfabric_version: v1.13.x - - config_name: PMI simple - sos_config: --enable-pmi-simple - libfabric_version: v2.1.x - - config_name: Deprecated tests - sos_config: --enable-pmi-simple --enable-deprecated-tests - libfabric_version: v2.1.x - - config_name: MR-Basic, AV-map, memcpy - sos_config: --enable-ofi-mr=basic --enable-av-map --disable-cxx --enable-memcpy - --enable-pmi-simple --with-hwloc=no - libfabric_version: v2.1.x - - config_name: PMI MPI - sos_config: --disable-fortran --enable-pmi-mpi CC=mpicc - libfabric_version: v2.1.x - - config_name: Remove Virtual Addressing (RVA) - sos_config: --disable-fortran --enable-error-checking --enable-remote-virtual-addressing - --disable-aslr-check --enable-pmi-simple - libfabric_version: v2.1.x - - config_name: heap use malloc - env_setup: export SHMEM_SYMMETRIC_HEAP_USE_MALLOC=1 - sos_config: --disable-threads --enable-error-checking --enable-pmi-simple --with-hwloc=no - libfabric_version: v2.1.x - # too slow, times out on Github (but passes on another Ubuntu 20.04 system)... - #- config_name: CMA, MR Basic, RVA - # sos_config: --disable-fortran --with-cma --enable-error-checking --enable-profiling - # --enable-ofi-mr=basic --enable-av-map --enable-remote-virtual-addressing - # --enable-pmi-simple - # libfabric_version: v2.1.x - - config_name: XPMEM RVA - xpmem_version: master - sos_config: --with-xpmem=${XPMEM_INSTALL_DIR} --enable-error-checking - --enable-remote-virtual-addressing --enable-pmi-simple - --enable-hard-polling - libfabric_version: v2.1.x - - config_name: XPMEM shared atomics - xpmem_version: master - sos_config: --with-xpmem=${XPMEM_INSTALL_DIR} --enable-shr-atomics - --enable-error-checking --enable-pmi-simple - --enable-hard-polling - libfabric_version: v2.1.x - - config_name: RVA, thread completion - sos_config: --enable-error-checking --enable-remote-virtual-addressing - --enable-thread-completion --enable-pmi-simple - libfabric_version: v2.1.x - - config_name: huge pages, zero bounce - env_setup: SHMEM_SYMMETRIC_HEAP_USE_HUGE_PAGES=1 SHMEM_BOUNCE_SIZE=0 - sos_config: --enable-error-checking --enable-remote-virtual-addressing - --enable-pmi-simple --enable-ofi-fence --with-hwloc=no - libfabric_version: v2.1.x - - config_name: auto algorithms - env_setup: export SHMEM_BARRIER_ALGORITHM=auto; - export SHMEM_BCAST_ALGORITHM=auto; - export SHMEM_REDUCE_ALGORITHM=auto; - export SHMEM_COLLECT_ALGORITHM=auto; - export SHMEM_FCOLLECT_ALGORITHM=auto - sos_config: --enable-error-checking --enable-remote-virtual-addressing - --enable-thread-completion --enable-pmi-simple - libfabric_version: v2.1.x - - config_name: linear algorithms - env_setup: export SHMEM_BARRIER_ALGORITHM=linear; - export SHMEM_BCAST_ALGORITHM=linear; - export SHMEM_REDUCE_ALGORITHM=linear; - export SHMEM_COLLECT_ALGORITHM=linear; - export SHMEM_FCOLLECT_ALGORITHM=linear - sos_config: --enable-error-checking --enable-remote-virtual-addressing - --enable-pmi-simple --with-hwloc=no - libfabric_version: v2.1.x - - config_name: tree algorithms - env_setup: export SHMEM_BARRIER_ALGORITHM=tree; - export SHMEM_BCAST_ALGORITHM=tree; - export SHMEM_REDUCE_ALGORITHM=tree; - export SHMEM_OFI_STX_MAX=0 - sos_config: --enable-error-checking --enable-remote-virtual-addressing - --enable-pmi-simple --enable-manual-progress - libfabric_version: v2.1.x - - config_name: dissem/recdbl algorithms - env_setup: export SHMEM_BARRIER_ALGORITHM=dissem; - export SHMEM_REDUCE_ALGORITHM=recdbl; - export SHMEM_FCOLLECT_ALGORITHM=recdbl; - export SHMEM_OFI_STX_AUTO=1 - sos_config: --enable-error-checking --enable-remote-virtual-addressing - --enable-pmi-simple --enable-manual-progress --enable-hard-polling --with-hwloc=no - libfabric_version: v2.1.x - - config_name: ring reduce algorithm - env_setup: export SHMEM_REDUCE_ALGORITHM=ring - export SHMEM_SCAN_ALGORITHM=ring - sos_config: --enable-error-checking --enable-pmi-simple - libfabric_version: v2.1.x - - config_name: ring fcollect algorithm, tx/rx single poll limit - env_setup: export SHMEM_FCOLLECT_ALGORITHM=ring; - export SHMEM_OFI_TX_POLL_LIMIT=1; - export SHMEM_OFI_RX_POLL_LIMIT=1; - export SHMEM_OFI_STX_THRESHOLD=1024 - sos_config: --enable-error-checking --enable-remote-virtual-addressing - --enable-pmi-simple - libfabric_version: v2.1.x - - config_name: STX random - env_setup: export SHMEM_OFI_STX_MAX=8; - export SHMEM_OFI_STX_ALLOCATOR=random - sos_config: --enable-error-checking --enable-remote-virtual-addressing - --enable-pmi-simple - libfabric_version: v2.1.x - - config_name: RPM build - env_setup: export SOS_CHECK_TARBALL_RPM=1 - sos_config: --enable-pmi-simple - rpm_build: true - libfabric_version: v2.1.x - - config_name: Lengthy tests - sos_config: --enable-lengthy-tests --enable-pmi-simple - libfabric_version: v2.1.x - - config_name: Manpages - sos_config: --enable-manpages --enable-pmi-simple - libfabric_version: v2.1.x - - config_name: Without OFI inject - sos_config: --disable-ofi-inject --enable-pmi-simple - libfabric_version: v2.1.x - - config_name: Without Non-fetch AMO - sos_config: --disable-nonfetch-amo --enable-pmi-simple - libfabric_version: v2.1.x - - name: OFI ${{ matrix.libfabric_version }} (${{ matrix.config_name }}) - - steps: - - name: Checking OS version - run: | - echo "OS_NAME=$(lsb_release -si)-$(lsb_release -sr)" >> $GITHUB_ENV - - name: Save Config Parameters - run: | - DEMANGLE_CONFIG=$(echo ${{ matrix.sos_config }} | sed 's/ //g' | sed -E 's/(-)\1+/-/g' | sed -E 's/[^=a-zA-Z0-9._\-]/\+/g') - echo "config param is $DEMANGLE_CONFIG" - echo "CONFIG_PARAM=$DEMANGLE_CONFIG" >> $GITHUB_ENV - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: 'true' - - name: Install dependencies - run: | - sudo apt-get -y update - sudo apt-get install -y gfortran rpm mpich libmpich-dev libhwloc-dev gdb - sudo sysctl -w kernel.yama.ptrace_scope=0 - sudo sysctl -w kernel.randomize_va_space=0 - - # LIBFABRIC - - name: Cache libfabric install - id: cache-libfabric - uses: actions/cache@v4 - with: - path: ${{ env.LIBFABRIC_INSTALL_DIR }} - key: libfabric-${{ matrix.libfabric_version }}-${{ env.OS_NAME }} - - name: Checkout libfabric - if: steps.cache-libfabric.outputs.cache-hit != 'true' - uses: actions/checkout@v4 - with: - repository: ofiwg/libfabric - path: repos/libfabric - ref: ${{ matrix.libfabric_version }} - - name: Build libfabric - if: steps.cache-libfabric.outputs.cache-hit != 'true' - run: | - cd repos/libfabric - ./autogen.sh - mkdir build; cd build - ../configure --prefix=${LIBFABRIC_INSTALL_DIR} - make -j - make install - - # XPMEM - - name: Cache XPMEM install - if: ${{ matrix.xpmem_version }} - id: cache-xpmem - uses: actions/cache@v4 - with: - path: ${{ env.XPMEM_INSTALL_DIR }} - key: xpmem-${{ matrix.xpmem_version }}-${{ env.OS_NAME }} - - name: Checkout XPMEM - if: ${{ matrix.xpmem_version }} - uses: actions/checkout@v4 - with: - repository: openucx/xpmem - path: repos/xpmem - ref: ${{ matrix.xpmem_version }} - - name: Build XPMEM - if: ${{ matrix.xpmem_version }} - run: | - cd repos/xpmem - sudo apt-get -y install linux-headers-`uname -r` - ./autogen.sh - ./configure --prefix=${XPMEM_INSTALL_DIR} - make -j - sudo make install - sudo insmod ${XPMEM_INSTALL_DIR}/lib/modules/`uname -r`/kernel/xpmem/xpmem.ko - sudo chown `whoami` /dev/xpmem - - # SOS - - name: Build SOS (${{ matrix.sos_config }}) - run: | - git submodule update --remote - ./autogen.sh - mkdir build; cd build - ../configure --prefix=${SOS_INSTALL_DIR} --with-ofi=${LIBFABRIC_INSTALL_DIR} ${{ matrix.sos_config }} - make -j - make install - - name: Configure Core Analysis (${{ matrix.sos_config }}) - run: | - bash ${GITHUB_WORKSPACE}/.github/scripts/scan_core.sh init - - name: Test SOS (${{ matrix.sos_config }}) - run: | - cd build - make check TESTS= -j - ${{ matrix.env_setup }} - ulimit -c unlimited - ulimit -a - SHMEM_DEBUG=1 SHMEM_INFO=1 SHMEM_OFI_PROVIDER=sockets FI_PROVIDER=sockets \ - make VERBOSE=1 TEST_RUNNER="${SOS_PM} -np 2 timeout --signal=ABRT 15m" check - cat modules/tests-sos/test/unit/hello.log - - name: Scanning for Core Dumps (${{ matrix.sos_config }}) - if: always() - run: | - bash ${GITHUB_WORKSPACE}/.github/scripts/scan_core.sh scan - - name: Test RPM (${{ matrix.rpm_build }}) - if: ${{ matrix.rpm_build }} - run: | - # FIXME: RPM with a prefix not working on Ubuntu 20.04 (related to _docdir?) - #./configure --with-ofi=${LIBFABRIC_INSTALL_DIR} ${{ matrix.sos_config }} --enable-rpm-prefix - #make dist - #rpmbuild -ta ./sandia-openshmem-*.tar.gz --define "configargs --with-ofi=${LIBFABRIC_INSTALL_DIR} ${{ matrix.sos_config }}" --define "_prefix /usr/shmem" - #make clean - ./configure --with-ofi=${LIBFABRIC_INSTALL_DIR} ${{ matrix.sos_config }} - make dist - rpmbuild -ta ./sandia-openshmem-*.tar.gz --define "configargs --with-ofi=${LIBFABRIC_INSTALL_DIR} ${{ matrix.sos_config }}" - # Sanity check distribution tarball - tar zxvf sandia-openshmem-*.tar.gz - sos_version=$(cat configure.ac | grep AC_INIT | awk -F"]" '{print substr($2, 4, length($2)-2)}') - cd "sandia-openshmem-"$sos_version - ./autogen.sh - mkdir build - cd build - ../configure --with-ofi=${LIBFABRIC_INSTALL_DIR} ${{ matrix.sos_config }} - make -j check TESTS= - ${SOS_PM_PRE} - ulimit -c unlimited - ulimit -a - SHMEM_DEBUG=1 SHMEM_INFO=1 make VERBOSE=1 TEST_RUNNER="${SOS_PM} -np 2" check - ${SOS_PM_POST} - - name: Archive Artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: debug-artifacts-ofi-${{ matrix.libfabric_version }}-${{ env.CONFIG_PARAM }}-${{ github.run_number }}.${{ github.run_attempt }} - path: ${{ github.workspace }}/archives - if-no-files-found: ignore - -# PMIx: -# runs-on: ubuntu-24.04 -# strategy: -# fail-fast: false -# matrix: -# include: -# - sos_transport: libfabric -# libfabric_version: v1.7.x -# sos_transport_config: --with-ofi=${LIBFABRIC_INSTALL_DIR} --without-ucx --without-portals4 -# run_all_tests: true -# libevent_version: release-2.1.10-stable -# pmix_version: v4.1.1rc2 -# prrte_version: v2.0 -# sos_pm: prun -# sos_pm_pre: prte --daemonize --host localhost:4 -# sos_pm_post: pterm -# - sos_transport: ucx -# ucx_version: v1.9.0 -# sos_transport_config: --with-ucx=${UCX_INSTALL_DIR} --without-ofi --without-portals4 -# run_all_tests: true -# libevent_version: release-2.1.10-stable -# pmix_version: v4.1.1rc2 -# prrte_version: v2.0 -# sos_pm: prun -# sos_pm_pre: prte --daemonize --host localhost:4 -# sos_pm_post: pterm -# - sos_transport: portals4 -# portals4_version: master -# sos_transport_config: --with-portals4=${PORTALS4_INSTALL_DIR} --without-ofi --without-ucx -# run_all_tests: false -# libevent_version: release-2.1.10-stable -# pmix_version: v4.1.1rc2 -# prrte_version: v2.0 -# sos_pm: prun -# sos_pm_pre: prte --daemonize --host localhost:4 -# sos_pm_post: pterm -# -# name: PMIx (${{ matrix.sos_transport }}) -# -# steps: -# - name: Checking OS version -# run: | -# echo "OS_NAME=$(lsb_release -si)-$(ls_release -sr)" >> $GITHUB_ENV -# - uses: actions/checkout@v2 -# - name: Install dependencies -# run: | -# sudo apt-get -y update -# sudo apt-get install -y gfortran libhwloc-dev libev-dev libev-libevent-dev -# -# # LIBFABRIC -# - name: Cache libfabric install -# if: ${{ matrix.sos_transport == 'libfabric' }} -# id: cache-libfabric -# uses: actions/cache@v4 -# with: -# path: ${{ env.LIBFABRIC_INSTALL_DIR }} -# key: libfabric-${{ matrix.libfabric_version }}-${{ env.OS_NAME }} -# - name: Checkout libfabric -# if: ${{ matrix.sos_transport == 'libfabric' && steps.cache-libfabric.outputs.cache-hit != 'true' }} -# uses: actions/checkout@v2 -# with: -# repository: ofiwg/libfabric -# path: repos/libfabric -# ref: ${{ matrix.libfabric_version }} -# - name: Build libfabric -# if: ${{ matrix.sos_transport == 'libfabric' && steps.cache-libfabric.outputs.cache-hit != 'true' }} -# run: | -# cd repos/libfabric -# ./autogen.sh -# mkdir build; cd build -# ../configure --prefix=${LIBFABRIC_INSTALL_DIR} -# make -j -# make install -# -# # UCX -# - name: Cache UCX install -# if: ${{ matrix.sos_transport == 'ucx' }} -# id: cache-ucx -# uses: actions/cache@v4 -# with: -# path: ${{ env.UCX_INSTALL_DIR }} -# key: ucx-${{ matrix.ucx_version }}-${{ env.OS_NAME }} -# - name: Checkout UCX -# if: ${{ matrix.sos_transport == 'ucx' && steps.cache-ucx.outputs.cache-hit != 'true' }} -# uses: actions/checkout@v2 -# with: -# repository: openucx/ucx -# path: repos/ucx -# ref: ${{ matrix.ucx_version }} -# - name: Build UCX -# if: ${{ matrix.sos_transport == 'ucx' && steps.cache-ucx.outputs.cache-hit != 'true' }} -# run: | -# cd repos/ucx -# ./autogen.sh -# mkdir build; cd build -# ../configure --prefix=${UCX_INSTALL_DIR} --enable-mt --disable-numa --without-java -# make -j -# make install -# -# ## Portals4 -# - name: Cache Portals4 install -# if: ${{ matrix.sos_transport == 'portals4' }} -# id: cache-portals4 -# uses: actions/cache@v4 -# with: -# path: ${{ env.PORTALS4_INSTALL_DIR }} -# key: portals4-${{ matrix.portals4_version }}-${{ env.OS_NAME }} -# - name: Checkout Portals4 -# if: ${{ matrix.sos_transport == 'portals4' && steps.cache-portals4.outputs.cache-hit != 'true' }} -# uses: actions/checkout@v2 -# with: -# repository: regrant/portals4 -# path: repos/portals4 -# ref: ${{ matrix.portals4_version }} -# - name: Build Portals4 -# if: ${{ matrix.sos_transport == 'portals4' && steps.cache-portals4.outputs.cache-hit != 'true' }} -# run: | -# cd repos/portals4 -# ./autogen.sh -# mkdir build; cd build -# ../configure --prefix=${PORTALS4_INSTALL_DIR} --enable-zero-mrs --enable-reliable-udp --disable-pmi-from-portals ${SOS_BUILD_STATIC_OPTS} -# make -j -# make install -# -# # Libevent -# - name: Cache libevent install -# id: cache-libevent -# uses: actions/cache@v4 -# with: -# path: ${{ env.LIBEVENT_INSTALL_DIR }} -# key: libevent-${{ matrix.libevent_version }}-${{ env.OS_NAME }} -# - name: Checkout libevent -# if: steps.cache-libevent.outputs.cache-hit != 'true' -# uses: actions/checkout@v2 -# with: -# repository: libevent/libevent -# path: repos/libevent -# ref: ${{ matrix.libevent_version }} -# - name: Build libevent -# if: steps.cache-libevent.outputs.cache-hit != 'true' -# run: | -# cd repos/libevent -# ./autogen.sh -# ./configure --prefix=${LIBEVENT_INSTALL_DIR} -# make clean all install -# -# # PMIx -# - name: Cache PMIx install -# id: cache-pmix -# uses: actions/cache@v4 -# with: -# path: ${{ env.PMIX_INSTALL_DIR }} -# key: pmix-${{ matrix.pmix_version }}-${{ env.OS_NAME }} -# - name: Checkout PMIx -# if: steps.cache-pmix.outputs.cache-hit != 'true' -# uses: actions/checkout@v2 -# with: -# repository: openpmix/pmix -# path: repos/pmix -# ref: ${{ matrix.pmix_version }} -# - name: Build PMIx -# if: steps.cache-pmix.outputs.cache-hit != 'true' -# run: | -# cd repos/pmix -# ./autogen.pl -# ./configure --prefix=${PMIX_INSTALL_DIR} --with-libevent=${LIBEVENT_INSTALL_DIR} --without-libev --disable-debug CFLAGS=-O3 ${SOS_BUILD_STATIC_OPTS} -# make install -# -# # Build PRRTE -# - name: Cache PRRTE install -# id: cache-prrte -# uses: actions/cache@v4 -# with: -# path: ${{ env.PRRTE_INSTALL_DIR }} -# key: prrte-${{ matrix.prrte_version}}-${{ env.OS_NAME }} -# - name: Checkout PRRTE -# if: steps.cache-prrte.outputs.cache-hit != 'true' -# uses: actions/checkout@v2 -# with: -# repository: openpmix/prrte -# path: repos/prrte -# ref: ${{ matrix.prrte_version }} -# - name: Build PRRTE -# if: steps.cache-prrte.outputs.cache-hit != 'true' -# run: | -# cd repos/prrte -# ./autogen.pl -# ./configure --prefix=${PRRTE_INSTALL_DIR} --with-pmix=${PMIX_INSTALL_DIR} --without-slurm --with-libevent=${LIBEVENT_INSTALL_DIR} --without-libev ${SOS_BUILD_STATIC_OPTS} -# make install -# -# # SOS -# - name: Build SOS -# run: | -# ./autogen.sh -# mkdir build; cd build -# ../configure --prefix=${SOS_INSTALL_DIR} ${{ matrix.sos_transport_config }} --with-pmix=${PMIX_INSTALL_DIR} -# make -j -# make install -# - name: Test SOS (all tests) -# if: ${{ matrix.sos_transport == 'libfabric' || matrix.sos_transport == 'ucx' }} -# run: | -# cd build -# make check TESTS= -j -# export PATH=${PRRTE_INSTALL_DIR}/bin:$PATH -# ${{ matrix.sos_pm_pre }} -# SHMEM_INFO=1 make VERBOSE=1 TEST_RUNNER="${{ matrix.sos_pm }} -np 2" check -# cat modules/tests-sos/test/unit/hello.log -# ${{ matrix.sos_pm_post }} -# - name: Test SOS (hello) -# if: ${{ matrix.sos_transport == 'portals4' }} -# run: | -# cd build -# make check TESTS= -j -# export PATH=${PRRTE_INSTALL_DIR}/bin:$PATH -# ${{ matrix.sos_pm_pre }} -# SHMEM_INFO=1 ${{ matrix.sos_pm }} -np 1 modules/tests-sos/test/unit/hello -# ${{ matrix.sos_pm_post }} -# - UCX: - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - include: - - config_name: ucx-1.18.0 - ucx_version: v1.18.0 - xpmem_version: master - sos_config: [--enable-pmi-simple --disable-fortran, - --with-cma --enable-error-checking --enable-profiling - --enable-pmi-simple --disable-fortran --with-hwloc=no, - --with-xpmem --enable-error-checking --enable-pmi-simple --with-hwloc=no --enable-hard-polling] - steps: - - name: Checking OS version - run: | - echo "OS_NAME=$(lsb_release -si)-$(lsb_release -sr)" >> $GITHUB_ENV - - name: Save Config Parameters - run: | - DEMANGLE_CONFIG=$(echo ${{ matrix.sos_config }} | sed 's/ //g' | sed -E 's/(-)\1+/-/g' | sed -E 's/[^=a-zA-Z0-9._\-]/\+/g') - echo "config param is $DEMANGLE_CONFIG" - echo "CONFIG_PARAM=$DEMANGLE_CONFIG" >> $GITHUB_ENV - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: 'true' - - name: Install dependencies - run: | - sudo apt-get -y update - sudo apt-get install -y gfortran mpich libmpich-dev gdb - sudo sysctl -w kernel.yama.ptrace_scope=0 - sudo sysctl -w kernel.randomize_va_space=0 - - # XPMEM - - name: Checkout XPMEM - uses: actions/checkout@v4 - with: - repository: openucx/xpmem - path: repos/xpmem - ref: ${{ matrix.xpmem_version }} - - name: Build XPMEM - run: | - cd repos/xpmem - sudo apt-get -y install linux-headers-`uname -r` - ./autogen.sh - ./configure --prefix=/usr - make -j - sudo make install - sudo insmod /usr/lib/modules/`uname -r`/kernel/xpmem/xpmem.ko - sudo chown `whoami` /dev/xpmem - - # UCX - - name: Cache UCX install - id: cache-ucx - uses: actions/cache@v4 - with: - path: ${{ env.UCX_INSTALL_DIR }} - key: ucx-${{ matrix.ucx_version}}-${{ env.OS_NAME }} - - name: Checkout UCX - if: steps.cache-ucx.outputs.cache-hit != 'true' - uses: actions/checkout@v4 - with: - repository: openucx/ucx - path: repos/ucx - ref: ${{ matrix.ucx_version }} - - name: Build UCX - if: steps.cache-ucx.outputs.cache-hit != 'true' - run: | - cd repos/ucx - ./autogen.sh - mkdir build; cd build - ../configure --prefix=${UCX_INSTALL_DIR} --enable-mt --disable-numa --without-java - make -j - make install - - # SOS - - name: Build SOS (${{ matrix.sos_config }}) - run: | - git submodule update --remote - ./autogen.sh - mkdir build; cd build - ../configure --prefix=${SOS_INSTALL_DIR} --with-ucx=${UCX_INSTALL_DIR} ${{ matrix.sos_config }} - make -j - make install - - name: Configure Core Analysis (${{ matrix.sos_config }}) - run: | - bash ${GITHUB_WORKSPACE}/.github/scripts/scan_core.sh init - - name: Test SOS (${{ matrix.sos_config }}) - continue-on-error: true - run: | - cd build - make check TESTS= -j - ulimit -c unlimited - ulimit -a - SHMEM_DEBUG=1 SHMEM_INFO=1 make VERBOSE=1 TEST_RUNNER="${SOS_PM} -np 2 timeout --signal=ABRT 15m" check - cat modules/tests-sos/test/unit/hello.log - - name: Scanning for Core Dumps (${{ matrix.sos_config }}) - if: always() - run: | - bash ${GITHUB_WORKSPACE}/.github/scripts/scan_core.sh scan - - name: Archive Artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: debug-artifacts-${{ matrix.config_name }}-${{ env.CONFIG_PARAM }}-${{ github.run_number }}.${{ github.run_attempt }} - path: ${{ github.workspace }}/archives - if-no-files-found: ignore - - Portals4: - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - include: - - config_name: XPMEM with Shared Atomics - sos_config: --with-xpmem --enable-shr-atomics --enable-error-checking --enable-pmi-simple --enable-hard-polling - portals4_version: master - xpmem_version: master - -# - name: Heap use malloc -# env_setup: export SHMEM_SYMMETRIC_HEAP_USE_MALLOC=1 -# sos_config: --disable-threads --enable-error-checking --enable-pmi-simple --with-hwloc=no -# - name: Heap use huge pages, zero bounce -# env_setup: export SHMEM_SYMMETRIC_HEAP_USE_HUGE_PAGES=1; export SHMEM_BOUNCE_SIZE=0 -# sos_config: --enable-error-checking --enable-remote-virtual-addressing --enable-pmi-simple --enable-ofi-fence -# sos_config: [--enable-pmi-simple --with-hwloc=no, -# --disable-fortran --enable-error-checking --enable-remote-virtual-addressing -# --disable-aslr-check --enable-pmi-simple, -# --with-cma --enable-error-checking --enable-profiling -# --enable-remote-virtual-addressing --enable-pmi-simple, -# --enable-pmi-mpi CC=mpicc --disable-fortran --with-hwloc=no] - steps: - - name: Checking OS version - run: | - echo "OS_NAME=$(lsb_release -si)-$(lsb_release -sr)" >> $GITHUB_ENV - - name: Save Config Parameters - run: | - DEMANGLE_CONFIG=$(echo ${{ matrix.sos_config }} | sed 's/ //g' | sed -E 's/(-)\1+/-/g' | sed -E 's/[^=a-zA-Z0-9._\-]/\+/g') - echo "config param is $DEMANGLE_CONFIG" - echo "CONFIG_PARAM=$DEMANGLE_CONFIG" >> $GITHUB_ENV - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: 'true' - - name: Install dependencies - run: | - sudo apt-get -y update - sudo apt-get install -y gfortran mpich libmpich-dev libev-dev libev-libevent-dev libhwloc-dev gdb - sudo sysctl -w kernel.yama.ptrace_scope=0 - sudo sysctl -w kernel.randomize_va_space=0 - - # XPMEM - - name: Checkout XPMEM - uses: actions/checkout@v4 - with: - repository: openucx/xpmem - path: repos/xpmem - ref: ${{ matrix.xpmem_version }} - - name: Build XPMEM - run: | - cd repos/xpmem - sudo apt-get -y install linux-headers-`uname -r` - ./autogen.sh - ./configure --prefix=/usr - make -j - sudo make install - sudo insmod /usr/lib/modules/`uname -r`/kernel/xpmem/xpmem.ko - sudo chown `whoami` /dev/xpmem - - # Portals4 - - name: Cache Portals4 install - id: cache-portals4 - uses: actions/cache@v4 - with: - path: ${{ env.PORTALS4_INSTALL_DIR }} - key: portals4-${{ matrix.portals4_version}}-${{ env.OS_NAME }} - - name: Checkout Portals4 - if: steps.cache-portals4.outputs.cache-hit != 'true' - uses: actions/checkout@v4 - with: - repository: regrant/portals4 - path: repos/portals4 - ref: ${{ matrix.portals4_version }} - - name: Build Portals4 - if: steps.cache-portals4.outputs.cache-hit != 'true' - run: | - cd repos/portals4 - ./autogen.sh - sed -i 's/^struct ptl_abort_state abort_state;/static struct ptl_abort_state abort_state;/' src/ib/ptl_gbl.h - mkdir build; cd build - ../configure --prefix=${PORTALS4_INSTALL_DIR} --enable-zero-mrs --enable-reliable-udp --disable-pmi-from-portals - make -j - make install - - # SOS - - name: Build SOS (${{ matrix.name }}) - run: | - git submodule update --remote - ./autogen.sh - mkdir build; cd build - ../configure --prefix=${SOS_INSTALL_DIR} --with-portals4=${PORTALS4_INSTALL_DIR} ${{ matrix.sos_config }} - make -j - make install - - name: Configure Core Analysis (${{ matrix.sos_config }}) - run: | - bash ${GITHUB_WORKSPACE}/.github/scripts/scan_core.sh init - - name: Test SOS (${{ matrix.name }}) - continue-on-error: true - run: | - cd build - make check TESTS= -j - ulimit -c unlimited - ulimit -a - SHMEM_DEBUG=1 SHMEM_INFO=1 make VERBOSE=1 TEST_RUNNER="${SOS_PM} -np 2 timeout --signal=ABRT 15m" check - ${SOS_PM} -np 1 modules/tests-sos/test/unit/hello - - name: Archive Artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: debug-artifacts-${{ matrix.config_name }}-${{ env.CONFIG_PARAM }}-${{ github.run_number }}.${{ github.run_attempt }} - path: ${{ github.workspace }}/archives - if-no-files-found: ignore - - XPMEM_Only: - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - include: - - config_name: transport_none - xpmem_version: master - sos_config: [--with-xpmem --enable-shr-atomics --enable-error-checking --enable-pmi-simple --enable-hard-polling] - - steps: - - name: Checking OS version - run: | - echo "OS_NAME=$(lsb_release -si)-$(lsb_release -sr)" >> $GITHUB_ENV - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: 'true' - - name: Install dependencies - run: | - sudo apt-get -y update - sudo apt-get install -y gfortran mpich libmpich-dev libev-dev libev-libevent-dev libhwloc-dev gdb - sudo sysctl -w kernel.yama.ptrace_scope=0 - sudo sysctl -w kernel.randomize_va_space=0 - - # XPMEM - - name: Checkout XPMEM - uses: actions/checkout@v4 - with: - repository: openucx/xpmem - path: repos/xpmem - ref: ${{ matrix.xpmem_version }} - - name: Build XPMEM - run: | - cd repos/xpmem - sudo apt-get -y install linux-headers-`uname -r` - ./autogen.sh - ./configure --prefix=/usr - make -j - sudo make install - sudo insmod /usr/lib/modules/`uname -r`/kernel/xpmem/xpmem.ko - sudo chown `whoami` /dev/xpmem - - # SOS - - name: Build SOS (${{ matrix.name }}) - run: | - git submodule update --remote - ./autogen.sh - mkdir build; cd build - ../configure --prefix=${SOS_INSTALL_DIR} ${{ matrix.sos_config }} - make -j - make install - - name: Configure Core Analysis (${{ matrix.sos_config }}) - run: | - bash ${GITHUB_WORKSPACE}/.github/scripts/scan_core.sh init - - name: Test SOS (${{ matrix.name }}) - run: | - cd build - make check TESTS= -j - ulimit -c unlimited - ulimit -a - SHMEM_DEBUG=1 SHMEM_INFO=1 make VERBOSE=1 TEST_RUNNER="${SOS_PM} -np 2 timeout --signal=ABRT 15m" check - ${SOS_PM} -np 1 modules/tests-sos/test/unit/hello - - name: Scanning for Core Dumps (${{ matrix.sos_config }}) - if: always() - run: | - bash ${GITHUB_WORKSPACE}/.github/scripts/scan_core.sh scan - - name: Archive Artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: debug-artifacts-${{ matrix.config_name }}-${{ env.CONFIG_PARAM }}-${{ github.run_number }}.${{ github.run_attempt }} - path: ${{ github.workspace }}/archives - if-no-files-found: ignore From f00c97b94352af9c8e7abf58ebfd9332f73e93a0 Mon Sep 17 00:00:00 2001 From: "Mark F. Brown" Date: Tue, 20 Jan 2026 21:39:12 -0500 Subject: [PATCH 03/14] Improved OFI Transport Wait Replaced complex polling with simpler OFI counter wait Issue #1217 Signed-off-by: Mark F. Brown --- src/transport_ofi.c | 4 ++++ src/transport_ofi.h | 47 +++------------------------------------------ 2 files changed, 7 insertions(+), 44 deletions(-) diff --git a/src/transport_ofi.c b/src/transport_ofi.c index f8f04d65e..f0bd68b8f 100644 --- a/src/transport_ofi.c +++ b/src/transport_ofi.c @@ -1731,6 +1731,7 @@ static int shmem_transport_ofi_ctx_init(shmem_transport_ctx_t *ctx, int id) cntr_put_attr.events = FI_CNTR_EVENTS_COMP; cntr_get_attr.events = FI_CNTR_EVENTS_COMP; +#if 0 /* Set FI_WAIT based on the put and get polling limits defined above */ if (shmem_transport_ofi_put_poll_limit < 0) { cntr_put_attr.wait_obj = FI_WAIT_NONE; @@ -1742,6 +1743,9 @@ static int shmem_transport_ofi_ctx_init(shmem_transport_ctx_t *ctx, int id) } else { cntr_get_attr.wait_obj = FI_WAIT_UNSPEC; } +#endif + cntr_put_attr.wait_obj = FI_WAIT_UNSPEC; + cntr_get_attr.wait_obj = FI_WAIT_UNSPEC; /* Allow provider to choose CQ size, since we are using FI_RM_ENABLED. * Context format is used to return bounce buffer pointers in the event diff --git a/src/transport_ofi.h b/src/transport_ofi.h index f7e2ebf32..d7d972fff 100644 --- a/src/transport_ofi.h +++ b/src/transport_ofi.h @@ -504,28 +504,8 @@ void shmem_transport_put_quiet(shmem_transport_ctx_t* ctx) * reverse order: first the fid_cntr event counter, then the put issued * counter. We'll want to preserve this property in the future. */ - uint64_t success, fail, cnt, cnt_new; - long poll_count = 0; - while (poll_count < shmem_transport_ofi_put_poll_limit || - shmem_transport_ofi_put_poll_limit < 0) { - success = fi_cntr_read(ctx->put_cntr); - fail = fi_cntr_readerr(ctx->put_cntr); - cnt = SHMEM_TRANSPORT_OFI_CNTR_READ(&ctx->pending_put_cntr); - - shmem_transport_probe(); - - if (success < cnt && fail == 0) { - SHMEM_TRANSPORT_OFI_CTX_UNLOCK(ctx); - SPINLOCK_BODY(); - SHMEM_TRANSPORT_OFI_CTX_LOCK(ctx); - } else if (fail) { - RAISE_ERROR_MSG("Operations completed in error (%" PRIu64 ")\n", fail); - } else { - SHMEM_TRANSPORT_OFI_CTX_UNLOCK(ctx); - return; - } - poll_count++; - } + uint64_t cnt, cnt_new; + cnt_new = SHMEM_TRANSPORT_OFI_CNTR_READ(&ctx->pending_put_cntr); do { cnt = cnt_new; @@ -965,31 +945,10 @@ void shmem_transport_get_wait(shmem_transport_ctx_t* ctx) * reverse order: first the fid_cntr event counter, then the get issued * counter. We'll want to preserve this property in the future. */ - uint64_t success, fail, cnt, cnt_new; - long poll_count = 0; + uint64_t cnt, cnt_new; SHMEM_TRANSPORT_OFI_CTX_LOCK(ctx); - while (poll_count < shmem_transport_ofi_get_poll_limit || - shmem_transport_ofi_get_poll_limit < 0) { - success = fi_cntr_read(ctx->get_cntr); - fail = fi_cntr_readerr(ctx->get_cntr); - cnt = SHMEM_TRANSPORT_OFI_CNTR_READ(&ctx->pending_get_cntr); - - shmem_transport_probe(); - - if (success < cnt && fail == 0) { - SHMEM_TRANSPORT_OFI_CTX_UNLOCK(ctx); - SPINLOCK_BODY(); - SHMEM_TRANSPORT_OFI_CTX_LOCK(ctx); - } else if (fail) { - RAISE_ERROR_MSG("Operations completed in error (%" PRIu64 ")\n", fail); - } else { - SHMEM_TRANSPORT_OFI_CTX_UNLOCK(ctx); - return; - } - poll_count++; - } cnt_new = SHMEM_TRANSPORT_OFI_CNTR_READ(&ctx->pending_get_cntr); do { cnt = cnt_new; From 646d09953fc3ddbaf1f3bb6387b10d7fcd494bd4 Mon Sep 17 00:00:00 2001 From: "Mark F. Brown" Date: Tue, 27 Jan 2026 13:20:36 -0500 Subject: [PATCH 04/14] Allocate Bounce Buffer from Symmetric Heap Issue #1221 Signed-off-by: Mark F. Brown --- src/init.c | 4 +++ src/shmem_env_defs.h | 6 +++- src/shmem_free_list.c | 71 +++++++++++++++++++++++++++++----------- src/shmem_free_list.h | 15 ++++++--- src/shmem_internal.h | 5 ++- src/symmetric_heap_c.c | 3 +- src/transport_ofi.c | 9 +++-- src/transport_ofi.h | 8 +++-- src/transport_portals4.c | 6 ++-- 9 files changed, 93 insertions(+), 34 deletions(-) diff --git a/src/init.c b/src/init.c index 01ca23dfd..2a9189914 100644 --- a/src/init.c +++ b/src/init.c @@ -382,6 +382,10 @@ shmem_internal_heap_postinit(void) shmem_internal_heap_base, shmem_internal_heap_length, shmem_internal_data_base, shmem_internal_data_length); + if (shmem_internal_params.BOUNCE_MLOCK) { + DEBUG_MSG("Bounce buffer locking enabled\n"); + } + #ifdef HAVE_SCHED_GETAFFINITY #ifdef USE_HWLOC ret = hwloc_topology_init(&shmem_internal_topology); diff --git a/src/shmem_env_defs.h b/src/shmem_env_defs.h index c0a27a306..b76e5e95e 100644 --- a/src/shmem_env_defs.h +++ b/src/shmem_env_defs.h @@ -46,7 +46,7 @@ SHMEM_INTERNAL_ENV_DEF(SYMMETRIC_HEAP_USE_MALLOC, bool, false, SHMEM_INTERNAL_EN "Allocate the symmetric heap using malloc") SHMEM_INTERNAL_ENV_DEF(BOUNCE_SIZE, size, DEFAULT_BOUNCE_SIZE, SHMEM_INTERNAL_ENV_CAT_OTHER, "Maximum message size to bounce buffer") -SHMEM_INTERNAL_ENV_DEF(MAX_BOUNCE_BUFFERS, long, 128, SHMEM_INTERNAL_ENV_CAT_OTHER, +SHMEM_INTERNAL_ENV_DEF(MAX_BOUNCE_BUFFERS, size, 128, SHMEM_INTERNAL_ENV_CAT_OTHER, "Maximum number of bounce buffers per context") SHMEM_INTERNAL_ENV_DEF(TRAP_ON_ABORT, bool, false, SHMEM_INTERNAL_ENV_CAT_OTHER, "Generate trap if the program aborts or calls shmem_global_exit") @@ -126,3 +126,7 @@ SHMEM_INTERNAL_ENV_DEF(MPI_THREAD_LEVEL, string, "MPI_THREAD_SINGLE", SHMEM_INTE SHMEM_INTERNAL_ENV_DEF(BACKTRACE, string, "", SHMEM_INTERNAL_ENV_CAT_OTHER, "Specify the mechanism to use for backtracing on failure") +SHMEM_INTERNAL_ENV_DEF(BOUNCE_MLOCK, bool, false, SHMEM_INTERNAL_ENV_CAT_OTHER, + "Lock bounce buffer memory preventing buffers from being paged out to swap") +SHMEM_INTERNAL_ENV_DEF(BOUNCE_SHEAP, bool, true, SHMEM_INTERNAL_ENV_CAT_OTHER, + "Allocate bounce buffers using symmetric heap, if supported by transport") diff --git a/src/shmem_free_list.c b/src/shmem_free_list.c index 50fd8a70f..bbb37934a 100644 --- a/src/shmem_free_list.c +++ b/src/shmem_free_list.c @@ -22,10 +22,12 @@ #include "shmem.h" #include "shmem_free_list.h" +#define NUM_ELEMENTS 2 shmem_free_list_t* -shmem_free_list_init(unsigned int element_size, - shmem_free_list_item_init_fn_t init_fn) +shmem_free_list_init(size_t element_size, + shmem_free_list_item_init_fn_t init_fn, + size_t max_pool_cnt) { int ret; shmem_free_list_t *fl = (shmem_free_list_t*) calloc(1, sizeof(shmem_free_list_t)); @@ -34,6 +36,18 @@ shmem_free_list_init(unsigned int element_size, fl->element_size = element_size; fl->init_fn = init_fn; fl->nalloc = 0; + fl->alloc_size = sizeof(shmem_free_list_alloc_t) + NUM_ELEMENTS * fl->element_size; + fl->pool_size = 0; + fl->pool_ofs = 0; + fl->pool = NULL; + if (max_pool_cnt) { + /* preallocate pool with shmem malloc */ + /* memory must be reserved as a memory pool to prevent address conflicts between PEs */ + fl->pool_size = fl->alloc_size * max_pool_cnt; + fl->pool = shmem_internal_shmalloc(fl->pool_size); + } + /* if pool count is zero allocate with bounce buffers with malloc */ + SHMEM_MUTEX_INIT(fl->lock); ret = shmem_free_list_more(fl); if (0 != ret) { @@ -48,13 +62,20 @@ shmem_free_list_init(unsigned int element_size, void shmem_free_list_destroy(shmem_free_list_t *fl) { - shmem_free_list_alloc_t *alloc, *next; - - alloc = fl->allocs; - while (NULL != alloc) { - next = alloc->next; - free(alloc); - alloc = next; + if (fl->pool) { + /* allocated with shmem malloc */ + shmem_internal_free(fl->pool); + fl->pool = NULL; + fl->pool_ofs = 0; + } else { + /* allocated with malloc */ + shmem_free_list_alloc_t *alloc, *next; + alloc = fl->allocs; + while (NULL != alloc) { + next = alloc->next; + free(alloc); + alloc = next; + } } SHMEM_MUTEX_DESTROY(fl->lock); @@ -64,25 +85,35 @@ shmem_free_list_destroy(shmem_free_list_t *fl) int shmem_free_list_more(shmem_free_list_t *fl) { - size_t page_size = 4096 - sizeof(shmem_free_list_alloc_t); - int num_elements = (fl->element_size < page_size) ? page_size / fl->element_size : 1; shmem_free_list_item_t *item, *first, *next, *last = NULL; shmem_free_list_alloc_t *header; char *buf; - int i; - - num_elements = 2; - - buf = malloc(sizeof(shmem_free_list_alloc_t) + - num_elements * fl->element_size); - if (NULL == buf) return 1; + uint64_t i; + + if (fl->pool) { + if (fl->pool_ofs >= fl->pool_size) { + fprintf(stderr, "[%d] pool memory exhausted\n", shmem_internal_my_pe); + return 1; + } + + buf = &fl->pool[fl->pool_ofs]; + fl->pool_ofs += fl->alloc_size; + + if (shmem_internal_params.BOUNCE_MLOCK) { + mlock(buf, sizeof(shmem_free_list_alloc_t) + NUM_ELEMENTS * fl->element_size); + } + } else { + buf = malloc(sizeof(shmem_free_list_alloc_t) + + NUM_ELEMENTS * fl->element_size); + if (NULL == buf) return 1; + } header = (shmem_free_list_alloc_t*) buf; first = item = (shmem_free_list_item_t*) (header + 1); - for (i = 0 ; i < num_elements ; ++i) { + for (i = 0 ; i < NUM_ELEMENTS ; ++i) { fl->init_fn(item); next = (shmem_free_list_item_t*)((char*)item + fl->element_size); - if (i == num_elements - 1) { + if (i == NUM_ELEMENTS - 1) { item->next = NULL; last = item; } else { diff --git a/src/shmem_free_list.h b/src/shmem_free_list.h index f73a42576..f2c69e7d8 100644 --- a/src/shmem_free_list.h +++ b/src/shmem_free_list.h @@ -17,7 +17,8 @@ #define SHMEM_FREE_QUEUE_H #include - +#include +#include #include "shmem_internal.h" struct shmem_free_list_item_t { @@ -33,9 +34,12 @@ typedef struct shmem_free_list_alloc_t shmem_free_list_alloc_t; typedef void (*shmem_free_list_item_init_fn_t)(shmem_free_list_item_t *item); struct shmem_free_list_t { - uint32_t element_size; + size_t element_size; uint64_t nalloc; - + uint64_t alloc_size; + uint64_t pool_size; + char *pool; + size_t pool_ofs; shmem_free_list_item_init_fn_t init_fn; shmem_free_list_alloc_t *allocs; shmem_free_list_item_t* head; @@ -45,8 +49,9 @@ struct shmem_free_list_t { }; typedef struct shmem_free_list_t shmem_free_list_t; -shmem_free_list_t* shmem_free_list_init(unsigned int element_size, - shmem_free_list_item_init_fn_t init_fn); +shmem_free_list_t* shmem_free_list_init(size_t element_size, + shmem_free_list_item_init_fn_t init_fn, + size_t max_pool_cnt); void shmem_free_list_destroy(shmem_free_list_t *fl); int shmem_free_list_more(shmem_free_list_t *fl); diff --git a/src/shmem_internal.h b/src/shmem_internal.h index 465dc736a..f937af99f 100644 --- a/src/shmem_internal.h +++ b/src/shmem_internal.h @@ -56,7 +56,10 @@ extern unsigned int shmem_internal_rand_seed; extern hwloc_topology_t shmem_internal_topology; #endif -#define SHMEM_INTERNAL_HEAP_OVERHEAD (1024*1024) +#define SHMEM_INTERNAL_HEAP_OVERHEAD (10*1024*1024) +/* Note: this is an estimate */ +#define SHMEM_MAX_BOUNCE_BUFFER_OVERHEAD (2*1024*1024) + #define SHMEM_INTERNAL_DIAG_STRLEN 1024 #define SHMEM_INTERNAL_DIAG_WRAPLEN 72 diff --git a/src/symmetric_heap_c.c b/src/symmetric_heap_c.c index 30b319ea9..4da76c8e8 100644 --- a/src/symmetric_heap_c.c +++ b/src/symmetric_heap_c.c @@ -236,7 +236,8 @@ shmem_internal_symmetric_init(void) { /* add library overhead such that the max can be shmalloc()'ed */ shmem_internal_heap_length = shmem_internal_params.SYMMETRIC_SIZE + - SHMEM_INTERNAL_HEAP_OVERHEAD; + SHMEM_INTERNAL_HEAP_OVERHEAD + + SHMEM_MAX_BOUNCE_BUFFER_OVERHEAD; if (!shmem_internal_params.SYMMETRIC_HEAP_USE_MALLOC) { shmem_internal_heap_base = diff --git a/src/transport_ofi.c b/src/transport_ofi.c index f0bd68b8f..b3c2fe88b 100644 --- a/src/transport_ofi.c +++ b/src/transport_ofi.c @@ -103,7 +103,7 @@ long shmem_transport_ofi_get_poll_limit; size_t shmem_transport_ofi_max_buffered_send; size_t shmem_transport_ofi_max_msg_size; size_t shmem_transport_ofi_bounce_buffer_size; -long shmem_transport_ofi_max_bounce_buffers; +size_t shmem_transport_ofi_max_bounce_buffers; size_t shmem_transport_ofi_addrlen; #ifdef ENABLE_MR_RMA_EVENT int shmem_transport_ofi_mr_rma_event; @@ -1808,10 +1808,15 @@ static int shmem_transport_ofi_ctx_init(shmem_transport_ctx_t *ctx, int id) shmem_transport_ofi_bounce_buffer_size > 0 && shmem_transport_ofi_max_bounce_buffers > 0) { + size_t max_bb_pool_cnt = 0; + if (shmem_internal_params.BOUNCE_SHEAP) { + max_bb_pool_cnt = shmem_transport_ofi_max_bounce_buffers; + } ctx->bounce_buffers = shmem_free_list_init(sizeof(shmem_transport_ofi_bounce_buffer_t) + shmem_transport_ofi_bounce_buffer_size, - init_bounce_buffer); + init_bounce_buffer, + max_bb_pool_cnt); } else { ctx->options &= ~SHMEMX_CTX_BOUNCE_BUFFER; diff --git a/src/transport_ofi.h b/src/transport_ofi.h index d7d972fff..48cdeeafc 100644 --- a/src/transport_ofi.h +++ b/src/transport_ofi.h @@ -69,7 +69,7 @@ extern long shmem_transport_ofi_get_poll_limit; extern size_t shmem_transport_ofi_max_buffered_send; extern size_t shmem_transport_ofi_max_msg_size; extern size_t shmem_transport_ofi_bounce_buffer_size; -extern long shmem_transport_ofi_max_bounce_buffers; +extern size_t shmem_transport_ofi_max_bounce_buffers; extern pthread_mutex_t shmem_transport_ofi_progress_lock; @@ -436,7 +436,7 @@ void shmem_transport_ofi_drain_cq(shmem_transport_ctx_t *ctx) (shmem_transport_ofi_bounce_buffer_t *) frag); ctx->completed_bb_cntr++; } else { - RAISE_ERROR_STR("Unrecognized completion object"); + RAISE_ERROR_MSG("[%d] Unrecognized completion object %p %x mtofs %p\n", shmem_internal_my_pe, frag, frag->mytype, &frag->mytype); } } @@ -473,6 +473,10 @@ shmem_transport_ofi_bounce_buffer_t * create_bounce_buffer(shmem_transport_ctx_t if (NULL == buff) RAISE_ERROR_STR("Bounce buffer allocation failed"); + if (buff->frag.mytype != SHMEM_TRANSPORT_OFI_TYPE_BOUNCE) { + RAISE_ERROR_STR("Bounce buffer allocation failed"); + } + shmem_internal_assert(buff->frag.mytype == SHMEM_TRANSPORT_OFI_TYPE_BOUNCE); memcpy(buff->data, source, len); diff --git a/src/transport_portals4.c b/src/transport_portals4.c index f9298e7f9..695bc7519 100644 --- a/src/transport_portals4.c +++ b/src/transport_portals4.c @@ -446,11 +446,13 @@ shmem_transport_init(void) shmem_transport_portals4_bounce_buffers = shmem_free_list_init(sizeof(shmem_transport_portals4_bounce_buffer_t) + shmem_transport_portals4_bounce_buffer_size, - init_bounce_buffer); + init_bounce_buffer, + 0); shmem_transport_portals4_long_frags = shmem_free_list_init(sizeof(shmem_transport_portals4_long_frag_t), - init_long_frag); + init_long_frag, + 0); /* Initialize network */ ni_req_limits.max_entries = 1024; From db8cf246e0735d7547baff577d05567f1bc9486d Mon Sep 17 00:00:00 2001 From: Bryan Morgan Date: Tue, 12 May 2026 10:44:24 -0700 Subject: [PATCH 05/14] transport/ofi: zero max_buffered_send before fi_getinfo hint Let the provider advertise its natural inject_size rather than requiring it to be at least sizeof(long double). The returned inject_size is adopted immediately after fi_getinfo. --- src/transport_ofi.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/transport_ofi.c b/src/transport_ofi.c index b3c2fe88b..c0f417859 100644 --- a/src/transport_ofi.c +++ b/src/transport_ofi.c @@ -1470,7 +1470,9 @@ int query_for_fabric(struct fabric_info *info) struct fi_fabric_attr fabric_attr = {0}; struct fi_ep_attr ep_attr = {0}; - shmem_transport_ofi_max_buffered_send = sizeof(long double); + /* Hint 0 = no minimum inject requirement; provider returns its natural + * inject_size, which is adopted below after fi_getinfo. */ + shmem_transport_ofi_max_buffered_send = 0; fabric_attr.prov_name = info->prov_name; From 98af1b76627be3f833bd639a3d901f4d90b30372 Mon Sep 17 00:00:00 2001 From: Bryan Morgan Date: Thu, 30 Apr 2026 15:59:50 -0700 Subject: [PATCH 06/14] barrier: three-phase hierarchical barrier via XPMEM + NIC dissemination MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds --enable-hierarchical-barrier, a three-phase barrier that keeps intranode traffic off the NIC by using CPU atomics over XPMEM for gather/fanout and restricts NIC puts to the internode phase (node roots only). Phase 1 (intranode gather): local PEs signal up a k-ary tree. Each PE writes to its OWN up-slot in local_pSync; the parent reads each child's slot individually. Slots are padded to one cache line (HIER_SLOT_STRIDE=8 longs, 64 bytes) so no two PEs share a line, eliminating the MESI serialization that would occur if all children wrote to a single counter. Signal values increment monotonically via hier_sense, avoiding explicit slot resets between calls (sense alternation). Phase 2 (internode dissemination): node roots run a put-based binary dissemination across the NIC. After each round the slot is reset via a CPU store rather than a self-put, saving ceil(log2(N_nodes)) NIC round-trips per barrier (12 at 4096 nodes). Phase 3 (intranode fanout): node root CPU-stores an ack into each child's down-slot; children relay down the k-ary tree with reset-before-signal ordering. Down-slots are in the upper half of local_pSync, laid out with the same per-PE cache-line padding as up-slots. AUTO selection activates when local PE count >= SHMEM_HIER_BARRIER_THRESHOLD (default 2). Also selectable via SHMEM_BARRIER_ALGORITHM=hierarchical. New infrastructure: - src/shr_transport.h — XPMEM CPU pointer mapping; self-access returns the address directly without an XPMEM lookup - src/runtime_util.c — global hostname exchange so each PE can identify its node root - configure.ac — --enable-hierarchical-barrier requires --with-xpmem and a network transport --- configure.ac | 38 ++++ src/collectives.c | 410 +++++++++++++++++++++++++++++++++++++++- src/init.c | 18 ++ src/runtime-mpi.c | 59 ++++++ src/runtime-pmi.c | 47 +++++ src/runtime-pmi2.c | 49 +++++ src/runtime-pmix.c | 71 +++++-- src/runtime.h | 6 + src/runtime_util.c | 65 +++++++ src/shmem_collectives.h | 50 ++++- src/shmem_comm.h | 29 ++- src/shmem_env_defs.h | 6 +- src/shmem_internal.h | 11 ++ src/shr_transport.h4 | 16 +- src/transport_ofi.c | 4 + 15 files changed, 854 insertions(+), 25 deletions(-) diff --git a/configure.ac b/configure.ac index 19f854db0..197072c1a 100644 --- a/configure.ac +++ b/configure.ac @@ -327,6 +327,13 @@ AC_ARG_ENABLE([shr-atomics], AS_IF([test "$enable_shr_atomics" = "yes"], [AC_DEFINE([USE_SHR_ATOMICS], [1], [If defined, the shared memory layer will perform processor atomics.])]) +AC_ARG_ENABLE([hierarchical-barrier], + [AS_HELP_STRING([--enable-hierarchical-barrier], + [Enable the hierarchical barrier: intranode phase uses CPU atomics, + internode phase uses NIC puts. Requires an on-node transport (XPMEM + or CMA) and a network transport (OFI, Portals4, or UCX). + (default: disabled)])]) + AC_ARG_ENABLE([manpages], [AS_HELP_STRING([--enable-manpages], [Include man pages in the installation (default:disabled)])]) @@ -550,6 +557,36 @@ if test "$enable_shr_atomics" != "no" -a "$transport_xpmem" = "yes" -a "$transpo AC_DEFINE([USE_SHR_ATOMICS], [1], [If defined, the shared memory layer will perform processor atomics.]) fi +dnl Hierarchical barrier: only when explicitly requested. Validate that an +dnl on-node transport (XPMEM or CMA) and a network transport are both present; +dnl error if dependencies are missing. +transport_hierarchical_barrier="no" +if test "$enable_hierarchical_barrier" = "yes"; then + dnl Check on-node transport dependency + if test "$transport_xpmem" != "yes" -a "$transport_cma" != "yes"; then + AC_MSG_ERROR([--enable-hierarchical-barrier requires an on-node transport; configure with --with-xpmem or --with-cma]) + dnl Check network transport dependency + elif test "$transport_ofi" != "yes" -a "$transport_portals4" != "yes" -a "$transport_ucx" != "yes"; then + AC_MSG_ERROR([--enable-hierarchical-barrier requires a network transport; configure with --with-ofi, --with-portals4, or --with-ucx]) + else + transport_hierarchical_barrier="yes" + fi +fi +if test "$transport_hierarchical_barrier" = "yes"; then + AC_DEFINE([USE_HIERARCHICAL_BARRIER], [1], + [If defined, enables the hierarchical barrier: intranode CPU atomics + internode NIC puts.]) + dnl The hierarchical barrier uses shared-memory puts/gets (XPMEM/CMA) for + dnl intranode data movement. USE_SHR_ATOMICS is defined so that the + dnl shmem_shr_transport_atomic() function bodies are compiled; the routing + dnl logic in shmem_shr_transport_use_atomic() gates them behind a runtime + dnl check (shr_size == num_pes), so user-visible AMOs only go through CPU + dnl atomics when all PEs are on one node. In the multi-node case they + dnl always use the NIC, preserving the single-coherency-domain invariant. + AC_DEFINE([USE_SHR_ATOMICS], [1], + [If defined, the shared memory layer will perform processor atomics.]) + transport_shr_atomics="yes" +fi + AC_ARG_ENABLE([pmi-simple], [AS_HELP_STRING([--enable-pmi-simple], [Use MPICH simple PMI-1 library for process management])]) AC_ARG_ENABLE([pmi-mpi], [AS_HELP_STRING([--enable-pmi-mpi], @@ -1049,6 +1086,7 @@ echo " XPMEM: $transport_xpmem" echo " CMA: $transport_cma" echo " memcpy (self): $transport_memcpy" echo " Shr. atomics: $transport_shr_atomics" +echo " Hier. barrier: $transport_hierarchical_barrier" echo "" echo "Global Options:" if test "$enable_remote_virtual_addressing" = "yes"; then diff --git a/src/collectives.c b/src/collectives.c index 89c9bd273..b3964c952 100644 --- a/src/collectives.c +++ b/src/collectives.c @@ -15,6 +15,7 @@ #include "config.h" #include +#include #define SHMEM_INTERNAL_INCLUDE #include "shmem.h" @@ -30,13 +31,53 @@ coll_type_t shmem_internal_collect_type = AUTO; coll_type_t shmem_internal_fcollect_type = AUTO; long *shmem_internal_barrier_all_psync; long *shmem_internal_sync_all_psync; +#ifdef USE_HIERARCHICAL_BARRIER +long *shmem_internal_barrier_all_local_psync; +long *shmem_internal_sync_all_local_psync; +long *shmem_internal_hierarchical_local_psync; + +/* Layout of local_pSync — two cache-line-padded arrays, one slot per PE: + * + * up-slot for PE r: local_pSync[r * HIER_SLOT_STRIDE] + * down-slot for PE r: local_pSync[shr_size * HIER_SLOT_STRIDE + r * HIER_SLOT_STRIDE] + * + * Phase 1 (gather): each PE writes signal to its OWN up-slot once it has + * collected all its children; parent reads children's up-slots. + * One writer per slot — no MESI contention on the parent's cache line. + * + * Phase 3 (fanout): parent writes signal to each child's down-slot. + * One writer per slot — same property. + * + * Sense-alternation (hier_sense) avoids explicit resets between calls. + * + * Total allocation: 2 * shr_size * HIER_SLOT_STRIDE longs. */ +#define HIER_SLOT_STRIDE 8 /* 8 longs = 64 bytes = 1 cache line */ + +static long hier_sense = 0; + +/* Per-phase timing accumulators (root PE: all three phases; + * non-root PEs: phase1_us = gather wait, phase3_us = fanout wait). */ +static double hier_phase1_us = 0.0; +static double hier_phase2_us = 0.0; +static double hier_phase3_us = 0.0; +static long hier_call_count = 0; + +static inline double +hier_now_us(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ts.tv_sec * 1e6 + ts.tv_nsec * 1e-3; +} +#endif char *coll_type_str[] = { "AUTO", "LINEAR", "TREE", "DISSEM", "RING", - "RECDBL" }; + "RECDBL", + "HIERARCHICAL" }; static int *full_tree_children; static int full_tree_num_children; @@ -134,6 +175,44 @@ shmem_internal_collectives_init(void) for (i = 0; i < SHMEM_BARRIER_SYNC_SIZE; i++) shmem_internal_sync_all_psync[i] = SHMEM_SYNC_VALUE; +#ifdef USE_HIERARCHICAL_BARRIER + /* Allocate local (intranode, CPU-atomic) pSync arrays for the hierarchical + * barrier. Each PE owns HIER_SLOT_STRIDE longs (one cache line) within + * the array, indexed by shr_rank. The stride prevents false sharing: + * PE r's slot is at local_pSync[r * HIER_SLOT_STRIDE]. + * These arrays are touched only by on-node CPU stores/loads via XPMEM; + * the global pSync arrays above are touched only by NIC puts. */ + int shr_size = shmem_internal_get_shr_size(); + int local_psync_len = 2 * shr_size * HIER_SLOT_STRIDE; + + shmem_internal_barrier_all_local_psync = + shmem_internal_shmalloc(sizeof(long) * local_psync_len); + if (NULL == shmem_internal_barrier_all_local_psync) return -1; + + for (i = 0; i < local_psync_len; i++) { + shmem_internal_barrier_all_local_psync[i] = SHMEM_SYNC_VALUE; + } + + shmem_internal_sync_all_local_psync = + shmem_internal_shmalloc(sizeof(long) * local_psync_len); + if (NULL == shmem_internal_sync_all_local_psync) return -1; + + for (i = 0; i < local_psync_len; i++) { + shmem_internal_sync_all_local_psync[i] = SHMEM_SYNC_VALUE; + } + + /* Shared local pSync for general barriers/syncs. Safe to share because + * barriers are serialized — a PE cannot enter a new barrier until it has + * exited the previous one. */ + shmem_internal_hierarchical_local_psync = + shmem_internal_shmalloc(sizeof(long) * local_psync_len); + if (NULL == shmem_internal_hierarchical_local_psync) return -1; + + for (i = 0; i < local_psync_len; i++) { + shmem_internal_hierarchical_local_psync[i] = SHMEM_SYNC_VALUE; + } +#endif + /* initialize the binomial tree for collective operations over entire tree */ full_tree_num_children = 0; @@ -176,6 +255,10 @@ shmem_internal_collectives_init(void) shmem_internal_barrier_type = TREE; } else if (0 == strcmp(type, "dissem")) { shmem_internal_barrier_type = DISSEM; +#ifdef USE_HIERARCHICAL_BARRIER + } else if (0 == strcmp(type, "hierarchical")) { + shmem_internal_barrier_type = HIERARCHICAL; +#endif } else { RAISE_WARN_MSG("Ignoring bad barrier algorithm '%s'\n", type); } @@ -420,6 +503,331 @@ shmem_internal_sync_dissem(int PE_start, int PE_stride, int PE_size, long *pSync } +/***************************************** + * + * HIERARCHICAL BARRIER/SYNC + * + * Three-phase algorithm for USE_HIERARCHICAL_BARRIER builds: + * + * Phase 1 (intranode gather, CPU stores/loads via XPMEM): + * All local PEs run an intranode dissemination barrier using plain + * (non-atomic) stores and acquire-loads. Each PE owns one cache-line-padded + * slot (HIER_SLOT_STRIDE longs apart) so no two PEs share a cache line. + * Sense alternation (hier_sense) removes the need for explicit slot resets. + * ceil(log2(local_count)) rounds; each round: store signal to partner's slot, + * spin-load own slot until partner's signal arrives. + * + * Phase 2 (internode, NIC puts, root PEs only): + * Root PEs run a dissemination barrier among themselves using NIC puts + * across per-round pSync slots (ceil(log2(N)) rounds). A shmem_quiet + * after the last round ensures all outbound puts are retired before + * phase 3. + * + * Phase 3 (intranode fanout, CPU stores/loads via XPMEM): + * Same dissemination algorithm as phase 1, run in reverse sense so that the + * "all clear" propagates back to all local PEs without NIC involvement. + * + *****************************************/ +#ifdef USE_HIERARCHICAL_BARRIER + +/* Count and list the PEs in the active set that reside on this node. */ +static void +shmem_internal_build_local_set(int PE_start, int PE_stride, int PE_size, + int *local_pes, int *local_count) +{ + int i, pe; + *local_count = 0; + + for (i = 0, pe = PE_start; i < PE_size; i++, pe += PE_stride) { + if (shmem_internal_get_shr_rank(pe) != -1) { + local_pes[(*local_count)++] = pe; + } + } +} + +/* Build an active set of root PEs (the lowest-ranked PE on each node) from + * the original active set. shmem_runtime_get_node_rank() only returns the + * node-local rank for on-node PEs; for off-node PEs it returns -1. We + * therefore use shmem_runtime_is_node_root_pe() which reflects each PE's + * absolute rank within its own node, computed during init via global exchange. + */ +static void +shmem_internal_build_root_active_set(int PE_start, int PE_stride, int PE_size, + int *root_pes, int *root_count) +{ + int i, pe; + *root_count = 0; + + for (i = 0, pe = PE_start; i < PE_size; i++, pe += PE_stride) { + if (shmem_runtime_is_node_root_pe(pe)) { + root_pes[(*root_count)++] = pe; + } + } +} + +/* CPU atomic load of the long at `target` via local mapped pointer. */ +static inline long +shmem_internal_cpu_atomic_load_long(long *target, int noderank) +{ + void *remote_ptr; + long val; + shmem_shr_transport_ptr(target, noderank, &remote_ptr); + __atomic_load((long *)remote_ptr, &val, __ATOMIC_ACQUIRE); + return val; +} + +/* CPU atomic store of `val` to the long at `target` via local mapped pointer. */ +static inline void +shmem_internal_cpu_atomic_store_long(long *target, int noderank, long val) +{ + void *remote_ptr; + shmem_shr_transport_ptr(target, noderank, &remote_ptr); + __atomic_store((long *)remote_ptr, &val, __ATOMIC_RELEASE); +} + +void +shmem_internal_sync_hierarchical(int PE_start, int PE_stride, int PE_size, + long *pSync, long *local_pSync) +{ + int node_root_pe = shmem_internal_get_node_root_pe(); + int is_root = (shmem_internal_my_pe == node_root_pe); + int my_shr_rank = shmem_runtime_get_node_rank(shmem_internal_my_pe); + long one = 1; + + if (PE_size == 1) return; + + /* Collect local and root PE sets for this active set */ + int *local_pes = alloca(sizeof(int) * PE_size); + int local_count = 0; + shmem_internal_build_local_set(PE_start, PE_stride, PE_size, + local_pes, &local_count); + + int *root_pes = alloca(sizeof(int) * PE_size); + int root_count = 0; + shmem_internal_build_root_active_set(PE_start, PE_stride, PE_size, + root_pes, &root_count); + + /* Assign virtual indices 0..local_count-1, rotating so node_root_pe = vidx 0. + * Precompute tree parent shr_rank and children shr_ranks. */ + int my_local_idx = -1; + int root_local_idx = 0; + for (int i = 0; i < local_count; i++) { + if (local_pes[i] == shmem_internal_my_pe) my_local_idx = i; + if (local_pes[i] == node_root_pe) root_local_idx = i; + } + int my_vidx = (my_local_idx >= 0) + ? (my_local_idx - root_local_idx + local_count) % local_count + : -1; + + int tree_parent_shr = -1; + int tree_nchildren = 0; + int *tree_child_shr = alloca(sizeof(int) * tree_radix); + if (my_vidx >= 0) { + if (my_vidx > 0) { + tree_parent_shr = shmem_runtime_get_node_rank( + local_pes[((my_vidx - 1) / tree_radix + root_local_idx) % local_count]); + } + for (int j = 1; j <= tree_radix; j++) { + int cv = my_vidx * tree_radix + j; + if (cv < local_count) { + tree_child_shr[tree_nchildren++] = shmem_runtime_get_node_rank( + local_pes[(cv + root_local_idx) % local_count]); + } + } + } + + /* Sense-alternating signal — monotonically increasing, no slot resets needed. + * up-slot for PE r: local_pSync[r * HIER_SLOT_STRIDE] + * down-slot for PE r: local_pSync[shr_size * HIER_SLOT_STRIDE + r * HIER_SLOT_STRIDE] */ + long signal = SHMEM_SYNC_VALUE + 1 + hier_sense; + int shr_size = shmem_internal_get_shr_size(); + long *up_pSync = local_pSync; + long *down_pSync = local_pSync + (long)(shr_size * HIER_SLOT_STRIDE); + + /* Resolve own up-slot and down-slot mapped pointers once. */ + void *my_up_raw, *my_down_raw; + shmem_shr_transport_ptr(&up_pSync[my_shr_rank * HIER_SLOT_STRIDE], my_shr_rank, &my_up_raw); + shmem_shr_transport_ptr(&down_pSync[my_shr_rank * HIER_SLOT_STRIDE], my_shr_rank, &my_down_raw); + volatile long *my_up_slot = (volatile long *)my_up_raw; + volatile long *my_down_slot = (volatile long *)my_down_raw; + + /* ---- Degenerate case: all active PEs on one node ---- */ + if (root_count <= 1 || local_count == PE_size) { + if (my_vidx < 0) return; + + double t0 = shmem_internal_params.HIER_BARRIER_DEBUG ? hier_now_us() : 0.0; + + /* Phase 1: gather up tree — wait for children's up-slots, then signal parent */ + for (int c = 0; c < tree_nchildren; c++) { + void *child_up_raw; + shmem_shr_transport_ptr(&up_pSync[tree_child_shr[c] * HIER_SLOT_STRIDE], + tree_child_shr[c], &child_up_raw); + volatile long *child_up = (volatile long *)child_up_raw; + long cur; + do { + __atomic_load(child_up, &cur, __ATOMIC_ACQUIRE); + if (cur != signal) { SPINLOCK_BODY(); } + } while (cur != signal); + } + if (my_vidx > 0) { + void *parent_up_raw; + shmem_shr_transport_ptr(&up_pSync[my_shr_rank * HIER_SLOT_STRIDE], + my_shr_rank, &parent_up_raw); + __atomic_store((long *)parent_up_raw, &signal, __ATOMIC_RELEASE); + } + + double t1 = shmem_internal_params.HIER_BARRIER_DEBUG ? hier_now_us() : 0.0; + + /* Phase 3: fanout — root stores to children's down-slots, non-roots wait then relay */ + if (my_vidx == 0) { + for (int c = 0; c < tree_nchildren; c++) { + shmem_internal_cpu_atomic_store_long( + &down_pSync[tree_child_shr[c] * HIER_SLOT_STRIDE], + tree_child_shr[c], signal); + } + } else { + long cur; + do { + __atomic_load(my_down_slot, &cur, __ATOMIC_ACQUIRE); + if (cur != signal) { SPINLOCK_BODY(); } + } while (cur != signal); + for (int c = 0; c < tree_nchildren; c++) { + shmem_internal_cpu_atomic_store_long( + &down_pSync[tree_child_shr[c] * HIER_SLOT_STRIDE], + tree_child_shr[c], signal); + } + } + + if (shmem_internal_params.HIER_BARRIER_DEBUG) { + double t2 = hier_now_us(); + hier_phase1_us += t1 - t0; + hier_phase3_us += t2 - t1; + hier_call_count++; + } + + hier_sense++; + return; + } + + /* ---- Normal multi-node case ---- */ + + double mn_t0 = shmem_internal_params.HIER_BARRIER_DEBUG ? hier_now_us() : 0.0; + + /* Phase 1: intranode gather (k-ary tree, bottom-up). + * Each PE waits on each child's up-slot, then writes its own up-slot. + * One writer per up-slot — no cache-line contention on the parent. */ + if (my_vidx >= 0) { + for (int c = 0; c < tree_nchildren; c++) { + void *child_up_raw; + shmem_shr_transport_ptr(&up_pSync[tree_child_shr[c] * HIER_SLOT_STRIDE], + tree_child_shr[c], &child_up_raw); + volatile long *child_up = (volatile long *)child_up_raw; + long cur; + do { + __atomic_load(child_up, &cur, __ATOMIC_ACQUIRE); + if (cur != signal) { SPINLOCK_BODY(); } + } while (cur != signal); + } + if (my_vidx > 0) { + /* Signal parent by writing to OWN up-slot (parent reads it). */ + __atomic_store((long *)my_up_raw, &signal, __ATOMIC_RELEASE); + } + } + + double mn_t1 = shmem_internal_params.HIER_BARRIER_DEBUG ? hier_now_us() : 0.0; + + if (is_root) { + /* ---- Phase 2: internode barrier (NIC puts, root PEs only) ---- */ + if (root_count > 1) { + int my_root_idx = -1; + for (int i = 0; i < root_count; i++) { + if (root_pes[i] == shmem_internal_my_pe) { my_root_idx = i; break; } + } + shmem_internal_assert(my_root_idx >= 0); + + int num_rounds = 0; + { int n = root_count - 1; while (n > 0) { n >>= 1; num_rounds++; } } + shmem_internal_assert(num_rounds <= SHMEM_BARRIER_SYNC_SIZE); + + for (int r = 0; r < num_rounds; r++) { + int partner_idx = (my_root_idx + (1 << r)) % root_count; + int partner_pe = root_pes[partner_idx]; + shmem_internal_put_scalar(SHMEM_CTX_DEFAULT, &pSync[r], &one, + sizeof(one), partner_pe); + SHMEM_WAIT(&pSync[r], SHMEM_SYNC_VALUE); + __atomic_store_n(&pSync[r], SHMEM_SYNC_VALUE, __ATOMIC_RELEASE); + } + } + + shmem_internal_quiet(SHMEM_CTX_DEFAULT); + + double mn_t2 = shmem_internal_params.HIER_BARRIER_DEBUG ? hier_now_us() : 0.0; + + /* ---- Phase 3: intranode fanout (k-ary tree, top-down via down-slots) ---- */ + for (int c = 0; c < tree_nchildren; c++) { + shmem_internal_cpu_atomic_store_long( + &down_pSync[tree_child_shr[c] * HIER_SLOT_STRIDE], + tree_child_shr[c], signal); + } + + if (shmem_internal_params.HIER_BARRIER_DEBUG) { + double mn_t3 = hier_now_us(); + hier_phase1_us += mn_t1 - mn_t0; + hier_phase2_us += mn_t2 - mn_t1; + hier_phase3_us += mn_t3 - mn_t2; + hier_call_count++; + } + + } else if (my_vidx >= 0) { + /* Non-root: wait on own down-slot for parent's signal, then relay to children. */ + long cur; + do { + __atomic_load(my_down_slot, &cur, __ATOMIC_ACQUIRE); + if (cur != signal) { SPINLOCK_BODY(); } + } while (cur != signal); + + for (int c = 0; c < tree_nchildren; c++) { + shmem_internal_cpu_atomic_store_long( + &down_pSync[tree_child_shr[c] * HIER_SLOT_STRIDE], + tree_child_shr[c], signal); + } + + if (shmem_internal_params.HIER_BARRIER_DEBUG) { + double mn_t3 = hier_now_us(); + hier_phase1_us += mn_t1 - mn_t0; + hier_phase3_us += mn_t3 - mn_t1; + hier_call_count++; + } + } + + if (my_vidx >= 0) { hier_sense++; } + /* PEs absent from local_pes (my_vidx < 0) fall through silently. */ +} + +void +shmem_internal_hier_barrier_print_stats(void) +{ + if (!shmem_internal_params.HIER_BARRIER_DEBUG) return; + if (hier_call_count == 0) return; + + /* Each PE prints its own per-phase averages. At scale the output can be + * large; the caller should fence/barrier before this so the lines don't + * interleave badly, but we keep this simple intentionally. */ + fprintf(stderr, + "[PE %d] hier_barrier calls=%ld " + "phase1(gather)=%.2f us phase2(internode)=%.2f us " + "phase3(fanout)=%.2f us total=%.2f us\n", + shmem_internal_my_pe, + hier_call_count, + hier_phase1_us / hier_call_count, + hier_phase2_us / hier_call_count, + hier_phase3_us / hier_call_count, + (hier_phase1_us + hier_phase2_us + hier_phase3_us) / hier_call_count); +} + +#endif /* USE_HIERARCHICAL_BARRIER */ + + /***************************************** * * BROADCAST diff --git a/src/init.c b/src/init.c index 2a9189914..dd7ce3815 100644 --- a/src/init.c +++ b/src/init.c @@ -147,6 +147,10 @@ shmem_internal_shutdown(void) shmem_internal_finalized = 1; +#ifdef USE_HIERARCHICAL_BARRIER + shmem_internal_hier_barrier_print_stats(); +#endif + shmem_internal_team_fini(); shmem_transport_fini(); @@ -524,6 +528,20 @@ shmem_internal_heap_postinit(void) atexit(shmem_internal_shutdown_atexit); shmem_internal_initialized = 1; +#ifdef USE_HIERARCHICAL_BARRIER + if (shmem_internal_my_pe == 0) { + const char *effective_barrier = + (shmem_internal_barrier_type == AUTO && + shmem_internal_get_shr_size() >= shmem_internal_params.HIER_BARRIER_THRESHOLD) + ? "HIERARCHICAL (auto-selected)" + : coll_type_str[shmem_internal_barrier_type]; + + DEBUG_MSG("Hierarchical barrier enabled: intranode CPU atomics + internode NIC puts\n" + RAISE_PE_PREFIX "Barrier algorithm: %s\n", + shmem_internal_my_pe, effective_barrier); + } +#endif + /* finish up */ #ifndef USE_PMIX shmem_runtime_barrier(); diff --git a/src/runtime-mpi.c b/src/runtime-mpi.c index 3713ec3fd..606c44a31 100644 --- a/src/runtime-mpi.c +++ b/src/runtime-mpi.c @@ -38,6 +38,7 @@ static int kv_length = 0; static int initialized_mpi = 0; static int node_size; static int *node_ranks; +static int *is_node_root = NULL; char* kv_store_me; char* kv_store_all; @@ -103,6 +104,10 @@ shmem_runtime_init(int enable_node_ranks) if (size > 1 && enable_node_ranks) { node_ranks = malloc(size * sizeof(int)); if (NULL == node_ranks) return 8; +#ifdef USE_HIERARCHICAL_BARRIER + is_node_root = malloc(size * sizeof(int)); + if (NULL == is_node_root) return 9; +#endif } return 0; @@ -117,6 +122,7 @@ shmem_runtime_fini(void) if (node_ranks) { MPI_Comm_free(&SHMEM_RUNTIME_SHARED); free(node_ranks); + free(is_node_root); } MPI_Comm_free(&SHMEM_RUNTIME_WORLD); @@ -202,11 +208,44 @@ shmem_runtime_get_node_size(void) return node_size; } +int +shmem_runtime_get_node_root_pe(void) +{ + int i; + + if (size == 1) { + return 0; + } + + for (i = 0; i < size; i++) { + if (node_ranks[i] == 0) + return i; + } + + /* Should not be reached */ + return 0; +} + +int +shmem_runtime_is_node_root_pe(int pe) +{ + shmem_internal_assert(pe < size && pe >= 0); + + if (size == 1) + return 1; + + if (NULL == is_node_root) + return 0; + + return is_node_root[pe]; +} + int shmem_runtime_exchange(void) { if (size == 1) { kv_store_all = kv_store_me; + if (is_node_root) is_node_root[0] = 1; return 0; } @@ -231,6 +270,26 @@ shmem_runtime_exchange(void) MPI_Group_free(&world_group); MPI_Group_free(&node_group); free(world_ranks); + +#ifdef USE_HIERARCHICAL_BARRIER + /* Exchange absolute node ranks to identify node roots across all nodes. + * node_ranks[pe] is the rank of PE pe in the CALLING PE's node group; + * for off-node PEs it is MPI_UNDEFINED. We need each PE's rank in its + * OWN node group, so gather those now. */ + int my_node_rank; + MPI_Comm_rank(SHMEM_RUNTIME_SHARED, &my_node_rank); + + int *abs_node_ranks = malloc(size * sizeof(int)); + if (NULL == abs_node_ranks) return 2; + + MPI_Allgather(&my_node_rank, 1, MPI_INT, + abs_node_ranks, 1, MPI_INT, SHMEM_RUNTIME_WORLD); + + for (int i = 0; i < size; i++) + is_node_root[i] = (abs_node_ranks[i] == 0) ? 1 : 0; + + free(abs_node_ranks); +#endif } int chunkSize = kv_length * sizeof(char) * MAX_KV_LENGTH; diff --git a/src/runtime-pmi.c b/src/runtime-pmi.c index 07805a2cc..dc884e183 100644 --- a/src/runtime-pmi.c +++ b/src/runtime-pmi.c @@ -38,6 +38,7 @@ static char *kvs_name, *kvs_key, *kvs_value; static int max_name_len, max_key_len, max_val_len; static int initialized_pmi = 0; static int *location_array = NULL; +static int *is_node_root = NULL; #define SINGLETON_KEY_LEN 128 #define SINGLETON_VAL_LEN 1024 @@ -96,6 +97,10 @@ shmem_runtime_init(int enable_node_ranks) if (enable_node_ranks) { location_array = malloc(sizeof(int) * size); if (NULL == location_array) return 10; +#ifdef USE_HIERARCHICAL_BARRIER + is_node_root = malloc(sizeof(int) * size); + if (NULL == is_node_root) return 11; +#endif } } else { @@ -120,6 +125,7 @@ int shmem_runtime_fini(void) { free(location_array); + free(is_node_root); free(kvs_name); free(kvs_key); free(kvs_value); @@ -194,6 +200,40 @@ shmem_runtime_get_node_size(void) } +int +shmem_runtime_get_node_root_pe(void) +{ + int i; + + if (size == 1) { + return 0; + } + + for (i = 0; i < size; i++) { + if (location_array[i] == 0) + return i; + } + + /* Should not be reached */ + return 0; +} + + +int +shmem_runtime_is_node_root_pe(int pe) +{ + shmem_internal_assert(pe < size && pe >= 0); + + if (size == 1) + return 1; + + if (NULL == is_node_root) + return 0; + + return is_node_root[pe]; +} + + int shmem_runtime_exchange(void) { @@ -226,6 +266,13 @@ shmem_runtime_exchange(void) return 7; } } + if (is_node_root) { + ret = shmem_runtime_util_populate_global_node_roots(is_node_root, size); + if (0 != ret) { + RETURN_ERROR_MSG("Global node root mapping failed (%d)\n", ret); + return 8; + } + } return 0; } diff --git a/src/runtime-pmi2.c b/src/runtime-pmi2.c index 4460e5625..75efda9bf 100644 --- a/src/runtime-pmi2.c +++ b/src/runtime-pmi2.c @@ -42,6 +42,7 @@ static char *kvs_name, *kvs_key, *kvs_value; static int max_name_len, max_key_len, max_val_len; static int initialized_pmi = 0; static int *location_array = NULL; +static int *is_node_root = NULL; int @@ -77,6 +78,10 @@ shmem_runtime_init(int enable_node_ranks) if (enable_node_ranks) { location_array = malloc(sizeof(int) * size); if (NULL == location_array) return 8; +#ifdef USE_HIERARCHICAL_BARRIER + is_node_root = malloc(sizeof(int) * size); + if (NULL == is_node_root) return 9; +#endif } return 0; @@ -89,6 +94,9 @@ shmem_runtime_fini(void) if (location_array) { free(location_array); } + if (is_node_root) { + free(is_node_root); + } if (initialized_pmi == 1) { PMI2_Finalize(); @@ -145,6 +153,40 @@ shmem_runtime_get_node_size(void) } +int +shmem_runtime_get_node_root_pe(void) +{ + int i; + + if (size == 1) { + return 0; + } + + for (i = 0; i < size; i++) { + if (location_array[i] == 0) + return i; + } + + /* Should not be reached */ + return 0; +} + + +int +shmem_runtime_is_node_root_pe(int pe) +{ + shmem_internal_assert(pe < size && pe >= 0); + + if (size == 1) + return 1; + + if (NULL == is_node_root) + return 0; + + return is_node_root[pe]; +} + + int shmem_runtime_exchange(void) { @@ -169,6 +211,13 @@ shmem_runtime_exchange(void) return 7; } } + if (is_node_root) { + ret = shmem_runtime_util_populate_global_node_roots(is_node_root, size); + if (0 != ret) { + RETURN_ERROR_MSG("Global node root mapping failed (%d)\n", ret); + return 8; + } + } return 0; } diff --git a/src/runtime-pmix.c b/src/runtime-pmix.c index 1a0f043b1..7c904a083 100644 --- a/src/runtime-pmix.c +++ b/src/runtime-pmix.c @@ -37,6 +37,7 @@ static pmix_proc_t myproc; static uint32_t size; static uint32_t node_size = 0; static int *node_ranks = NULL; +static int *is_node_root = NULL; int shmem_runtime_init(int enable_node_ranks) @@ -69,6 +70,13 @@ shmem_runtime_init(int enable_node_ranks) RETURN_ERROR_MSG_PREINIT("Out of memory allocating node_ranks\n"); return 1; } +#ifdef USE_HIERARCHICAL_BARRIER + is_node_root = (int *)malloc(size * sizeof(int)); + if (NULL == is_node_root) { + RETURN_ERROR_MSG_PREINIT("Out of memory allocating is_node_root\n"); + return 2; + } +#endif } return PMIX_SUCCESS; @@ -82,6 +90,8 @@ shmem_runtime_fini(void) if (node_ranks) free(node_ranks); + if (is_node_root) + free(is_node_root); if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) { RETURN_ERROR_MSG_PREINIT("PMIx_Finalize failed (%d)\n", rc); @@ -143,13 +153,36 @@ shmem_runtime_get_node_size(void) return (int) node_size; } -// static void opcbfunc(pmix_status_t status, void *cbdata) -// { -// bool *active = (bool*)cbdata; -// fprintf(stderr, "%s:%d completed fence_nb", myproc.nspace, myproc.rank); -// *active = false; -// } +int +shmem_runtime_get_node_root_pe(void) +{ + int i; + + for (i = 0; i < (int) size; i++) { + if (node_ranks[i] == 0) + return i; + } + + /* Should not be reached */ + return 0; +} + + +int +shmem_runtime_is_node_root_pe(int pe) +{ + shmem_internal_assert(pe < (int) size && pe >= 0); + + if (size == 1) + return 1; + + if (NULL == is_node_root) + return 0; + + return is_node_root[pe]; +} + int shmem_runtime_exchange(void) @@ -157,7 +190,6 @@ shmem_runtime_exchange(void) pmix_status_t rc; pmix_info_t info; bool wantit=true; - //bool active = true; if (node_ranks) { pmix_proc_t proc; @@ -198,6 +230,16 @@ shmem_runtime_exchange(void) } else { RETURN_ERROR_MSG_PREINIT("PMIX_LOCAL_PEERS is not properly initiated (%d)\n", rc); } + + /* Publish hostname so shmem_runtime_util_populate_global_node_roots can + * determine which PEs are node roots across all nodes. */ + if (is_node_root) { + int ret = shmem_runtime_util_put_hostname(); + if (ret != 0) { + RETURN_ERROR_MSG("PMIx hostname put failed (%d)\n", ret); + return ret; + } + } } /* commit any values we "put" */ @@ -206,21 +248,22 @@ shmem_runtime_exchange(void) return rc; } - /* execute a fence, directing that all info be exchanged */ PMIX_INFO_CONSTRUCT(&info); PMIX_INFO_LOAD(&info, PMIX_COLLECT_DATA, &wantit, PMIX_BOOL); - // Future optimization for when fabrics are ready to support the non-block- - // ing capabilities. The commented out call function above, the commented - // variable "bool active," and the PMIx_Fence_nb if-statement are here for - // when that is ready. Current implementations will cause the test to hang. - - // if (PMIX_SUCCESS != (rc = PMIx_Fence_nb(NULL, 0, &info, 1, opcbfunc, &active))) { if (PMIX_SUCCESS != (rc = PMIx_Fence(NULL, 0, &info, 1))) { RETURN_ERROR_MSG("PMIx_Fence failed (%d)\n", rc); } PMIX_INFO_DESTRUCT(&info); + if (is_node_root) { + int ret = shmem_runtime_util_populate_global_node_roots(is_node_root, (int) size); + if (ret != 0) { + RETURN_ERROR_MSG("Global node root mapping failed (%d)\n", ret); + return ret; + } + } + return rc; } diff --git a/src/runtime.h b/src/runtime.h index ee0c65de2..ab62079c3 100644 --- a/src/runtime.h +++ b/src/runtime.h @@ -31,6 +31,7 @@ int shmem_runtime_get_size(void); * (use shmem_internal_get_shr_rank/size for such queries). */ int shmem_runtime_get_node_rank(int pe); int shmem_runtime_get_node_size(void); +int shmem_runtime_get_node_root_pe(void); int shmem_runtime_exchange(void); int shmem_runtime_put(char *key, void *value, size_t valuelen); @@ -41,6 +42,11 @@ void shmem_runtime_barrier(void); /* Utility functions used to implement the runtime layer */ int shmem_runtime_util_put_hostname(void); int shmem_runtime_util_populate_node(int *location_array, int size, int *node_size); +int shmem_runtime_util_populate_global_node_roots(int *is_node_root, int size); + +/* Returns 1 if PE pe is the lowest-ranked PE on its own node, 0 otherwise. + * Valid after shmem_runtime_exchange(). */ +int shmem_runtime_is_node_root_pe(int pe); int shmem_runtime_util_encode(const void *inval, int invallen, char *outval, int outvallen); int shmem_runtime_util_decode(const char *inval, void *outval, size_t outvallen); diff --git a/src/runtime_util.c b/src/runtime_util.c index bd9ccc992..e53831eec 100644 --- a/src/runtime_util.c +++ b/src/runtime_util.c @@ -164,3 +164,68 @@ int shmem_runtime_util_populate_node(int *location_array, int size, int *node_si return 0; } + + +/* Populate is_node_root[pe] = 1 if PE pe is the lowest-ranked (first) PE on + * its node, 0 otherwise. This determines which PEs act as internode + * representatives in the hierarchical barrier. + * + * Must be called after shmem_runtime_util_put_hostname and a runtime exchange, + * so that every PE's hostname is readable via shmem_runtime_get. */ +int +shmem_runtime_util_populate_global_node_roots(int *is_node_root, int size) +{ + int ret = 0; + char **hostnames = (char **) malloc(size * sizeof(char *)); + size_t *hlens = (size_t *) malloc(size * sizeof(size_t)); + + if (!hostnames || !hlens) { + free(hostnames); + free(hlens); + RETURN_ERROR_MSG("Out of memory allocating hostname arrays\n"); + return 1; + } + + for (int pe = 0; pe < size; pe++) { + hostnames[pe] = NULL; + } + + for (int pe = 0; pe < size; pe++) { + ret = shmem_runtime_get(pe, "hostname_len", &hlens[pe], sizeof(size_t)); + if (ret != 0) { + RETURN_ERROR_MSG("Failed to get hostname_len for PE %d (%d)\n", pe, ret); + goto cleanup; + } + hostnames[pe] = (char *) malloc(hlens[pe] + 1); + if (!hostnames[pe]) { + RETURN_ERROR_MSG("Out of memory for hostname of PE %d\n", pe); + ret = 2; + goto cleanup; + } + ret = shmem_runtime_get(pe, "hostname", hostnames[pe], hlens[pe]); + if (ret != 0) { + RETURN_ERROR_MSG("Failed to get hostname for PE %d (%d)\n", pe, ret); + goto cleanup; + } + } + + /* PE pe is a node root if no earlier PE (q < pe) has the same hostname. */ + for (int pe = 0; pe < size; pe++) { + is_node_root[pe] = 1; + for (int q = 0; q < pe; q++) { + if (hlens[pe] == hlens[q] && + memcmp(hostnames[pe], hostnames[q], hlens[pe]) == 0) { + is_node_root[pe] = 0; + break; + } + } + } + +cleanup: + for (int pe = 0; pe < size; pe++) { + free(hostnames[pe]); + } + free(hostnames); + free(hlens); + return ret; +} diff --git a/src/shmem_collectives.h b/src/shmem_collectives.h index d6a57ca76..43fc26fec 100644 --- a/src/shmem_collectives.h +++ b/src/shmem_collectives.h @@ -25,7 +25,8 @@ enum coll_type_t { TREE, DISSEM, RING, - RECDBL + RECDBL, + HIERARCHICAL }; typedef enum coll_type_t coll_type_t; @@ -33,6 +34,11 @@ extern char *coll_type_str[]; extern long *shmem_internal_barrier_all_psync; extern long *shmem_internal_sync_all_psync; +#ifdef USE_HIERARCHICAL_BARRIER +extern long *shmem_internal_barrier_all_local_psync; +extern long *shmem_internal_sync_all_local_psync; +extern long *shmem_internal_hierarchical_local_psync; +#endif extern coll_type_t shmem_internal_barrier_type; extern coll_type_t shmem_internal_bcast_type; @@ -44,6 +50,11 @@ extern coll_type_t shmem_internal_fcollect_type; void shmem_internal_sync_linear(int PE_start, int PE_stride, int PE_size, long *pSync); void shmem_internal_sync_tree(int PE_start, int PE_stride, int PE_size, long *pSync); void shmem_internal_sync_dissem(int PE_start, int PE_stride, int PE_size, long *pSync); +#ifdef USE_HIERARCHICAL_BARRIER +void shmem_internal_sync_hierarchical(int PE_start, int PE_stride, int PE_size, + long *pSync, long *local_pSync); +void shmem_internal_hier_barrier_print_stats(void); +#endif static inline void @@ -58,6 +69,14 @@ shmem_internal_sync(int PE_start, int PE_stride, int PE_size, long *pSync) switch (shmem_internal_barrier_type) { case AUTO: +#ifdef USE_HIERARCHICAL_BARRIER + if (shmem_internal_get_shr_size() >= shmem_internal_params.HIER_BARRIER_THRESHOLD) { + shmem_internal_sync_hierarchical(PE_start, PE_stride, PE_size, + pSync, + shmem_internal_hierarchical_local_psync); + break; + } +#endif if (PE_size < shmem_internal_params.COLL_CROSSOVER) { shmem_internal_sync_linear(PE_start, PE_stride, PE_size, pSync); } else { @@ -73,6 +92,13 @@ shmem_internal_sync(int PE_start, int PE_stride, int PE_size, long *pSync) case DISSEM: shmem_internal_sync_dissem(PE_start, PE_stride, PE_size, pSync); break; +#ifdef USE_HIERARCHICAL_BARRIER + case HIERARCHICAL: + shmem_internal_sync_hierarchical(PE_start, PE_stride, PE_size, + pSync, + shmem_internal_hierarchical_local_psync); + break; +#endif default: RAISE_ERROR_MSG("Illegal barrier/sync type (%d)\n", shmem_internal_barrier_type); @@ -88,6 +114,17 @@ static inline void shmem_internal_sync_all(void) { +#ifdef USE_HIERARCHICAL_BARRIER + if (shmem_internal_barrier_type == AUTO && + shmem_internal_get_shr_size() >= shmem_internal_params.HIER_BARRIER_THRESHOLD) { + shmem_internal_sync_hierarchical(0, 1, shmem_internal_num_pes, + shmem_internal_sync_all_psync, + shmem_internal_sync_all_local_psync); + shmem_internal_membar_acq_rel(); + shmem_transport_syncmem(); + return; + } +#endif shmem_internal_sync(0, 1, shmem_internal_num_pes, shmem_internal_sync_all_psync); } @@ -106,6 +143,17 @@ void shmem_internal_barrier_all(void) { shmem_internal_quiet(SHMEM_CTX_DEFAULT); +#ifdef USE_HIERARCHICAL_BARRIER + if (shmem_internal_barrier_type == AUTO && + shmem_internal_get_shr_size() >= shmem_internal_params.HIER_BARRIER_THRESHOLD) { + shmem_internal_sync_hierarchical(0, 1, shmem_internal_num_pes, + shmem_internal_barrier_all_psync, + shmem_internal_barrier_all_local_psync); + shmem_internal_membar_acq_rel(); + shmem_transport_syncmem(); + return; + } +#endif shmem_internal_sync(0, 1, shmem_internal_num_pes, shmem_internal_barrier_all_psync); } diff --git a/src/shmem_comm.h b/src/shmem_comm.h index 3a3381834..6ec13549d 100644 --- a/src/shmem_comm.h +++ b/src/shmem_comm.h @@ -80,12 +80,29 @@ shmem_internal_put_signal_nbi(shmem_ctx_t ctx, void *target, const void *source, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe) { if (len == 0) { - if (sig_op == SHMEM_SIGNAL_ADD) - shmem_transport_atomic((shmem_transport_ctx_t *) ctx, sig_addr, &signal, sizeof(uint64_t), - pe, SHM_INTERNAL_SUM, SHM_INTERNAL_UINT64); - else - shmem_transport_atomic_set((shmem_transport_ctx_t *) ctx, sig_addr, &signal, - sizeof(uint64_t), pe, SHM_INTERNAL_UINT64); + /* Signal-only (no data): use shmem_shr_transport_use_atomic() to pick + * the right plane. In the multi-node case this always resolves to the + * NIC (preserving FIFO ordering with any prior in-flight NIC puts and + * avoiding the CPU/NIC coherency hazard). In the all-PEs-on-one-node + * case it resolves to CPU atomics, consistent with data puts. */ + if (sig_op == SHMEM_SIGNAL_ADD) { + if (shmem_shr_transport_use_atomic(ctx, sig_addr, sizeof(uint64_t), + pe, SHM_INTERNAL_UINT64)) + shmem_shr_transport_atomic(ctx, sig_addr, &signal, sizeof(uint64_t), + pe, SHM_INTERNAL_SUM, SHM_INTERNAL_UINT64); + else + shmem_transport_atomic((shmem_transport_ctx_t *) ctx, sig_addr, &signal, + sizeof(uint64_t), pe, SHM_INTERNAL_SUM, + SHM_INTERNAL_UINT64); + } else { + if (shmem_shr_transport_use_atomic(ctx, sig_addr, sizeof(uint64_t), + pe, SHM_INTERNAL_UINT64)) + shmem_shr_transport_atomic_set(ctx, sig_addr, &signal, sizeof(uint64_t), + pe, SHM_INTERNAL_UINT64); + else + shmem_transport_atomic_set((shmem_transport_ctx_t *) ctx, sig_addr, &signal, + sizeof(uint64_t), pe, SHM_INTERNAL_UINT64); + } return; } diff --git a/src/shmem_env_defs.h b/src/shmem_env_defs.h index b76e5e95e..c6a2847ce 100644 --- a/src/shmem_env_defs.h +++ b/src/shmem_env_defs.h @@ -58,7 +58,11 @@ SHMEM_INTERNAL_ENV_DEF(COLL_SIZE_CROSSOVER, size, 16384, SHMEM_INTERNAL_ENV_CAT_ SHMEM_INTERNAL_ENV_DEF(COLL_RADIX, long, 4, SHMEM_INTERNAL_ENV_CAT_COLLECTIVES, "Radix for tree-based collectives") SHMEM_INTERNAL_ENV_DEF(BARRIER_ALGORITHM, string, "auto", SHMEM_INTERNAL_ENV_CAT_COLLECTIVES, - "Algorithm for barrier. Options are auto, linear, tree, dissem") + "Algorithm for barrier. Options are auto, linear, tree, dissem, hierarchical") +SHMEM_INTERNAL_ENV_DEF(HIER_BARRIER_THRESHOLD, long, 2, SHMEM_INTERNAL_ENV_CAT_COLLECTIVES, + "Minimum local PE count per node to auto-select the hierarchical barrier") +SHMEM_INTERNAL_ENV_DEF(HIER_BARRIER_DEBUG, bool, 0, SHMEM_INTERNAL_ENV_CAT_COLLECTIVES, + "Print per-phase hierarchical barrier timing at finalize") SHMEM_INTERNAL_ENV_DEF(BCAST_ALGORITHM, string, "auto", SHMEM_INTERNAL_ENV_CAT_COLLECTIVES, "Algorithm for broadcast. Options are auto, linear, tree") SHMEM_INTERNAL_ENV_DEF(REDUCE_ALGORITHM, string, "auto", SHMEM_INTERNAL_ENV_CAT_COLLECTIVES, diff --git a/src/shmem_internal.h b/src/shmem_internal.h index f937af99f..cda7e4ae5 100644 --- a/src/shmem_internal.h +++ b/src/shmem_internal.h @@ -549,6 +549,17 @@ static inline int shmem_internal_get_shr_size(void) #endif } +/* Return the global PE rank of the lowest-ranked PE on this node (node_rank==0). + * Used as the internode representative in hierarchical collectives. */ +static inline int shmem_internal_get_node_root_pe(void) +{ +#ifdef USE_ON_NODE_COMMS + return shmem_runtime_get_node_root_pe(); +#else + return shmem_internal_my_pe; +#endif +} + static inline double shmem_internal_wtime(void) { double wtime = 0.0; diff --git a/src/shr_transport.h4 b/src/shr_transport.h4 index 9379ef2e5..4282d6ae4 100644 --- a/src/shr_transport.h4 +++ b/src/shr_transport.h4 @@ -122,12 +122,24 @@ shmem_shr_transport_use_read(shmem_ctx_t ctx, void *target, const void *source, /* Each OpenSHMEM AMO has only one symmetric pointer. Check whether shared * transport AMOs are in use with respect to the given symmetric target * pointer and datatype. For a given datatype, all atomic operations must - * use the same transport; therefore, op is not needed in this check. */ + * use the same transport; therefore, op is not needed in this check. + * + * Under USE_HIERARCHICAL_BARRIER, CPU atomics and NIC AMOs are not coherent + * with each other; mixing them on the same variable produces lost updates. + * CPU atomics are therefore only safe when ALL PEs are on the same node — + * in that case no NIC AMOs will ever be issued to any variable, so there is + * only one coherency domain in play. */ static inline int shmem_shr_transport_use_atomic(shmem_ctx_t ctx, void *target, size_t len, int pe, shm_internal_datatype_t datatype) { -#if USE_SHR_ATOMICS +#if USE_HIERARCHICAL_BARRIER + /* Use CPU atomics only when all PEs are local (no off-node peers). */ + if (shmem_internal_get_shr_size() == shmem_internal_num_pes) { + return -1 != shmem_internal_get_shr_rank(pe); + } + return 0; +#elif USE_SHR_ATOMICS return -1 != shmem_internal_get_shr_rank(pe); #else return 0; diff --git a/src/transport_ofi.c b/src/transport_ofi.c index c0f417859..b52c772fe 100644 --- a/src/transport_ofi.c +++ b/src/transport_ofi.c @@ -1652,6 +1652,10 @@ int query_for_fabric(struct fabric_info *info) #endif #ifndef DISABLE_OFI_INJECT + DEBUG_MSG(RAISE_PE_PREFIX "tx_attr->inject_size (provider): %zu, requested: %zu\n", + shmem_internal_my_pe, + info->p_info->tx_attr->inject_size, + shmem_transport_ofi_max_buffered_send); shmem_internal_assertp(info->p_info->tx_attr->inject_size >= shmem_transport_ofi_max_buffered_send); shmem_transport_ofi_max_buffered_send = info->p_info->tx_attr->inject_size; #else From dbe498c777aee85c870b0389df4cb59e1d672c27 Mon Sep 17 00:00:00 2001 From: Bryan Morgan Date: Thu, 25 Jun 2026 12:04:34 -0700 Subject: [PATCH 07/14] collectives: per-team sense, persistent scratch, and cleanup for hierarchical barrier Three coupled refinements to the hierarchical barrier: Per-team sense state. A static global hier_sense caused a signal mismatch when PEs participated in different barrier teams: only PEs in the active set incremented it, so PEs that skipped a subset barrier carried a stale sense value into the next TEAM_WORLD barrier. Move hier_sense into shmem_internal_team_t; TEAM_WORLD barriers (PE_start=0, PE_stride=1, PE_size=num_pes) use team-local state, and subset barriers use a static fallback. All PEs in TEAM_WORLD now stay synchronized across interleaved team/subset barrier sequences. Persistent active-set scratch. The barrier built its local_pes/root_pes arrays fresh on every call. An alloca-per-call risked stack overflow at scale (PE_size up to 100K+ is ~400KB per array), while malloc/free per-call serialized the hot path and tanked throughput. Allocate both once in shmem_internal_collectives_init, sized to num_pes, and reuse. Safe to share across calls because barriers are serialized -- a PE cannot enter a new barrier until it has exited the previous one, the same invariant the shared hierarchical_local_psync relies on. Dead-code removal. Drop three unused symbols that produced -Wunused warnings: shmem_internal_cpu_atomic_load_long (never called), tree_parent_shr (computed but never read -- the Phase 1 up signal writes to the PE's own up-slot, which the parent reads directly), and my_up_slot (only the down-slot is read in the spin loops; my_up_raw is still used for the parent store). --- src/collectives.c | 53 ++++++++++++++++++++++++++--------------------- src/shmem_team.c | 9 ++++++++ src/shmem_team.h | 3 +++ 3 files changed, 41 insertions(+), 24 deletions(-) diff --git a/src/collectives.c b/src/collectives.c index b3964c952..5b58b9d63 100644 --- a/src/collectives.c +++ b/src/collectives.c @@ -36,6 +36,15 @@ long *shmem_internal_barrier_all_local_psync; long *shmem_internal_sync_all_local_psync; long *shmem_internal_hierarchical_local_psync; +/* Persistent scratch for the active-set PE lists built on every hierarchical + * barrier. Allocated once at init, sized to num_pes (the maximum any active + * set can hold), and reused across calls. Safe to share: barriers are + * serialized — a PE cannot enter a new barrier until it has exited the + * previous one (same invariant the shared hierarchical_local_psync relies + * on). */ +static int *hier_local_pes = NULL; +static int *hier_root_pes = NULL; + /* Layout of local_pSync — two cache-line-padded arrays, one slot per PE: * * up-slot for PE r: local_pSync[r * HIER_SLOT_STRIDE] @@ -53,8 +62,6 @@ long *shmem_internal_hierarchical_local_psync; * Total allocation: 2 * shr_size * HIER_SLOT_STRIDE longs. */ #define HIER_SLOT_STRIDE 8 /* 8 longs = 64 bytes = 1 cache line */ -static long hier_sense = 0; - /* Per-phase timing accumulators (root PE: all three phases; * non-root PEs: phase1_us = gather wait, phase3_us = fanout wait). */ static double hier_phase1_us = 0.0; @@ -211,6 +218,12 @@ shmem_internal_collectives_init(void) for (i = 0; i < local_psync_len; i++) { shmem_internal_hierarchical_local_psync[i] = SHMEM_SYNC_VALUE; } + + /* Persistent per-barrier scratch (see declaration above). Sized to + * num_pes so it fits the largest possible active set. */ + hier_local_pes = malloc(sizeof(int) * shmem_internal_num_pes); + hier_root_pes = malloc(sizeof(int) * shmem_internal_num_pes); + if (NULL == hier_local_pes || NULL == hier_root_pes) return -1; #endif /* initialize the binomial tree for collective operations over @@ -565,17 +578,6 @@ shmem_internal_build_root_active_set(int PE_start, int PE_stride, int PE_size, } } -/* CPU atomic load of the long at `target` via local mapped pointer. */ -static inline long -shmem_internal_cpu_atomic_load_long(long *target, int noderank) -{ - void *remote_ptr; - long val; - shmem_shr_transport_ptr(target, noderank, &remote_ptr); - __atomic_load((long *)remote_ptr, &val, __ATOMIC_ACQUIRE); - return val; -} - /* CPU atomic store of `val` to the long at `target` via local mapped pointer. */ static inline void shmem_internal_cpu_atomic_store_long(long *target, int noderank, long val) @@ -596,13 +598,22 @@ shmem_internal_sync_hierarchical(int PE_start, int PE_stride, int PE_size, if (PE_size == 1) return; + /* Determine sense state: for TEAM_WORLD (0, 1, num_pes) use per-team state; else static fallback. */ + long *sense_ptr; + if (PE_start == 0 && PE_stride == 1 && PE_size == shmem_internal_num_pes) { + sense_ptr = &shmem_internal_team_world.hier_sense; + } else { + static long fallback_sense = 0; + sense_ptr = &fallback_sense; + } + /* Collect local and root PE sets for this active set */ - int *local_pes = alloca(sizeof(int) * PE_size); + int *local_pes = hier_local_pes; int local_count = 0; shmem_internal_build_local_set(PE_start, PE_stride, PE_size, local_pes, &local_count); - int *root_pes = alloca(sizeof(int) * PE_size); + int *root_pes = hier_root_pes; int root_count = 0; shmem_internal_build_root_active_set(PE_start, PE_stride, PE_size, root_pes, &root_count); @@ -619,14 +630,9 @@ shmem_internal_sync_hierarchical(int PE_start, int PE_stride, int PE_size, ? (my_local_idx - root_local_idx + local_count) % local_count : -1; - int tree_parent_shr = -1; int tree_nchildren = 0; int *tree_child_shr = alloca(sizeof(int) * tree_radix); if (my_vidx >= 0) { - if (my_vidx > 0) { - tree_parent_shr = shmem_runtime_get_node_rank( - local_pes[((my_vidx - 1) / tree_radix + root_local_idx) % local_count]); - } for (int j = 1; j <= tree_radix; j++) { int cv = my_vidx * tree_radix + j; if (cv < local_count) { @@ -639,7 +645,7 @@ shmem_internal_sync_hierarchical(int PE_start, int PE_stride, int PE_size, /* Sense-alternating signal — monotonically increasing, no slot resets needed. * up-slot for PE r: local_pSync[r * HIER_SLOT_STRIDE] * down-slot for PE r: local_pSync[shr_size * HIER_SLOT_STRIDE + r * HIER_SLOT_STRIDE] */ - long signal = SHMEM_SYNC_VALUE + 1 + hier_sense; + long signal = SHMEM_SYNC_VALUE + 1 + *sense_ptr; int shr_size = shmem_internal_get_shr_size(); long *up_pSync = local_pSync; long *down_pSync = local_pSync + (long)(shr_size * HIER_SLOT_STRIDE); @@ -648,7 +654,6 @@ shmem_internal_sync_hierarchical(int PE_start, int PE_stride, int PE_size, void *my_up_raw, *my_down_raw; shmem_shr_transport_ptr(&up_pSync[my_shr_rank * HIER_SLOT_STRIDE], my_shr_rank, &my_up_raw); shmem_shr_transport_ptr(&down_pSync[my_shr_rank * HIER_SLOT_STRIDE], my_shr_rank, &my_down_raw); - volatile long *my_up_slot = (volatile long *)my_up_raw; volatile long *my_down_slot = (volatile long *)my_down_raw; /* ---- Degenerate case: all active PEs on one node ---- */ @@ -705,7 +710,7 @@ shmem_internal_sync_hierarchical(int PE_start, int PE_stride, int PE_size, hier_call_count++; } - hier_sense++; + (*sense_ptr)++; return; } @@ -800,7 +805,7 @@ shmem_internal_sync_hierarchical(int PE_start, int PE_stride, int PE_size, } } - if (my_vidx >= 0) { hier_sense++; } + if (my_vidx >= 0) { (*sense_ptr)++; } /* PEs absent from local_pes (my_vidx < 0) fall through silently. */ } diff --git a/src/shmem_team.c b/src/shmem_team.c index d77f28096..3241b94e0 100644 --- a/src/shmem_team.c +++ b/src/shmem_team.c @@ -85,6 +85,9 @@ int shmem_internal_team_init(void) memset(&shmem_internal_team_world.config, 0, sizeof(shmem_team_config_t)); for (size_t i = 0; i < N_PSYNCS_PER_TEAM; i++) shmem_internal_team_world.psync_avail[i] = 1; +#ifdef USE_HIERARCHICAL_BARRIER + shmem_internal_team_world.hier_sense = 0; +#endif SHMEM_TEAM_WORLD = (shmem_team_t) &shmem_internal_team_world; /* Initialize SHMEM_TEAM_SHARED */ @@ -95,6 +98,9 @@ int shmem_internal_team_init(void) memset(&shmem_internal_team_shared.config, 0, sizeof(shmem_team_config_t)); for (size_t i = 0; i < N_PSYNCS_PER_TEAM; i++) shmem_internal_team_shared.psync_avail[i] = 1; +#ifdef USE_HIERARCHICAL_BARRIER + shmem_internal_team_shared.hier_sense = 0; +#endif SHMEM_TEAM_SHARED = (shmem_team_t) &shmem_internal_team_shared; /* Initialize SHMEM_TEAM_NODE */ @@ -105,6 +111,9 @@ int shmem_internal_team_init(void) memset(&shmem_internal_team_node.config, 0, sizeof(shmem_team_config_t)); for (size_t i = 0; i < N_PSYNCS_PER_TEAM; i++) shmem_internal_team_node.psync_avail[i] = 1; +#ifdef USE_HIERARCHICAL_BARRIER + shmem_internal_team_node.hier_sense = 0; +#endif SHMEMX_TEAM_NODE = (shmem_team_t) &shmem_internal_team_node; if (shmem_internal_params.TEAM_SHARED_ONLY_SELF) { diff --git a/src/shmem_team.h b/src/shmem_team.h index f6ddf6555..654fd105a 100644 --- a/src/shmem_team.h +++ b/src/shmem_team.h @@ -26,6 +26,9 @@ struct shmem_internal_team_t { long config_mask; size_t contexts_len; struct shmem_transport_ctx_t **contexts; +#ifdef USE_HIERARCHICAL_BARRIER + long hier_sense; +#endif }; typedef struct shmem_internal_team_t shmem_internal_team_t; From 9eab54d4d08ab239949763a7b10c53f67413e148 Mon Sep 17 00:00:00 2001 From: Bryan Morgan Date: Wed, 24 Jun 2026 08:45:07 -0700 Subject: [PATCH 08/14] transport/ofi: enable CXI hybrid local MR descriptor mode Cray SHMEM calls FI_CXI_DOM_OPS_3 enable_hybrid_mr_desc right after fi_domain(); SOS does not. Without it, the CXI provider performs internal memory registration on every fi_write/fi_writemsg/fi_fetch_ atomicmsg call where the desc field is non-NULL, even when the buffer already lives in a provider-registered region. Each MR cache lookup adds latency to large RDMA puts. With hybrid mode enabled, the provider trusts a non-NULL desc and skips its internal registration walk, dropping per-call overhead on the put data path. The call must occur before any endpoints are created; the domain only propagates the setting to child endpoints at creation time. Reintroduces the provider-name infrastructure to gate this: the provider name is captured via strdup in query_for_fabric (it must outlive fi_freeinfo) and freed in shmem_transport_fini. An inline shmem_transport_ofi_check_provider helper does an exact strcmp so that the call is attempted only on a plain cxi provider and layered providers do not inherit it. The dom_ops struct fields are mirrored locally so the build does not require rdma/fi_cxi_ext.h. Gated by SHMEM_OFI_CXI_HYBRID_MR_DESC=true (default) for A/B testing without rebuilding. --- src/shmem_env_defs.h | 2 ++ src/transport_ofi.c | 42 ++++++++++++++++++++++++++++++++++++++++++ src/transport_ofi.h | 12 ++++++++++++ 3 files changed, 56 insertions(+) diff --git a/src/shmem_env_defs.h b/src/shmem_env_defs.h index c6a2847ce..94ea2750a 100644 --- a/src/shmem_env_defs.h +++ b/src/shmem_env_defs.h @@ -99,6 +99,8 @@ SHMEM_INTERNAL_ENV_DEF(OFI_DOMAIN, string, "auto", SHMEM_INTERNAL_ENV_CAT_TRANSP "Fabric domain that should be used by the OFI transport") SHMEM_INTERNAL_ENV_DEF(OFI_TX_POLL_LIMIT, long, DEFAULT_POLL_LIMIT, SHMEM_INTERNAL_ENV_CAT_TRANSPORT, "Put completion poll limit") +SHMEM_INTERNAL_ENV_DEF(OFI_CXI_HYBRID_MR_DESC, bool, true, SHMEM_INTERNAL_ENV_CAT_TRANSPORT, + "Enable CXI hybrid local MR descriptor mode (skips internal MR registration when desc is non-NULL); ignored on non-CXI providers") SHMEM_INTERNAL_ENV_DEF(OFI_RX_POLL_LIMIT, long, DEFAULT_POLL_LIMIT, SHMEM_INTERNAL_ENV_CAT_TRANSPORT, "Get completion poll limit") SHMEM_INTERNAL_ENV_DEF(OFI_STX_MAX, long, 1, SHMEM_INTERNAL_ENV_CAT_TRANSPORT, diff --git a/src/transport_ofi.c b/src/transport_ofi.c index b52c772fe..ba7099d11 100644 --- a/src/transport_ofi.c +++ b/src/transport_ofi.c @@ -208,9 +208,12 @@ struct shmem_internal_tid shmem_transport_ofi_gettid(void) } #define SHMEM_TRANSPORT_OFI_PROV_SOCKETS "sockets" +#define SHMEM_TRANSPORT_OFI_PROV_CXI "cxi" static struct fabric_info shmem_transport_ofi_info = {0}; +char *shmem_transport_ofi_prov_name = NULL; + static size_t shmem_transport_ofi_grow_size = 128; #define SHMEM_TRANSPORT_CTX_DEFAULT_ID -1 @@ -1341,6 +1344,39 @@ int allocate_fabric_resources(struct fabric_info *info) &shmem_transport_ofi_domainfd,NULL); OFI_CHECK_RETURN_STR(ret, "domain initialization failed"); + /* CXI provider: enable hybrid local MR descriptor mode. When enabled, + * libfabric will skip its internal MR registration if a non-NULL desc is + * passed (and proceed without registration if desc is NULL). This avoids + * per-call MR cache lookups for source buffers in fi_write/fi_writemsg. + * Must be done BEFORE any endpoints are created (the provider only + * propagates this setting to child endpoints at creation time). + * Gated on a CXI provider check so non-CXI providers never attempt the + * fi_open_ops call and do not produce spurious startup warnings. */ + if (shmem_transport_ofi_check_provider(SHMEM_TRANSPORT_OFI_PROV_CXI)) { + if (shmem_internal_params.OFI_CXI_HYBRID_MR_DESC) { + struct cxi_dom_ops_v3_local { + int (*cntr_read)(struct fid *, unsigned int, uint64_t *, struct timespec *); + int (*topology)(struct fid *, unsigned int *, unsigned int *, unsigned int *); + int (*enable_hybrid_mr_desc)(struct fid *, bool); + } *cxi_dom_ops = NULL; + int hret = fi_open_ops(&shmem_transport_ofi_domainfd->fid, + "dom_ops_v3", 0, (void **)&cxi_dom_ops, NULL); + if (hret == 0 && cxi_dom_ops && cxi_dom_ops->enable_hybrid_mr_desc) { + hret = cxi_dom_ops->enable_hybrid_mr_desc(&shmem_transport_ofi_domainfd->fid, true); + if (hret == 0) { + DEBUG_STR("CXI: hybrid local MR descriptor mode enabled"); + } else { + DEBUG_MSG("CXI: enable_hybrid_mr_desc failed (%s)\n", fi_strerror(-hret)); + } + } else { + DEBUG_MSG("CXI: hybrid MR desc not available (fi_open_ops returned %d / %s)\n", + hret, hret ? fi_strerror(-hret) : "no ops struct"); + } + } else { + DEBUG_STR("CXI: hybrid local MR descriptor mode disabled (SHMEM_OFI_CXI_HYBRID_MR_DESC=0)"); + } + } + /* AV table set-up for PE mapping */ #ifdef USE_AV_MAP @@ -1678,6 +1714,10 @@ int query_for_fabric(struct fabric_info *info) shmem_transport_ofi_stx_max, num_nics); + /* Store provider name for runtime checks. strdup so the pointer remains + * valid after fi_freeinfo() is called in shmem_transport_fini(). */ + shmem_transport_ofi_prov_name = strdup(info->p_info->fabric_attr->prov_name); + return ret; } @@ -2288,6 +2328,8 @@ int shmem_transport_fini(void) free(addr_table); #endif + free(shmem_transport_ofi_prov_name); + shmem_transport_ofi_prov_name = NULL; fi_freeinfo(shmem_transport_ofi_info.fabrics); SHMEM_MUTEX_DESTROY(shmem_transport_ofi_lock); diff --git a/src/transport_ofi.h b/src/transport_ofi.h index 48cdeeafc..c8a91e8ce 100644 --- a/src/transport_ofi.h +++ b/src/transport_ofi.h @@ -532,6 +532,18 @@ int shmem_transport_quiet(shmem_transport_ctx_t* ctx) return 0; } +/* Check if the current OFI provider exactly matches the given name. + * Uses strcmp so that layered providers (e.g. "foo;ofi_rxm") do not + * incorrectly inherit provider-specific fast paths. + * Returns 1 if provider matches, 0 otherwise. */ +extern char *shmem_transport_ofi_prov_name; +static inline int +shmem_transport_ofi_check_provider(const char *name) +{ + return (shmem_transport_ofi_prov_name && + strcmp(shmem_transport_ofi_prov_name, name) == 0); +} + static inline int shmem_transport_fence(shmem_transport_ctx_t* ctx) From 52858899d05fc47ed072455e61a2eac9f758da1c Mon Sep 17 00:00:00 2001 From: Bryan Morgan Date: Wed, 24 Jun 2026 10:14:39 -0700 Subject: [PATCH 09/14] symmetric heap: tiered huge page allocation without THP fallback Reintroduce the symmetric heap huge page allocator gated on SHMEM_SYMMETRIC_HEAP_USE_HUGE_PAGES (default off). When enabled, the heap is backed by explicitly reserved huge pages via two paths, tried in order: 1. hugetlbfs file mapping (when a matching hugetlbfs mount exists) 2. anonymous MAP_HUGETLB with an explicit 2MB page size (works with nr_overcommit_hugepages) When neither reserved-huge-page path succeeds, the heap falls back to regular pages. There is intentionally no transparent-huge-page (madvise) tier: only explicitly reserved huge pages are used, otherwise regular pages. The regular-page fallback is a single fixed-address mmap at requested_base, identical to the mapping used when huge pages are disabled, so the symmetric heap keeps the same virtual address on every PE whether or not the huge-page flag is set. mmap_alloc now reports the actual mapped size back to the caller, since the hugetlbfs path rounds the length up to a huge-page boundary; that rounded size is used for munmap and transport registration. Disabled by default, so the regular-page path is byte-for-byte the prior behavior unless the flag is set. --- src/symmetric_heap_c.c | 99 +++++++++++++++++++++++++++++++++++------- 1 file changed, 84 insertions(+), 15 deletions(-) diff --git a/src/symmetric_heap_c.c b/src/symmetric_heap_c.c index 4da76c8e8..51f4c6090 100644 --- a/src/symmetric_heap_c.c +++ b/src/symmetric_heap_c.c @@ -162,15 +162,16 @@ shmem_internal_get_next(intptr_t incr) /* alloc VM space starting @ '_end' + 1GB */ #define ONEGIG (1024UL*1024UL*1024UL) -static void *mmap_alloc(size_t bytes) +static void *mmap_alloc(size_t bytes, size_t *mapped_bytes) { char *file_name = NULL; int fd = 0; char *directory = NULL; - void *requested_base = - (void*) (((unsigned long) shmem_internal_data_base + - shmem_internal_data_length + 2 * ONEGIG) & ~(ONEGIG - 1)); - void *ret; + void *requested_base = (void*) (((unsigned long) shmem_internal_data_base + shmem_internal_data_length + 2 * ONEGIG) & ~(ONEGIG - 1)); + void *ret = MAP_FAILED; + size_t hugetlbfs_bytes = 0; /* Rounded size for hugetlbfs, 0 if not used */ + + *mapped_bytes = bytes; /* default: actual mapped size equals requested size */ #ifdef __linux__ /* huge page support only on Linux for now, default is to use 2MB large pages */ @@ -191,24 +192,89 @@ static void *mmap_alloc(size_t bytes) sprintf(file_name, "%s/%s.%d", directory, basename, getpid()); fd = open(file_name, O_CREAT | O_RDWR, 0755); if (fd < 0) { - RAISE_WARN_STR("file open failed, cannot use huge pages"); + DEBUG_MSG("file open failed, will fall back to anonymous MAP_HUGETLB"); + free(directory); + free(file_name); + directory = NULL; + file_name = NULL; fd = 0; } else { - /* have to round up by the pagesize being used */ - bytes = CEILING(bytes, shmem_internal_params.SYMMETRIC_HEAP_PAGE_SIZE); + /* Round up by the pagesize for hugetlbfs file */ + hugetlbfs_bytes = CEILING(bytes, shmem_internal_params.SYMMETRIC_HEAP_PAGE_SIZE); } } } } } -#endif /* __linux__ */ + DEBUG_MSG("mmap_alloc: bytes=%zu, hugetlbfs_bytes=%zu, fd=%d", + bytes, hugetlbfs_bytes, fd); + + if (fd) { + /* Map the hugetlbfs file directly; MAP_ANON must not be used here + * because MAP_ANONYMOUS causes the kernel to ignore the fd, which + * would silently fall back to regular pages. */ + if (ftruncate(fd, hugetlbfs_bytes) == -1) { + DEBUG_MSG("ftruncate on hugetlbfs file failed (%s), falling back", strerror(errno)); + unlink(file_name); + close(fd); + free(directory); + free(file_name); + directory = NULL; + file_name = NULL; + fd = 0; + } else { + ret = mmap(requested_base, hugetlbfs_bytes, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_HUGETLB, fd, 0); + if (ret != MAP_FAILED) { + DEBUG_MSG("Allocated symmetric heap via hugetlbfs file: %zu bytes", hugetlbfs_bytes); + *mapped_bytes = hugetlbfs_bytes; + } + unlink(file_name); + close(fd); + free(directory); + free(file_name); + directory = NULL; + file_name = NULL; + fd = 0; + } + } + + /* If hugetlbfs file mapping failed or was not attempted, continue with the + * fallback chain. Note: fd is always zeroed after the hugetlbfs block + * (cleanup on both success and failure paths), so testing fd == 0 here + * would always be true and would incorrectly enter the fallback after a + * successful hugetlbfs allocation. Test ret only. */ + if (ret == MAP_FAILED) { + if (shmem_internal_params.SYMMETRIC_HEAP_USE_HUGE_PAGES) { + /* Try anonymous MAP_HUGETLB (works with nr_overcommit_hugepages). + * Explicitly request 2MB pages via MAP_HUGE_SHIFT (21 << MAP_HUGE_SHIFT = 2^21 = 2MB). */ + ret = mmap(requested_base, bytes, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE | MAP_HUGETLB | (21 << MAP_HUGE_SHIFT), -1, 0); + if (ret == MAP_FAILED) { + DEBUG_MSG("mmap(MAP_HUGETLB) failed (%s), falling back to regular pages", strerror(errno)); + } else { + DEBUG_MSG("Allocated symmetric heap via anonymous MAP_HUGETLB (2MB pages): %zu bytes", bytes); + } + } + + /* Regular pages: used when huge pages are not requested, or when the + * reserved huge-page paths above are unavailable. Identical to the + * non-huge-page mapping: a single fixed-address map at requested_base + * so the symmetric heap keeps the same virtual address on every PE. */ + if (ret == MAP_FAILED) { + ret = mmap(requested_base, bytes, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, fd, 0); + if (ret != MAP_FAILED) { + DEBUG_MSG("Allocated symmetric heap via regular pages (4KB): %zu bytes", bytes); + } + } + } +#else ret = mmap(requested_base, bytes, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, fd, 0); +#endif /* __linux__ */ if (ret == MAP_FAILED) { RAISE_WARN_MSG("Unable to allocate sym. heap, size %zuB: %s\n" RAISE_PE_PREFIX @@ -216,17 +282,16 @@ static void *mmap_alloc(size_t bytes) bytes, strerror(errno), shmem_internal_my_pe); ret = NULL; } - if (fd) { - if (file_name) - unlink(file_name); - close(fd); - } + /* Cleanup any remaining allocations (will be NULL if already freed above) */ if (directory) { free(directory); } if (file_name) { free(file_name); } + if (fd > 0) { + close(fd); + } return ret; } @@ -240,9 +305,13 @@ shmem_internal_symmetric_init(void) SHMEM_MAX_BOUNCE_BUFFER_OVERHEAD; if (!shmem_internal_params.SYMMETRIC_HEAP_USE_MALLOC) { + size_t mapped_length = shmem_internal_heap_length; shmem_internal_heap_base = shmem_internal_heap_curr = - mmap_alloc(shmem_internal_heap_length); + mmap_alloc(shmem_internal_heap_length, &mapped_length); + /* Use the actual mapped size for munmap and transport registration. + * On the hugetlbfs path this may be rounded up to a huge-page boundary. */ + shmem_internal_heap_length = mapped_length; } else { shmem_internal_heap_base = shmem_internal_heap_curr = From 71521f4ced364bb6afb5cc2ec7278e50dd6ee4c8 Mon Sep 17 00:00:00 2001 From: Bryan Morgan Date: Wed, 24 Jun 2026 18:25:24 -0700 Subject: [PATCH 10/14] collectives: per-team sense counters and node-id root sets for hierarchical barrier Give the hierarchical barrier a per-team sense counter to eliminate the overlapping-team deadlock. shmem_internal_sync_hierarchical gains a hier_sense_ptr parameter: non-NULL supplies a caller-owned per-team counter; NULL retains the prior selection (TEAM_WORLD uses its own counter, other active sets share a static fallback). New shmem_internal_sync_for_team / shmem_internal_barrier_for_team wrappers pass &team->hier_sense, and shmem_team_sync, the team psync-recycle barrier, and the two team-split barriers route through them. sync_all / barrier_all pass &team_world.hier_sense explicitly. Add shmem_internal_sync_no_hier for the deprecated shmem_barrier / shmem_sync APIs: they carry no team context, so they never select the hierarchical algorithm and fall through to linear/tree by COLL_CROSSOVER. Build root active sets from a global node_id array instead of is_node_root. shmem_runtime_get_node_id(pe) returns the global PE number of the lowest-ranked PE on pe's node, computed once at init across all runtimes (mpi, pmi, pmi2, pmix). The root representative for each node is the lowest-ranked active PE on it (local_pes[0]), and the degenerate-case guard is reduced to local_count == PE_size. --- src/collectives.c | 59 +++++++++++-------- src/collectives_c.c4 | 7 ++- src/runtime-mpi.c | 28 +++++++++ src/runtime-pmi.c | 21 ++++++- src/runtime-pmi2.c | 23 +++++++- src/runtime-pmix.c | 25 +++++++- src/runtime.h | 7 ++- src/runtime_util.c | 14 +++-- src/shmem_collectives.h | 122 ++++++++++++++++++++++++++++++++++++++-- src/shmem_team.c | 8 +-- 10 files changed, 270 insertions(+), 44 deletions(-) diff --git a/src/collectives.c b/src/collectives.c index 5b58b9d63..16f029ece 100644 --- a/src/collectives.c +++ b/src/collectives.c @@ -558,12 +558,10 @@ shmem_internal_build_local_set(int PE_start, int PE_stride, int PE_size, } } -/* Build an active set of root PEs (the lowest-ranked PE on each node) from - * the original active set. shmem_runtime_get_node_rank() only returns the - * node-local rank for on-node PEs; for off-node PEs it returns -1. We - * therefore use shmem_runtime_is_node_root_pe() which reflects each PE's - * absolute rank within its own node, computed during init via global exchange. - */ +/* Build an active set of root PEs — one representative per node, defined as + * the lowest-ranked active PE on each participating node. Uses the global + * node_id array (computed once at init) so all PEs deterministically compute + * the same list without requiring communication. */ static void shmem_internal_build_root_active_set(int PE_start, int PE_stride, int PE_size, int *root_pes, int *root_count) @@ -571,8 +569,19 @@ shmem_internal_build_root_active_set(int PE_start, int PE_stride, int PE_size, int i, pe; *root_count = 0; + /* Iterate in ascending PE order. The first active PE encountered for each + * node_id is the representative for that node. */ for (i = 0, pe = PE_start; i < PE_size; i++, pe += PE_stride) { - if (shmem_runtime_is_node_root_pe(pe)) { + int pe_node_id = shmem_runtime_get_node_id(pe); + int already_represented = 0; + int j, qpe; + for (j = 0, qpe = PE_start; j < i; j++, qpe += PE_stride) { + if (shmem_runtime_get_node_id(qpe) == pe_node_id) { + already_represented = 1; + break; + } + } + if (!already_represented) { root_pes[(*root_count)++] = pe; } } @@ -589,18 +598,23 @@ shmem_internal_cpu_atomic_store_long(long *target, int noderank, long val) void shmem_internal_sync_hierarchical(int PE_start, int PE_stride, int PE_size, - long *pSync, long *local_pSync) + long *pSync, long *local_pSync, + long *hier_sense_ptr) { - int node_root_pe = shmem_internal_get_node_root_pe(); - int is_root = (shmem_internal_my_pe == node_root_pe); int my_shr_rank = shmem_runtime_get_node_rank(shmem_internal_my_pe); long one = 1; if (PE_size == 1) return; - /* Determine sense state: for TEAM_WORLD (0, 1, num_pes) use per-team state; else static fallback. */ + /* Determine sense state. Caller may supply a per-team sense pointer + * (hier_sense_ptr != NULL); this ensures each team maintains an independent + * sense counter and prevents deadlocks when overlapping teams are synced in + * different sequences. NULL falls back to the internal selection: TEAM_WORLD + * uses its own counter; all other active sets share a static fallback. */ long *sense_ptr; - if (PE_start == 0 && PE_stride == 1 && PE_size == shmem_internal_num_pes) { + if (hier_sense_ptr != NULL) { + sense_ptr = hier_sense_ptr; + } else if (PE_start == 0 && PE_stride == 1 && PE_size == shmem_internal_num_pes) { sense_ptr = &shmem_internal_team_world.hier_sense; } else { static long fallback_sense = 0; @@ -618,17 +632,16 @@ shmem_internal_sync_hierarchical(int PE_start, int PE_stride, int PE_size, shmem_internal_build_root_active_set(PE_start, PE_stride, PE_size, root_pes, &root_count); - /* Assign virtual indices 0..local_count-1, rotating so node_root_pe = vidx 0. - * Precompute tree parent shr_rank and children shr_ranks. */ - int my_local_idx = -1; - int root_local_idx = 0; + /* Assign virtual index 0..local_count-1. build_local_set fills local_pes + * in ascending PE order, so local_pes[0] is always the lowest-ranked active + * PE on this node and is the active-set representative for phase 2. */ + int my_local_idx = -1; for (int i = 0; i < local_count; i++) { - if (local_pes[i] == shmem_internal_my_pe) my_local_idx = i; - if (local_pes[i] == node_root_pe) root_local_idx = i; + if (local_pes[i] == shmem_internal_my_pe) { my_local_idx = i; break; } } - int my_vidx = (my_local_idx >= 0) - ? (my_local_idx - root_local_idx + local_count) % local_count - : -1; + int my_vidx = my_local_idx; /* -1 if not in active set on this node */ + int active_root_pe = (local_count > 0) ? local_pes[0] : -1; + int is_root = (shmem_internal_my_pe == active_root_pe); int tree_nchildren = 0; int *tree_child_shr = alloca(sizeof(int) * tree_radix); @@ -637,7 +650,7 @@ shmem_internal_sync_hierarchical(int PE_start, int PE_stride, int PE_size, int cv = my_vidx * tree_radix + j; if (cv < local_count) { tree_child_shr[tree_nchildren++] = shmem_runtime_get_node_rank( - local_pes[(cv + root_local_idx) % local_count]); + local_pes[cv]); } } } @@ -657,7 +670,7 @@ shmem_internal_sync_hierarchical(int PE_start, int PE_stride, int PE_size, volatile long *my_down_slot = (volatile long *)my_down_raw; /* ---- Degenerate case: all active PEs on one node ---- */ - if (root_count <= 1 || local_count == PE_size) { + if (local_count == PE_size) { if (my_vidx < 0) return; double t0 = shmem_internal_params.HIER_BARRIER_DEBUG ? hier_now_us() : 0.0; diff --git a/src/collectives_c.c4 b/src/collectives_c.c4 index b6e2f9993..c55418b37 100644 --- a/src/collectives_c.c4 +++ b/src/collectives_c.c4 @@ -181,7 +181,8 @@ shmem_barrier(int PE_start, int logPE_stride, int PE_size, long *pSync) SHMEM_ERR_CHECK_ACTIVE_SET(PE_start, 1 << logPE_stride, PE_size); SHMEM_ERR_CHECK_SYMMETRIC(pSync, sizeof(long)*SHMEM_BARRIER_SYNC_SIZE); - shmem_internal_barrier(PE_start, 1 << logPE_stride, PE_size, pSync); + shmem_internal_quiet(SHMEM_CTX_DEFAULT); + shmem_internal_sync_no_hier(PE_start, 1 << logPE_stride, PE_size, pSync); } @@ -200,7 +201,7 @@ shmem_sync(int PE_start, int logPE_stride, int PE_size, long *pSync) SHMEM_ERR_CHECK_ACTIVE_SET(PE_start, 1 << logPE_stride, PE_size); SHMEM_ERR_CHECK_SYMMETRIC(pSync, sizeof(long)*SHMEM_BARRIER_SYNC_SIZE); - shmem_internal_sync(PE_start, 1 << logPE_stride, PE_size, pSync); + shmem_internal_sync_no_hier(PE_start, 1 << logPE_stride, PE_size, pSync); } /* Team-based Collective Routines */ @@ -213,7 +214,7 @@ shmem_team_sync(shmem_team_t team) shmem_internal_team_t *myteam = (shmem_internal_team_t *)team; long *psync = shmem_internal_team_choose_psync(myteam, SYNC); - shmem_internal_sync(myteam->start, myteam->stride, myteam->size, psync); + shmem_internal_sync_for_team(myteam, psync); shmem_internal_team_release_psyncs(myteam, SYNC); return 0; } diff --git a/src/runtime-mpi.c b/src/runtime-mpi.c index 606c44a31..c5cd56202 100644 --- a/src/runtime-mpi.c +++ b/src/runtime-mpi.c @@ -39,6 +39,7 @@ static int initialized_mpi = 0; static int node_size; static int *node_ranks; static int *is_node_root = NULL; +static int *node_id_array = NULL; char* kv_store_me; char* kv_store_all; @@ -107,6 +108,8 @@ shmem_runtime_init(int enable_node_ranks) #ifdef USE_HIERARCHICAL_BARRIER is_node_root = malloc(size * sizeof(int)); if (NULL == is_node_root) return 9; + node_id_array = malloc(size * sizeof(int)); + if (NULL == node_id_array) return 10; #endif } @@ -123,6 +126,7 @@ shmem_runtime_fini(void) MPI_Comm_free(&SHMEM_RUNTIME_SHARED); free(node_ranks); free(is_node_root); + free(node_id_array); } MPI_Comm_free(&SHMEM_RUNTIME_WORLD); @@ -240,6 +244,21 @@ shmem_runtime_is_node_root_pe(int pe) return is_node_root[pe]; } + +int +shmem_runtime_get_node_id(int pe) +{ + shmem_internal_assert(pe < size && pe >= 0); + + if (size == 1) + return 0; + + if (NULL == node_id_array) + return pe; + + return node_id_array[pe]; +} + int shmem_runtime_exchange(void) { @@ -289,6 +308,15 @@ shmem_runtime_exchange(void) is_node_root[i] = (abs_node_ranks[i] == 0) ? 1 : 0; free(abs_node_ranks); + + /* Compute node_id_array: each PE's node is identified by the global + * rank of its lowest-ranked process. Bcast rank-0's world rank within + * the shared communicator so every PE on the same node gets the same + * value, then Allgather so all PEs know every other PE's node root. */ + int my_node_root_grank = rank; + MPI_Bcast(&my_node_root_grank, 1, MPI_INT, 0, SHMEM_RUNTIME_SHARED); + MPI_Allgather(&my_node_root_grank, 1, MPI_INT, + node_id_array, 1, MPI_INT, SHMEM_RUNTIME_WORLD); #endif } diff --git a/src/runtime-pmi.c b/src/runtime-pmi.c index dc884e183..94a322a24 100644 --- a/src/runtime-pmi.c +++ b/src/runtime-pmi.c @@ -39,6 +39,7 @@ static int max_name_len, max_key_len, max_val_len; static int initialized_pmi = 0; static int *location_array = NULL; static int *is_node_root = NULL; +static int *node_id_array = NULL; #define SINGLETON_KEY_LEN 128 #define SINGLETON_VAL_LEN 1024 @@ -100,6 +101,8 @@ shmem_runtime_init(int enable_node_ranks) #ifdef USE_HIERARCHICAL_BARRIER is_node_root = malloc(sizeof(int) * size); if (NULL == is_node_root) return 11; + node_id_array = malloc(sizeof(int) * size); + if (NULL == node_id_array) return 12; #endif } } @@ -126,6 +129,7 @@ shmem_runtime_fini(void) { free(location_array); free(is_node_root); + free(node_id_array); free(kvs_name); free(kvs_key); free(kvs_value); @@ -234,6 +238,21 @@ shmem_runtime_is_node_root_pe(int pe) } +int +shmem_runtime_get_node_id(int pe) +{ + shmem_internal_assert(pe < size && pe >= 0); + + if (size == 1) + return 0; + + if (NULL == node_id_array) + return pe; + + return node_id_array[pe]; +} + + int shmem_runtime_exchange(void) { @@ -267,7 +286,7 @@ shmem_runtime_exchange(void) } } if (is_node_root) { - ret = shmem_runtime_util_populate_global_node_roots(is_node_root, size); + ret = shmem_runtime_util_populate_global_node_roots(is_node_root, node_id_array, size); if (0 != ret) { RETURN_ERROR_MSG("Global node root mapping failed (%d)\n", ret); return 8; diff --git a/src/runtime-pmi2.c b/src/runtime-pmi2.c index 75efda9bf..d1b885e23 100644 --- a/src/runtime-pmi2.c +++ b/src/runtime-pmi2.c @@ -43,6 +43,7 @@ static int max_name_len, max_key_len, max_val_len; static int initialized_pmi = 0; static int *location_array = NULL; static int *is_node_root = NULL; +static int *node_id_array = NULL; int @@ -81,6 +82,8 @@ shmem_runtime_init(int enable_node_ranks) #ifdef USE_HIERARCHICAL_BARRIER is_node_root = malloc(sizeof(int) * size); if (NULL == is_node_root) return 9; + node_id_array = malloc(sizeof(int) * size); + if (NULL == node_id_array) return 10; #endif } @@ -97,6 +100,9 @@ shmem_runtime_fini(void) if (is_node_root) { free(is_node_root); } + if (node_id_array) { + free(node_id_array); + } if (initialized_pmi == 1) { PMI2_Finalize(); @@ -187,6 +193,21 @@ shmem_runtime_is_node_root_pe(int pe) } +int +shmem_runtime_get_node_id(int pe) +{ + shmem_internal_assert(pe < size && pe >= 0); + + if (size == 1) + return 0; + + if (NULL == node_id_array) + return pe; + + return node_id_array[pe]; +} + + int shmem_runtime_exchange(void) { @@ -212,7 +233,7 @@ shmem_runtime_exchange(void) } } if (is_node_root) { - ret = shmem_runtime_util_populate_global_node_roots(is_node_root, size); + ret = shmem_runtime_util_populate_global_node_roots(is_node_root, node_id_array, size); if (0 != ret) { RETURN_ERROR_MSG("Global node root mapping failed (%d)\n", ret); return 8; diff --git a/src/runtime-pmix.c b/src/runtime-pmix.c index 7c904a083..f91c12a3c 100644 --- a/src/runtime-pmix.c +++ b/src/runtime-pmix.c @@ -38,6 +38,7 @@ static uint32_t size; static uint32_t node_size = 0; static int *node_ranks = NULL; static int *is_node_root = NULL; +static int *node_id_array = NULL; int shmem_runtime_init(int enable_node_ranks) @@ -76,6 +77,11 @@ shmem_runtime_init(int enable_node_ranks) RETURN_ERROR_MSG_PREINIT("Out of memory allocating is_node_root\n"); return 2; } + node_id_array = (int *)malloc(size * sizeof(int)); + if (NULL == node_id_array) { + RETURN_ERROR_MSG_PREINIT("Out of memory allocating node_id_array\n"); + return 3; + } #endif } @@ -92,6 +98,8 @@ shmem_runtime_fini(void) free(node_ranks); if (is_node_root) free(is_node_root); + if (node_id_array) + free(node_id_array); if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) { RETURN_ERROR_MSG_PREINIT("PMIx_Finalize failed (%d)\n", rc); @@ -184,6 +192,21 @@ shmem_runtime_is_node_root_pe(int pe) } +int +shmem_runtime_get_node_id(int pe) +{ + shmem_internal_assert(pe < (int) size && pe >= 0); + + if (size == 1) + return 0; + + if (NULL == node_id_array) + return pe; + + return node_id_array[pe]; +} + + int shmem_runtime_exchange(void) { @@ -257,7 +280,7 @@ shmem_runtime_exchange(void) PMIX_INFO_DESTRUCT(&info); if (is_node_root) { - int ret = shmem_runtime_util_populate_global_node_roots(is_node_root, (int) size); + int ret = shmem_runtime_util_populate_global_node_roots(is_node_root, node_id_array, (int) size); if (ret != 0) { RETURN_ERROR_MSG("Global node root mapping failed (%d)\n", ret); return ret; diff --git a/src/runtime.h b/src/runtime.h index ab62079c3..243cecc06 100644 --- a/src/runtime.h +++ b/src/runtime.h @@ -42,12 +42,17 @@ void shmem_runtime_barrier(void); /* Utility functions used to implement the runtime layer */ int shmem_runtime_util_put_hostname(void); int shmem_runtime_util_populate_node(int *location_array, int size, int *node_size); -int shmem_runtime_util_populate_global_node_roots(int *is_node_root, int size); +int shmem_runtime_util_populate_global_node_roots(int *is_node_root, int *node_id_out, int size); /* Returns 1 if PE pe is the lowest-ranked PE on its own node, 0 otherwise. * Valid after shmem_runtime_exchange(). */ int shmem_runtime_is_node_root_pe(int pe); +/* Returns the global PE number of the lowest-ranked PE on pe's node. + * All PEs on the same physical node share the same node_id value. + * Valid after shmem_runtime_exchange(). */ +int shmem_runtime_get_node_id(int pe); + int shmem_runtime_util_encode(const void *inval, int invallen, char *outval, int outvallen); int shmem_runtime_util_decode(const char *inval, void *outval, size_t outvallen); #endif diff --git a/src/runtime_util.c b/src/runtime_util.c index e53831eec..0160731de 100644 --- a/src/runtime_util.c +++ b/src/runtime_util.c @@ -167,13 +167,14 @@ int shmem_runtime_util_populate_node(int *location_array, int size, int *node_si /* Populate is_node_root[pe] = 1 if PE pe is the lowest-ranked (first) PE on - * its node, 0 otherwise. This determines which PEs act as internode - * representatives in the hierarchical barrier. + * its node, 0 otherwise. Optionally populate node_id_out[pe] with the global + * PE number of the lowest-ranked PE on pe's node (the same value for all PEs + * that share a node). node_id_out may be NULL. * * Must be called after shmem_runtime_util_put_hostname and a runtime exchange, * so that every PE's hostname is readable via shmem_runtime_get. */ int -shmem_runtime_util_populate_global_node_roots(int *is_node_root, int size) +shmem_runtime_util_populate_global_node_roots(int *is_node_root, int *node_id_out, int size) { int ret = 0; char **hostnames = (char **) malloc(size * sizeof(char *)); @@ -209,13 +210,18 @@ shmem_runtime_util_populate_global_node_roots(int *is_node_root, int size) } } - /* PE pe is a node root if no earlier PE (q < pe) has the same hostname. */ + /* PE pe is a node root if no earlier PE (q < pe) has the same hostname. + * node_id_out[pe] (when non-NULL) is set to the global PE number of the + * lowest-ranked PE on pe's node, which serves as a stable node identifier + * that all PEs can compute identically from global data. */ for (int pe = 0; pe < size; pe++) { is_node_root[pe] = 1; + if (node_id_out) node_id_out[pe] = pe; for (int q = 0; q < pe; q++) { if (hlens[pe] == hlens[q] && memcmp(hostnames[pe], hostnames[q], hlens[pe]) == 0) { is_node_root[pe] = 0; + if (node_id_out) node_id_out[pe] = node_id_out[q]; break; } } diff --git a/src/shmem_collectives.h b/src/shmem_collectives.h index 43fc26fec..c8a4c9242 100644 --- a/src/shmem_collectives.h +++ b/src/shmem_collectives.h @@ -17,6 +17,9 @@ #define SHMEM_COLLECTIVES_H #include "shmem_synchronization.h" +#ifdef USE_HIERARCHICAL_BARRIER +#include "shmem_team.h" +#endif enum coll_type_t { @@ -52,7 +55,8 @@ void shmem_internal_sync_tree(int PE_start, int PE_stride, int PE_size, long *pS void shmem_internal_sync_dissem(int PE_start, int PE_stride, int PE_size, long *pSync); #ifdef USE_HIERARCHICAL_BARRIER void shmem_internal_sync_hierarchical(int PE_start, int PE_stride, int PE_size, - long *pSync, long *local_pSync); + long *pSync, long *local_pSync, + long *hier_sense_ptr); void shmem_internal_hier_barrier_print_stats(void); #endif @@ -60,6 +64,12 @@ static inline void shmem_internal_sync(int PE_start, int PE_stride, int PE_size, long *pSync) { + /* Passes NULL as hier_sense_ptr: the hierarchical barrier then uses a + * shared static sense counter, which is only valid when all callers of a + * given active set sync in lock-step (the intra-collective syncs here do). + * Team-aware callers use shmem_internal_sync_for_team / + * shmem_internal_barrier_for_team to supply a per-team sense counter; + * deprecated shmem_barrier/shmem_sync use shmem_internal_sync_no_hier. */ if (shmem_internal_params.BARRIERS_FLUSH) { fflush(stdout); fflush(stderr); @@ -73,7 +83,8 @@ shmem_internal_sync(int PE_start, int PE_stride, int PE_size, long *pSync) if (shmem_internal_get_shr_size() >= shmem_internal_params.HIER_BARRIER_THRESHOLD) { shmem_internal_sync_hierarchical(PE_start, PE_stride, PE_size, pSync, - shmem_internal_hierarchical_local_psync); + shmem_internal_hierarchical_local_psync, + NULL); break; } #endif @@ -96,7 +107,8 @@ shmem_internal_sync(int PE_start, int PE_stride, int PE_size, long *pSync) case HIERARCHICAL: shmem_internal_sync_hierarchical(PE_start, PE_stride, PE_size, pSync, - shmem_internal_hierarchical_local_psync); + shmem_internal_hierarchical_local_psync, + NULL); break; #endif default: @@ -110,6 +122,51 @@ shmem_internal_sync(int PE_start, int PE_stride, int PE_size, long *pSync) } +/* Sync for the deprecated active-set APIs (shmem_barrier, shmem_sync). + * Never uses the hierarchical algorithm: the deprecated APIs carry no team + * context, so there is no per-active-set sense counter available. Falls + * through to the next-best algorithm selected at configure/runtime. */ +static inline +void +shmem_internal_sync_no_hier(int PE_start, int PE_stride, int PE_size, long *pSync) +{ + if (shmem_internal_params.BARRIERS_FLUSH) { + fflush(stdout); + fflush(stderr); + } + + if (PE_size == 1) return; + + switch (shmem_internal_barrier_type) { + case AUTO: + case HIERARCHICAL: + /* Fall through to CROSSOVER-based selection: hierarchical is unsafe + * here because no per-active-set sense pointer is available. */ + if (PE_size < shmem_internal_params.COLL_CROSSOVER) { + shmem_internal_sync_linear(PE_start, PE_stride, PE_size, pSync); + } else { + shmem_internal_sync_tree(PE_start, PE_stride, PE_size, pSync); + } + break; + case LINEAR: + shmem_internal_sync_linear(PE_start, PE_stride, PE_size, pSync); + break; + case TREE: + shmem_internal_sync_tree(PE_start, PE_stride, PE_size, pSync); + break; + case DISSEM: + shmem_internal_sync_dissem(PE_start, PE_stride, PE_size, pSync); + break; + default: + RAISE_ERROR_MSG("Illegal barrier/sync type (%d)\n", + shmem_internal_barrier_type); + } + + shmem_internal_membar_acq_rel(); + shmem_transport_syncmem(); +} + + static inline void shmem_internal_sync_all(void) @@ -119,7 +176,8 @@ shmem_internal_sync_all(void) shmem_internal_get_shr_size() >= shmem_internal_params.HIER_BARRIER_THRESHOLD) { shmem_internal_sync_hierarchical(0, 1, shmem_internal_num_pes, shmem_internal_sync_all_psync, - shmem_internal_sync_all_local_psync); + shmem_internal_sync_all_local_psync, + &shmem_internal_team_world.hier_sense); shmem_internal_membar_acq_rel(); shmem_transport_syncmem(); return; @@ -148,7 +206,8 @@ shmem_internal_barrier_all(void) shmem_internal_get_shr_size() >= shmem_internal_params.HIER_BARRIER_THRESHOLD) { shmem_internal_sync_hierarchical(0, 1, shmem_internal_num_pes, shmem_internal_barrier_all_psync, - shmem_internal_barrier_all_local_psync); + shmem_internal_barrier_all_local_psync, + &shmem_internal_team_world.hier_sense); shmem_internal_membar_acq_rel(); shmem_transport_syncmem(); return; @@ -158,6 +217,59 @@ shmem_internal_barrier_all(void) } +#ifdef USE_HIERARCHICAL_BARRIER +/* Team-aware sync: passes team->hier_sense to the hierarchical barrier so each + * team maintains an independent sense counter. Falls back to the generic + * shmem_internal_sync for non-hierarchical algorithms. */ +static inline +void +shmem_internal_sync_for_team(shmem_internal_team_t *team, long *pSync) +{ + int use_hier = (shmem_internal_barrier_type == HIERARCHICAL) || + (shmem_internal_barrier_type == AUTO && + shmem_internal_get_shr_size() >= shmem_internal_params.HIER_BARRIER_THRESHOLD); + if (use_hier) { + if (shmem_internal_params.BARRIERS_FLUSH) { + fflush(stdout); + fflush(stderr); + } + if (team->size == 1) return; + shmem_internal_sync_hierarchical(team->start, team->stride, team->size, + pSync, + shmem_internal_hierarchical_local_psync, + &team->hier_sense); + shmem_internal_membar_acq_rel(); + shmem_transport_syncmem(); + return; + } + shmem_internal_sync(team->start, team->stride, team->size, pSync); +} + +/* Team-aware barrier: quiet + team-aware sync. */ +static inline +void +shmem_internal_barrier_for_team(shmem_internal_team_t *team, long *pSync) +{ + shmem_internal_quiet(SHMEM_CTX_DEFAULT); + shmem_internal_sync_for_team(team, pSync); +} +#else +static inline +void +shmem_internal_sync_for_team(shmem_internal_team_t *team, long *pSync) +{ + shmem_internal_sync(team->start, team->stride, team->size, pSync); +} + +static inline +void +shmem_internal_barrier_for_team(shmem_internal_team_t *team, long *pSync) +{ + shmem_internal_barrier(team->start, team->stride, team->size, pSync); +} +#endif + + void shmem_internal_bcast_linear(void *target, const void *source, size_t len, int PE_root, int PE_start, int PE_stride, int PE_size, long *pSync, int complete); diff --git a/src/shmem_team.c b/src/shmem_team.c index 3241b94e0..a4430d2a4 100644 --- a/src/shmem_team.c +++ b/src/shmem_team.c @@ -419,7 +419,7 @@ int shmem_internal_team_split_strided(shmem_internal_team_t *parent_team, int PE * during psync allocation between back-to-back team creations. */ psync = shmem_internal_team_choose_psync(parent_team, SYNC); - shmem_internal_barrier(parent_team->start, parent_team->stride, parent_team->size, psync); + shmem_internal_barrier_for_team(parent_team, psync); shmem_internal_team_release_psyncs(parent_team, SYNC); @@ -457,8 +457,6 @@ int shmem_internal_team_split_2d(shmem_internal_team_t *parent_team, int xrange, xrange = parent_team->size; } - const int parent_start = parent_team->start; - const int parent_stride = parent_team->stride; const int parent_size = parent_team->size; const int num_xteams = ceil( parent_size / (float)xrange ); const int num_yteams = xrange; @@ -506,7 +504,7 @@ int shmem_internal_team_split_2d(shmem_internal_team_t *parent_team, int xrange, long *psync = shmem_internal_team_choose_psync(parent_team, SYNC); - shmem_internal_barrier(parent_start, parent_stride, parent_size, psync); + shmem_internal_barrier_for_team(parent_team, psync); shmem_internal_team_release_psyncs(parent_team, SYNC); @@ -566,7 +564,7 @@ long * shmem_internal_team_choose_psync(shmem_internal_team_t *team, shmem_inter shmem_internal_quiet(SHMEM_CTX_DEFAULT); size_t psync = team->psync_idx * SHMEM_SYNC_SIZE; - shmem_internal_sync(team->start, team->stride, team->size, + shmem_internal_sync_for_team(team, &shmem_internal_psync_barrier_pool[psync]); for (int i = 0; i < N_PSYNCS_PER_TEAM; i++) { From 33cc4bece96e82cb8ec5dc7dd5053a45c52f870e Mon Sep 17 00:00:00 2001 From: Bryan Morgan Date: Thu, 25 Jun 2026 08:45:20 -0700 Subject: [PATCH 11/14] collectives: cache per-team hierarchical-barrier topology Each hierarchical barrier rebuilt the active-set topology from scratch: the local PE set, the per-node root set, virtual indices, tree children (in shr-rank space), and the phase-2 dissemination round count. For a fixed team this is all invariant, yet it ran on every call -- including an O(PE_size^2) root-set scan, an O(PE_size) local scan, and a shmem_runtime_get_node_rank lookup per tree child. Cache it per team. Add shmem_internal_hier_cache_t to the team struct, built once on the first barrier (shmem_internal_hier_compute) and reused thereafter. shmem_internal_sync_hierarchical takes a hier_cache pointer: team-aware callers (sync_all, barrier_all, sync_for_team) pass &team->hier_cache; callers with no team context pass NULL and build into the shared per-call scratch as before. After the first call, per-barrier topology cost drops to a few field loads. Only topology is cached, not XPMEM mapped pointers: those depend on the local_pSync array (sync_all and barrier_all share a team but pass different arrays) and are cheap base+offset arithmetic to re-derive. Also make build_root_active_set O(PE_size) via a node_id-keyed seen-flag buffer (hier_node_seen) instead of the nested O(PE_size^2) scan, so even the one-time first-barrier build is linear at scale. The buffer is self-cleaning (resets only touched entries) to avoid an O(PE_size) clear per call. Cache persistence is allocation-failure safe: if any of the three exact-size buffers fails to allocate, the barrier falls back to the scratch topology for that call without persisting (valid stays 0), so it still completes correctly and retries the build next time. The topology cache is zeroed for the three predefined teams (world, shared, node; split teams are calloc'd) and freed in shmem_internal_team_destroy. configure.ac: --enable-hierarchical-barrier now requires XPMEM specifically, not "XPMEM or CMA". The intranode phase does CPU atomics on peers' slots through shmem_shr_transport_ptr, which only XPMEM provides; a CMA-only build would have configured successfully then failed at the first barrier. --- configure.ac | 23 +++-- src/collectives.c | 203 +++++++++++++++++++++++++++++----------- src/shmem_collectives.h | 17 ++-- src/shmem_team.c | 7 ++ src/shmem_team.h | 25 +++++ 5 files changed, 206 insertions(+), 69 deletions(-) diff --git a/configure.ac b/configure.ac index 197072c1a..53a0493e1 100644 --- a/configure.ac +++ b/configure.ac @@ -330,9 +330,10 @@ AS_IF([test "$enable_shr_atomics" = "yes"], AC_ARG_ENABLE([hierarchical-barrier], [AS_HELP_STRING([--enable-hierarchical-barrier], [Enable the hierarchical barrier: intranode phase uses CPU atomics, - internode phase uses NIC puts. Requires an on-node transport (XPMEM - or CMA) and a network transport (OFI, Portals4, or UCX). - (default: disabled)])]) + internode phase uses NIC puts. Requires XPMEM (the intranode + phase does CPU atomics on peers' slots via mapped pointers, which + only XPMEM provides) and a network transport (OFI, Portals4, or + UCX). (default: disabled)])]) AC_ARG_ENABLE([manpages], [AS_HELP_STRING([--enable-manpages], @@ -557,14 +558,16 @@ if test "$enable_shr_atomics" != "no" -a "$transport_xpmem" = "yes" -a "$transpo AC_DEFINE([USE_SHR_ATOMICS], [1], [If defined, the shared memory layer will perform processor atomics.]) fi -dnl Hierarchical barrier: only when explicitly requested. Validate that an -dnl on-node transport (XPMEM or CMA) and a network transport are both present; -dnl error if dependencies are missing. +dnl Hierarchical barrier: only when explicitly requested. Validate that XPMEM +dnl and a network transport are both present; error if dependencies are missing. +dnl XPMEM (not CMA) is required because the intranode phase performs CPU atomics +dnl on peers' pSync slots through directly mapped pointers (shmem_shr_transport_ptr), +dnl which only XPMEM can provide; CMA's copy-based path cannot. transport_hierarchical_barrier="no" if test "$enable_hierarchical_barrier" = "yes"; then dnl Check on-node transport dependency - if test "$transport_xpmem" != "yes" -a "$transport_cma" != "yes"; then - AC_MSG_ERROR([--enable-hierarchical-barrier requires an on-node transport; configure with --with-xpmem or --with-cma]) + if test "$transport_xpmem" != "yes"; then + AC_MSG_ERROR([--enable-hierarchical-barrier requires XPMEM; configure with --with-xpmem]) dnl Check network transport dependency elif test "$transport_ofi" != "yes" -a "$transport_portals4" != "yes" -a "$transport_ucx" != "yes"; then AC_MSG_ERROR([--enable-hierarchical-barrier requires a network transport; configure with --with-ofi, --with-portals4, or --with-ucx]) @@ -575,8 +578,8 @@ fi if test "$transport_hierarchical_barrier" = "yes"; then AC_DEFINE([USE_HIERARCHICAL_BARRIER], [1], [If defined, enables the hierarchical barrier: intranode CPU atomics + internode NIC puts.]) - dnl The hierarchical barrier uses shared-memory puts/gets (XPMEM/CMA) for - dnl intranode data movement. USE_SHR_ATOMICS is defined so that the + dnl The hierarchical barrier uses XPMEM-mapped pointers for intranode data + dnl movement (CPU atomics on peers' slots). USE_SHR_ATOMICS is defined so that the dnl shmem_shr_transport_atomic() function bodies are compiled; the routing dnl logic in shmem_shr_transport_use_atomic() gates them behind a runtime dnl check (shr_size == num_pes), so user-visible AMOs only go through CPU diff --git a/src/collectives.c b/src/collectives.c index 16f029ece..3cb67dc99 100644 --- a/src/collectives.c +++ b/src/collectives.c @@ -45,6 +45,11 @@ long *shmem_internal_hierarchical_local_psync; static int *hier_local_pes = NULL; static int *hier_root_pes = NULL; +/* Seen-flags for the root-set build, indexed by node_id (a global PE number in + * [0, num_pes)). Lets build_root_active_set run in O(PE_size) instead of + * O(PE_size^2). Allocated once at init, sized to num_pes. */ +static char *hier_node_seen = NULL; + /* Layout of local_pSync — two cache-line-padded arrays, one slot per PE: * * up-slot for PE r: local_pSync[r * HIER_SLOT_STRIDE] @@ -223,7 +228,11 @@ shmem_internal_collectives_init(void) * num_pes so it fits the largest possible active set. */ hier_local_pes = malloc(sizeof(int) * shmem_internal_num_pes); hier_root_pes = malloc(sizeof(int) * shmem_internal_num_pes); - if (NULL == hier_local_pes || NULL == hier_root_pes) return -1; + /* calloc: seen-flags must start cleared; build_root_active_set leaves them + * clean afterward by resetting only the entries it touched. */ + hier_node_seen = calloc(shmem_internal_num_pes, sizeof(char)); + if (NULL == hier_local_pes || NULL == hier_root_pes || + NULL == hier_node_seen) return -1; #endif /* initialize the binomial tree for collective operations over @@ -561,30 +570,33 @@ shmem_internal_build_local_set(int PE_start, int PE_stride, int PE_size, /* Build an active set of root PEs — one representative per node, defined as * the lowest-ranked active PE on each participating node. Uses the global * node_id array (computed once at init) so all PEs deterministically compute - * the same list without requiring communication. */ + * the same list without requiring communication. + * + * O(PE_size): a seen-flag keyed by node_id (a global PE number) records which + * nodes already have a representative. Iterating in ascending PE order, the + * first active PE seen for a node_id is its representative. seen_buf holds + * num_pes entries and must enter all-zero; a second pass resets exactly the + * entries this build touched, preserving that invariant for the next call. */ static void shmem_internal_build_root_active_set(int PE_start, int PE_stride, int PE_size, - int *root_pes, int *root_count) + int *root_pes, int *root_count, + char *seen_buf) { int i, pe; *root_count = 0; - /* Iterate in ascending PE order. The first active PE encountered for each - * node_id is the representative for that node. */ for (i = 0, pe = PE_start; i < PE_size; i++, pe += PE_stride) { - int pe_node_id = shmem_runtime_get_node_id(pe); - int already_represented = 0; - int j, qpe; - for (j = 0, qpe = PE_start; j < i; j++, qpe += PE_stride) { - if (shmem_runtime_get_node_id(qpe) == pe_node_id) { - already_represented = 1; - break; - } - } - if (!already_represented) { + int nid = shmem_runtime_get_node_id(pe); + if (!seen_buf[nid]) { + seen_buf[nid] = 1; root_pes[(*root_count)++] = pe; } } + + /* Reset only the touched entries so seen_buf is clean for the next build. */ + for (i = 0; i < *root_count; i++) { + seen_buf[shmem_runtime_get_node_id(root_pes[i])] = 0; + } } /* CPU atomic store of `val` to the long at `target` via local mapped pointer. */ @@ -596,10 +608,74 @@ shmem_internal_cpu_atomic_store_long(long *target, int noderank, long val) __atomic_store((long *)remote_ptr, &val, __ATOMIC_RELEASE); } +/* Compute the full hierarchical-barrier topology for an active set into a + * cache struct. The PE/root lists are written into the caller-supplied + * buffers (local_buf >= on-node active count, root_buf >= node count, + * child_buf >= tree_radix); the cache's pointers are set to reference them. + * Every field here is invariant for a fixed (PE_start, PE_stride, PE_size) and + * node layout, so a team can compute this once and reuse it. */ +static void +shmem_internal_hier_compute(int PE_start, int PE_stride, int PE_size, + int *local_buf, int *root_buf, int *child_buf, + shmem_internal_hier_cache_t *c) +{ + c->local_pes = local_buf; + c->root_pes = root_buf; + c->tree_child_shr = child_buf; + + shmem_internal_build_local_set(PE_start, PE_stride, PE_size, + c->local_pes, &c->local_count); + shmem_internal_build_root_active_set(PE_start, PE_stride, PE_size, + c->root_pes, &c->root_count, + hier_node_seen); + + /* build_local_set fills local_pes in ascending PE order, so local_pes[0] is + * the lowest-ranked active PE on this node — the active-set representative + * for phase 2. my_vidx is this PE's index in that list (-1 if absent). */ + c->my_vidx = -1; + for (int i = 0; i < c->local_count; i++) { + if (c->local_pes[i] == shmem_internal_my_pe) { c->my_vidx = i; break; } + } + c->active_root_pe = (c->local_count > 0) ? c->local_pes[0] : -1; + c->is_root = (shmem_internal_my_pe == c->active_root_pe); + + c->tree_nchildren = 0; + if (c->my_vidx >= 0) { + for (int j = 1; j <= tree_radix; j++) { + int cv = c->my_vidx * tree_radix + j; + if (cv < c->local_count) { + c->tree_child_shr[c->tree_nchildren++] = + shmem_runtime_get_node_rank(c->local_pes[cv]); + } + } + } + + c->my_root_idx = -1; + for (int i = 0; i < c->root_count; i++) { + if (c->root_pes[i] == shmem_internal_my_pe) { c->my_root_idx = i; break; } + } + + c->num_rounds = 0; + { int n = c->root_count - 1; while (n > 0) { n >>= 1; c->num_rounds++; } } +} + +/* Free the malloc'd arrays held by a persisted topology cache. */ +void +shmem_internal_hier_cache_free(shmem_internal_hier_cache_t *c) +{ + if (c == NULL) return; + free(c->local_pes); + free(c->root_pes); + free(c->tree_child_shr); + c->local_pes = c->root_pes = c->tree_child_shr = NULL; + c->valid = 0; +} + void shmem_internal_sync_hierarchical(int PE_start, int PE_stride, int PE_size, long *pSync, long *local_pSync, - long *hier_sense_ptr) + long *hier_sense_ptr, + shmem_internal_hier_cache_t *hier_cache) { int my_shr_rank = shmem_runtime_get_node_rank(shmem_internal_my_pe); long one = 1; @@ -621,40 +697,66 @@ shmem_internal_sync_hierarchical(int PE_start, int PE_stride, int PE_size, sense_ptr = &fallback_sense; } - /* Collect local and root PE sets for this active set */ - int *local_pes = hier_local_pes; - int local_count = 0; - shmem_internal_build_local_set(PE_start, PE_stride, PE_size, - local_pes, &local_count); - - int *root_pes = hier_root_pes; - int root_count = 0; - shmem_internal_build_root_active_set(PE_start, PE_stride, PE_size, - root_pes, &root_count); - - /* Assign virtual index 0..local_count-1. build_local_set fills local_pes - * in ascending PE order, so local_pes[0] is always the lowest-ranked active - * PE on this node and is the active-set representative for phase 2. */ - int my_local_idx = -1; - for (int i = 0; i < local_count; i++) { - if (local_pes[i] == shmem_internal_my_pe) { my_local_idx = i; break; } - } - int my_vidx = my_local_idx; /* -1 if not in active set on this node */ - int active_root_pe = (local_count > 0) ? local_pes[0] : -1; - int is_root = (shmem_internal_my_pe == active_root_pe); - - int tree_nchildren = 0; - int *tree_child_shr = alloca(sizeof(int) * tree_radix); - if (my_vidx >= 0) { - for (int j = 1; j <= tree_radix; j++) { - int cv = my_vidx * tree_radix + j; - if (cv < local_count) { - tree_child_shr[tree_nchildren++] = shmem_runtime_get_node_rank( - local_pes[cv]); + /* Resolve the active-set topology. A team supplies a cache (hier_cache): + * built once on the first barrier, reused thereafter — at scale this + * removes the O(PE_size) / O(PE_size^2) set scans and per-child runtime + * lookups from the hot path. Callers with no team context (hier_cache == + * NULL) build into the shared per-call scratch every time. */ + shmem_internal_hier_cache_t scratch_cache; + shmem_internal_hier_cache_t *topo; + if (hier_cache != NULL && hier_cache->valid) { + topo = hier_cache; + } else { + shmem_internal_hier_compute(PE_start, PE_stride, PE_size, + hier_local_pes, hier_root_pes, + alloca(sizeof(int) * tree_radix), + &scratch_cache); + if (hier_cache != NULL) { + /* Persist exact-size copies in the team cache for reuse. If any + * allocation fails, fall back to using the scratch topology for + * this call without persisting — the barrier still completes + * correctly, just without the caching optimization. */ + int lc = scratch_cache.local_count, rc = scratch_cache.root_count; + int nc = scratch_cache.tree_nchildren; + int *cl = malloc(sizeof(int) * (lc > 0 ? lc : 1)); + int *cr = malloc(sizeof(int) * (rc > 0 ? rc : 1)); + int *cc = malloc(sizeof(int) * (nc > 0 ? nc : 1)); + if (cl == NULL || cr == NULL || cc == NULL) { + free(cl); + free(cr); + free(cc); + topo = &scratch_cache; + } else { + memcpy(cl, scratch_cache.local_pes, sizeof(int) * lc); + memcpy(cr, scratch_cache.root_pes, sizeof(int) * rc); + memcpy(cc, scratch_cache.tree_child_shr, sizeof(int) * nc); + hier_cache->local_pes = cl; + hier_cache->root_pes = cr; + hier_cache->tree_child_shr = cc; + hier_cache->local_count = lc; + hier_cache->root_count = rc; + hier_cache->tree_nchildren = nc; + hier_cache->my_vidx = scratch_cache.my_vidx; + hier_cache->active_root_pe = scratch_cache.active_root_pe; + hier_cache->is_root = scratch_cache.is_root; + hier_cache->my_root_idx = scratch_cache.my_root_idx; + hier_cache->num_rounds = scratch_cache.num_rounds; + hier_cache->valid = 1; + topo = hier_cache; } + } else { + topo = &scratch_cache; } } + int local_count = topo->local_count; + int *root_pes = topo->root_pes; + int root_count = topo->root_count; + int my_vidx = topo->my_vidx; + int is_root = topo->is_root; + int *tree_child_shr = topo->tree_child_shr; + int tree_nchildren = topo->tree_nchildren; + /* Sense-alternating signal — monotonically increasing, no slot resets needed. * up-slot for PE r: local_pSync[r * HIER_SLOT_STRIDE] * down-slot for PE r: local_pSync[shr_size * HIER_SLOT_STRIDE + r * HIER_SLOT_STRIDE] */ @@ -757,14 +859,9 @@ shmem_internal_sync_hierarchical(int PE_start, int PE_stride, int PE_size, if (is_root) { /* ---- Phase 2: internode barrier (NIC puts, root PEs only) ---- */ if (root_count > 1) { - int my_root_idx = -1; - for (int i = 0; i < root_count; i++) { - if (root_pes[i] == shmem_internal_my_pe) { my_root_idx = i; break; } - } + int my_root_idx = topo->my_root_idx; + int num_rounds = topo->num_rounds; shmem_internal_assert(my_root_idx >= 0); - - int num_rounds = 0; - { int n = root_count - 1; while (n > 0) { n >>= 1; num_rounds++; } } shmem_internal_assert(num_rounds <= SHMEM_BARRIER_SYNC_SIZE); for (int r = 0; r < num_rounds; r++) { diff --git a/src/shmem_collectives.h b/src/shmem_collectives.h index c8a4c9242..e898210bf 100644 --- a/src/shmem_collectives.h +++ b/src/shmem_collectives.h @@ -56,7 +56,9 @@ void shmem_internal_sync_dissem(int PE_start, int PE_stride, int PE_size, long * #ifdef USE_HIERARCHICAL_BARRIER void shmem_internal_sync_hierarchical(int PE_start, int PE_stride, int PE_size, long *pSync, long *local_pSync, - long *hier_sense_ptr); + long *hier_sense_ptr, + shmem_internal_hier_cache_t *hier_cache); +void shmem_internal_hier_cache_free(shmem_internal_hier_cache_t *hier_cache); void shmem_internal_hier_barrier_print_stats(void); #endif @@ -84,7 +86,7 @@ shmem_internal_sync(int PE_start, int PE_stride, int PE_size, long *pSync) shmem_internal_sync_hierarchical(PE_start, PE_stride, PE_size, pSync, shmem_internal_hierarchical_local_psync, - NULL); + NULL, NULL); break; } #endif @@ -108,7 +110,7 @@ shmem_internal_sync(int PE_start, int PE_stride, int PE_size, long *pSync) shmem_internal_sync_hierarchical(PE_start, PE_stride, PE_size, pSync, shmem_internal_hierarchical_local_psync, - NULL); + NULL, NULL); break; #endif default: @@ -177,7 +179,8 @@ shmem_internal_sync_all(void) shmem_internal_sync_hierarchical(0, 1, shmem_internal_num_pes, shmem_internal_sync_all_psync, shmem_internal_sync_all_local_psync, - &shmem_internal_team_world.hier_sense); + &shmem_internal_team_world.hier_sense, + &shmem_internal_team_world.hier_cache); shmem_internal_membar_acq_rel(); shmem_transport_syncmem(); return; @@ -207,7 +210,8 @@ shmem_internal_barrier_all(void) shmem_internal_sync_hierarchical(0, 1, shmem_internal_num_pes, shmem_internal_barrier_all_psync, shmem_internal_barrier_all_local_psync, - &shmem_internal_team_world.hier_sense); + &shmem_internal_team_world.hier_sense, + &shmem_internal_team_world.hier_cache); shmem_internal_membar_acq_rel(); shmem_transport_syncmem(); return; @@ -237,7 +241,8 @@ shmem_internal_sync_for_team(shmem_internal_team_t *team, long *pSync) shmem_internal_sync_hierarchical(team->start, team->stride, team->size, pSync, shmem_internal_hierarchical_local_psync, - &team->hier_sense); + &team->hier_sense, + &team->hier_cache); shmem_internal_membar_acq_rel(); shmem_transport_syncmem(); return; diff --git a/src/shmem_team.c b/src/shmem_team.c index a4430d2a4..f5cb69dac 100644 --- a/src/shmem_team.c +++ b/src/shmem_team.c @@ -87,6 +87,7 @@ int shmem_internal_team_init(void) shmem_internal_team_world.psync_avail[i] = 1; #ifdef USE_HIERARCHICAL_BARRIER shmem_internal_team_world.hier_sense = 0; + memset(&shmem_internal_team_world.hier_cache, 0, sizeof(shmem_internal_hier_cache_t)); #endif SHMEM_TEAM_WORLD = (shmem_team_t) &shmem_internal_team_world; @@ -100,6 +101,7 @@ int shmem_internal_team_init(void) shmem_internal_team_shared.psync_avail[i] = 1; #ifdef USE_HIERARCHICAL_BARRIER shmem_internal_team_shared.hier_sense = 0; + memset(&shmem_internal_team_shared.hier_cache, 0, sizeof(shmem_internal_hier_cache_t)); #endif SHMEM_TEAM_SHARED = (shmem_team_t) &shmem_internal_team_shared; @@ -113,6 +115,7 @@ int shmem_internal_team_init(void) shmem_internal_team_node.psync_avail[i] = 1; #ifdef USE_HIERARCHICAL_BARRIER shmem_internal_team_node.hier_sense = 0; + memset(&shmem_internal_team_node.hier_cache, 0, sizeof(shmem_internal_hier_cache_t)); #endif SHMEMX_TEAM_NODE = (shmem_team_t) &shmem_internal_team_node; @@ -534,6 +537,10 @@ void shmem_internal_team_destroy(shmem_internal_team_t *team) shmem_internal_team_pool[team->psync_idx] = NULL; free(team->contexts); +#ifdef USE_HIERARCHICAL_BARRIER + shmem_internal_hier_cache_free(&team->hier_cache); +#endif + if (team != &shmem_internal_team_world && team != &shmem_internal_team_shared && team != &shmem_internal_team_node) { free(team); diff --git a/src/shmem_team.h b/src/shmem_team.h index 654fd105a..042dc4e8d 100644 --- a/src/shmem_team.h +++ b/src/shmem_team.h @@ -17,6 +17,30 @@ #define N_PSYNCS_PER_TEAM 2 +#ifdef USE_HIERARCHICAL_BARRIER +/* Cached hierarchical-barrier topology for a fixed active set. All fields are + * invariant for a given (start, stride, size) and the node layout (both fixed + * for the life of a team), so they are built once on the first barrier and + * reused. Holds only topology — NOT XPMEM mapped pointers, which depend on the + * local_pSync array (sync_all and barrier_all share a team but pass different + * arrays) and are cheap base+offset arithmetic to re-derive per call. */ +struct shmem_internal_hier_cache_t { + int valid; + int *local_pes; /* active PEs on this node, ascending */ + int local_count; + int *root_pes; /* one representative (lowest active PE) per node */ + int root_count; + int my_vidx; /* my index in local_pes, or -1 if not in the set */ + int active_root_pe; /* local_pes[0], this node's representative */ + int is_root; /* my_pe == active_root_pe */ + int *tree_child_shr; /* shr_ranks of my intranode tree children */ + int tree_nchildren; + int my_root_idx; /* my index in root_pes, or -1 if not a root */ + int num_rounds; /* phase-2 dissemination rounds (log2(root_count)) */ +}; +typedef struct shmem_internal_hier_cache_t shmem_internal_hier_cache_t; +#endif + struct shmem_internal_team_t { int my_pe; int start, stride, size; @@ -28,6 +52,7 @@ struct shmem_internal_team_t { struct shmem_transport_ctx_t **contexts; #ifdef USE_HIERARCHICAL_BARRIER long hier_sense; + shmem_internal_hier_cache_t hier_cache; #endif }; typedef struct shmem_internal_team_t shmem_internal_team_t; From d5b40ee464fe046ec762d56d7d7c9cea34fe3e88 Mon Sep 17 00:00:00 2001 From: Bryan Morgan Date: Thu, 25 Jun 2026 11:58:33 -0700 Subject: [PATCH 12/14] collectives: gate AUTO hierarchical barrier on multi-node, job-global min PPN AUTO barrier selection chose the hierarchical algorithm whenever the local node had >= HIER_BARRIER_THRESHOLD PEs. Two problems: 1. On a single-node job the hierarchical barrier runs its degenerate intranode-only path, which is strictly worse than linear/tree: under USE_HIERARCHICAL_BARRIER the shr transport already routes single-node atomics through CPU atomics (shmem_shr_transport_use_atomic), so linear/tree run over shared memory with none of the hierarchical scaffolding. Now require the job to span more than one node (shr_size < num_pes) before AUTO selects hierarchical. 2. The threshold was checked against the local node size, which differs per PE on a heterogeneous-PPN job -- a large node could select hierarchical while a small node selects tree, diverging within one collective. Gate instead on a job-global minimum PEs-per-node, computed once in collectives_init from the global node_id array (already exchanged at init, no new communication): tally PEs per node, take the min over populated nodes. Both terms of the predicate are now job-global, so every PE makes the same selection. Consolidated into shmem_internal_hier_barrier_available(), used at all AUTO selection sites (sync, sync_all, barrier_all, sync_for_team) and the init-time barrier-algorithm diagnostic. Explicit BARRIER_ALGORITHM=hierarchical is unaffected and always honored. --- src/collectives.c | 25 +++++++++++++++++++++++++ src/init.c | 2 +- src/shmem_collectives.h | 31 +++++++++++++++++++++++++++---- 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/src/collectives.c b/src/collectives.c index 3cb67dc99..0ad125cef 100644 --- a/src/collectives.c +++ b/src/collectives.c @@ -50,6 +50,13 @@ static int *hier_root_pes = NULL; * O(PE_size^2). Allocated once at init, sized to num_pes. */ static char *hier_node_seen = NULL; +/* Job-global minimum PEs-per-node, computed once at init from the global + * node_id array. AUTO hierarchical-barrier selection gates on this (not the + * local node size) so every PE makes the same decision even on a job with + * heterogeneous PPN; otherwise PEs on a large node could pick hierarchical + * while PEs on a small node pick tree, diverging within one collective. */ +int shmem_internal_hier_min_ppn = 0; + /* Layout of local_pSync — two cache-line-padded arrays, one slot per PE: * * up-slot for PE r: local_pSync[r * HIER_SLOT_STRIDE] @@ -233,6 +240,24 @@ shmem_internal_collectives_init(void) hier_node_seen = calloc(shmem_internal_num_pes, sizeof(char)); if (NULL == hier_local_pes || NULL == hier_root_pes || NULL == hier_node_seen) return -1; + + /* Compute the job-global minimum PEs-per-node from the global node_id array + * (already exchanged at init — no communication needed here). node_id[pe] + * is the global PE number of the lowest-ranked PE on pe's node, so it both + * identifies the node and is a valid index in [0, num_pes); tally PE counts + * per node, then take the minimum over nodes that actually have PEs. */ + { + int *ppn_count = calloc(shmem_internal_num_pes, sizeof(int)); + if (NULL == ppn_count) return -1; + for (i = 0; i < shmem_internal_num_pes; i++) + ppn_count[shmem_runtime_get_node_id(i)]++; + shmem_internal_hier_min_ppn = shmem_internal_num_pes; + for (i = 0; i < shmem_internal_num_pes; i++) { + if (ppn_count[i] > 0 && ppn_count[i] < shmem_internal_hier_min_ppn) + shmem_internal_hier_min_ppn = ppn_count[i]; + } + free(ppn_count); + } #endif /* initialize the binomial tree for collective operations over diff --git a/src/init.c b/src/init.c index dd7ce3815..c76cd0e00 100644 --- a/src/init.c +++ b/src/init.c @@ -532,7 +532,7 @@ shmem_internal_heap_postinit(void) if (shmem_internal_my_pe == 0) { const char *effective_barrier = (shmem_internal_barrier_type == AUTO && - shmem_internal_get_shr_size() >= shmem_internal_params.HIER_BARRIER_THRESHOLD) + shmem_internal_hier_barrier_available()) ? "HIERARCHICAL (auto-selected)" : coll_type_str[shmem_internal_barrier_type]; diff --git a/src/shmem_collectives.h b/src/shmem_collectives.h index e898210bf..99a91feaa 100644 --- a/src/shmem_collectives.h +++ b/src/shmem_collectives.h @@ -60,6 +60,29 @@ void shmem_internal_sync_hierarchical(int PE_start, int PE_stride, int PE_size, shmem_internal_hier_cache_t *hier_cache); void shmem_internal_hier_cache_free(shmem_internal_hier_cache_t *hier_cache); void shmem_internal_hier_barrier_print_stats(void); + +/* Job-global minimum PEs-per-node, computed once in collectives_init. */ +extern int shmem_internal_hier_min_ppn; + +/* Whether AUTO selection should use the hierarchical barrier. Requires the + * job to span more than one node (some PE off-node, i.e. shr_size < num_pes) + * AND at least HIER_BARRIER_THRESHOLD PEs on the smallest node. Both terms are + * job-global (shr_size < num_pes holds on every PE of a multi-node job; + * hier_min_ppn is reduced over all nodes), so every PE makes the same decision + * even when PPN is heterogeneous — gating on the local node size could let a + * large node pick hierarchical while a small node picks tree, diverging within + * one collective. On a single-node job the hierarchical barrier degenerates to + * its intranode-only path, which is strictly worse than linear/tree: under + * USE_HIERARCHICAL_BARRIER the shr transport already routes single-node atomics + * through CPU atomics (see shmem_shr_transport_use_atomic), so linear/tree run + * over shared memory with none of the hierarchical scaffolding. Explicit + * BARRIER_ALGORITHM=hierarchical bypasses this and is always honored. */ +static inline int +shmem_internal_hier_barrier_available(void) +{ + return shmem_internal_get_shr_size() < shmem_internal_num_pes && + shmem_internal_hier_min_ppn >= shmem_internal_params.HIER_BARRIER_THRESHOLD; +} #endif static inline @@ -82,7 +105,7 @@ shmem_internal_sync(int PE_start, int PE_stride, int PE_size, long *pSync) switch (shmem_internal_barrier_type) { case AUTO: #ifdef USE_HIERARCHICAL_BARRIER - if (shmem_internal_get_shr_size() >= shmem_internal_params.HIER_BARRIER_THRESHOLD) { + if (shmem_internal_hier_barrier_available()) { shmem_internal_sync_hierarchical(PE_start, PE_stride, PE_size, pSync, shmem_internal_hierarchical_local_psync, @@ -175,7 +198,7 @@ shmem_internal_sync_all(void) { #ifdef USE_HIERARCHICAL_BARRIER if (shmem_internal_barrier_type == AUTO && - shmem_internal_get_shr_size() >= shmem_internal_params.HIER_BARRIER_THRESHOLD) { + shmem_internal_hier_barrier_available()) { shmem_internal_sync_hierarchical(0, 1, shmem_internal_num_pes, shmem_internal_sync_all_psync, shmem_internal_sync_all_local_psync, @@ -206,7 +229,7 @@ shmem_internal_barrier_all(void) shmem_internal_quiet(SHMEM_CTX_DEFAULT); #ifdef USE_HIERARCHICAL_BARRIER if (shmem_internal_barrier_type == AUTO && - shmem_internal_get_shr_size() >= shmem_internal_params.HIER_BARRIER_THRESHOLD) { + shmem_internal_hier_barrier_available()) { shmem_internal_sync_hierarchical(0, 1, shmem_internal_num_pes, shmem_internal_barrier_all_psync, shmem_internal_barrier_all_local_psync, @@ -231,7 +254,7 @@ shmem_internal_sync_for_team(shmem_internal_team_t *team, long *pSync) { int use_hier = (shmem_internal_barrier_type == HIERARCHICAL) || (shmem_internal_barrier_type == AUTO && - shmem_internal_get_shr_size() >= shmem_internal_params.HIER_BARRIER_THRESHOLD); + shmem_internal_hier_barrier_available()); if (use_hier) { if (shmem_internal_params.BARRIERS_FLUSH) { fflush(stdout); From 3ba9445e7044d7276868eac5d7216dcf58321bee Mon Sep 17 00:00:00 2001 From: Bryan Morgan Date: Fri, 26 Jun 2026 03:16:00 -0700 Subject: [PATCH 13/14] transport/ofi: gate CXI hybrid MR desc on a configure feature probe configure checks for rdma/fi_cxi_ext.h and, when present, probes struct fi_cxi_dom_ops for the enable_hybrid_mr_desc member (added in a later libfabric). The hybrid local MR descriptor optimization is compiled in only when both HAVE_RDMA_FI_CXI_EXT_H and HAVE_STRUCT_FI_CXI_DOM_OPS_ENABLE_HYBRID_MR_DESC are defined, using the provider's real struct fi_cxi_dom_ops and the FI_CXI_DOM_OPS_3 macro. There is no hand-rolled struct mirror, so the build never risks an ABI mismatch against the provider's actual dom_ops layout. When support is absent and the runtime requests hybrid mode, PE 0 warns once and the request is ignored. --- configure.ac | 23 +++++++++++++++++++++++ src/transport_ofi.c | 21 ++++++++++++++------- src/transport_ofi.h | 15 +++++++++++++++ 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/configure.ac b/configure.ac index 53a0493e1..a98b37ee1 100644 --- a/configure.ac +++ b/configure.ac @@ -959,6 +959,29 @@ if test "$transport_ofi" = "yes" ; then LIBS="$LIBS $ofi_LIBS" WRAPPER_COMPILER_EXTRA_LDFLAGS="$ofi_LDFLAGS" WRAPPER_COMPILER_EXTRA_LIBS="$ofi_LIBS" + dnl CXI provider extensions (struct fi_cxi_dom_ops, enable_hybrid_mr_desc). + dnl Checked with the OFI include path in CPPFLAGS. fi_cxi_ext.h is not + dnl self-contained (it uses fabric/fi_domain types without including them), + dnl so it is probed with those prerequisite headers in the 4th argument to + dnl avoid a "present but cannot be compiled" warning. AC_CHECK_HEADERS + dnl defines HAVE_RDMA_FI_CXI_EXT_H when the header compiles; only then do we + dnl probe for the enable_hybrid_mr_desc member, which was added to struct + dnl fi_cxi_dom_ops in a later libfabric. AC_CHECK_MEMBER defines + dnl HAVE_STRUCT_FI_CXI_DOM_OPS_ENABLE_HYBRID_MR_DESC on success. The hybrid + dnl MR descriptor optimization is compiled in only when BOTH are defined. + AC_CHECK_HEADERS([rdma/fi_cxi_ext.h], + [AC_CHECK_MEMBER([struct fi_cxi_dom_ops.enable_hybrid_mr_desc], + [AC_DEFINE([HAVE_STRUCT_FI_CXI_DOM_OPS_ENABLE_HYBRID_MR_DESC], [1], + [Define to 1 if struct fi_cxi_dom_ops has the enable_hybrid_mr_desc member.])], + [], + [[#include + #include + #include + #include ]])], + [], + [[#include + #include + #include ]]) fi if test "$transport_ucx" = "yes" ; then diff --git a/src/transport_ofi.c b/src/transport_ofi.c index ba7099d11..c848a10b1 100644 --- a/src/transport_ofi.c +++ b/src/transport_ofi.c @@ -1351,16 +1351,16 @@ int allocate_fabric_resources(struct fabric_info *info) * Must be done BEFORE any endpoints are created (the provider only * propagates this setting to child endpoints at creation time). * Gated on a CXI provider check so non-CXI providers never attempt the - * fi_open_ops call and do not produce spurious startup warnings. */ + * fi_open_ops call and do not produce spurious startup warnings. The + * optimization is compiled in only when configure found both the CXI + * extension header and the enable_hybrid_mr_desc member; otherwise it is + * disabled and a request to enable it is reported once on PE 0. */ if (shmem_transport_ofi_check_provider(SHMEM_TRANSPORT_OFI_PROV_CXI)) { if (shmem_internal_params.OFI_CXI_HYBRID_MR_DESC) { - struct cxi_dom_ops_v3_local { - int (*cntr_read)(struct fid *, unsigned int, uint64_t *, struct timespec *); - int (*topology)(struct fid *, unsigned int *, unsigned int *, unsigned int *); - int (*enable_hybrid_mr_desc)(struct fid *, bool); - } *cxi_dom_ops = NULL; +#ifdef SHMEM_TRANSPORT_OFI_HAVE_CXI_HYBRID_MR_DESC + struct fi_cxi_dom_ops *cxi_dom_ops = NULL; int hret = fi_open_ops(&shmem_transport_ofi_domainfd->fid, - "dom_ops_v3", 0, (void **)&cxi_dom_ops, NULL); + FI_CXI_DOM_OPS_3, 0, (void **)&cxi_dom_ops, NULL); if (hret == 0 && cxi_dom_ops && cxi_dom_ops->enable_hybrid_mr_desc) { hret = cxi_dom_ops->enable_hybrid_mr_desc(&shmem_transport_ofi_domainfd->fid, true); if (hret == 0) { @@ -1372,6 +1372,13 @@ int allocate_fabric_resources(struct fabric_info *info) DEBUG_MSG("CXI: hybrid MR desc not available (fi_open_ops returned %d / %s)\n", hret, hret ? fi_strerror(-hret) : "no ops struct"); } +#else + if (shmem_internal_my_pe == 0) { + RAISE_WARN_STR("CXI hybrid MR desc requested (SHMEM_OFI_CXI_HYBRID_MR_DESC=1) " + "but SOS was built without CXI extension support " + "(missing rdma/fi_cxi_ext.h or enable_hybrid_mr_desc); ignoring"); + } +#endif } else { DEBUG_STR("CXI: hybrid local MR descriptor mode disabled (SHMEM_OFI_CXI_HYBRID_MR_DESC=0)"); } diff --git a/src/transport_ofi.h b/src/transport_ofi.h index c8a91e8ce..aba23efc0 100644 --- a/src/transport_ofi.h +++ b/src/transport_ofi.h @@ -23,6 +23,21 @@ #include #include #include +#ifdef HAVE_RDMA_FI_CXI_EXT_H +#include /* fi_cxi_ext.h uses bool but does not include stdbool.h */ +#include +#endif +/* The CXI hybrid local MR descriptor optimization needs the CXI extension + * header AND a struct fi_cxi_dom_ops that actually exposes the + * enable_hybrid_mr_desc member (added in a later libfabric). configure probes + * both (HAVE_RDMA_FI_CXI_EXT_H, HAVE_STRUCT_FI_CXI_DOM_OPS_ENABLE_HYBRID_MR_DESC); + * the feature is compiled in only when both are present, otherwise it is + * disabled entirely — no hand-rolled struct mirror, so we never risk an ABI + * mismatch against the provider's real layout. */ +#if defined(HAVE_RDMA_FI_CXI_EXT_H) && \ + defined(HAVE_STRUCT_FI_CXI_DOM_OPS_ENABLE_HYBRID_MR_DESC) +#define SHMEM_TRANSPORT_OFI_HAVE_CXI_HYBRID_MR_DESC 1 +#endif #include #include #include From ccdba1ea7cfb307dbaaee40b0b4af8e3b7b357e9 Mon Sep 17 00:00:00 2001 From: Bryan Morgan Date: Sat, 27 Jun 2026 07:03:25 -0700 Subject: [PATCH 14/14] transport/ofi: make CXI ext probe self-contained and surface result fi_cxi_ext.h references struct fid / uint64_t / bool / struct timespec without including the headers that define them. Whether those resolve during the configure probe depends on what a given libfabric's fabric.h pulls in transitively, which varies across clusters and versions, so the header could be present yet reported as unusable and silently disable the hybrid MR descriptor optimization. Supply the full prerequisite set (stdbool.h, time.h, stdint.h, rdma/fabric.h, rdma/fi_domain.h) in the AC_CHECK_HEADERS/AC_CHECK_MEMBER test bodies so the probe compiles independently of fabric.h's transitive includes. Also report CXI hybrid MR desc support in the configure summary so a misdetection is visible at configure time rather than only as a runtime warning on CXI hardware. Also lower the huge-page snprintf-failure message in symmetric_heap_c.c from a warning to a debug message, matching the other fallback-path messages in mmap_alloc. --- configure.ac | 27 ++++++++++++++++++++++----- src/symmetric_heap_c.c | 2 +- src/transport_ofi.c | 8 +++----- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/configure.ac b/configure.ac index a98b37ee1..1a51ee1a6 100644 --- a/configure.ac +++ b/configure.ac @@ -961,11 +961,16 @@ if test "$transport_ofi" = "yes" ; then WRAPPER_COMPILER_EXTRA_LIBS="$ofi_LIBS" dnl CXI provider extensions (struct fi_cxi_dom_ops, enable_hybrid_mr_desc). dnl Checked with the OFI include path in CPPFLAGS. fi_cxi_ext.h is not - dnl self-contained (it uses fabric/fi_domain types without including them), - dnl so it is probed with those prerequisite headers in the 4th argument to - dnl avoid a "present but cannot be compiled" warning. AC_CHECK_HEADERS - dnl defines HAVE_RDMA_FI_CXI_EXT_H when the header compiles; only then do we - dnl probe for the enable_hybrid_mr_desc member, which was added to struct + dnl self-contained: it references struct fid / uint64_t / bool / struct + dnl timespec without including the headers that define them. We therefore + dnl supply the full prerequisite set (stdbool.h, time.h, stdint.h, + dnl rdma/fabric.h, rdma/fi_domain.h) in the 4th argument so the probe + dnl compiles independently of whatever a given libfabric's fabric.h happens + dnl to pull in transitively (which varies across clusters/versions). Without + dnl this the header can be present yet reported as unusable, silently + dnl disabling the optimization. AC_CHECK_HEADERS defines + dnl HAVE_RDMA_FI_CXI_EXT_H when the header compiles; only then do we probe + dnl for the enable_hybrid_mr_desc member, which was added to struct dnl fi_cxi_dom_ops in a later libfabric. AC_CHECK_MEMBER defines dnl HAVE_STRUCT_FI_CXI_DOM_OPS_ENABLE_HYBRID_MR_DESC on success. The hybrid dnl MR descriptor optimization is compiled in only when BOTH are defined. @@ -975,13 +980,23 @@ if test "$transport_ofi" = "yes" ; then [Define to 1 if struct fi_cxi_dom_ops has the enable_hybrid_mr_desc member.])], [], [[#include + #include + #include #include #include #include ]])], [], [[#include + #include + #include #include #include ]]) + + dnl Surface the result in the configure summary so a misdetection is visible + dnl at configure time rather than only as a runtime warning on CXI hardware. + cxi_hybrid_mr_desc="no" + AS_IF([test "x$ac_cv_member_struct_fi_cxi_dom_ops_enable_hybrid_mr_desc" = "xyes"], + [cxi_hybrid_mr_desc="yes"]) fi if test "$transport_ucx" = "yes" ; then @@ -1147,6 +1162,8 @@ if test "$transport_ofi" = "yes"; then echo " Addr. vector: table" fi + echo " CXI hybrid MR desc: $cxi_hybrid_mr_desc" + echo "" fi diff --git a/src/symmetric_heap_c.c b/src/symmetric_heap_c.c index 51f4c6090..40e689704 100644 --- a/src/symmetric_heap_c.c +++ b/src/symmetric_heap_c.c @@ -185,7 +185,7 @@ static void *mmap_alloc(size_t bytes, size_t *mapped_bytes) int size = snprintf(NULL, 0, "%s/%s.%d", directory, basename, getpid()); if (size < 0) { - RAISE_WARN_STR("snprintf returned error, cannot use huge pages"); + DEBUG_MSG("snprintf returned error, cannot use huge pages"); } else { file_name = malloc(size + 1); if (file_name) { diff --git a/src/transport_ofi.c b/src/transport_ofi.c index c848a10b1..67da22ffe 100644 --- a/src/transport_ofi.c +++ b/src/transport_ofi.c @@ -1373,11 +1373,9 @@ int allocate_fabric_resources(struct fabric_info *info) hret, hret ? fi_strerror(-hret) : "no ops struct"); } #else - if (shmem_internal_my_pe == 0) { - RAISE_WARN_STR("CXI hybrid MR desc requested (SHMEM_OFI_CXI_HYBRID_MR_DESC=1) " - "but SOS was built without CXI extension support " - "(missing rdma/fi_cxi_ext.h or enable_hybrid_mr_desc); ignoring"); - } + DEBUG_STR("CXI hybrid MR desc requested (SHMEM_OFI_CXI_HYBRID_MR_DESC=1) " + "but SOS was built without CXI extension support " + "(missing rdma/fi_cxi_ext.h or enable_hybrid_mr_desc); ignoring"); #endif } else { DEBUG_STR("CXI: hybrid local MR descriptor mode disabled (SHMEM_OFI_CXI_HYBRID_MR_DESC=0)");