diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index ff4795a7..00000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,804 +0,0 @@ -name: SOS-github-actions - -on: [push, pull_request] - -defaults: - run: - shell: bash - -env: - SOS_INSTALL_DIR: ${{ github.workspace }}/install/sos - LIBFABRIC_INSTALL_DIR: ${{ github.workspace }}/install/libfabric - UCX_INSTALL_DIR: ${{ github.workspace }}/install/ucx - PORTALS4_INSTALL_DIR: ${{ github.workspace }}/install/portals4 - LIBEVENT_INSTALL_DIR: ${{ github.workspace }}/install/libevent - PRRTE_INSTALL_DIR: ${{ github.workspace }}/install/prrte - PMIX_INSTALL_DIR: ${{ github.workspace }}/install/pmix - SOS_BUILD_STATIC_OPTS: --disable-shared --enable-static - XPMEM_INSTALL_DIR: ${{ github.workspace }}/install/xpmem - SOS_PM: mpiexec.hydra - SOS_PM_POST: true - -jobs: - OFI: - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - include: - - config_name: PMI simple - sos_config: --enable-pmi-simple - libfabric_version: v1.7.x - - config_name: PMI simple - sos_config: --enable-pmi-simple - libfabric_version: v1.9.x - - config_name: PMI simple - sos_config: --enable-pmi-simple - libfabric_version: v1.11.x - - config_name: PMI simple - sos_config: --enable-pmi-simple - libfabric_version: v1.13.x - - config_name: PMI simple - sos_config: --enable-pmi-simple - libfabric_version: v2.1.x - - config_name: Deprecated tests - sos_config: --enable-pmi-simple --enable-deprecated-tests - libfabric_version: v2.1.x - - config_name: MR-Basic, AV-map, memcpy - sos_config: --enable-ofi-mr=basic --enable-av-map --disable-cxx --enable-memcpy - --enable-pmi-simple --with-hwloc=no - libfabric_version: v2.1.x - - config_name: PMI MPI - sos_config: --disable-fortran --enable-pmi-mpi CC=mpicc - libfabric_version: v2.1.x - - config_name: Remove Virtual Addressing (RVA) - sos_config: --disable-fortran --enable-error-checking --enable-remote-virtual-addressing - --disable-aslr-check --enable-pmi-simple - libfabric_version: v2.1.x - - config_name: heap use malloc - env_setup: export SHMEM_SYMMETRIC_HEAP_USE_MALLOC=1 - sos_config: --disable-threads --enable-error-checking --enable-pmi-simple --with-hwloc=no - libfabric_version: v2.1.x - # too slow, times out on Github (but passes on another Ubuntu 20.04 system)... - #- config_name: CMA, MR Basic, RVA - # sos_config: --disable-fortran --with-cma --enable-error-checking --enable-profiling - # --enable-ofi-mr=basic --enable-av-map --enable-remote-virtual-addressing - # --enable-pmi-simple - # libfabric_version: v2.1.x - - config_name: XPMEM RVA - xpmem_version: master - sos_config: --with-xpmem=${XPMEM_INSTALL_DIR} --enable-error-checking - --enable-remote-virtual-addressing --enable-pmi-simple - libfabric_version: v2.1.x - - config_name: XPMEM shared atomics - xpmem_version: master - sos_config: --with-xpmem=${XPMEM_INSTALL_DIR} --enable-shr-atomics - --enable-error-checking --enable-pmi-simple - libfabric_version: v2.1.x - - config_name: RVA, thread completion - sos_config: --enable-error-checking --enable-remote-virtual-addressing - --enable-thread-completion --enable-pmi-simple - libfabric_version: v2.1.x - - config_name: huge pages, zero bounce - env_setup: SHMEM_SYMMETRIC_HEAP_USE_HUGE_PAGES=1 SHMEM_BOUNCE_SIZE=0 - sos_config: --enable-error-checking --enable-remote-virtual-addressing - --enable-pmi-simple --enable-ofi-fence --with-hwloc=no - libfabric_version: v2.1.x - - config_name: auto algorithms - env_setup: export SHMEM_BARRIER_ALGORITHM=auto; - export SHMEM_BCAST_ALGORITHM=auto; - export SHMEM_REDUCE_ALGORITHM=auto; - export SHMEM_COLLECT_ALGORITHM=auto; - export SHMEM_FCOLLECT_ALGORITHM=auto - sos_config: --enable-error-checking --enable-remote-virtual-addressing - --enable-thread-completion --enable-pmi-simple - libfabric_version: v2.1.x - - config_name: linear algorithms - env_setup: export SHMEM_BARRIER_ALGORITHM=linear; - export SHMEM_BCAST_ALGORITHM=linear; - export SHMEM_REDUCE_ALGORITHM=linear; - export SHMEM_COLLECT_ALGORITHM=linear; - export SHMEM_FCOLLECT_ALGORITHM=linear - sos_config: --enable-error-checking --enable-remote-virtual-addressing - --enable-pmi-simple --with-hwloc=no - libfabric_version: v2.1.x - - config_name: tree algorithms - env_setup: export SHMEM_BARRIER_ALGORITHM=tree; - export SHMEM_BCAST_ALGORITHM=tree; - export SHMEM_REDUCE_ALGORITHM=tree; - export SHMEM_OFI_STX_MAX=0 - sos_config: --enable-error-checking --enable-remote-virtual-addressing - --enable-pmi-simple --enable-manual-progress - libfabric_version: v2.1.x - - config_name: dissem/recdbl algorithms - env_setup: export SHMEM_BARRIER_ALGORITHM=dissem; - export SHMEM_REDUCE_ALGORITHM=recdbl; - export SHMEM_FCOLLECT_ALGORITHM=recdbl; - export SHMEM_OFI_STX_AUTO=1 - sos_config: --enable-error-checking --enable-remote-virtual-addressing - --enable-pmi-simple --enable-manual-progress --enable-hard-polling --with-hwloc=no - libfabric_version: v2.1.x - - config_name: ring reduce algorithm - env_setup: export SHMEM_REDUCE_ALGORITHM=ring - export SHMEM_SCAN_ALGORITHM=ring - sos_config: --enable-error-checking --enable-pmi-simple - libfabric_version: v2.1.x - - config_name: ring fcollect algorithm, tx/rx single poll limit - env_setup: export SHMEM_FCOLLECT_ALGORITHM=ring; - export SHMEM_OFI_TX_POLL_LIMIT=1; - export SHMEM_OFI_RX_POLL_LIMIT=1; - export SHMEM_OFI_STX_THRESHOLD=1024 - sos_config: --enable-error-checking --enable-remote-virtual-addressing - --enable-pmi-simple - libfabric_version: v2.1.x - - config_name: STX random - env_setup: export SHMEM_OFI_STX_MAX=8; - export SHMEM_OFI_STX_ALLOCATOR=random - sos_config: --enable-error-checking --enable-remote-virtual-addressing - --enable-pmi-simple - libfabric_version: v2.1.x - - config_name: RPM build - env_setup: export SOS_CHECK_TARBALL_RPM=1 - sos_config: --enable-pmi-simple - rpm_build: true - libfabric_version: v2.1.x - - config_name: Lengthy tests - sos_config: --enable-lengthy-tests --enable-pmi-simple - libfabric_version: v2.1.x - - config_name: Manpages - sos_config: --enable-manpages --enable-pmi-simple - libfabric_version: v2.1.x - - config_name: Without OFI inject - sos_config: --disable-ofi-inject --enable-pmi-simple - libfabric_version: v2.1.x - - config_name: Without Non-fetch AMO - sos_config: --disable-nonfetch-amo --enable-pmi-simple - libfabric_version: v2.1.x - - name: OFI ${{ matrix.libfabric_version }} (${{ matrix.config_name }}) - - steps: - - name: Checking OS version - run: | - echo "OS_NAME=$(lsb_release -si)-$(lsb_release -sr)" >> $GITHUB_ENV - - name: Save Config Parameters - run: | - DEMANGLE_CONFIG=$(echo ${{ matrix.sos_config }} | sed 's/ //g' | sed -E 's/(-)\1+/-/g' | sed -E 's/[^=a-zA-Z0-9._\-]/\+/g') - echo "config param is $DEMANGLE_CONFIG" - echo "CONFIG_PARAM=$DEMANGLE_CONFIG" >> $GITHUB_ENV - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: 'true' - - name: Install dependencies - run: | - sudo apt-get -y update - sudo apt-get install -y gfortran rpm mpich libmpich-dev libhwloc-dev gdb - sudo sysctl -w kernel.yama.ptrace_scope=0 - sudo sysctl -w kernel.randomize_va_space=0 - - # LIBFABRIC - - name: Cache libfabric install - id: cache-libfabric - uses: actions/cache@v4 - with: - path: ${{ env.LIBFABRIC_INSTALL_DIR }} - key: libfabric-${{ matrix.libfabric_version }}-${{ env.OS_NAME }} - - name: Checkout libfabric - if: steps.cache-libfabric.outputs.cache-hit != 'true' - uses: actions/checkout@v4 - with: - repository: ofiwg/libfabric - path: repos/libfabric - ref: ${{ matrix.libfabric_version }} - - name: Build libfabric - if: steps.cache-libfabric.outputs.cache-hit != 'true' - run: | - cd repos/libfabric - ./autogen.sh - mkdir build; cd build - ../configure --prefix=${LIBFABRIC_INSTALL_DIR} - make -j - make install - - # XPMEM - - name: Cache XPMEM install - if: ${{ matrix.xpmem_version }} - id: cache-xpmem - uses: actions/cache@v4 - with: - path: ${{ env.XPMEM_INSTALL_DIR }} - key: xpmem-${{ matrix.xpmem_version }}-${{ env.OS_NAME }} - - name: Checkout XPMEM - if: ${{ matrix.xpmem_version }} - uses: actions/checkout@v4 - with: - repository: openucx/xpmem - path: repos/xpmem - ref: ${{ matrix.xpmem_version }} - - name: Build XPMEM - if: ${{ matrix.xpmem_version }} - run: | - cd repos/xpmem - sudo apt-get -y install linux-headers-`uname -r` - ./autogen.sh - ./configure --prefix=${XPMEM_INSTALL_DIR} - make -j - sudo make install - sudo insmod ${XPMEM_INSTALL_DIR}/lib/modules/`uname -r`/kernel/xpmem/xpmem.ko - sudo chown `whoami` /dev/xpmem - - # SOS - - name: Build SOS (${{ matrix.sos_config }}) - run: | - git submodule update --remote - ./autogen.sh - mkdir build; cd build - ../configure --prefix=${SOS_INSTALL_DIR} --with-ofi=${LIBFABRIC_INSTALL_DIR} ${{ matrix.sos_config }} - make -j - make install - - name: Configure Core Analysis (${{ matrix.sos_config }}) - run: | - bash ${GITHUB_WORKSPACE}/.github/scripts/scan_core.sh init - - name: Test SOS (${{ matrix.sos_config }}) - run: | - cd build - make check TESTS= -j - ${{ matrix.env_setup }} - ulimit -c unlimited - ulimit -a - SHMEM_DEBUG=1 SHMEM_INFO=1 SHMEM_OFI_PROVIDER=sockets FI_PROVIDER=sockets \ - make VERBOSE=1 TEST_RUNNER="${SOS_PM} -np 2 timeout --signal=ABRT 15m" check - cat modules/tests-sos/test/unit/hello.log - - name: Scanning for Core Dumps (${{ matrix.sos_config }}) - if: always() - run: | - bash ${GITHUB_WORKSPACE}/.github/scripts/scan_core.sh scan - - name: Test RPM (${{ matrix.rpm_build }}) - if: ${{ matrix.rpm_build }} - run: | - # FIXME: RPM with a prefix not working on Ubuntu 20.04 (related to _docdir?) - #./configure --with-ofi=${LIBFABRIC_INSTALL_DIR} ${{ matrix.sos_config }} --enable-rpm-prefix - #make dist - #rpmbuild -ta ./sandia-openshmem-*.tar.gz --define "configargs --with-ofi=${LIBFABRIC_INSTALL_DIR} ${{ matrix.sos_config }}" --define "_prefix /usr/shmem" - #make clean - ./configure --with-ofi=${LIBFABRIC_INSTALL_DIR} ${{ matrix.sos_config }} - make dist - rpmbuild -ta ./sandia-openshmem-*.tar.gz --define "configargs --with-ofi=${LIBFABRIC_INSTALL_DIR} ${{ matrix.sos_config }}" - # Sanity check distribution tarball - tar zxvf sandia-openshmem-*.tar.gz - sos_version=$(cat configure.ac | grep AC_INIT | awk -F"]" '{print substr($2, 4, length($2)-2)}') - cd "sandia-openshmem-"$sos_version - ./autogen.sh - mkdir build - cd build - ../configure --with-ofi=${LIBFABRIC_INSTALL_DIR} ${{ matrix.sos_config }} - make -j check TESTS= - ${SOS_PM_PRE} - ulimit -c unlimited - ulimit -a - SHMEM_DEBUG=1 SHMEM_INFO=1 make VERBOSE=1 TEST_RUNNER="${SOS_PM} -np 2" check - ${SOS_PM_POST} - - name: Archive Artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: debug-artifacts-ofi-${{ matrix.libfabric_version }}-${{ env.CONFIG_PARAM }}-${{ github.run_number }}.${{ github.run_attempt }} - path: ${{ github.workspace }}/archives - if-no-files-found: ignore - -# PMIx: -# runs-on: ubuntu-24.04 -# strategy: -# fail-fast: false -# matrix: -# include: -# - sos_transport: libfabric -# libfabric_version: v1.7.x -# sos_transport_config: --with-ofi=${LIBFABRIC_INSTALL_DIR} --without-ucx --without-portals4 -# run_all_tests: true -# libevent_version: release-2.1.10-stable -# pmix_version: v4.1.1rc2 -# prrte_version: v2.0 -# sos_pm: prun -# sos_pm_pre: prte --daemonize --host localhost:4 -# sos_pm_post: pterm -# - sos_transport: ucx -# ucx_version: v1.9.0 -# sos_transport_config: --with-ucx=${UCX_INSTALL_DIR} --without-ofi --without-portals4 -# run_all_tests: true -# libevent_version: release-2.1.10-stable -# pmix_version: v4.1.1rc2 -# prrte_version: v2.0 -# sos_pm: prun -# sos_pm_pre: prte --daemonize --host localhost:4 -# sos_pm_post: pterm -# - sos_transport: portals4 -# portals4_version: master -# sos_transport_config: --with-portals4=${PORTALS4_INSTALL_DIR} --without-ofi --without-ucx -# run_all_tests: false -# libevent_version: release-2.1.10-stable -# pmix_version: v4.1.1rc2 -# prrte_version: v2.0 -# sos_pm: prun -# sos_pm_pre: prte --daemonize --host localhost:4 -# sos_pm_post: pterm -# -# name: PMIx (${{ matrix.sos_transport }}) -# -# steps: -# - name: Checking OS version -# run: | -# echo "OS_NAME=$(lsb_release -si)-$(ls_release -sr)" >> $GITHUB_ENV -# - uses: actions/checkout@v2 -# - name: Install dependencies -# run: | -# sudo apt-get -y update -# sudo apt-get install -y gfortran libhwloc-dev libev-dev libev-libevent-dev -# -# # LIBFABRIC -# - name: Cache libfabric install -# if: ${{ matrix.sos_transport == 'libfabric' }} -# id: cache-libfabric -# uses: actions/cache@v4 -# with: -# path: ${{ env.LIBFABRIC_INSTALL_DIR }} -# key: libfabric-${{ matrix.libfabric_version }}-${{ env.OS_NAME }} -# - name: Checkout libfabric -# if: ${{ matrix.sos_transport == 'libfabric' && steps.cache-libfabric.outputs.cache-hit != 'true' }} -# uses: actions/checkout@v2 -# with: -# repository: ofiwg/libfabric -# path: repos/libfabric -# ref: ${{ matrix.libfabric_version }} -# - name: Build libfabric -# if: ${{ matrix.sos_transport == 'libfabric' && steps.cache-libfabric.outputs.cache-hit != 'true' }} -# run: | -# cd repos/libfabric -# ./autogen.sh -# mkdir build; cd build -# ../configure --prefix=${LIBFABRIC_INSTALL_DIR} -# make -j -# make install -# -# # UCX -# - name: Cache UCX install -# if: ${{ matrix.sos_transport == 'ucx' }} -# id: cache-ucx -# uses: actions/cache@v4 -# with: -# path: ${{ env.UCX_INSTALL_DIR }} -# key: ucx-${{ matrix.ucx_version }}-${{ env.OS_NAME }} -# - name: Checkout UCX -# if: ${{ matrix.sos_transport == 'ucx' && steps.cache-ucx.outputs.cache-hit != 'true' }} -# uses: actions/checkout@v2 -# with: -# repository: openucx/ucx -# path: repos/ucx -# ref: ${{ matrix.ucx_version }} -# - name: Build UCX -# if: ${{ matrix.sos_transport == 'ucx' && steps.cache-ucx.outputs.cache-hit != 'true' }} -# run: | -# cd repos/ucx -# ./autogen.sh -# mkdir build; cd build -# ../configure --prefix=${UCX_INSTALL_DIR} --enable-mt --disable-numa --without-java -# make -j -# make install -# -# ## Portals4 -# - name: Cache Portals4 install -# if: ${{ matrix.sos_transport == 'portals4' }} -# id: cache-portals4 -# uses: actions/cache@v4 -# with: -# path: ${{ env.PORTALS4_INSTALL_DIR }} -# key: portals4-${{ matrix.portals4_version }}-${{ env.OS_NAME }} -# - name: Checkout Portals4 -# if: ${{ matrix.sos_transport == 'portals4' && steps.cache-portals4.outputs.cache-hit != 'true' }} -# uses: actions/checkout@v2 -# with: -# repository: regrant/portals4 -# path: repos/portals4 -# ref: ${{ matrix.portals4_version }} -# - name: Build Portals4 -# if: ${{ matrix.sos_transport == 'portals4' && steps.cache-portals4.outputs.cache-hit != 'true' }} -# run: | -# cd repos/portals4 -# ./autogen.sh -# mkdir build; cd build -# ../configure --prefix=${PORTALS4_INSTALL_DIR} --enable-zero-mrs --enable-reliable-udp --disable-pmi-from-portals ${SOS_BUILD_STATIC_OPTS} -# make -j -# make install -# -# # Libevent -# - name: Cache libevent install -# id: cache-libevent -# uses: actions/cache@v4 -# with: -# path: ${{ env.LIBEVENT_INSTALL_DIR }} -# key: libevent-${{ matrix.libevent_version }}-${{ env.OS_NAME }} -# - name: Checkout libevent -# if: steps.cache-libevent.outputs.cache-hit != 'true' -# uses: actions/checkout@v2 -# with: -# repository: libevent/libevent -# path: repos/libevent -# ref: ${{ matrix.libevent_version }} -# - name: Build libevent -# if: steps.cache-libevent.outputs.cache-hit != 'true' -# run: | -# cd repos/libevent -# ./autogen.sh -# ./configure --prefix=${LIBEVENT_INSTALL_DIR} -# make clean all install -# -# # PMIx -# - name: Cache PMIx install -# id: cache-pmix -# uses: actions/cache@v4 -# with: -# path: ${{ env.PMIX_INSTALL_DIR }} -# key: pmix-${{ matrix.pmix_version }}-${{ env.OS_NAME }} -# - name: Checkout PMIx -# if: steps.cache-pmix.outputs.cache-hit != 'true' -# uses: actions/checkout@v2 -# with: -# repository: openpmix/pmix -# path: repos/pmix -# ref: ${{ matrix.pmix_version }} -# - name: Build PMIx -# if: steps.cache-pmix.outputs.cache-hit != 'true' -# run: | -# cd repos/pmix -# ./autogen.pl -# ./configure --prefix=${PMIX_INSTALL_DIR} --with-libevent=${LIBEVENT_INSTALL_DIR} --without-libev --disable-debug CFLAGS=-O3 ${SOS_BUILD_STATIC_OPTS} -# make install -# -# # Build PRRTE -# - name: Cache PRRTE install -# id: cache-prrte -# uses: actions/cache@v4 -# with: -# path: ${{ env.PRRTE_INSTALL_DIR }} -# key: prrte-${{ matrix.prrte_version}}-${{ env.OS_NAME }} -# - name: Checkout PRRTE -# if: steps.cache-prrte.outputs.cache-hit != 'true' -# uses: actions/checkout@v2 -# with: -# repository: openpmix/prrte -# path: repos/prrte -# ref: ${{ matrix.prrte_version }} -# - name: Build PRRTE -# if: steps.cache-prrte.outputs.cache-hit != 'true' -# run: | -# cd repos/prrte -# ./autogen.pl -# ./configure --prefix=${PRRTE_INSTALL_DIR} --with-pmix=${PMIX_INSTALL_DIR} --without-slurm --with-libevent=${LIBEVENT_INSTALL_DIR} --without-libev ${SOS_BUILD_STATIC_OPTS} -# make install -# -# # SOS -# - name: Build SOS -# run: | -# ./autogen.sh -# mkdir build; cd build -# ../configure --prefix=${SOS_INSTALL_DIR} ${{ matrix.sos_transport_config }} --with-pmix=${PMIX_INSTALL_DIR} -# make -j -# make install -# - name: Test SOS (all tests) -# if: ${{ matrix.sos_transport == 'libfabric' || matrix.sos_transport == 'ucx' }} -# run: | -# cd build -# make check TESTS= -j -# export PATH=${PRRTE_INSTALL_DIR}/bin:$PATH -# ${{ matrix.sos_pm_pre }} -# SHMEM_INFO=1 make VERBOSE=1 TEST_RUNNER="${{ matrix.sos_pm }} -np 2" check -# cat modules/tests-sos/test/unit/hello.log -# ${{ matrix.sos_pm_post }} -# - name: Test SOS (hello) -# if: ${{ matrix.sos_transport == 'portals4' }} -# run: | -# cd build -# make check TESTS= -j -# export PATH=${PRRTE_INSTALL_DIR}/bin:$PATH -# ${{ matrix.sos_pm_pre }} -# SHMEM_INFO=1 ${{ matrix.sos_pm }} -np 1 modules/tests-sos/test/unit/hello -# ${{ matrix.sos_pm_post }} -# - UCX: - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - include: - - config_name: ucx-1.18.0 - ucx_version: v1.18.0 - xpmem_version: master - sos_config: [--enable-pmi-simple --disable-fortran, - --with-cma --enable-error-checking --enable-profiling - --enable-pmi-simple --disable-fortran --with-hwloc=no, - --with-xpmem --enable-error-checking --enable-pmi-simple --with-hwloc=no] - steps: - - name: Checking OS version - run: | - echo "OS_NAME=$(lsb_release -si)-$(lsb_release -sr)" >> $GITHUB_ENV - - name: Save Config Parameters - run: | - DEMANGLE_CONFIG=$(echo ${{ matrix.sos_config }} | sed 's/ //g' | sed -E 's/(-)\1+/-/g' | sed -E 's/[^=a-zA-Z0-9._\-]/\+/g') - echo "config param is $DEMANGLE_CONFIG" - echo "CONFIG_PARAM=$DEMANGLE_CONFIG" >> $GITHUB_ENV - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: 'true' - - name: Install dependencies - run: | - sudo apt-get -y update - sudo apt-get install -y gfortran mpich libmpich-dev gdb - sudo sysctl -w kernel.yama.ptrace_scope=0 - sudo sysctl -w kernel.randomize_va_space=0 - - # XPMEM - - name: Checkout XPMEM - uses: actions/checkout@v4 - with: - repository: openucx/xpmem - path: repos/xpmem - ref: ${{ matrix.xpmem_version }} - - name: Build XPMEM - run: | - cd repos/xpmem - sudo apt-get -y install linux-headers-`uname -r` - ./autogen.sh - ./configure --prefix=/usr - make -j - sudo make install - sudo insmod /usr/lib/modules/`uname -r`/kernel/xpmem/xpmem.ko - sudo chown `whoami` /dev/xpmem - - # UCX - - name: Cache UCX install - id: cache-ucx - uses: actions/cache@v4 - with: - path: ${{ env.UCX_INSTALL_DIR }} - key: ucx-${{ matrix.ucx_version}}-${{ env.OS_NAME }} - - name: Checkout UCX - if: steps.cache-ucx.outputs.cache-hit != 'true' - uses: actions/checkout@v4 - with: - repository: openucx/ucx - path: repos/ucx - ref: ${{ matrix.ucx_version }} - - name: Build UCX - if: steps.cache-ucx.outputs.cache-hit != 'true' - run: | - cd repos/ucx - ./autogen.sh - mkdir build; cd build - ../configure --prefix=${UCX_INSTALL_DIR} --enable-mt --disable-numa --without-java - make -j - make install - - # SOS - - name: Build SOS (${{ matrix.sos_config }}) - run: | - git submodule update --remote - ./autogen.sh - mkdir build; cd build - ../configure --prefix=${SOS_INSTALL_DIR} --with-ucx=${UCX_INSTALL_DIR} ${{ matrix.sos_config }} - make -j - make install - - name: Configure Core Analysis (${{ matrix.sos_config }}) - run: | - bash ${GITHUB_WORKSPACE}/.github/scripts/scan_core.sh init - - name: Test SOS (${{ matrix.sos_config }}) - continue-on-error: true - run: | - cd build - make check TESTS= -j - ulimit -c unlimited - ulimit -a - SHMEM_DEBUG=1 SHMEM_INFO=1 make VERBOSE=1 TEST_RUNNER="${SOS_PM} -np 2 timeout --signal=ABRT 15m" check - cat modules/tests-sos/test/unit/hello.log - - name: Scanning for Core Dumps (${{ matrix.sos_config }}) - if: always() - run: | - bash ${GITHUB_WORKSPACE}/.github/scripts/scan_core.sh scan - - name: Archive Artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: debug-artifacts-${{ matrix.config_name }}-${{ env.CONFIG_PARAM }}-${{ github.run_number }}.${{ github.run_attempt }} - path: ${{ github.workspace }}/archives - if-no-files-found: ignore - - Portals4: - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - include: - - config_name: XPMEM with Shared Atomics - sos_config: --with-xpmem --enable-shr-atomics --enable-error-checking --enable-pmi-simple - portals4_version: master - xpmem_version: master - -# - name: Heap use malloc -# env_setup: export SHMEM_SYMMETRIC_HEAP_USE_MALLOC=1 -# sos_config: --disable-threads --enable-error-checking --enable-pmi-simple --with-hwloc=no -# - name: Heap use huge pages, zero bounce -# env_setup: export SHMEM_SYMMETRIC_HEAP_USE_HUGE_PAGES=1; export SHMEM_BOUNCE_SIZE=0 -# sos_config: --enable-error-checking --enable-remote-virtual-addressing --enable-pmi-simple --enable-ofi-fence -# sos_config: [--enable-pmi-simple --with-hwloc=no, -# --disable-fortran --enable-error-checking --enable-remote-virtual-addressing -# --disable-aslr-check --enable-pmi-simple, -# --with-cma --enable-error-checking --enable-profiling -# --enable-remote-virtual-addressing --enable-pmi-simple, -# --enable-pmi-mpi CC=mpicc --disable-fortran --with-hwloc=no] - steps: - - name: Checking OS version - run: | - echo "OS_NAME=$(lsb_release -si)-$(lsb_release -sr)" >> $GITHUB_ENV - - name: Save Config Parameters - run: | - DEMANGLE_CONFIG=$(echo ${{ matrix.sos_config }} | sed 's/ //g' | sed -E 's/(-)\1+/-/g' | sed -E 's/[^=a-zA-Z0-9._\-]/\+/g') - echo "config param is $DEMANGLE_CONFIG" - echo "CONFIG_PARAM=$DEMANGLE_CONFIG" >> $GITHUB_ENV - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: 'true' - - name: Install dependencies - run: | - sudo apt-get -y update - sudo apt-get install -y gfortran mpich libmpich-dev libev-dev libev-libevent-dev libhwloc-dev gdb - sudo sysctl -w kernel.yama.ptrace_scope=0 - sudo sysctl -w kernel.randomize_va_space=0 - - # XPMEM - - name: Checkout XPMEM - uses: actions/checkout@v4 - with: - repository: openucx/xpmem - path: repos/xpmem - ref: ${{ matrix.xpmem_version }} - - name: Build XPMEM - run: | - cd repos/xpmem - sudo apt-get -y install linux-headers-`uname -r` - ./autogen.sh - ./configure --prefix=/usr - make -j - sudo make install - sudo insmod /usr/lib/modules/`uname -r`/kernel/xpmem/xpmem.ko - sudo chown `whoami` /dev/xpmem - - # Portals4 - - name: Cache Portals4 install - id: cache-portals4 - uses: actions/cache@v4 - with: - path: ${{ env.PORTALS4_INSTALL_DIR }} - key: portals4-${{ matrix.portals4_version}}-${{ env.OS_NAME }} - - name: Checkout Portals4 - if: steps.cache-portals4.outputs.cache-hit != 'true' - uses: actions/checkout@v4 - with: - repository: regrant/portals4 - path: repos/portals4 - ref: ${{ matrix.portals4_version }} - - name: Build Portals4 - if: steps.cache-portals4.outputs.cache-hit != 'true' - run: | - cd repos/portals4 - ./autogen.sh - sed -i 's/^struct ptl_abort_state abort_state;/static struct ptl_abort_state abort_state;/' src/ib/ptl_gbl.h - mkdir build; cd build - ../configure --prefix=${PORTALS4_INSTALL_DIR} --enable-zero-mrs --enable-reliable-udp --disable-pmi-from-portals - make -j - make install - - # SOS - - name: Build SOS (${{ matrix.name }}) - run: | - git submodule update --remote - ./autogen.sh - mkdir build; cd build - ../configure --prefix=${SOS_INSTALL_DIR} --with-portals4=${PORTALS4_INSTALL_DIR} ${{ matrix.sos_config }} - make -j - make install - - name: Configure Core Analysis (${{ matrix.sos_config }}) - run: | - bash ${GITHUB_WORKSPACE}/.github/scripts/scan_core.sh init - - name: Test SOS (${{ matrix.name }}) - continue-on-error: true - run: | - cd build - make check TESTS= -j - ulimit -c unlimited - ulimit -a - SHMEM_DEBUG=1 SHMEM_INFO=1 make VERBOSE=1 TEST_RUNNER="${SOS_PM} -np 2 timeout --signal=ABRT 15m" check - ${SOS_PM} -np 1 modules/tests-sos/test/unit/hello - - name: Archive Artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: debug-artifacts-${{ matrix.config_name }}-${{ env.CONFIG_PARAM }}-${{ github.run_number }}.${{ github.run_attempt }} - path: ${{ github.workspace }}/archives - if-no-files-found: ignore - - XPMEM_Only: - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - include: - - config_name: transport_none - xpmem_version: master - sos_config: [--with-xpmem --enable-shr-atomics --enable-error-checking --enable-pmi-simple] - - steps: - - name: Checking OS version - run: | - echo "OS_NAME=$(lsb_release -si)-$(lsb_release -sr)" >> $GITHUB_ENV - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: 'true' - - name: Install dependencies - run: | - sudo apt-get -y update - sudo apt-get install -y gfortran mpich libmpich-dev libev-dev libev-libevent-dev libhwloc-dev gdb - sudo sysctl -w kernel.yama.ptrace_scope=0 - sudo sysctl -w kernel.randomize_va_space=0 - - # XPMEM - - name: Checkout XPMEM - uses: actions/checkout@v4 - with: - repository: openucx/xpmem - path: repos/xpmem - ref: ${{ matrix.xpmem_version }} - - name: Build XPMEM - run: | - cd repos/xpmem - sudo apt-get -y install linux-headers-`uname -r` - ./autogen.sh - ./configure --prefix=/usr - make -j - sudo make install - sudo insmod /usr/lib/modules/`uname -r`/kernel/xpmem/xpmem.ko - sudo chown `whoami` /dev/xpmem - - # SOS - - name: Build SOS (${{ matrix.name }}) - run: | - git submodule update --remote - ./autogen.sh - mkdir build; cd build - ../configure --prefix=${SOS_INSTALL_DIR} ${{ matrix.sos_config }} - make -j - make install - - name: Configure Core Analysis (${{ matrix.sos_config }}) - run: | - bash ${GITHUB_WORKSPACE}/.github/scripts/scan_core.sh init - - name: Test SOS (${{ matrix.name }}) - run: | - cd build - make check TESTS= -j - ulimit -c unlimited - ulimit -a - SHMEM_DEBUG=1 SHMEM_INFO=1 make VERBOSE=1 TEST_RUNNER="${SOS_PM} -np 2 timeout --signal=ABRT 15m" check - ${SOS_PM} -np 1 modules/tests-sos/test/unit/hello - - name: Scanning for Core Dumps (${{ matrix.sos_config }}) - if: always() - run: | - bash ${GITHUB_WORKSPACE}/.github/scripts/scan_core.sh scan - - name: Archive Artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: debug-artifacts-${{ matrix.config_name }}-${{ env.CONFIG_PARAM }}-${{ github.run_number }}.${{ github.run_attempt }} - path: ${{ github.workspace }}/archives - if-no-files-found: ignore diff --git a/configure.ac b/configure.ac index 0145c53f..1a51ee1a 100644 --- a/configure.ac +++ b/configure.ac @@ -327,6 +327,14 @@ AC_ARG_ENABLE([shr-atomics], AS_IF([test "$enable_shr_atomics" = "yes"], [AC_DEFINE([USE_SHR_ATOMICS], [1], [If defined, the shared memory layer will perform processor atomics.])]) +AC_ARG_ENABLE([hierarchical-barrier], + [AS_HELP_STRING([--enable-hierarchical-barrier], + [Enable the hierarchical barrier: intranode phase uses CPU atomics, + internode phase uses NIC puts. Requires XPMEM (the intranode + phase does CPU atomics on peers' slots via mapped pointers, which + only XPMEM provides) and a network transport (OFI, Portals4, or + UCX). (default: disabled)])]) + AC_ARG_ENABLE([manpages], [AS_HELP_STRING([--enable-manpages], [Include man pages in the installation (default:disabled)])]) @@ -537,7 +545,6 @@ AM_CONDITIONAL([USE_CMA], [test "$transport_cma" = "yes"]) AS_IF([test "$transport_xpmem" = "yes" -o "$transport_cma" = "yes"], [AC_DEFINE([USE_ON_NODE_COMMS], [1], [Define if any on-node comm transport is available]) - AC_DEFINE([ENABLE_HARD_POLLING], [1], [Enable hard polling]) ]) if test "$enable_shr_atomics" = "yes"; then @@ -551,6 +558,38 @@ if test "$enable_shr_atomics" != "no" -a "$transport_xpmem" = "yes" -a "$transpo AC_DEFINE([USE_SHR_ATOMICS], [1], [If defined, the shared memory layer will perform processor atomics.]) fi +dnl Hierarchical barrier: only when explicitly requested. Validate that XPMEM +dnl and a network transport are both present; error if dependencies are missing. +dnl XPMEM (not CMA) is required because the intranode phase performs CPU atomics +dnl on peers' pSync slots through directly mapped pointers (shmem_shr_transport_ptr), +dnl which only XPMEM can provide; CMA's copy-based path cannot. +transport_hierarchical_barrier="no" +if test "$enable_hierarchical_barrier" = "yes"; then + dnl Check on-node transport dependency + if test "$transport_xpmem" != "yes"; then + AC_MSG_ERROR([--enable-hierarchical-barrier requires XPMEM; configure with --with-xpmem]) + dnl Check network transport dependency + elif test "$transport_ofi" != "yes" -a "$transport_portals4" != "yes" -a "$transport_ucx" != "yes"; then + AC_MSG_ERROR([--enable-hierarchical-barrier requires a network transport; configure with --with-ofi, --with-portals4, or --with-ucx]) + else + transport_hierarchical_barrier="yes" + fi +fi +if test "$transport_hierarchical_barrier" = "yes"; then + AC_DEFINE([USE_HIERARCHICAL_BARRIER], [1], + [If defined, enables the hierarchical barrier: intranode CPU atomics + internode NIC puts.]) + dnl The hierarchical barrier uses XPMEM-mapped pointers for intranode data + dnl movement (CPU atomics on peers' slots). USE_SHR_ATOMICS is defined so that the + dnl shmem_shr_transport_atomic() function bodies are compiled; the routing + dnl logic in shmem_shr_transport_use_atomic() gates them behind a runtime + dnl check (shr_size == num_pes), so user-visible AMOs only go through CPU + dnl atomics when all PEs are on one node. In the multi-node case they + dnl always use the NIC, preserving the single-coherency-domain invariant. + AC_DEFINE([USE_SHR_ATOMICS], [1], + [If defined, the shared memory layer will perform processor atomics.]) + transport_shr_atomics="yes" +fi + AC_ARG_ENABLE([pmi-simple], [AS_HELP_STRING([--enable-pmi-simple], [Use MPICH simple PMI-1 library for process management])]) AC_ARG_ENABLE([pmi-mpi], [AS_HELP_STRING([--enable-pmi-mpi], @@ -920,6 +959,44 @@ if test "$transport_ofi" = "yes" ; then LIBS="$LIBS $ofi_LIBS" WRAPPER_COMPILER_EXTRA_LDFLAGS="$ofi_LDFLAGS" WRAPPER_COMPILER_EXTRA_LIBS="$ofi_LIBS" + dnl CXI provider extensions (struct fi_cxi_dom_ops, enable_hybrid_mr_desc). + dnl Checked with the OFI include path in CPPFLAGS. fi_cxi_ext.h is not + dnl self-contained: it references struct fid / uint64_t / bool / struct + dnl timespec without including the headers that define them. We therefore + dnl supply the full prerequisite set (stdbool.h, time.h, stdint.h, + dnl rdma/fabric.h, rdma/fi_domain.h) in the 4th argument so the probe + dnl compiles independently of whatever a given libfabric's fabric.h happens + dnl to pull in transitively (which varies across clusters/versions). Without + dnl this the header can be present yet reported as unusable, silently + dnl disabling the optimization. AC_CHECK_HEADERS defines + dnl HAVE_RDMA_FI_CXI_EXT_H when the header compiles; only then do we probe + dnl for the enable_hybrid_mr_desc member, which was added to struct + dnl fi_cxi_dom_ops in a later libfabric. AC_CHECK_MEMBER defines + dnl HAVE_STRUCT_FI_CXI_DOM_OPS_ENABLE_HYBRID_MR_DESC on success. The hybrid + dnl MR descriptor optimization is compiled in only when BOTH are defined. + AC_CHECK_HEADERS([rdma/fi_cxi_ext.h], + [AC_CHECK_MEMBER([struct fi_cxi_dom_ops.enable_hybrid_mr_desc], + [AC_DEFINE([HAVE_STRUCT_FI_CXI_DOM_OPS_ENABLE_HYBRID_MR_DESC], [1], + [Define to 1 if struct fi_cxi_dom_ops has the enable_hybrid_mr_desc member.])], + [], + [[#include + #include + #include + #include + #include + #include ]])], + [], + [[#include + #include + #include + #include + #include ]]) + + dnl Surface the result in the configure summary so a misdetection is visible + dnl at configure time rather than only as a runtime warning on CXI hardware. + cxi_hybrid_mr_desc="no" + AS_IF([test "x$ac_cv_member_struct_fi_cxi_dom_ops_enable_hybrid_mr_desc" = "xyes"], + [cxi_hybrid_mr_desc="yes"]) fi if test "$transport_ucx" = "yes" ; then @@ -1050,6 +1127,7 @@ echo " XPMEM: $transport_xpmem" echo " CMA: $transport_cma" echo " memcpy (self): $transport_memcpy" echo " Shr. atomics: $transport_shr_atomics" +echo " Hier. barrier: $transport_hierarchical_barrier" echo "" echo "Global Options:" if test "$enable_remote_virtual_addressing" = "yes"; then @@ -1084,6 +1162,8 @@ if test "$transport_ofi" = "yes"; then echo " Addr. vector: table" fi + echo " CXI hybrid MR desc: $cxi_hybrid_mr_desc" + echo "" fi diff --git a/src/collectives.c b/src/collectives.c index 89c9bd27..0ad125ce 100644 --- a/src/collectives.c +++ b/src/collectives.c @@ -15,6 +15,7 @@ #include "config.h" #include +#include #define SHMEM_INTERNAL_INCLUDE #include "shmem.h" @@ -30,13 +31,72 @@ coll_type_t shmem_internal_collect_type = AUTO; coll_type_t shmem_internal_fcollect_type = AUTO; long *shmem_internal_barrier_all_psync; long *shmem_internal_sync_all_psync; +#ifdef USE_HIERARCHICAL_BARRIER +long *shmem_internal_barrier_all_local_psync; +long *shmem_internal_sync_all_local_psync; +long *shmem_internal_hierarchical_local_psync; + +/* Persistent scratch for the active-set PE lists built on every hierarchical + * barrier. Allocated once at init, sized to num_pes (the maximum any active + * set can hold), and reused across calls. Safe to share: barriers are + * serialized — a PE cannot enter a new barrier until it has exited the + * previous one (same invariant the shared hierarchical_local_psync relies + * on). */ +static int *hier_local_pes = NULL; +static int *hier_root_pes = NULL; + +/* Seen-flags for the root-set build, indexed by node_id (a global PE number in + * [0, num_pes)). Lets build_root_active_set run in O(PE_size) instead of + * O(PE_size^2). Allocated once at init, sized to num_pes. */ +static char *hier_node_seen = NULL; + +/* Job-global minimum PEs-per-node, computed once at init from the global + * node_id array. AUTO hierarchical-barrier selection gates on this (not the + * local node size) so every PE makes the same decision even on a job with + * heterogeneous PPN; otherwise PEs on a large node could pick hierarchical + * while PEs on a small node pick tree, diverging within one collective. */ +int shmem_internal_hier_min_ppn = 0; + +/* Layout of local_pSync — two cache-line-padded arrays, one slot per PE: + * + * up-slot for PE r: local_pSync[r * HIER_SLOT_STRIDE] + * down-slot for PE r: local_pSync[shr_size * HIER_SLOT_STRIDE + r * HIER_SLOT_STRIDE] + * + * Phase 1 (gather): each PE writes signal to its OWN up-slot once it has + * collected all its children; parent reads children's up-slots. + * One writer per slot — no MESI contention on the parent's cache line. + * + * Phase 3 (fanout): parent writes signal to each child's down-slot. + * One writer per slot — same property. + * + * Sense-alternation (hier_sense) avoids explicit resets between calls. + * + * Total allocation: 2 * shr_size * HIER_SLOT_STRIDE longs. */ +#define HIER_SLOT_STRIDE 8 /* 8 longs = 64 bytes = 1 cache line */ + +/* Per-phase timing accumulators (root PE: all three phases; + * non-root PEs: phase1_us = gather wait, phase3_us = fanout wait). */ +static double hier_phase1_us = 0.0; +static double hier_phase2_us = 0.0; +static double hier_phase3_us = 0.0; +static long hier_call_count = 0; + +static inline double +hier_now_us(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ts.tv_sec * 1e6 + ts.tv_nsec * 1e-3; +} +#endif char *coll_type_str[] = { "AUTO", "LINEAR", "TREE", "DISSEM", "RING", - "RECDBL" }; + "RECDBL", + "HIERARCHICAL" }; static int *full_tree_children; static int full_tree_num_children; @@ -134,6 +194,72 @@ shmem_internal_collectives_init(void) for (i = 0; i < SHMEM_BARRIER_SYNC_SIZE; i++) shmem_internal_sync_all_psync[i] = SHMEM_SYNC_VALUE; +#ifdef USE_HIERARCHICAL_BARRIER + /* Allocate local (intranode, CPU-atomic) pSync arrays for the hierarchical + * barrier. Each PE owns HIER_SLOT_STRIDE longs (one cache line) within + * the array, indexed by shr_rank. The stride prevents false sharing: + * PE r's slot is at local_pSync[r * HIER_SLOT_STRIDE]. + * These arrays are touched only by on-node CPU stores/loads via XPMEM; + * the global pSync arrays above are touched only by NIC puts. */ + int shr_size = shmem_internal_get_shr_size(); + int local_psync_len = 2 * shr_size * HIER_SLOT_STRIDE; + + shmem_internal_barrier_all_local_psync = + shmem_internal_shmalloc(sizeof(long) * local_psync_len); + if (NULL == shmem_internal_barrier_all_local_psync) return -1; + + for (i = 0; i < local_psync_len; i++) { + shmem_internal_barrier_all_local_psync[i] = SHMEM_SYNC_VALUE; + } + + shmem_internal_sync_all_local_psync = + shmem_internal_shmalloc(sizeof(long) * local_psync_len); + if (NULL == shmem_internal_sync_all_local_psync) return -1; + + for (i = 0; i < local_psync_len; i++) { + shmem_internal_sync_all_local_psync[i] = SHMEM_SYNC_VALUE; + } + + /* Shared local pSync for general barriers/syncs. Safe to share because + * barriers are serialized — a PE cannot enter a new barrier until it has + * exited the previous one. */ + shmem_internal_hierarchical_local_psync = + shmem_internal_shmalloc(sizeof(long) * local_psync_len); + if (NULL == shmem_internal_hierarchical_local_psync) return -1; + + for (i = 0; i < local_psync_len; i++) { + shmem_internal_hierarchical_local_psync[i] = SHMEM_SYNC_VALUE; + } + + /* Persistent per-barrier scratch (see declaration above). Sized to + * num_pes so it fits the largest possible active set. */ + hier_local_pes = malloc(sizeof(int) * shmem_internal_num_pes); + hier_root_pes = malloc(sizeof(int) * shmem_internal_num_pes); + /* calloc: seen-flags must start cleared; build_root_active_set leaves them + * clean afterward by resetting only the entries it touched. */ + hier_node_seen = calloc(shmem_internal_num_pes, sizeof(char)); + if (NULL == hier_local_pes || NULL == hier_root_pes || + NULL == hier_node_seen) return -1; + + /* Compute the job-global minimum PEs-per-node from the global node_id array + * (already exchanged at init — no communication needed here). node_id[pe] + * is the global PE number of the lowest-ranked PE on pe's node, so it both + * identifies the node and is a valid index in [0, num_pes); tally PE counts + * per node, then take the minimum over nodes that actually have PEs. */ + { + int *ppn_count = calloc(shmem_internal_num_pes, sizeof(int)); + if (NULL == ppn_count) return -1; + for (i = 0; i < shmem_internal_num_pes; i++) + ppn_count[shmem_runtime_get_node_id(i)]++; + shmem_internal_hier_min_ppn = shmem_internal_num_pes; + for (i = 0; i < shmem_internal_num_pes; i++) { + if (ppn_count[i] > 0 && ppn_count[i] < shmem_internal_hier_min_ppn) + shmem_internal_hier_min_ppn = ppn_count[i]; + } + free(ppn_count); + } +#endif + /* initialize the binomial tree for collective operations over entire tree */ full_tree_num_children = 0; @@ -176,6 +302,10 @@ shmem_internal_collectives_init(void) shmem_internal_barrier_type = TREE; } else if (0 == strcmp(type, "dissem")) { shmem_internal_barrier_type = DISSEM; +#ifdef USE_HIERARCHICAL_BARRIER + } else if (0 == strcmp(type, "hierarchical")) { + shmem_internal_barrier_type = HIERARCHICAL; +#endif } else { RAISE_WARN_MSG("Ignoring bad barrier algorithm '%s'\n", type); } @@ -420,6 +550,424 @@ shmem_internal_sync_dissem(int PE_start, int PE_stride, int PE_size, long *pSync } +/***************************************** + * + * HIERARCHICAL BARRIER/SYNC + * + * Three-phase algorithm for USE_HIERARCHICAL_BARRIER builds: + * + * Phase 1 (intranode gather, CPU stores/loads via XPMEM): + * All local PEs run an intranode dissemination barrier using plain + * (non-atomic) stores and acquire-loads. Each PE owns one cache-line-padded + * slot (HIER_SLOT_STRIDE longs apart) so no two PEs share a cache line. + * Sense alternation (hier_sense) removes the need for explicit slot resets. + * ceil(log2(local_count)) rounds; each round: store signal to partner's slot, + * spin-load own slot until partner's signal arrives. + * + * Phase 2 (internode, NIC puts, root PEs only): + * Root PEs run a dissemination barrier among themselves using NIC puts + * across per-round pSync slots (ceil(log2(N)) rounds). A shmem_quiet + * after the last round ensures all outbound puts are retired before + * phase 3. + * + * Phase 3 (intranode fanout, CPU stores/loads via XPMEM): + * Same dissemination algorithm as phase 1, run in reverse sense so that the + * "all clear" propagates back to all local PEs without NIC involvement. + * + *****************************************/ +#ifdef USE_HIERARCHICAL_BARRIER + +/* Count and list the PEs in the active set that reside on this node. */ +static void +shmem_internal_build_local_set(int PE_start, int PE_stride, int PE_size, + int *local_pes, int *local_count) +{ + int i, pe; + *local_count = 0; + + for (i = 0, pe = PE_start; i < PE_size; i++, pe += PE_stride) { + if (shmem_internal_get_shr_rank(pe) != -1) { + local_pes[(*local_count)++] = pe; + } + } +} + +/* Build an active set of root PEs — one representative per node, defined as + * the lowest-ranked active PE on each participating node. Uses the global + * node_id array (computed once at init) so all PEs deterministically compute + * the same list without requiring communication. + * + * O(PE_size): a seen-flag keyed by node_id (a global PE number) records which + * nodes already have a representative. Iterating in ascending PE order, the + * first active PE seen for a node_id is its representative. seen_buf holds + * num_pes entries and must enter all-zero; a second pass resets exactly the + * entries this build touched, preserving that invariant for the next call. */ +static void +shmem_internal_build_root_active_set(int PE_start, int PE_stride, int PE_size, + int *root_pes, int *root_count, + char *seen_buf) +{ + int i, pe; + *root_count = 0; + + for (i = 0, pe = PE_start; i < PE_size; i++, pe += PE_stride) { + int nid = shmem_runtime_get_node_id(pe); + if (!seen_buf[nid]) { + seen_buf[nid] = 1; + root_pes[(*root_count)++] = pe; + } + } + + /* Reset only the touched entries so seen_buf is clean for the next build. */ + for (i = 0; i < *root_count; i++) { + seen_buf[shmem_runtime_get_node_id(root_pes[i])] = 0; + } +} + +/* CPU atomic store of `val` to the long at `target` via local mapped pointer. */ +static inline void +shmem_internal_cpu_atomic_store_long(long *target, int noderank, long val) +{ + void *remote_ptr; + shmem_shr_transport_ptr(target, noderank, &remote_ptr); + __atomic_store((long *)remote_ptr, &val, __ATOMIC_RELEASE); +} + +/* Compute the full hierarchical-barrier topology for an active set into a + * cache struct. The PE/root lists are written into the caller-supplied + * buffers (local_buf >= on-node active count, root_buf >= node count, + * child_buf >= tree_radix); the cache's pointers are set to reference them. + * Every field here is invariant for a fixed (PE_start, PE_stride, PE_size) and + * node layout, so a team can compute this once and reuse it. */ +static void +shmem_internal_hier_compute(int PE_start, int PE_stride, int PE_size, + int *local_buf, int *root_buf, int *child_buf, + shmem_internal_hier_cache_t *c) +{ + c->local_pes = local_buf; + c->root_pes = root_buf; + c->tree_child_shr = child_buf; + + shmem_internal_build_local_set(PE_start, PE_stride, PE_size, + c->local_pes, &c->local_count); + shmem_internal_build_root_active_set(PE_start, PE_stride, PE_size, + c->root_pes, &c->root_count, + hier_node_seen); + + /* build_local_set fills local_pes in ascending PE order, so local_pes[0] is + * the lowest-ranked active PE on this node — the active-set representative + * for phase 2. my_vidx is this PE's index in that list (-1 if absent). */ + c->my_vidx = -1; + for (int i = 0; i < c->local_count; i++) { + if (c->local_pes[i] == shmem_internal_my_pe) { c->my_vidx = i; break; } + } + c->active_root_pe = (c->local_count > 0) ? c->local_pes[0] : -1; + c->is_root = (shmem_internal_my_pe == c->active_root_pe); + + c->tree_nchildren = 0; + if (c->my_vidx >= 0) { + for (int j = 1; j <= tree_radix; j++) { + int cv = c->my_vidx * tree_radix + j; + if (cv < c->local_count) { + c->tree_child_shr[c->tree_nchildren++] = + shmem_runtime_get_node_rank(c->local_pes[cv]); + } + } + } + + c->my_root_idx = -1; + for (int i = 0; i < c->root_count; i++) { + if (c->root_pes[i] == shmem_internal_my_pe) { c->my_root_idx = i; break; } + } + + c->num_rounds = 0; + { int n = c->root_count - 1; while (n > 0) { n >>= 1; c->num_rounds++; } } +} + +/* Free the malloc'd arrays held by a persisted topology cache. */ +void +shmem_internal_hier_cache_free(shmem_internal_hier_cache_t *c) +{ + if (c == NULL) return; + free(c->local_pes); + free(c->root_pes); + free(c->tree_child_shr); + c->local_pes = c->root_pes = c->tree_child_shr = NULL; + c->valid = 0; +} + +void +shmem_internal_sync_hierarchical(int PE_start, int PE_stride, int PE_size, + long *pSync, long *local_pSync, + long *hier_sense_ptr, + shmem_internal_hier_cache_t *hier_cache) +{ + int my_shr_rank = shmem_runtime_get_node_rank(shmem_internal_my_pe); + long one = 1; + + if (PE_size == 1) return; + + /* Determine sense state. Caller may supply a per-team sense pointer + * (hier_sense_ptr != NULL); this ensures each team maintains an independent + * sense counter and prevents deadlocks when overlapping teams are synced in + * different sequences. NULL falls back to the internal selection: TEAM_WORLD + * uses its own counter; all other active sets share a static fallback. */ + long *sense_ptr; + if (hier_sense_ptr != NULL) { + sense_ptr = hier_sense_ptr; + } else if (PE_start == 0 && PE_stride == 1 && PE_size == shmem_internal_num_pes) { + sense_ptr = &shmem_internal_team_world.hier_sense; + } else { + static long fallback_sense = 0; + sense_ptr = &fallback_sense; + } + + /* Resolve the active-set topology. A team supplies a cache (hier_cache): + * built once on the first barrier, reused thereafter — at scale this + * removes the O(PE_size) / O(PE_size^2) set scans and per-child runtime + * lookups from the hot path. Callers with no team context (hier_cache == + * NULL) build into the shared per-call scratch every time. */ + shmem_internal_hier_cache_t scratch_cache; + shmem_internal_hier_cache_t *topo; + if (hier_cache != NULL && hier_cache->valid) { + topo = hier_cache; + } else { + shmem_internal_hier_compute(PE_start, PE_stride, PE_size, + hier_local_pes, hier_root_pes, + alloca(sizeof(int) * tree_radix), + &scratch_cache); + if (hier_cache != NULL) { + /* Persist exact-size copies in the team cache for reuse. If any + * allocation fails, fall back to using the scratch topology for + * this call without persisting — the barrier still completes + * correctly, just without the caching optimization. */ + int lc = scratch_cache.local_count, rc = scratch_cache.root_count; + int nc = scratch_cache.tree_nchildren; + int *cl = malloc(sizeof(int) * (lc > 0 ? lc : 1)); + int *cr = malloc(sizeof(int) * (rc > 0 ? rc : 1)); + int *cc = malloc(sizeof(int) * (nc > 0 ? nc : 1)); + if (cl == NULL || cr == NULL || cc == NULL) { + free(cl); + free(cr); + free(cc); + topo = &scratch_cache; + } else { + memcpy(cl, scratch_cache.local_pes, sizeof(int) * lc); + memcpy(cr, scratch_cache.root_pes, sizeof(int) * rc); + memcpy(cc, scratch_cache.tree_child_shr, sizeof(int) * nc); + hier_cache->local_pes = cl; + hier_cache->root_pes = cr; + hier_cache->tree_child_shr = cc; + hier_cache->local_count = lc; + hier_cache->root_count = rc; + hier_cache->tree_nchildren = nc; + hier_cache->my_vidx = scratch_cache.my_vidx; + hier_cache->active_root_pe = scratch_cache.active_root_pe; + hier_cache->is_root = scratch_cache.is_root; + hier_cache->my_root_idx = scratch_cache.my_root_idx; + hier_cache->num_rounds = scratch_cache.num_rounds; + hier_cache->valid = 1; + topo = hier_cache; + } + } else { + topo = &scratch_cache; + } + } + + int local_count = topo->local_count; + int *root_pes = topo->root_pes; + int root_count = topo->root_count; + int my_vidx = topo->my_vidx; + int is_root = topo->is_root; + int *tree_child_shr = topo->tree_child_shr; + int tree_nchildren = topo->tree_nchildren; + + /* Sense-alternating signal — monotonically increasing, no slot resets needed. + * up-slot for PE r: local_pSync[r * HIER_SLOT_STRIDE] + * down-slot for PE r: local_pSync[shr_size * HIER_SLOT_STRIDE + r * HIER_SLOT_STRIDE] */ + long signal = SHMEM_SYNC_VALUE + 1 + *sense_ptr; + int shr_size = shmem_internal_get_shr_size(); + long *up_pSync = local_pSync; + long *down_pSync = local_pSync + (long)(shr_size * HIER_SLOT_STRIDE); + + /* Resolve own up-slot and down-slot mapped pointers once. */ + void *my_up_raw, *my_down_raw; + shmem_shr_transport_ptr(&up_pSync[my_shr_rank * HIER_SLOT_STRIDE], my_shr_rank, &my_up_raw); + shmem_shr_transport_ptr(&down_pSync[my_shr_rank * HIER_SLOT_STRIDE], my_shr_rank, &my_down_raw); + volatile long *my_down_slot = (volatile long *)my_down_raw; + + /* ---- Degenerate case: all active PEs on one node ---- */ + if (local_count == PE_size) { + if (my_vidx < 0) return; + + double t0 = shmem_internal_params.HIER_BARRIER_DEBUG ? hier_now_us() : 0.0; + + /* Phase 1: gather up tree — wait for children's up-slots, then signal parent */ + for (int c = 0; c < tree_nchildren; c++) { + void *child_up_raw; + shmem_shr_transport_ptr(&up_pSync[tree_child_shr[c] * HIER_SLOT_STRIDE], + tree_child_shr[c], &child_up_raw); + volatile long *child_up = (volatile long *)child_up_raw; + long cur; + do { + __atomic_load(child_up, &cur, __ATOMIC_ACQUIRE); + if (cur != signal) { SPINLOCK_BODY(); } + } while (cur != signal); + } + if (my_vidx > 0) { + void *parent_up_raw; + shmem_shr_transport_ptr(&up_pSync[my_shr_rank * HIER_SLOT_STRIDE], + my_shr_rank, &parent_up_raw); + __atomic_store((long *)parent_up_raw, &signal, __ATOMIC_RELEASE); + } + + double t1 = shmem_internal_params.HIER_BARRIER_DEBUG ? hier_now_us() : 0.0; + + /* Phase 3: fanout — root stores to children's down-slots, non-roots wait then relay */ + if (my_vidx == 0) { + for (int c = 0; c < tree_nchildren; c++) { + shmem_internal_cpu_atomic_store_long( + &down_pSync[tree_child_shr[c] * HIER_SLOT_STRIDE], + tree_child_shr[c], signal); + } + } else { + long cur; + do { + __atomic_load(my_down_slot, &cur, __ATOMIC_ACQUIRE); + if (cur != signal) { SPINLOCK_BODY(); } + } while (cur != signal); + for (int c = 0; c < tree_nchildren; c++) { + shmem_internal_cpu_atomic_store_long( + &down_pSync[tree_child_shr[c] * HIER_SLOT_STRIDE], + tree_child_shr[c], signal); + } + } + + if (shmem_internal_params.HIER_BARRIER_DEBUG) { + double t2 = hier_now_us(); + hier_phase1_us += t1 - t0; + hier_phase3_us += t2 - t1; + hier_call_count++; + } + + (*sense_ptr)++; + return; + } + + /* ---- Normal multi-node case ---- */ + + double mn_t0 = shmem_internal_params.HIER_BARRIER_DEBUG ? hier_now_us() : 0.0; + + /* Phase 1: intranode gather (k-ary tree, bottom-up). + * Each PE waits on each child's up-slot, then writes its own up-slot. + * One writer per up-slot — no cache-line contention on the parent. */ + if (my_vidx >= 0) { + for (int c = 0; c < tree_nchildren; c++) { + void *child_up_raw; + shmem_shr_transport_ptr(&up_pSync[tree_child_shr[c] * HIER_SLOT_STRIDE], + tree_child_shr[c], &child_up_raw); + volatile long *child_up = (volatile long *)child_up_raw; + long cur; + do { + __atomic_load(child_up, &cur, __ATOMIC_ACQUIRE); + if (cur != signal) { SPINLOCK_BODY(); } + } while (cur != signal); + } + if (my_vidx > 0) { + /* Signal parent by writing to OWN up-slot (parent reads it). */ + __atomic_store((long *)my_up_raw, &signal, __ATOMIC_RELEASE); + } + } + + double mn_t1 = shmem_internal_params.HIER_BARRIER_DEBUG ? hier_now_us() : 0.0; + + if (is_root) { + /* ---- Phase 2: internode barrier (NIC puts, root PEs only) ---- */ + if (root_count > 1) { + int my_root_idx = topo->my_root_idx; + int num_rounds = topo->num_rounds; + shmem_internal_assert(my_root_idx >= 0); + shmem_internal_assert(num_rounds <= SHMEM_BARRIER_SYNC_SIZE); + + for (int r = 0; r < num_rounds; r++) { + int partner_idx = (my_root_idx + (1 << r)) % root_count; + int partner_pe = root_pes[partner_idx]; + shmem_internal_put_scalar(SHMEM_CTX_DEFAULT, &pSync[r], &one, + sizeof(one), partner_pe); + SHMEM_WAIT(&pSync[r], SHMEM_SYNC_VALUE); + __atomic_store_n(&pSync[r], SHMEM_SYNC_VALUE, __ATOMIC_RELEASE); + } + } + + shmem_internal_quiet(SHMEM_CTX_DEFAULT); + + double mn_t2 = shmem_internal_params.HIER_BARRIER_DEBUG ? hier_now_us() : 0.0; + + /* ---- Phase 3: intranode fanout (k-ary tree, top-down via down-slots) ---- */ + for (int c = 0; c < tree_nchildren; c++) { + shmem_internal_cpu_atomic_store_long( + &down_pSync[tree_child_shr[c] * HIER_SLOT_STRIDE], + tree_child_shr[c], signal); + } + + if (shmem_internal_params.HIER_BARRIER_DEBUG) { + double mn_t3 = hier_now_us(); + hier_phase1_us += mn_t1 - mn_t0; + hier_phase2_us += mn_t2 - mn_t1; + hier_phase3_us += mn_t3 - mn_t2; + hier_call_count++; + } + + } else if (my_vidx >= 0) { + /* Non-root: wait on own down-slot for parent's signal, then relay to children. */ + long cur; + do { + __atomic_load(my_down_slot, &cur, __ATOMIC_ACQUIRE); + if (cur != signal) { SPINLOCK_BODY(); } + } while (cur != signal); + + for (int c = 0; c < tree_nchildren; c++) { + shmem_internal_cpu_atomic_store_long( + &down_pSync[tree_child_shr[c] * HIER_SLOT_STRIDE], + tree_child_shr[c], signal); + } + + if (shmem_internal_params.HIER_BARRIER_DEBUG) { + double mn_t3 = hier_now_us(); + hier_phase1_us += mn_t1 - mn_t0; + hier_phase3_us += mn_t3 - mn_t1; + hier_call_count++; + } + } + + if (my_vidx >= 0) { (*sense_ptr)++; } + /* PEs absent from local_pes (my_vidx < 0) fall through silently. */ +} + +void +shmem_internal_hier_barrier_print_stats(void) +{ + if (!shmem_internal_params.HIER_BARRIER_DEBUG) return; + if (hier_call_count == 0) return; + + /* Each PE prints its own per-phase averages. At scale the output can be + * large; the caller should fence/barrier before this so the lines don't + * interleave badly, but we keep this simple intentionally. */ + fprintf(stderr, + "[PE %d] hier_barrier calls=%ld " + "phase1(gather)=%.2f us phase2(internode)=%.2f us " + "phase3(fanout)=%.2f us total=%.2f us\n", + shmem_internal_my_pe, + hier_call_count, + hier_phase1_us / hier_call_count, + hier_phase2_us / hier_call_count, + hier_phase3_us / hier_call_count, + (hier_phase1_us + hier_phase2_us + hier_phase3_us) / hier_call_count); +} + +#endif /* USE_HIERARCHICAL_BARRIER */ + + /***************************************** * * BROADCAST diff --git a/src/collectives_c.c4 b/src/collectives_c.c4 index b6e2f999..c55418b3 100644 --- a/src/collectives_c.c4 +++ b/src/collectives_c.c4 @@ -181,7 +181,8 @@ shmem_barrier(int PE_start, int logPE_stride, int PE_size, long *pSync) SHMEM_ERR_CHECK_ACTIVE_SET(PE_start, 1 << logPE_stride, PE_size); SHMEM_ERR_CHECK_SYMMETRIC(pSync, sizeof(long)*SHMEM_BARRIER_SYNC_SIZE); - shmem_internal_barrier(PE_start, 1 << logPE_stride, PE_size, pSync); + shmem_internal_quiet(SHMEM_CTX_DEFAULT); + shmem_internal_sync_no_hier(PE_start, 1 << logPE_stride, PE_size, pSync); } @@ -200,7 +201,7 @@ shmem_sync(int PE_start, int logPE_stride, int PE_size, long *pSync) SHMEM_ERR_CHECK_ACTIVE_SET(PE_start, 1 << logPE_stride, PE_size); SHMEM_ERR_CHECK_SYMMETRIC(pSync, sizeof(long)*SHMEM_BARRIER_SYNC_SIZE); - shmem_internal_sync(PE_start, 1 << logPE_stride, PE_size, pSync); + shmem_internal_sync_no_hier(PE_start, 1 << logPE_stride, PE_size, pSync); } /* Team-based Collective Routines */ @@ -213,7 +214,7 @@ shmem_team_sync(shmem_team_t team) shmem_internal_team_t *myteam = (shmem_internal_team_t *)team; long *psync = shmem_internal_team_choose_psync(myteam, SYNC); - shmem_internal_sync(myteam->start, myteam->stride, myteam->size, psync); + shmem_internal_sync_for_team(myteam, psync); shmem_internal_team_release_psyncs(myteam, SYNC); return 0; } diff --git a/src/init.c b/src/init.c index 01ca23df..c76cd0e0 100644 --- a/src/init.c +++ b/src/init.c @@ -147,6 +147,10 @@ shmem_internal_shutdown(void) shmem_internal_finalized = 1; +#ifdef USE_HIERARCHICAL_BARRIER + shmem_internal_hier_barrier_print_stats(); +#endif + shmem_internal_team_fini(); shmem_transport_fini(); @@ -382,6 +386,10 @@ shmem_internal_heap_postinit(void) shmem_internal_heap_base, shmem_internal_heap_length, shmem_internal_data_base, shmem_internal_data_length); + if (shmem_internal_params.BOUNCE_MLOCK) { + DEBUG_MSG("Bounce buffer locking enabled\n"); + } + #ifdef HAVE_SCHED_GETAFFINITY #ifdef USE_HWLOC ret = hwloc_topology_init(&shmem_internal_topology); @@ -520,6 +528,20 @@ shmem_internal_heap_postinit(void) atexit(shmem_internal_shutdown_atexit); shmem_internal_initialized = 1; +#ifdef USE_HIERARCHICAL_BARRIER + if (shmem_internal_my_pe == 0) { + const char *effective_barrier = + (shmem_internal_barrier_type == AUTO && + shmem_internal_hier_barrier_available()) + ? "HIERARCHICAL (auto-selected)" + : coll_type_str[shmem_internal_barrier_type]; + + DEBUG_MSG("Hierarchical barrier enabled: intranode CPU atomics + internode NIC puts\n" + RAISE_PE_PREFIX "Barrier algorithm: %s\n", + shmem_internal_my_pe, effective_barrier); + } +#endif + /* finish up */ #ifndef USE_PMIX shmem_runtime_barrier(); diff --git a/src/runtime-mpi.c b/src/runtime-mpi.c index 3713ec3f..c5cd5620 100644 --- a/src/runtime-mpi.c +++ b/src/runtime-mpi.c @@ -38,6 +38,8 @@ static int kv_length = 0; static int initialized_mpi = 0; static int node_size; static int *node_ranks; +static int *is_node_root = NULL; +static int *node_id_array = NULL; char* kv_store_me; char* kv_store_all; @@ -103,6 +105,12 @@ shmem_runtime_init(int enable_node_ranks) if (size > 1 && enable_node_ranks) { node_ranks = malloc(size * sizeof(int)); if (NULL == node_ranks) return 8; +#ifdef USE_HIERARCHICAL_BARRIER + is_node_root = malloc(size * sizeof(int)); + if (NULL == is_node_root) return 9; + node_id_array = malloc(size * sizeof(int)); + if (NULL == node_id_array) return 10; +#endif } return 0; @@ -117,6 +125,8 @@ shmem_runtime_fini(void) if (node_ranks) { MPI_Comm_free(&SHMEM_RUNTIME_SHARED); free(node_ranks); + free(is_node_root); + free(node_id_array); } MPI_Comm_free(&SHMEM_RUNTIME_WORLD); @@ -202,11 +212,59 @@ shmem_runtime_get_node_size(void) return node_size; } +int +shmem_runtime_get_node_root_pe(void) +{ + int i; + + if (size == 1) { + return 0; + } + + for (i = 0; i < size; i++) { + if (node_ranks[i] == 0) + return i; + } + + /* Should not be reached */ + return 0; +} + +int +shmem_runtime_is_node_root_pe(int pe) +{ + shmem_internal_assert(pe < size && pe >= 0); + + if (size == 1) + return 1; + + if (NULL == is_node_root) + return 0; + + return is_node_root[pe]; +} + + +int +shmem_runtime_get_node_id(int pe) +{ + shmem_internal_assert(pe < size && pe >= 0); + + if (size == 1) + return 0; + + if (NULL == node_id_array) + return pe; + + return node_id_array[pe]; +} + int shmem_runtime_exchange(void) { if (size == 1) { kv_store_all = kv_store_me; + if (is_node_root) is_node_root[0] = 1; return 0; } @@ -231,6 +289,35 @@ shmem_runtime_exchange(void) MPI_Group_free(&world_group); MPI_Group_free(&node_group); free(world_ranks); + +#ifdef USE_HIERARCHICAL_BARRIER + /* Exchange absolute node ranks to identify node roots across all nodes. + * node_ranks[pe] is the rank of PE pe in the CALLING PE's node group; + * for off-node PEs it is MPI_UNDEFINED. We need each PE's rank in its + * OWN node group, so gather those now. */ + int my_node_rank; + MPI_Comm_rank(SHMEM_RUNTIME_SHARED, &my_node_rank); + + int *abs_node_ranks = malloc(size * sizeof(int)); + if (NULL == abs_node_ranks) return 2; + + MPI_Allgather(&my_node_rank, 1, MPI_INT, + abs_node_ranks, 1, MPI_INT, SHMEM_RUNTIME_WORLD); + + for (int i = 0; i < size; i++) + is_node_root[i] = (abs_node_ranks[i] == 0) ? 1 : 0; + + free(abs_node_ranks); + + /* Compute node_id_array: each PE's node is identified by the global + * rank of its lowest-ranked process. Bcast rank-0's world rank within + * the shared communicator so every PE on the same node gets the same + * value, then Allgather so all PEs know every other PE's node root. */ + int my_node_root_grank = rank; + MPI_Bcast(&my_node_root_grank, 1, MPI_INT, 0, SHMEM_RUNTIME_SHARED); + MPI_Allgather(&my_node_root_grank, 1, MPI_INT, + node_id_array, 1, MPI_INT, SHMEM_RUNTIME_WORLD); +#endif } int chunkSize = kv_length * sizeof(char) * MAX_KV_LENGTH; diff --git a/src/runtime-pmi.c b/src/runtime-pmi.c index 07805a2c..94a322a2 100644 --- a/src/runtime-pmi.c +++ b/src/runtime-pmi.c @@ -38,6 +38,8 @@ static char *kvs_name, *kvs_key, *kvs_value; static int max_name_len, max_key_len, max_val_len; static int initialized_pmi = 0; static int *location_array = NULL; +static int *is_node_root = NULL; +static int *node_id_array = NULL; #define SINGLETON_KEY_LEN 128 #define SINGLETON_VAL_LEN 1024 @@ -96,6 +98,12 @@ shmem_runtime_init(int enable_node_ranks) if (enable_node_ranks) { location_array = malloc(sizeof(int) * size); if (NULL == location_array) return 10; +#ifdef USE_HIERARCHICAL_BARRIER + is_node_root = malloc(sizeof(int) * size); + if (NULL == is_node_root) return 11; + node_id_array = malloc(sizeof(int) * size); + if (NULL == node_id_array) return 12; +#endif } } else { @@ -120,6 +128,8 @@ int shmem_runtime_fini(void) { free(location_array); + free(is_node_root); + free(node_id_array); free(kvs_name); free(kvs_key); free(kvs_value); @@ -194,6 +204,55 @@ shmem_runtime_get_node_size(void) } +int +shmem_runtime_get_node_root_pe(void) +{ + int i; + + if (size == 1) { + return 0; + } + + for (i = 0; i < size; i++) { + if (location_array[i] == 0) + return i; + } + + /* Should not be reached */ + return 0; +} + + +int +shmem_runtime_is_node_root_pe(int pe) +{ + shmem_internal_assert(pe < size && pe >= 0); + + if (size == 1) + return 1; + + if (NULL == is_node_root) + return 0; + + return is_node_root[pe]; +} + + +int +shmem_runtime_get_node_id(int pe) +{ + shmem_internal_assert(pe < size && pe >= 0); + + if (size == 1) + return 0; + + if (NULL == node_id_array) + return pe; + + return node_id_array[pe]; +} + + int shmem_runtime_exchange(void) { @@ -226,6 +285,13 @@ shmem_runtime_exchange(void) return 7; } } + if (is_node_root) { + ret = shmem_runtime_util_populate_global_node_roots(is_node_root, node_id_array, size); + if (0 != ret) { + RETURN_ERROR_MSG("Global node root mapping failed (%d)\n", ret); + return 8; + } + } return 0; } diff --git a/src/runtime-pmi2.c b/src/runtime-pmi2.c index 4460e562..d1b885e2 100644 --- a/src/runtime-pmi2.c +++ b/src/runtime-pmi2.c @@ -42,6 +42,8 @@ static char *kvs_name, *kvs_key, *kvs_value; static int max_name_len, max_key_len, max_val_len; static int initialized_pmi = 0; static int *location_array = NULL; +static int *is_node_root = NULL; +static int *node_id_array = NULL; int @@ -77,6 +79,12 @@ shmem_runtime_init(int enable_node_ranks) if (enable_node_ranks) { location_array = malloc(sizeof(int) * size); if (NULL == location_array) return 8; +#ifdef USE_HIERARCHICAL_BARRIER + is_node_root = malloc(sizeof(int) * size); + if (NULL == is_node_root) return 9; + node_id_array = malloc(sizeof(int) * size); + if (NULL == node_id_array) return 10; +#endif } return 0; @@ -89,6 +97,12 @@ shmem_runtime_fini(void) if (location_array) { free(location_array); } + if (is_node_root) { + free(is_node_root); + } + if (node_id_array) { + free(node_id_array); + } if (initialized_pmi == 1) { PMI2_Finalize(); @@ -145,6 +159,55 @@ shmem_runtime_get_node_size(void) } +int +shmem_runtime_get_node_root_pe(void) +{ + int i; + + if (size == 1) { + return 0; + } + + for (i = 0; i < size; i++) { + if (location_array[i] == 0) + return i; + } + + /* Should not be reached */ + return 0; +} + + +int +shmem_runtime_is_node_root_pe(int pe) +{ + shmem_internal_assert(pe < size && pe >= 0); + + if (size == 1) + return 1; + + if (NULL == is_node_root) + return 0; + + return is_node_root[pe]; +} + + +int +shmem_runtime_get_node_id(int pe) +{ + shmem_internal_assert(pe < size && pe >= 0); + + if (size == 1) + return 0; + + if (NULL == node_id_array) + return pe; + + return node_id_array[pe]; +} + + int shmem_runtime_exchange(void) { @@ -169,6 +232,13 @@ shmem_runtime_exchange(void) return 7; } } + if (is_node_root) { + ret = shmem_runtime_util_populate_global_node_roots(is_node_root, node_id_array, size); + if (0 != ret) { + RETURN_ERROR_MSG("Global node root mapping failed (%d)\n", ret); + return 8; + } + } return 0; } diff --git a/src/runtime-pmix.c b/src/runtime-pmix.c index 1a0f043b..f91c12a3 100644 --- a/src/runtime-pmix.c +++ b/src/runtime-pmix.c @@ -37,6 +37,8 @@ static pmix_proc_t myproc; static uint32_t size; static uint32_t node_size = 0; static int *node_ranks = NULL; +static int *is_node_root = NULL; +static int *node_id_array = NULL; int shmem_runtime_init(int enable_node_ranks) @@ -69,6 +71,18 @@ shmem_runtime_init(int enable_node_ranks) RETURN_ERROR_MSG_PREINIT("Out of memory allocating node_ranks\n"); return 1; } +#ifdef USE_HIERARCHICAL_BARRIER + is_node_root = (int *)malloc(size * sizeof(int)); + if (NULL == is_node_root) { + RETURN_ERROR_MSG_PREINIT("Out of memory allocating is_node_root\n"); + return 2; + } + node_id_array = (int *)malloc(size * sizeof(int)); + if (NULL == node_id_array) { + RETURN_ERROR_MSG_PREINIT("Out of memory allocating node_id_array\n"); + return 3; + } +#endif } return PMIX_SUCCESS; @@ -82,6 +96,10 @@ shmem_runtime_fini(void) if (node_ranks) free(node_ranks); + if (is_node_root) + free(is_node_root); + if (node_id_array) + free(node_id_array); if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) { RETURN_ERROR_MSG_PREINIT("PMIx_Finalize failed (%d)\n", rc); @@ -143,13 +161,51 @@ shmem_runtime_get_node_size(void) return (int) node_size; } -// static void opcbfunc(pmix_status_t status, void *cbdata) -// { -// bool *active = (bool*)cbdata; -// fprintf(stderr, "%s:%d completed fence_nb", myproc.nspace, myproc.rank); -// *active = false; -// } +int +shmem_runtime_get_node_root_pe(void) +{ + int i; + + for (i = 0; i < (int) size; i++) { + if (node_ranks[i] == 0) + return i; + } + + /* Should not be reached */ + return 0; +} + + +int +shmem_runtime_is_node_root_pe(int pe) +{ + shmem_internal_assert(pe < (int) size && pe >= 0); + + if (size == 1) + return 1; + + if (NULL == is_node_root) + return 0; + + return is_node_root[pe]; +} + + +int +shmem_runtime_get_node_id(int pe) +{ + shmem_internal_assert(pe < (int) size && pe >= 0); + + if (size == 1) + return 0; + + if (NULL == node_id_array) + return pe; + + return node_id_array[pe]; +} + int shmem_runtime_exchange(void) @@ -157,7 +213,6 @@ shmem_runtime_exchange(void) pmix_status_t rc; pmix_info_t info; bool wantit=true; - //bool active = true; if (node_ranks) { pmix_proc_t proc; @@ -198,6 +253,16 @@ shmem_runtime_exchange(void) } else { RETURN_ERROR_MSG_PREINIT("PMIX_LOCAL_PEERS is not properly initiated (%d)\n", rc); } + + /* Publish hostname so shmem_runtime_util_populate_global_node_roots can + * determine which PEs are node roots across all nodes. */ + if (is_node_root) { + int ret = shmem_runtime_util_put_hostname(); + if (ret != 0) { + RETURN_ERROR_MSG("PMIx hostname put failed (%d)\n", ret); + return ret; + } + } } /* commit any values we "put" */ @@ -206,21 +271,22 @@ shmem_runtime_exchange(void) return rc; } - /* execute a fence, directing that all info be exchanged */ PMIX_INFO_CONSTRUCT(&info); PMIX_INFO_LOAD(&info, PMIX_COLLECT_DATA, &wantit, PMIX_BOOL); - // Future optimization for when fabrics are ready to support the non-block- - // ing capabilities. The commented out call function above, the commented - // variable "bool active," and the PMIx_Fence_nb if-statement are here for - // when that is ready. Current implementations will cause the test to hang. - - // if (PMIX_SUCCESS != (rc = PMIx_Fence_nb(NULL, 0, &info, 1, opcbfunc, &active))) { if (PMIX_SUCCESS != (rc = PMIx_Fence(NULL, 0, &info, 1))) { RETURN_ERROR_MSG("PMIx_Fence failed (%d)\n", rc); } PMIX_INFO_DESTRUCT(&info); + if (is_node_root) { + int ret = shmem_runtime_util_populate_global_node_roots(is_node_root, node_id_array, (int) size); + if (ret != 0) { + RETURN_ERROR_MSG("Global node root mapping failed (%d)\n", ret); + return ret; + } + } + return rc; } diff --git a/src/runtime.h b/src/runtime.h index ee0c65de..243cecc0 100644 --- a/src/runtime.h +++ b/src/runtime.h @@ -31,6 +31,7 @@ int shmem_runtime_get_size(void); * (use shmem_internal_get_shr_rank/size for such queries). */ int shmem_runtime_get_node_rank(int pe); int shmem_runtime_get_node_size(void); +int shmem_runtime_get_node_root_pe(void); int shmem_runtime_exchange(void); int shmem_runtime_put(char *key, void *value, size_t valuelen); @@ -41,6 +42,16 @@ void shmem_runtime_barrier(void); /* Utility functions used to implement the runtime layer */ int shmem_runtime_util_put_hostname(void); int shmem_runtime_util_populate_node(int *location_array, int size, int *node_size); +int shmem_runtime_util_populate_global_node_roots(int *is_node_root, int *node_id_out, int size); + +/* Returns 1 if PE pe is the lowest-ranked PE on its own node, 0 otherwise. + * Valid after shmem_runtime_exchange(). */ +int shmem_runtime_is_node_root_pe(int pe); + +/* Returns the global PE number of the lowest-ranked PE on pe's node. + * All PEs on the same physical node share the same node_id value. + * Valid after shmem_runtime_exchange(). */ +int shmem_runtime_get_node_id(int pe); int shmem_runtime_util_encode(const void *inval, int invallen, char *outval, int outvallen); int shmem_runtime_util_decode(const char *inval, void *outval, size_t outvallen); diff --git a/src/runtime_util.c b/src/runtime_util.c index bd9ccc99..0160731d 100644 --- a/src/runtime_util.c +++ b/src/runtime_util.c @@ -164,3 +164,74 @@ int shmem_runtime_util_populate_node(int *location_array, int size, int *node_si return 0; } + + +/* Populate is_node_root[pe] = 1 if PE pe is the lowest-ranked (first) PE on + * its node, 0 otherwise. Optionally populate node_id_out[pe] with the global + * PE number of the lowest-ranked PE on pe's node (the same value for all PEs + * that share a node). node_id_out may be NULL. + * + * Must be called after shmem_runtime_util_put_hostname and a runtime exchange, + * so that every PE's hostname is readable via shmem_runtime_get. */ +int +shmem_runtime_util_populate_global_node_roots(int *is_node_root, int *node_id_out, int size) +{ + int ret = 0; + char **hostnames = (char **) malloc(size * sizeof(char *)); + size_t *hlens = (size_t *) malloc(size * sizeof(size_t)); + + if (!hostnames || !hlens) { + free(hostnames); + free(hlens); + RETURN_ERROR_MSG("Out of memory allocating hostname arrays\n"); + return 1; + } + + for (int pe = 0; pe < size; pe++) { + hostnames[pe] = NULL; + } + + for (int pe = 0; pe < size; pe++) { + ret = shmem_runtime_get(pe, "hostname_len", &hlens[pe], sizeof(size_t)); + if (ret != 0) { + RETURN_ERROR_MSG("Failed to get hostname_len for PE %d (%d)\n", pe, ret); + goto cleanup; + } + hostnames[pe] = (char *) malloc(hlens[pe] + 1); + if (!hostnames[pe]) { + RETURN_ERROR_MSG("Out of memory for hostname of PE %d\n", pe); + ret = 2; + goto cleanup; + } + ret = shmem_runtime_get(pe, "hostname", hostnames[pe], hlens[pe]); + if (ret != 0) { + RETURN_ERROR_MSG("Failed to get hostname for PE %d (%d)\n", pe, ret); + goto cleanup; + } + } + + /* PE pe is a node root if no earlier PE (q < pe) has the same hostname. + * node_id_out[pe] (when non-NULL) is set to the global PE number of the + * lowest-ranked PE on pe's node, which serves as a stable node identifier + * that all PEs can compute identically from global data. */ + for (int pe = 0; pe < size; pe++) { + is_node_root[pe] = 1; + if (node_id_out) node_id_out[pe] = pe; + for (int q = 0; q < pe; q++) { + if (hlens[pe] == hlens[q] && + memcmp(hostnames[pe], hostnames[q], hlens[pe]) == 0) { + is_node_root[pe] = 0; + if (node_id_out) node_id_out[pe] = node_id_out[q]; + break; + } + } + } + +cleanup: + for (int pe = 0; pe < size; pe++) { + free(hostnames[pe]); + } + free(hostnames); + free(hlens); + return ret; +} diff --git a/src/shmem_collectives.h b/src/shmem_collectives.h index d6a57ca7..99a91fea 100644 --- a/src/shmem_collectives.h +++ b/src/shmem_collectives.h @@ -17,6 +17,9 @@ #define SHMEM_COLLECTIVES_H #include "shmem_synchronization.h" +#ifdef USE_HIERARCHICAL_BARRIER +#include "shmem_team.h" +#endif enum coll_type_t { @@ -25,7 +28,8 @@ enum coll_type_t { TREE, DISSEM, RING, - RECDBL + RECDBL, + HIERARCHICAL }; typedef enum coll_type_t coll_type_t; @@ -33,6 +37,11 @@ extern char *coll_type_str[]; extern long *shmem_internal_barrier_all_psync; extern long *shmem_internal_sync_all_psync; +#ifdef USE_HIERARCHICAL_BARRIER +extern long *shmem_internal_barrier_all_local_psync; +extern long *shmem_internal_sync_all_local_psync; +extern long *shmem_internal_hierarchical_local_psync; +#endif extern coll_type_t shmem_internal_barrier_type; extern coll_type_t shmem_internal_bcast_type; @@ -44,11 +53,48 @@ extern coll_type_t shmem_internal_fcollect_type; void shmem_internal_sync_linear(int PE_start, int PE_stride, int PE_size, long *pSync); void shmem_internal_sync_tree(int PE_start, int PE_stride, int PE_size, long *pSync); void shmem_internal_sync_dissem(int PE_start, int PE_stride, int PE_size, long *pSync); +#ifdef USE_HIERARCHICAL_BARRIER +void shmem_internal_sync_hierarchical(int PE_start, int PE_stride, int PE_size, + long *pSync, long *local_pSync, + long *hier_sense_ptr, + shmem_internal_hier_cache_t *hier_cache); +void shmem_internal_hier_cache_free(shmem_internal_hier_cache_t *hier_cache); +void shmem_internal_hier_barrier_print_stats(void); + +/* Job-global minimum PEs-per-node, computed once in collectives_init. */ +extern int shmem_internal_hier_min_ppn; + +/* Whether AUTO selection should use the hierarchical barrier. Requires the + * job to span more than one node (some PE off-node, i.e. shr_size < num_pes) + * AND at least HIER_BARRIER_THRESHOLD PEs on the smallest node. Both terms are + * job-global (shr_size < num_pes holds on every PE of a multi-node job; + * hier_min_ppn is reduced over all nodes), so every PE makes the same decision + * even when PPN is heterogeneous — gating on the local node size could let a + * large node pick hierarchical while a small node picks tree, diverging within + * one collective. On a single-node job the hierarchical barrier degenerates to + * its intranode-only path, which is strictly worse than linear/tree: under + * USE_HIERARCHICAL_BARRIER the shr transport already routes single-node atomics + * through CPU atomics (see shmem_shr_transport_use_atomic), so linear/tree run + * over shared memory with none of the hierarchical scaffolding. Explicit + * BARRIER_ALGORITHM=hierarchical bypasses this and is always honored. */ +static inline int +shmem_internal_hier_barrier_available(void) +{ + return shmem_internal_get_shr_size() < shmem_internal_num_pes && + shmem_internal_hier_min_ppn >= shmem_internal_params.HIER_BARRIER_THRESHOLD; +} +#endif static inline void shmem_internal_sync(int PE_start, int PE_stride, int PE_size, long *pSync) { + /* Passes NULL as hier_sense_ptr: the hierarchical barrier then uses a + * shared static sense counter, which is only valid when all callers of a + * given active set sync in lock-step (the intra-collective syncs here do). + * Team-aware callers use shmem_internal_sync_for_team / + * shmem_internal_barrier_for_team to supply a per-team sense counter; + * deprecated shmem_barrier/shmem_sync use shmem_internal_sync_no_hier. */ if (shmem_internal_params.BARRIERS_FLUSH) { fflush(stdout); fflush(stderr); @@ -58,6 +104,15 @@ shmem_internal_sync(int PE_start, int PE_stride, int PE_size, long *pSync) switch (shmem_internal_barrier_type) { case AUTO: +#ifdef USE_HIERARCHICAL_BARRIER + if (shmem_internal_hier_barrier_available()) { + shmem_internal_sync_hierarchical(PE_start, PE_stride, PE_size, + pSync, + shmem_internal_hierarchical_local_psync, + NULL, NULL); + break; + } +#endif if (PE_size < shmem_internal_params.COLL_CROSSOVER) { shmem_internal_sync_linear(PE_start, PE_stride, PE_size, pSync); } else { @@ -73,6 +128,14 @@ shmem_internal_sync(int PE_start, int PE_stride, int PE_size, long *pSync) case DISSEM: shmem_internal_sync_dissem(PE_start, PE_stride, PE_size, pSync); break; +#ifdef USE_HIERARCHICAL_BARRIER + case HIERARCHICAL: + shmem_internal_sync_hierarchical(PE_start, PE_stride, PE_size, + pSync, + shmem_internal_hierarchical_local_psync, + NULL, NULL); + break; +#endif default: RAISE_ERROR_MSG("Illegal barrier/sync type (%d)\n", shmem_internal_barrier_type); @@ -84,10 +147,68 @@ shmem_internal_sync(int PE_start, int PE_stride, int PE_size, long *pSync) } +/* Sync for the deprecated active-set APIs (shmem_barrier, shmem_sync). + * Never uses the hierarchical algorithm: the deprecated APIs carry no team + * context, so there is no per-active-set sense counter available. Falls + * through to the next-best algorithm selected at configure/runtime. */ +static inline +void +shmem_internal_sync_no_hier(int PE_start, int PE_stride, int PE_size, long *pSync) +{ + if (shmem_internal_params.BARRIERS_FLUSH) { + fflush(stdout); + fflush(stderr); + } + + if (PE_size == 1) return; + + switch (shmem_internal_barrier_type) { + case AUTO: + case HIERARCHICAL: + /* Fall through to CROSSOVER-based selection: hierarchical is unsafe + * here because no per-active-set sense pointer is available. */ + if (PE_size < shmem_internal_params.COLL_CROSSOVER) { + shmem_internal_sync_linear(PE_start, PE_stride, PE_size, pSync); + } else { + shmem_internal_sync_tree(PE_start, PE_stride, PE_size, pSync); + } + break; + case LINEAR: + shmem_internal_sync_linear(PE_start, PE_stride, PE_size, pSync); + break; + case TREE: + shmem_internal_sync_tree(PE_start, PE_stride, PE_size, pSync); + break; + case DISSEM: + shmem_internal_sync_dissem(PE_start, PE_stride, PE_size, pSync); + break; + default: + RAISE_ERROR_MSG("Illegal barrier/sync type (%d)\n", + shmem_internal_barrier_type); + } + + shmem_internal_membar_acq_rel(); + shmem_transport_syncmem(); +} + + static inline void shmem_internal_sync_all(void) { +#ifdef USE_HIERARCHICAL_BARRIER + if (shmem_internal_barrier_type == AUTO && + shmem_internal_hier_barrier_available()) { + shmem_internal_sync_hierarchical(0, 1, shmem_internal_num_pes, + shmem_internal_sync_all_psync, + shmem_internal_sync_all_local_psync, + &shmem_internal_team_world.hier_sense, + &shmem_internal_team_world.hier_cache); + shmem_internal_membar_acq_rel(); + shmem_transport_syncmem(); + return; + } +#endif shmem_internal_sync(0, 1, shmem_internal_num_pes, shmem_internal_sync_all_psync); } @@ -106,10 +227,77 @@ void shmem_internal_barrier_all(void) { shmem_internal_quiet(SHMEM_CTX_DEFAULT); +#ifdef USE_HIERARCHICAL_BARRIER + if (shmem_internal_barrier_type == AUTO && + shmem_internal_hier_barrier_available()) { + shmem_internal_sync_hierarchical(0, 1, shmem_internal_num_pes, + shmem_internal_barrier_all_psync, + shmem_internal_barrier_all_local_psync, + &shmem_internal_team_world.hier_sense, + &shmem_internal_team_world.hier_cache); + shmem_internal_membar_acq_rel(); + shmem_transport_syncmem(); + return; + } +#endif shmem_internal_sync(0, 1, shmem_internal_num_pes, shmem_internal_barrier_all_psync); } +#ifdef USE_HIERARCHICAL_BARRIER +/* Team-aware sync: passes team->hier_sense to the hierarchical barrier so each + * team maintains an independent sense counter. Falls back to the generic + * shmem_internal_sync for non-hierarchical algorithms. */ +static inline +void +shmem_internal_sync_for_team(shmem_internal_team_t *team, long *pSync) +{ + int use_hier = (shmem_internal_barrier_type == HIERARCHICAL) || + (shmem_internal_barrier_type == AUTO && + shmem_internal_hier_barrier_available()); + if (use_hier) { + if (shmem_internal_params.BARRIERS_FLUSH) { + fflush(stdout); + fflush(stderr); + } + if (team->size == 1) return; + shmem_internal_sync_hierarchical(team->start, team->stride, team->size, + pSync, + shmem_internal_hierarchical_local_psync, + &team->hier_sense, + &team->hier_cache); + shmem_internal_membar_acq_rel(); + shmem_transport_syncmem(); + return; + } + shmem_internal_sync(team->start, team->stride, team->size, pSync); +} + +/* Team-aware barrier: quiet + team-aware sync. */ +static inline +void +shmem_internal_barrier_for_team(shmem_internal_team_t *team, long *pSync) +{ + shmem_internal_quiet(SHMEM_CTX_DEFAULT); + shmem_internal_sync_for_team(team, pSync); +} +#else +static inline +void +shmem_internal_sync_for_team(shmem_internal_team_t *team, long *pSync) +{ + shmem_internal_sync(team->start, team->stride, team->size, pSync); +} + +static inline +void +shmem_internal_barrier_for_team(shmem_internal_team_t *team, long *pSync) +{ + shmem_internal_barrier(team->start, team->stride, team->size, pSync); +} +#endif + + void shmem_internal_bcast_linear(void *target, const void *source, size_t len, int PE_root, int PE_start, int PE_stride, int PE_size, long *pSync, int complete); diff --git a/src/shmem_comm.h b/src/shmem_comm.h index 3a338183..6ec13549 100644 --- a/src/shmem_comm.h +++ b/src/shmem_comm.h @@ -80,12 +80,29 @@ shmem_internal_put_signal_nbi(shmem_ctx_t ctx, void *target, const void *source, uint64_t *sig_addr, uint64_t signal, int sig_op, int pe) { if (len == 0) { - if (sig_op == SHMEM_SIGNAL_ADD) - shmem_transport_atomic((shmem_transport_ctx_t *) ctx, sig_addr, &signal, sizeof(uint64_t), - pe, SHM_INTERNAL_SUM, SHM_INTERNAL_UINT64); - else - shmem_transport_atomic_set((shmem_transport_ctx_t *) ctx, sig_addr, &signal, - sizeof(uint64_t), pe, SHM_INTERNAL_UINT64); + /* Signal-only (no data): use shmem_shr_transport_use_atomic() to pick + * the right plane. In the multi-node case this always resolves to the + * NIC (preserving FIFO ordering with any prior in-flight NIC puts and + * avoiding the CPU/NIC coherency hazard). In the all-PEs-on-one-node + * case it resolves to CPU atomics, consistent with data puts. */ + if (sig_op == SHMEM_SIGNAL_ADD) { + if (shmem_shr_transport_use_atomic(ctx, sig_addr, sizeof(uint64_t), + pe, SHM_INTERNAL_UINT64)) + shmem_shr_transport_atomic(ctx, sig_addr, &signal, sizeof(uint64_t), + pe, SHM_INTERNAL_SUM, SHM_INTERNAL_UINT64); + else + shmem_transport_atomic((shmem_transport_ctx_t *) ctx, sig_addr, &signal, + sizeof(uint64_t), pe, SHM_INTERNAL_SUM, + SHM_INTERNAL_UINT64); + } else { + if (shmem_shr_transport_use_atomic(ctx, sig_addr, sizeof(uint64_t), + pe, SHM_INTERNAL_UINT64)) + shmem_shr_transport_atomic_set(ctx, sig_addr, &signal, sizeof(uint64_t), + pe, SHM_INTERNAL_UINT64); + else + shmem_transport_atomic_set((shmem_transport_ctx_t *) ctx, sig_addr, &signal, + sizeof(uint64_t), pe, SHM_INTERNAL_UINT64); + } return; } diff --git a/src/shmem_env_defs.h b/src/shmem_env_defs.h index c0a27a30..94ea2750 100644 --- a/src/shmem_env_defs.h +++ b/src/shmem_env_defs.h @@ -46,7 +46,7 @@ SHMEM_INTERNAL_ENV_DEF(SYMMETRIC_HEAP_USE_MALLOC, bool, false, SHMEM_INTERNAL_EN "Allocate the symmetric heap using malloc") SHMEM_INTERNAL_ENV_DEF(BOUNCE_SIZE, size, DEFAULT_BOUNCE_SIZE, SHMEM_INTERNAL_ENV_CAT_OTHER, "Maximum message size to bounce buffer") -SHMEM_INTERNAL_ENV_DEF(MAX_BOUNCE_BUFFERS, long, 128, SHMEM_INTERNAL_ENV_CAT_OTHER, +SHMEM_INTERNAL_ENV_DEF(MAX_BOUNCE_BUFFERS, size, 128, SHMEM_INTERNAL_ENV_CAT_OTHER, "Maximum number of bounce buffers per context") SHMEM_INTERNAL_ENV_DEF(TRAP_ON_ABORT, bool, false, SHMEM_INTERNAL_ENV_CAT_OTHER, "Generate trap if the program aborts or calls shmem_global_exit") @@ -58,7 +58,11 @@ SHMEM_INTERNAL_ENV_DEF(COLL_SIZE_CROSSOVER, size, 16384, SHMEM_INTERNAL_ENV_CAT_ SHMEM_INTERNAL_ENV_DEF(COLL_RADIX, long, 4, SHMEM_INTERNAL_ENV_CAT_COLLECTIVES, "Radix for tree-based collectives") SHMEM_INTERNAL_ENV_DEF(BARRIER_ALGORITHM, string, "auto", SHMEM_INTERNAL_ENV_CAT_COLLECTIVES, - "Algorithm for barrier. Options are auto, linear, tree, dissem") + "Algorithm for barrier. Options are auto, linear, tree, dissem, hierarchical") +SHMEM_INTERNAL_ENV_DEF(HIER_BARRIER_THRESHOLD, long, 2, SHMEM_INTERNAL_ENV_CAT_COLLECTIVES, + "Minimum local PE count per node to auto-select the hierarchical barrier") +SHMEM_INTERNAL_ENV_DEF(HIER_BARRIER_DEBUG, bool, 0, SHMEM_INTERNAL_ENV_CAT_COLLECTIVES, + "Print per-phase hierarchical barrier timing at finalize") SHMEM_INTERNAL_ENV_DEF(BCAST_ALGORITHM, string, "auto", SHMEM_INTERNAL_ENV_CAT_COLLECTIVES, "Algorithm for broadcast. Options are auto, linear, tree") SHMEM_INTERNAL_ENV_DEF(REDUCE_ALGORITHM, string, "auto", SHMEM_INTERNAL_ENV_CAT_COLLECTIVES, @@ -95,6 +99,8 @@ SHMEM_INTERNAL_ENV_DEF(OFI_DOMAIN, string, "auto", SHMEM_INTERNAL_ENV_CAT_TRANSP "Fabric domain that should be used by the OFI transport") SHMEM_INTERNAL_ENV_DEF(OFI_TX_POLL_LIMIT, long, DEFAULT_POLL_LIMIT, SHMEM_INTERNAL_ENV_CAT_TRANSPORT, "Put completion poll limit") +SHMEM_INTERNAL_ENV_DEF(OFI_CXI_HYBRID_MR_DESC, bool, true, SHMEM_INTERNAL_ENV_CAT_TRANSPORT, + "Enable CXI hybrid local MR descriptor mode (skips internal MR registration when desc is non-NULL); ignored on non-CXI providers") SHMEM_INTERNAL_ENV_DEF(OFI_RX_POLL_LIMIT, long, DEFAULT_POLL_LIMIT, SHMEM_INTERNAL_ENV_CAT_TRANSPORT, "Get completion poll limit") SHMEM_INTERNAL_ENV_DEF(OFI_STX_MAX, long, 1, SHMEM_INTERNAL_ENV_CAT_TRANSPORT, @@ -126,3 +132,7 @@ SHMEM_INTERNAL_ENV_DEF(MPI_THREAD_LEVEL, string, "MPI_THREAD_SINGLE", SHMEM_INTE SHMEM_INTERNAL_ENV_DEF(BACKTRACE, string, "", SHMEM_INTERNAL_ENV_CAT_OTHER, "Specify the mechanism to use for backtracing on failure") +SHMEM_INTERNAL_ENV_DEF(BOUNCE_MLOCK, bool, false, SHMEM_INTERNAL_ENV_CAT_OTHER, + "Lock bounce buffer memory preventing buffers from being paged out to swap") +SHMEM_INTERNAL_ENV_DEF(BOUNCE_SHEAP, bool, true, SHMEM_INTERNAL_ENV_CAT_OTHER, + "Allocate bounce buffers using symmetric heap, if supported by transport") diff --git a/src/shmem_free_list.c b/src/shmem_free_list.c index 50fd8a70..bbb37934 100644 --- a/src/shmem_free_list.c +++ b/src/shmem_free_list.c @@ -22,10 +22,12 @@ #include "shmem.h" #include "shmem_free_list.h" +#define NUM_ELEMENTS 2 shmem_free_list_t* -shmem_free_list_init(unsigned int element_size, - shmem_free_list_item_init_fn_t init_fn) +shmem_free_list_init(size_t element_size, + shmem_free_list_item_init_fn_t init_fn, + size_t max_pool_cnt) { int ret; shmem_free_list_t *fl = (shmem_free_list_t*) calloc(1, sizeof(shmem_free_list_t)); @@ -34,6 +36,18 @@ shmem_free_list_init(unsigned int element_size, fl->element_size = element_size; fl->init_fn = init_fn; fl->nalloc = 0; + fl->alloc_size = sizeof(shmem_free_list_alloc_t) + NUM_ELEMENTS * fl->element_size; + fl->pool_size = 0; + fl->pool_ofs = 0; + fl->pool = NULL; + if (max_pool_cnt) { + /* preallocate pool with shmem malloc */ + /* memory must be reserved as a memory pool to prevent address conflicts between PEs */ + fl->pool_size = fl->alloc_size * max_pool_cnt; + fl->pool = shmem_internal_shmalloc(fl->pool_size); + } + /* if pool count is zero allocate with bounce buffers with malloc */ + SHMEM_MUTEX_INIT(fl->lock); ret = shmem_free_list_more(fl); if (0 != ret) { @@ -48,13 +62,20 @@ shmem_free_list_init(unsigned int element_size, void shmem_free_list_destroy(shmem_free_list_t *fl) { - shmem_free_list_alloc_t *alloc, *next; - - alloc = fl->allocs; - while (NULL != alloc) { - next = alloc->next; - free(alloc); - alloc = next; + if (fl->pool) { + /* allocated with shmem malloc */ + shmem_internal_free(fl->pool); + fl->pool = NULL; + fl->pool_ofs = 0; + } else { + /* allocated with malloc */ + shmem_free_list_alloc_t *alloc, *next; + alloc = fl->allocs; + while (NULL != alloc) { + next = alloc->next; + free(alloc); + alloc = next; + } } SHMEM_MUTEX_DESTROY(fl->lock); @@ -64,25 +85,35 @@ shmem_free_list_destroy(shmem_free_list_t *fl) int shmem_free_list_more(shmem_free_list_t *fl) { - size_t page_size = 4096 - sizeof(shmem_free_list_alloc_t); - int num_elements = (fl->element_size < page_size) ? page_size / fl->element_size : 1; shmem_free_list_item_t *item, *first, *next, *last = NULL; shmem_free_list_alloc_t *header; char *buf; - int i; - - num_elements = 2; - - buf = malloc(sizeof(shmem_free_list_alloc_t) + - num_elements * fl->element_size); - if (NULL == buf) return 1; + uint64_t i; + + if (fl->pool) { + if (fl->pool_ofs >= fl->pool_size) { + fprintf(stderr, "[%d] pool memory exhausted\n", shmem_internal_my_pe); + return 1; + } + + buf = &fl->pool[fl->pool_ofs]; + fl->pool_ofs += fl->alloc_size; + + if (shmem_internal_params.BOUNCE_MLOCK) { + mlock(buf, sizeof(shmem_free_list_alloc_t) + NUM_ELEMENTS * fl->element_size); + } + } else { + buf = malloc(sizeof(shmem_free_list_alloc_t) + + NUM_ELEMENTS * fl->element_size); + if (NULL == buf) return 1; + } header = (shmem_free_list_alloc_t*) buf; first = item = (shmem_free_list_item_t*) (header + 1); - for (i = 0 ; i < num_elements ; ++i) { + for (i = 0 ; i < NUM_ELEMENTS ; ++i) { fl->init_fn(item); next = (shmem_free_list_item_t*)((char*)item + fl->element_size); - if (i == num_elements - 1) { + if (i == NUM_ELEMENTS - 1) { item->next = NULL; last = item; } else { diff --git a/src/shmem_free_list.h b/src/shmem_free_list.h index f73a4257..f2c69e7d 100644 --- a/src/shmem_free_list.h +++ b/src/shmem_free_list.h @@ -17,7 +17,8 @@ #define SHMEM_FREE_QUEUE_H #include - +#include +#include #include "shmem_internal.h" struct shmem_free_list_item_t { @@ -33,9 +34,12 @@ typedef struct shmem_free_list_alloc_t shmem_free_list_alloc_t; typedef void (*shmem_free_list_item_init_fn_t)(shmem_free_list_item_t *item); struct shmem_free_list_t { - uint32_t element_size; + size_t element_size; uint64_t nalloc; - + uint64_t alloc_size; + uint64_t pool_size; + char *pool; + size_t pool_ofs; shmem_free_list_item_init_fn_t init_fn; shmem_free_list_alloc_t *allocs; shmem_free_list_item_t* head; @@ -45,8 +49,9 @@ struct shmem_free_list_t { }; typedef struct shmem_free_list_t shmem_free_list_t; -shmem_free_list_t* shmem_free_list_init(unsigned int element_size, - shmem_free_list_item_init_fn_t init_fn); +shmem_free_list_t* shmem_free_list_init(size_t element_size, + shmem_free_list_item_init_fn_t init_fn, + size_t max_pool_cnt); void shmem_free_list_destroy(shmem_free_list_t *fl); int shmem_free_list_more(shmem_free_list_t *fl); diff --git a/src/shmem_internal.h b/src/shmem_internal.h index 465dc736..cda7e4ae 100644 --- a/src/shmem_internal.h +++ b/src/shmem_internal.h @@ -56,7 +56,10 @@ extern unsigned int shmem_internal_rand_seed; extern hwloc_topology_t shmem_internal_topology; #endif -#define SHMEM_INTERNAL_HEAP_OVERHEAD (1024*1024) +#define SHMEM_INTERNAL_HEAP_OVERHEAD (10*1024*1024) +/* Note: this is an estimate */ +#define SHMEM_MAX_BOUNCE_BUFFER_OVERHEAD (2*1024*1024) + #define SHMEM_INTERNAL_DIAG_STRLEN 1024 #define SHMEM_INTERNAL_DIAG_WRAPLEN 72 @@ -546,6 +549,17 @@ static inline int shmem_internal_get_shr_size(void) #endif } +/* Return the global PE rank of the lowest-ranked PE on this node (node_rank==0). + * Used as the internode representative in hierarchical collectives. */ +static inline int shmem_internal_get_node_root_pe(void) +{ +#ifdef USE_ON_NODE_COMMS + return shmem_runtime_get_node_root_pe(); +#else + return shmem_internal_my_pe; +#endif +} + static inline double shmem_internal_wtime(void) { double wtime = 0.0; diff --git a/src/shmem_team.c b/src/shmem_team.c index d77f2809..f5cb69da 100644 --- a/src/shmem_team.c +++ b/src/shmem_team.c @@ -85,6 +85,10 @@ int shmem_internal_team_init(void) memset(&shmem_internal_team_world.config, 0, sizeof(shmem_team_config_t)); for (size_t i = 0; i < N_PSYNCS_PER_TEAM; i++) shmem_internal_team_world.psync_avail[i] = 1; +#ifdef USE_HIERARCHICAL_BARRIER + shmem_internal_team_world.hier_sense = 0; + memset(&shmem_internal_team_world.hier_cache, 0, sizeof(shmem_internal_hier_cache_t)); +#endif SHMEM_TEAM_WORLD = (shmem_team_t) &shmem_internal_team_world; /* Initialize SHMEM_TEAM_SHARED */ @@ -95,6 +99,10 @@ int shmem_internal_team_init(void) memset(&shmem_internal_team_shared.config, 0, sizeof(shmem_team_config_t)); for (size_t i = 0; i < N_PSYNCS_PER_TEAM; i++) shmem_internal_team_shared.psync_avail[i] = 1; +#ifdef USE_HIERARCHICAL_BARRIER + shmem_internal_team_shared.hier_sense = 0; + memset(&shmem_internal_team_shared.hier_cache, 0, sizeof(shmem_internal_hier_cache_t)); +#endif SHMEM_TEAM_SHARED = (shmem_team_t) &shmem_internal_team_shared; /* Initialize SHMEM_TEAM_NODE */ @@ -105,6 +113,10 @@ int shmem_internal_team_init(void) memset(&shmem_internal_team_node.config, 0, sizeof(shmem_team_config_t)); for (size_t i = 0; i < N_PSYNCS_PER_TEAM; i++) shmem_internal_team_node.psync_avail[i] = 1; +#ifdef USE_HIERARCHICAL_BARRIER + shmem_internal_team_node.hier_sense = 0; + memset(&shmem_internal_team_node.hier_cache, 0, sizeof(shmem_internal_hier_cache_t)); +#endif SHMEMX_TEAM_NODE = (shmem_team_t) &shmem_internal_team_node; if (shmem_internal_params.TEAM_SHARED_ONLY_SELF) { @@ -410,7 +422,7 @@ int shmem_internal_team_split_strided(shmem_internal_team_t *parent_team, int PE * during psync allocation between back-to-back team creations. */ psync = shmem_internal_team_choose_psync(parent_team, SYNC); - shmem_internal_barrier(parent_team->start, parent_team->stride, parent_team->size, psync); + shmem_internal_barrier_for_team(parent_team, psync); shmem_internal_team_release_psyncs(parent_team, SYNC); @@ -448,8 +460,6 @@ int shmem_internal_team_split_2d(shmem_internal_team_t *parent_team, int xrange, xrange = parent_team->size; } - const int parent_start = parent_team->start; - const int parent_stride = parent_team->stride; const int parent_size = parent_team->size; const int num_xteams = ceil( parent_size / (float)xrange ); const int num_yteams = xrange; @@ -497,7 +507,7 @@ int shmem_internal_team_split_2d(shmem_internal_team_t *parent_team, int xrange, long *psync = shmem_internal_team_choose_psync(parent_team, SYNC); - shmem_internal_barrier(parent_start, parent_stride, parent_size, psync); + shmem_internal_barrier_for_team(parent_team, psync); shmem_internal_team_release_psyncs(parent_team, SYNC); @@ -527,6 +537,10 @@ void shmem_internal_team_destroy(shmem_internal_team_t *team) shmem_internal_team_pool[team->psync_idx] = NULL; free(team->contexts); +#ifdef USE_HIERARCHICAL_BARRIER + shmem_internal_hier_cache_free(&team->hier_cache); +#endif + if (team != &shmem_internal_team_world && team != &shmem_internal_team_shared && team != &shmem_internal_team_node) { free(team); @@ -557,7 +571,7 @@ long * shmem_internal_team_choose_psync(shmem_internal_team_t *team, shmem_inter shmem_internal_quiet(SHMEM_CTX_DEFAULT); size_t psync = team->psync_idx * SHMEM_SYNC_SIZE; - shmem_internal_sync(team->start, team->stride, team->size, + shmem_internal_sync_for_team(team, &shmem_internal_psync_barrier_pool[psync]); for (int i = 0; i < N_PSYNCS_PER_TEAM; i++) { diff --git a/src/shmem_team.h b/src/shmem_team.h index f6ddf655..042dc4e8 100644 --- a/src/shmem_team.h +++ b/src/shmem_team.h @@ -17,6 +17,30 @@ #define N_PSYNCS_PER_TEAM 2 +#ifdef USE_HIERARCHICAL_BARRIER +/* Cached hierarchical-barrier topology for a fixed active set. All fields are + * invariant for a given (start, stride, size) and the node layout (both fixed + * for the life of a team), so they are built once on the first barrier and + * reused. Holds only topology — NOT XPMEM mapped pointers, which depend on the + * local_pSync array (sync_all and barrier_all share a team but pass different + * arrays) and are cheap base+offset arithmetic to re-derive per call. */ +struct shmem_internal_hier_cache_t { + int valid; + int *local_pes; /* active PEs on this node, ascending */ + int local_count; + int *root_pes; /* one representative (lowest active PE) per node */ + int root_count; + int my_vidx; /* my index in local_pes, or -1 if not in the set */ + int active_root_pe; /* local_pes[0], this node's representative */ + int is_root; /* my_pe == active_root_pe */ + int *tree_child_shr; /* shr_ranks of my intranode tree children */ + int tree_nchildren; + int my_root_idx; /* my index in root_pes, or -1 if not a root */ + int num_rounds; /* phase-2 dissemination rounds (log2(root_count)) */ +}; +typedef struct shmem_internal_hier_cache_t shmem_internal_hier_cache_t; +#endif + struct shmem_internal_team_t { int my_pe; int start, stride, size; @@ -26,6 +50,10 @@ struct shmem_internal_team_t { long config_mask; size_t contexts_len; struct shmem_transport_ctx_t **contexts; +#ifdef USE_HIERARCHICAL_BARRIER + long hier_sense; + shmem_internal_hier_cache_t hier_cache; +#endif }; typedef struct shmem_internal_team_t shmem_internal_team_t; diff --git a/src/shr_transport.h4 b/src/shr_transport.h4 index 9379ef2e..4282d6ae 100644 --- a/src/shr_transport.h4 +++ b/src/shr_transport.h4 @@ -122,12 +122,24 @@ shmem_shr_transport_use_read(shmem_ctx_t ctx, void *target, const void *source, /* Each OpenSHMEM AMO has only one symmetric pointer. Check whether shared * transport AMOs are in use with respect to the given symmetric target * pointer and datatype. For a given datatype, all atomic operations must - * use the same transport; therefore, op is not needed in this check. */ + * use the same transport; therefore, op is not needed in this check. + * + * Under USE_HIERARCHICAL_BARRIER, CPU atomics and NIC AMOs are not coherent + * with each other; mixing them on the same variable produces lost updates. + * CPU atomics are therefore only safe when ALL PEs are on the same node — + * in that case no NIC AMOs will ever be issued to any variable, so there is + * only one coherency domain in play. */ static inline int shmem_shr_transport_use_atomic(shmem_ctx_t ctx, void *target, size_t len, int pe, shm_internal_datatype_t datatype) { -#if USE_SHR_ATOMICS +#if USE_HIERARCHICAL_BARRIER + /* Use CPU atomics only when all PEs are local (no off-node peers). */ + if (shmem_internal_get_shr_size() == shmem_internal_num_pes) { + return -1 != shmem_internal_get_shr_rank(pe); + } + return 0; +#elif USE_SHR_ATOMICS return -1 != shmem_internal_get_shr_rank(pe); #else return 0; diff --git a/src/symmetric_heap_c.c b/src/symmetric_heap_c.c index 30b319ea..40e68970 100644 --- a/src/symmetric_heap_c.c +++ b/src/symmetric_heap_c.c @@ -162,15 +162,16 @@ shmem_internal_get_next(intptr_t incr) /* alloc VM space starting @ '_end' + 1GB */ #define ONEGIG (1024UL*1024UL*1024UL) -static void *mmap_alloc(size_t bytes) +static void *mmap_alloc(size_t bytes, size_t *mapped_bytes) { char *file_name = NULL; int fd = 0; char *directory = NULL; - void *requested_base = - (void*) (((unsigned long) shmem_internal_data_base + - shmem_internal_data_length + 2 * ONEGIG) & ~(ONEGIG - 1)); - void *ret; + void *requested_base = (void*) (((unsigned long) shmem_internal_data_base + shmem_internal_data_length + 2 * ONEGIG) & ~(ONEGIG - 1)); + void *ret = MAP_FAILED; + size_t hugetlbfs_bytes = 0; /* Rounded size for hugetlbfs, 0 if not used */ + + *mapped_bytes = bytes; /* default: actual mapped size equals requested size */ #ifdef __linux__ /* huge page support only on Linux for now, default is to use 2MB large pages */ @@ -184,31 +185,96 @@ static void *mmap_alloc(size_t bytes) int size = snprintf(NULL, 0, "%s/%s.%d", directory, basename, getpid()); if (size < 0) { - RAISE_WARN_STR("snprintf returned error, cannot use huge pages"); + DEBUG_MSG("snprintf returned error, cannot use huge pages"); } else { file_name = malloc(size + 1); if (file_name) { sprintf(file_name, "%s/%s.%d", directory, basename, getpid()); fd = open(file_name, O_CREAT | O_RDWR, 0755); if (fd < 0) { - RAISE_WARN_STR("file open failed, cannot use huge pages"); + DEBUG_MSG("file open failed, will fall back to anonymous MAP_HUGETLB"); + free(directory); + free(file_name); + directory = NULL; + file_name = NULL; fd = 0; } else { - /* have to round up by the pagesize being used */ - bytes = CEILING(bytes, shmem_internal_params.SYMMETRIC_HEAP_PAGE_SIZE); + /* Round up by the pagesize for hugetlbfs file */ + hugetlbfs_bytes = CEILING(bytes, shmem_internal_params.SYMMETRIC_HEAP_PAGE_SIZE); } } } } } -#endif /* __linux__ */ + DEBUG_MSG("mmap_alloc: bytes=%zu, hugetlbfs_bytes=%zu, fd=%d", + bytes, hugetlbfs_bytes, fd); + + if (fd) { + /* Map the hugetlbfs file directly; MAP_ANON must not be used here + * because MAP_ANONYMOUS causes the kernel to ignore the fd, which + * would silently fall back to regular pages. */ + if (ftruncate(fd, hugetlbfs_bytes) == -1) { + DEBUG_MSG("ftruncate on hugetlbfs file failed (%s), falling back", strerror(errno)); + unlink(file_name); + close(fd); + free(directory); + free(file_name); + directory = NULL; + file_name = NULL; + fd = 0; + } else { + ret = mmap(requested_base, hugetlbfs_bytes, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_HUGETLB, fd, 0); + if (ret != MAP_FAILED) { + DEBUG_MSG("Allocated symmetric heap via hugetlbfs file: %zu bytes", hugetlbfs_bytes); + *mapped_bytes = hugetlbfs_bytes; + } + unlink(file_name); + close(fd); + free(directory); + free(file_name); + directory = NULL; + file_name = NULL; + fd = 0; + } + } + + /* If hugetlbfs file mapping failed or was not attempted, continue with the + * fallback chain. Note: fd is always zeroed after the hugetlbfs block + * (cleanup on both success and failure paths), so testing fd == 0 here + * would always be true and would incorrectly enter the fallback after a + * successful hugetlbfs allocation. Test ret only. */ + if (ret == MAP_FAILED) { + if (shmem_internal_params.SYMMETRIC_HEAP_USE_HUGE_PAGES) { + /* Try anonymous MAP_HUGETLB (works with nr_overcommit_hugepages). + * Explicitly request 2MB pages via MAP_HUGE_SHIFT (21 << MAP_HUGE_SHIFT = 2^21 = 2MB). */ + ret = mmap(requested_base, bytes, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE | MAP_HUGETLB | (21 << MAP_HUGE_SHIFT), -1, 0); + if (ret == MAP_FAILED) { + DEBUG_MSG("mmap(MAP_HUGETLB) failed (%s), falling back to regular pages", strerror(errno)); + } else { + DEBUG_MSG("Allocated symmetric heap via anonymous MAP_HUGETLB (2MB pages): %zu bytes", bytes); + } + } + + /* Regular pages: used when huge pages are not requested, or when the + * reserved huge-page paths above are unavailable. Identical to the + * non-huge-page mapping: a single fixed-address map at requested_base + * so the symmetric heap keeps the same virtual address on every PE. */ + if (ret == MAP_FAILED) { + ret = mmap(requested_base, bytes, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, fd, 0); + if (ret != MAP_FAILED) { + DEBUG_MSG("Allocated symmetric heap via regular pages (4KB): %zu bytes", bytes); + } + } + } +#else ret = mmap(requested_base, bytes, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, fd, 0); +#endif /* __linux__ */ if (ret == MAP_FAILED) { RAISE_WARN_MSG("Unable to allocate sym. heap, size %zuB: %s\n" RAISE_PE_PREFIX @@ -216,17 +282,16 @@ static void *mmap_alloc(size_t bytes) bytes, strerror(errno), shmem_internal_my_pe); ret = NULL; } - if (fd) { - if (file_name) - unlink(file_name); - close(fd); - } + /* Cleanup any remaining allocations (will be NULL if already freed above) */ if (directory) { free(directory); } if (file_name) { free(file_name); } + if (fd > 0) { + close(fd); + } return ret; } @@ -236,12 +301,17 @@ shmem_internal_symmetric_init(void) { /* add library overhead such that the max can be shmalloc()'ed */ shmem_internal_heap_length = shmem_internal_params.SYMMETRIC_SIZE + - SHMEM_INTERNAL_HEAP_OVERHEAD; + SHMEM_INTERNAL_HEAP_OVERHEAD + + SHMEM_MAX_BOUNCE_BUFFER_OVERHEAD; if (!shmem_internal_params.SYMMETRIC_HEAP_USE_MALLOC) { + size_t mapped_length = shmem_internal_heap_length; shmem_internal_heap_base = shmem_internal_heap_curr = - mmap_alloc(shmem_internal_heap_length); + mmap_alloc(shmem_internal_heap_length, &mapped_length); + /* Use the actual mapped size for munmap and transport registration. + * On the hugetlbfs path this may be rounded up to a huge-page boundary. */ + shmem_internal_heap_length = mapped_length; } else { shmem_internal_heap_base = shmem_internal_heap_curr = diff --git a/src/transport_ofi.c b/src/transport_ofi.c index f8f04d65..67da22ff 100644 --- a/src/transport_ofi.c +++ b/src/transport_ofi.c @@ -103,7 +103,7 @@ long shmem_transport_ofi_get_poll_limit; size_t shmem_transport_ofi_max_buffered_send; size_t shmem_transport_ofi_max_msg_size; size_t shmem_transport_ofi_bounce_buffer_size; -long shmem_transport_ofi_max_bounce_buffers; +size_t shmem_transport_ofi_max_bounce_buffers; size_t shmem_transport_ofi_addrlen; #ifdef ENABLE_MR_RMA_EVENT int shmem_transport_ofi_mr_rma_event; @@ -208,9 +208,12 @@ struct shmem_internal_tid shmem_transport_ofi_gettid(void) } #define SHMEM_TRANSPORT_OFI_PROV_SOCKETS "sockets" +#define SHMEM_TRANSPORT_OFI_PROV_CXI "cxi" static struct fabric_info shmem_transport_ofi_info = {0}; +char *shmem_transport_ofi_prov_name = NULL; + static size_t shmem_transport_ofi_grow_size = 128; #define SHMEM_TRANSPORT_CTX_DEFAULT_ID -1 @@ -1341,6 +1344,44 @@ int allocate_fabric_resources(struct fabric_info *info) &shmem_transport_ofi_domainfd,NULL); OFI_CHECK_RETURN_STR(ret, "domain initialization failed"); + /* CXI provider: enable hybrid local MR descriptor mode. When enabled, + * libfabric will skip its internal MR registration if a non-NULL desc is + * passed (and proceed without registration if desc is NULL). This avoids + * per-call MR cache lookups for source buffers in fi_write/fi_writemsg. + * Must be done BEFORE any endpoints are created (the provider only + * propagates this setting to child endpoints at creation time). + * Gated on a CXI provider check so non-CXI providers never attempt the + * fi_open_ops call and do not produce spurious startup warnings. The + * optimization is compiled in only when configure found both the CXI + * extension header and the enable_hybrid_mr_desc member; otherwise it is + * disabled and a request to enable it is reported once on PE 0. */ + if (shmem_transport_ofi_check_provider(SHMEM_TRANSPORT_OFI_PROV_CXI)) { + if (shmem_internal_params.OFI_CXI_HYBRID_MR_DESC) { +#ifdef SHMEM_TRANSPORT_OFI_HAVE_CXI_HYBRID_MR_DESC + struct fi_cxi_dom_ops *cxi_dom_ops = NULL; + int hret = fi_open_ops(&shmem_transport_ofi_domainfd->fid, + FI_CXI_DOM_OPS_3, 0, (void **)&cxi_dom_ops, NULL); + if (hret == 0 && cxi_dom_ops && cxi_dom_ops->enable_hybrid_mr_desc) { + hret = cxi_dom_ops->enable_hybrid_mr_desc(&shmem_transport_ofi_domainfd->fid, true); + if (hret == 0) { + DEBUG_STR("CXI: hybrid local MR descriptor mode enabled"); + } else { + DEBUG_MSG("CXI: enable_hybrid_mr_desc failed (%s)\n", fi_strerror(-hret)); + } + } else { + DEBUG_MSG("CXI: hybrid MR desc not available (fi_open_ops returned %d / %s)\n", + hret, hret ? fi_strerror(-hret) : "no ops struct"); + } +#else + DEBUG_STR("CXI hybrid MR desc requested (SHMEM_OFI_CXI_HYBRID_MR_DESC=1) " + "but SOS was built without CXI extension support " + "(missing rdma/fi_cxi_ext.h or enable_hybrid_mr_desc); ignoring"); +#endif + } else { + DEBUG_STR("CXI: hybrid local MR descriptor mode disabled (SHMEM_OFI_CXI_HYBRID_MR_DESC=0)"); + } + } + /* AV table set-up for PE mapping */ #ifdef USE_AV_MAP @@ -1470,7 +1511,9 @@ int query_for_fabric(struct fabric_info *info) struct fi_fabric_attr fabric_attr = {0}; struct fi_ep_attr ep_attr = {0}; - shmem_transport_ofi_max_buffered_send = sizeof(long double); + /* Hint 0 = no minimum inject requirement; provider returns its natural + * inject_size, which is adopted below after fi_getinfo. */ + shmem_transport_ofi_max_buffered_send = 0; fabric_attr.prov_name = info->prov_name; @@ -1650,6 +1693,10 @@ int query_for_fabric(struct fabric_info *info) #endif #ifndef DISABLE_OFI_INJECT + DEBUG_MSG(RAISE_PE_PREFIX "tx_attr->inject_size (provider): %zu, requested: %zu\n", + shmem_internal_my_pe, + info->p_info->tx_attr->inject_size, + shmem_transport_ofi_max_buffered_send); shmem_internal_assertp(info->p_info->tx_attr->inject_size >= shmem_transport_ofi_max_buffered_send); shmem_transport_ofi_max_buffered_send = info->p_info->tx_attr->inject_size; #else @@ -1672,6 +1719,10 @@ int query_for_fabric(struct fabric_info *info) shmem_transport_ofi_stx_max, num_nics); + /* Store provider name for runtime checks. strdup so the pointer remains + * valid after fi_freeinfo() is called in shmem_transport_fini(). */ + shmem_transport_ofi_prov_name = strdup(info->p_info->fabric_attr->prov_name); + return ret; } @@ -1731,6 +1782,7 @@ static int shmem_transport_ofi_ctx_init(shmem_transport_ctx_t *ctx, int id) cntr_put_attr.events = FI_CNTR_EVENTS_COMP; cntr_get_attr.events = FI_CNTR_EVENTS_COMP; +#if 0 /* Set FI_WAIT based on the put and get polling limits defined above */ if (shmem_transport_ofi_put_poll_limit < 0) { cntr_put_attr.wait_obj = FI_WAIT_NONE; @@ -1742,6 +1794,9 @@ static int shmem_transport_ofi_ctx_init(shmem_transport_ctx_t *ctx, int id) } else { cntr_get_attr.wait_obj = FI_WAIT_UNSPEC; } +#endif + cntr_put_attr.wait_obj = FI_WAIT_UNSPEC; + cntr_get_attr.wait_obj = FI_WAIT_UNSPEC; /* Allow provider to choose CQ size, since we are using FI_RM_ENABLED. * Context format is used to return bounce buffer pointers in the event @@ -1804,10 +1859,15 @@ static int shmem_transport_ofi_ctx_init(shmem_transport_ctx_t *ctx, int id) shmem_transport_ofi_bounce_buffer_size > 0 && shmem_transport_ofi_max_bounce_buffers > 0) { + size_t max_bb_pool_cnt = 0; + if (shmem_internal_params.BOUNCE_SHEAP) { + max_bb_pool_cnt = shmem_transport_ofi_max_bounce_buffers; + } ctx->bounce_buffers = shmem_free_list_init(sizeof(shmem_transport_ofi_bounce_buffer_t) + shmem_transport_ofi_bounce_buffer_size, - init_bounce_buffer); + init_bounce_buffer, + max_bb_pool_cnt); } else { ctx->options &= ~SHMEMX_CTX_BOUNCE_BUFFER; @@ -2273,6 +2333,8 @@ int shmem_transport_fini(void) free(addr_table); #endif + free(shmem_transport_ofi_prov_name); + shmem_transport_ofi_prov_name = NULL; fi_freeinfo(shmem_transport_ofi_info.fabrics); SHMEM_MUTEX_DESTROY(shmem_transport_ofi_lock); diff --git a/src/transport_ofi.h b/src/transport_ofi.h index f7e2ebf3..aba23efc 100644 --- a/src/transport_ofi.h +++ b/src/transport_ofi.h @@ -23,6 +23,21 @@ #include #include #include +#ifdef HAVE_RDMA_FI_CXI_EXT_H +#include /* fi_cxi_ext.h uses bool but does not include stdbool.h */ +#include +#endif +/* The CXI hybrid local MR descriptor optimization needs the CXI extension + * header AND a struct fi_cxi_dom_ops that actually exposes the + * enable_hybrid_mr_desc member (added in a later libfabric). configure probes + * both (HAVE_RDMA_FI_CXI_EXT_H, HAVE_STRUCT_FI_CXI_DOM_OPS_ENABLE_HYBRID_MR_DESC); + * the feature is compiled in only when both are present, otherwise it is + * disabled entirely — no hand-rolled struct mirror, so we never risk an ABI + * mismatch against the provider's real layout. */ +#if defined(HAVE_RDMA_FI_CXI_EXT_H) && \ + defined(HAVE_STRUCT_FI_CXI_DOM_OPS_ENABLE_HYBRID_MR_DESC) +#define SHMEM_TRANSPORT_OFI_HAVE_CXI_HYBRID_MR_DESC 1 +#endif #include #include #include @@ -69,7 +84,7 @@ extern long shmem_transport_ofi_get_poll_limit; extern size_t shmem_transport_ofi_max_buffered_send; extern size_t shmem_transport_ofi_max_msg_size; extern size_t shmem_transport_ofi_bounce_buffer_size; -extern long shmem_transport_ofi_max_bounce_buffers; +extern size_t shmem_transport_ofi_max_bounce_buffers; extern pthread_mutex_t shmem_transport_ofi_progress_lock; @@ -436,7 +451,7 @@ void shmem_transport_ofi_drain_cq(shmem_transport_ctx_t *ctx) (shmem_transport_ofi_bounce_buffer_t *) frag); ctx->completed_bb_cntr++; } else { - RAISE_ERROR_STR("Unrecognized completion object"); + RAISE_ERROR_MSG("[%d] Unrecognized completion object %p %x mtofs %p\n", shmem_internal_my_pe, frag, frag->mytype, &frag->mytype); } } @@ -473,6 +488,10 @@ shmem_transport_ofi_bounce_buffer_t * create_bounce_buffer(shmem_transport_ctx_t if (NULL == buff) RAISE_ERROR_STR("Bounce buffer allocation failed"); + if (buff->frag.mytype != SHMEM_TRANSPORT_OFI_TYPE_BOUNCE) { + RAISE_ERROR_STR("Bounce buffer allocation failed"); + } + shmem_internal_assert(buff->frag.mytype == SHMEM_TRANSPORT_OFI_TYPE_BOUNCE); memcpy(buff->data, source, len); @@ -504,28 +523,8 @@ void shmem_transport_put_quiet(shmem_transport_ctx_t* ctx) * reverse order: first the fid_cntr event counter, then the put issued * counter. We'll want to preserve this property in the future. */ - uint64_t success, fail, cnt, cnt_new; - long poll_count = 0; - while (poll_count < shmem_transport_ofi_put_poll_limit || - shmem_transport_ofi_put_poll_limit < 0) { - success = fi_cntr_read(ctx->put_cntr); - fail = fi_cntr_readerr(ctx->put_cntr); - cnt = SHMEM_TRANSPORT_OFI_CNTR_READ(&ctx->pending_put_cntr); - - shmem_transport_probe(); - - if (success < cnt && fail == 0) { - SHMEM_TRANSPORT_OFI_CTX_UNLOCK(ctx); - SPINLOCK_BODY(); - SHMEM_TRANSPORT_OFI_CTX_LOCK(ctx); - } else if (fail) { - RAISE_ERROR_MSG("Operations completed in error (%" PRIu64 ")\n", fail); - } else { - SHMEM_TRANSPORT_OFI_CTX_UNLOCK(ctx); - return; - } - poll_count++; - } + uint64_t cnt, cnt_new; + cnt_new = SHMEM_TRANSPORT_OFI_CNTR_READ(&ctx->pending_put_cntr); do { cnt = cnt_new; @@ -548,6 +547,18 @@ int shmem_transport_quiet(shmem_transport_ctx_t* ctx) return 0; } +/* Check if the current OFI provider exactly matches the given name. + * Uses strcmp so that layered providers (e.g. "foo;ofi_rxm") do not + * incorrectly inherit provider-specific fast paths. + * Returns 1 if provider matches, 0 otherwise. */ +extern char *shmem_transport_ofi_prov_name; +static inline int +shmem_transport_ofi_check_provider(const char *name) +{ + return (shmem_transport_ofi_prov_name && + strcmp(shmem_transport_ofi_prov_name, name) == 0); +} + static inline int shmem_transport_fence(shmem_transport_ctx_t* ctx) @@ -965,31 +976,10 @@ void shmem_transport_get_wait(shmem_transport_ctx_t* ctx) * reverse order: first the fid_cntr event counter, then the get issued * counter. We'll want to preserve this property in the future. */ - uint64_t success, fail, cnt, cnt_new; - long poll_count = 0; + uint64_t cnt, cnt_new; SHMEM_TRANSPORT_OFI_CTX_LOCK(ctx); - while (poll_count < shmem_transport_ofi_get_poll_limit || - shmem_transport_ofi_get_poll_limit < 0) { - success = fi_cntr_read(ctx->get_cntr); - fail = fi_cntr_readerr(ctx->get_cntr); - cnt = SHMEM_TRANSPORT_OFI_CNTR_READ(&ctx->pending_get_cntr); - - shmem_transport_probe(); - - if (success < cnt && fail == 0) { - SHMEM_TRANSPORT_OFI_CTX_UNLOCK(ctx); - SPINLOCK_BODY(); - SHMEM_TRANSPORT_OFI_CTX_LOCK(ctx); - } else if (fail) { - RAISE_ERROR_MSG("Operations completed in error (%" PRIu64 ")\n", fail); - } else { - SHMEM_TRANSPORT_OFI_CTX_UNLOCK(ctx); - return; - } - poll_count++; - } cnt_new = SHMEM_TRANSPORT_OFI_CNTR_READ(&ctx->pending_get_cntr); do { cnt = cnt_new; diff --git a/src/transport_portals4.c b/src/transport_portals4.c index f9298e7f..695bc751 100644 --- a/src/transport_portals4.c +++ b/src/transport_portals4.c @@ -446,11 +446,13 @@ shmem_transport_init(void) shmem_transport_portals4_bounce_buffers = shmem_free_list_init(sizeof(shmem_transport_portals4_bounce_buffer_t) + shmem_transport_portals4_bounce_buffer_size, - init_bounce_buffer); + init_bounce_buffer, + 0); shmem_transport_portals4_long_frags = shmem_free_list_init(sizeof(shmem_transport_portals4_long_frag_t), - init_long_frag); + init_long_frag, + 0); /* Initialize network */ ni_req_limits.max_entries = 1024;