Forest fire

.clang-format

@@ -0,0 +1,41 @@
+# MATAR style: Google base with adjustments for MATAR/Kokkos idioms
+BasedOnStyle: Google
+
+# --- deviations from stock Google, matching existing MATAR code ---
+IndentWidth: 4
+ColumnLimit: 150
+AccessModifierOffset: -4
+
+# Never reorder #includes: preserves author-intended ordering
+SortIncludes: Never
+
+# --- MATAR / Kokkos macro handling ---
+# Complete-statement macros: prevents mis-indentation of following lines
+StatementMacros:
+  - MATAR_INITIALIZE
+  - MATAR_FINALIZE
+  - MATAR_FENCE
+  - MATAR_MPI_INIT
+  - MATAR_MPI_FINALIZE
+  - MATAR_MPI_BARRIER
+
+# Declaration decorators
+AttributeMacros:
+  - KOKKOS_INLINE_FUNCTION
+  - KOKKOS_FUNCTION
+  - KOKKOS_LAMBDA
+  - KOKKOS_CLASS_LAMBDA
+
+# Keep the braced loop-body argument of parallel macros formatted as a block:
+# penalize packing so each index triple stays on its own line in multi-dim loops
+AlignAfterOpenBracket: Align
+AllowAllArgumentsOnNextLine: false
+BinPackArguments: false
+
+# Don't let clang-format merge short loop bodies onto one line inside macros
+AllowShortBlocksOnASingleLine: Never
+AllowShortLambdasOnASingleLine: None
+
+
+# MATAR convention: aligned consecutive assignments (deviation from Google)
+AlignConsecutiveAssignments: Consecutive

.clang-format-ignore

@@ -0,0 +1,5 @@
+# Files clang-format must not touch (gitignore-style globs, clang-format >= 18)
+
+# Heavy preprocessor code: #define continuations and variadic macro dispatch
+# tables that clang-format mangles badly. Format by hand.
+src/include/macros.h

.github/workflows/cmake.yml

@@ -6,82 +6,140 @@ on:
   pull_request:
     branches: [ "main" ]
 
-#env:
-#  # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
-#  BUILD_TYPE: Debug
+# The Kokkos release the bundled submodule is pinned to. Bump here and in the
+# submodule together; the verify step below fails the build if they disagree.
+env:
+  KOKKOS_VERSION: "5.2.1"
 
 jobs:
-  # This workflow contains a single job called "build"
-  build:
-    #
-    name: ${{ matrix.config.name }}
-    runs-on: ${{ matrix.config.os }}
+  # MATAR without Kokkos: host-only header library (double and float are the
+  # only precisions supported there).
+  build-no-kokkos:
+    name: Ubuntu_GCC_NO_KOKKOS_${{ matrix.real }}
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        config:
-        - {name: "Ubuntu_Latest_GCC_SERIAL", os: ubuntu-latest, build_type: "Debug", cc: "gcc", cxx: "g++", matar_config_args: "", kokkos_config_args: ""}
-        - {name: "Ubuntu_Latest_GCC_KOKKOS_SERIAL", os: ubuntu-latest, build_type: "Debug", cc: "gcc", cxx: "g++", matar_config_args: "-DKOKKOS=ON", kokkos_config_args: "-DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_TESTS=OFF -DBUILD_TESTING=OFF"}
+        real: [double, float]
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Configure
+      run: |
+        cmake -B build/no-kokkos \
+          -DCMAKE_BUILD_TYPE=Debug \
+          -DMATAR_ENABLE_KOKKOS=OFF \
+          -DMATAR_REAL=${{ matrix.real }} \
+          -DMATAR_BUILD_TESTS=ON \
+          -DMATAR_BUILD_EXAMPLES=ON
+
+    - name: Build
+      run: cmake --build build/no-kokkos -j2
+
+    - name: Test
+      run: ctest --test-dir build/no-kokkos --output-on-failure
+
+  # Bundled Kokkos submodule (the default path), plus an install round-trip
+  # verifying a downstream find_package(Matar) consumer.
+  build-bundled-kokkos:
+    name: Ubuntu_GCC_BUNDLED_KOKKOS
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Checkout submodules
+      run: git submodule update --init --recursive
+
+    - name: Verify bundled Kokkos version
+      run: |
+        ver=$(awk '/^set\(Kokkos_VERSION_MAJOR/{m=$2} /^set\(Kokkos_VERSION_MINOR/{n=$2} /^set\(Kokkos_VERSION_PATCH/{p=$2} \
+                   END{gsub(/\)/,"",m);gsub(/\)/,"",n);gsub(/\)/,"",p);print m"."n"."p}' \
+                  src/Kokkos/kokkos/CMakeLists.txt)
+        echo "Bundled Kokkos version: $ver (expected ${KOKKOS_VERSION})"
+        test "$ver" = "${KOKKOS_VERSION}" || {
+          echo "::error::Submodule is Kokkos $ver but ${KOKKOS_VERSION} is required; run 'git submodule update --init --recursive'"
+          exit 1
+        }
 
+    - name: Configure
+      run: cmake --preset serial
+
+    - name: Build
+      run: cmake --build --preset serial -j2
+
+    - name: Test
+      run: ctest --preset serial
+
+    - name: Install
+      run: cmake --install build/serial --prefix ${{ github.workspace }}/install
+
+    - name: Downstream find_package(Matar) smoke test
+      run: |
+        mkdir -p consumer
+        cat > consumer/main.cpp << 'EOF'
+        #include <matar.h>
+        int main(int argc, char* argv[]) {
+            MATAR_INITIALIZE(argc, argv);
+            { mtr::CArrayDevice<double> a(3, 3); }
+            MATAR_FINALIZE();
+            return 0;
+        }
+        EOF
+        cat > consumer/CMakeLists.txt << 'EOF'
+        cmake_minimum_required(VERSION 3.22)
+        project(MatarConsumer LANGUAGES CXX)
+        find_package(Matar REQUIRED)
+        add_executable(consumer main.cpp)
+        target_link_libraries(consumer matar::matar)
+        EOF
+        cmake -S consumer -B consumer/build -DCMAKE_PREFIX_PATH=${{ github.workspace }}/install
+        cmake --build consumer/build -j2
+        ./consumer/build/consumer
+
+  # External (pre-installed) Kokkos via MATAR_USE_EXTERNAL_KOKKOS.
+  build-external-kokkos:
+    name: Ubuntu_GCC_EXTERNAL_KOKKOS
+    runs-on: ubuntu-latest
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-
+    - uses: actions/checkout@v3
+
     - name: Checkout submodules
       run: git submodule update --init --recursive
-
-    # Kokkos configure, build, and install------------------------------------------
-    - if: contains(matrix.config.name, 'KOKKOS')
-      name: Configure Kokkos CMake
-      shell: bash
+
+    - name: Verify bundled Kokkos version
       run: |
-        cmake \
-          -S ${{ github.workspace }}/src/Kokkos/kokkos \
-          -B ${{ github.workspace }}/build-kokkos-serial/kokkos \
-          -D CMAKE_INSTALL_PREFIX=${{ github.workspace }}/build-kokkos-serial/kokkos \
-          ${{ matrix.config.kokkos_config_args }}
-
-    - if: contains(matrix.config.name, 'KOKKOS')
-      name: Build Kokkos
-      shell: bash
-      run: cmake --build ${{ github.workspace }}/build-kokkos-serial/kokkos
-
-    - if: contains(matrix.config.name, 'KOKKOS')
-      name: Install Kokkos
-      shell: bash
-      run: cmake --install ${{ github.workspace }}/build-kokkos-serial/kokkos
-
-    # End Kokkos configure, build, and install------------------------------------------
-
-
-    # MATAR configure and build------------------------------------------
-    - if: false == contains(matrix.config.name, 'KOKKOS')
-      name: Configure MATAR CMake Without Kokkos
-      shell: bash
+        ver=$(awk '/^set\(Kokkos_VERSION_MAJOR/{m=$2} /^set\(Kokkos_VERSION_MINOR/{n=$2} /^set\(Kokkos_VERSION_PATCH/{p=$2} \
+                   END{gsub(/\)/,"",m);gsub(/\)/,"",n);gsub(/\)/,"",p);print m"."n"."p}' \
+                  src/Kokkos/kokkos/CMakeLists.txt)
+        echo "Bundled Kokkos version: $ver (expected ${KOKKOS_VERSION})"
+        test "$ver" = "${KOKKOS_VERSION}" || {
+          echo "::error::Submodule is Kokkos $ver but ${KOKKOS_VERSION} is required; run 'git submodule update --init --recursive'"
+          exit 1
+        }
+
+    - name: Install Kokkos standalone
       run: |
-        cmake \
-          -B ${{ github.workspace }}/build-kokkos-serial/matar \
-          -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }} \
-          ${{ matrix.config.matar_config_args }}
-
-    - if: contains(matrix.config.name, 'KOKKOS')
-      name: Configure MATAR CMake With Kokkos
-      shell: bash
+        cmake -S src/Kokkos/kokkos -B build/kokkos \
+          -DCMAKE_BUILD_TYPE=Release \
+          -DCMAKE_CXX_STANDARD=20 \
+          -DKokkos_ENABLE_SERIAL=ON \
+          -DKokkos_ENABLE_TESTS=OFF \
+          -DBUILD_TESTING=OFF \
+          -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/kokkos-install
+        cmake --build build/kokkos -j2
+        cmake --install build/kokkos
+
+    - name: Configure MATAR against external Kokkos
       run: |
-        cmake \
-          -B ${{ github.workspace }}/build-kokkos-serial/matar \
-          -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }} \
-          ${{ matrix.config.matar_config_args }} \
-          -DKokkos_DIR=${{ github.workspace }}/build-kokkos-serial/kokkos/lib/cmake/Kokkos
-
-    - name: Build MATAR
-      shell: bash
-      run: cmake --build ${{github.workspace}}/build-kokkos-serial/matar
-    # End MATAR configure and build------------------------------------------
-
-    #- name: Test
-    #  working-directory: ${{github.workspace}}/build
-    #  # Execute tests defined by the CMake configuration.  
-    #  # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail
-    #  run: ctest -C ${{env.BUILD_TYPE}}
-
+        cmake -B build/external \
+          -DCMAKE_BUILD_TYPE=Debug \
+          -DMATAR_USE_EXTERNAL_KOKKOS=ON \
+          -DKokkos_ROOT=${{ github.workspace }}/kokkos-install \
+          -DMATAR_BUILD_TESTS=ON \
+          -DMATAR_BUILD_EXAMPLES=ON
+
+    - name: Build
+      run: cmake --build build/external -j2
+
+    - name: Test
+      run: ctest --test-dir build/external --output-on-failure

.github/workflows/test.yml

@@ -7,38 +7,63 @@ on:
   pull_request:
     branches: [ "main" ]
 
-#env:
-#  # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
-
+# The Kokkos release the bundled submodule is pinned to. Bump here and in the
+# submodule together; the verify step below fails the build if they disagree.
+env:
+  KOKKOS_VERSION: "5.2.1"
 
 jobs:
-  # This workflow contains a single job called "build"
   build:
-    #
     name: ${{ matrix.config.name }}
     runs-on: ${{ matrix.config.os }}
     strategy:
       fail-fast: false
       matrix:
         config:
-        - {name: "TEST_UBUNTU_SERIAL_DEBUG", os: ubuntu-latest, debug: "enabled", cc: "gcc", cxx: "g++", kokkos_backend: "serial"}
-        - {name: "TEST_UBUNTU_OPENMP_DEBUG", os: ubuntu-latest, debug: "enabled", cc: "gcc", cxx: "g++", kokkos_backend: "openmp"}
-        - {name: "TEST_UBUNTU_SERIAL_RELEASE", os: ubuntu-latest, debug: "disabled", cc: "gcc", cxx: "g++", kokkos_backend: "serial"}
-        - {name: "TEST_UBUNTU_OPENMP_RELEASE", os: ubuntu-latest, debug: "disabled", cc: "gcc", cxx: "g++", kokkos_backend: "openmp"}
-        # Kokkos + MPI: builds MATAR with MPI, registers MPICArrayKokkos_mpi_suite (mpirun from CTest).
-        - {name: "TEST_UBUNTU_SERIAL_MPI_DEBUG", os: ubuntu-latest, debug: "enabled", cc: "gcc", cxx: "g++", kokkos_backend: "serial_mpi"}
-        - {name: "TEST_MAC_SERIAL_DEBUG", os: macos-14, debug: "enabled", cc: "clang", cxx: "clang++", kokkos_backend: "serial"}
-        - {name: "TEST_MAC_SERIAL_RELEASE", os: macos-14, debug: "disabled", cc: "clang", cxx: "clang++", kokkos_backend: "serial"}
+        - {name: "TEST_UBUNTU_SERIAL_DEBUG",     os: ubuntu-latest, preset: "serial-debug"}
+        - {name: "TEST_UBUNTU_OPENMP_DEBUG",     os: ubuntu-latest, preset: "openmp-debug"}
+        - {name: "TEST_UBUNTU_SERIAL_RELEASE",   os: ubuntu-latest, preset: "serial"}
+        - {name: "TEST_UBUNTU_OPENMP_RELEASE",   os: ubuntu-latest, preset: "openmp"}
+        # Kokkos + MPI: registers MPICArrayKokkos_mpi_suite (mpirun from CTest).
+        - {name: "TEST_UBUNTU_SERIAL_MPI_DEBUG", os: ubuntu-latest, preset: "serial-mpi-debug"}
+        # Precision-tier matrix: every real_t definition testable without a GPU
+        # (half/bfloat16 run float-emulated on CPU backends)
+        - {name: "TEST_UBUNTU_SERIAL_FP32",      os: ubuntu-latest, preset: "serial-fp32"}
+        - {name: "TEST_UBUNTU_SERIAL_FP16",      os: ubuntu-latest, preset: "serial-fp16"}
+        - {name: "TEST_UBUNTU_SERIAL_BF16",      os: ubuntu-latest, preset: "serial-bf16"}
+        - {name: "TEST_UBUNTU_SERIAL_QUAD",      os: ubuntu-latest, preset: "serial-quad"}
+        - {name: "TEST_UBUNTU_SERIAL_MIXED",     os: ubuntu-latest, preset: "serial-mixed"}
+        # MPI at non-default tiers: fp16 proves tier plumbing (float-emulated);
+        # quad exercises the custom MPI datatype + custom MPI_Op reduction paths
+        - {name: "TEST_UBUNTU_SERIAL_MPI_FP16",  os: ubuntu-latest, preset: "serial-mpi-fp16"}
+        - {name: "TEST_UBUNTU_SERIAL_MPI_QUAD",  os: ubuntu-latest, preset: "serial-mpi-quad"}
+        # Host-side parallel macros via the MATAR_*_BACKEND front door.
+        # NOTE: CI has no GPU, so this verifies host-macro correctness and the
+        # option plumbing, not actual host/device overlap.
+        - {name: "TEST_UBUNTU_HOST_OPENMP",      os: ubuntu-latest, preset: "hostomp"}
+        - {name: "TEST_MAC_SERIAL_DEBUG",        os: macos-14,      preset: "serial-debug"}
+        - {name: "TEST_MAC_SERIAL_RELEASE",      os: macos-14,      preset: "serial"}
 
     steps:
     - name: Checkout repository
       uses: actions/checkout@v3
-      
+
     - name: Checkout submodules
       run: git submodule update --init --recursive
 
+    - name: Verify bundled Kokkos version
+      run: |
+        ver=$(awk '/^set\(Kokkos_VERSION_MAJOR/{m=$2} /^set\(Kokkos_VERSION_MINOR/{n=$2} /^set\(Kokkos_VERSION_PATCH/{p=$2} \
+                   END{gsub(/\)/,"",m);gsub(/\)/,"",n);gsub(/\)/,"",p);print m"."n"."p}' \
+                  src/Kokkos/kokkos/CMakeLists.txt)
+        echo "Bundled Kokkos version: $ver (expected ${KOKKOS_VERSION})"
+        test "$ver" = "${KOKKOS_VERSION}" || {
+          echo "::error::Submodule is Kokkos $ver but ${KOKKOS_VERSION} is required; run 'git submodule update --init --recursive'"
+          exit 1
+        }
+
     - name: Install Open MPI
-      if: contains(matrix.config.kokkos_backend, 'mpi') && matrix.config.os == 'ubuntu-latest'
+      if: contains(matrix.config.preset, 'mpi') && matrix.config.os == 'ubuntu-latest'
       env:
         DEBIAN_FRONTEND: noninteractive
       run: |
@@ -49,23 +74,11 @@ jobs:
           mpi-default-bin \
           mpi-default-dev
 
-    # Build MATAR tests using the build-matar.sh script
-    - name: Build MATAR Tests
-      shell: bash
-      run: |
-        cd ${{ github.workspace }}/scripts
-        source build-matar.sh \
-          --execution=test \
-          --kokkos_build_type=${{ matrix.config.kokkos_backend }} \
-          --build_action=full-app \
-          --machine=linux \
-          --debug=${{ matrix.config.debug }} \
-          --build_cores=1
+    - name: Configure
+      run: cmake --preset ${{ matrix.config.preset }}
 
-    # Run the tests
-    - name: Run Tests
-      shell: bash
-      run: |
-        cd ${{ github.workspace }}/build-matar-${{ matrix.config.kokkos_backend }}
-        ctest --output-on-failure
+    - name: Build
+      run: cmake --build --preset ${{ matrix.config.preset }} -j2
 
+    - name: Run Tests
+      run: ctest --preset ${{ matrix.config.preset }}

.gitignore

@@ -1,10 +1,13 @@
+build/
+CMakeUserPresets.json
 install-*
 benchmark/benchmark*
 build-matar-*
 install/*
 heffte/
 docs_doxygen/
 docs_sphinx/
-tutorial/getting_started/Example0/build_*
-tutorial/getting_started/Example0/install*
-examples/mesh_decomp/lib/*
+tutorial/getting_started/Example*/build_*
+tutorial/getting_started/Example*/install*
+examples/mesh_decomp/lib/*
+test/googletest/