Feature/do concurrent

.github/workflows/linux-amdflang-cmake.yaml

@@ -83,8 +83,10 @@ jobs:
 
       - name: Run ctests
         run: |
-          cd build/test
-          ctest || ctest --rerun-failed --output-on-failure
+          export WORKSPACE=/home/runner/work/SELF/SELF/
+          ctest --verbose \
+                --output-on-failure \
+                --test-dir ./build/
 
       - name: Push packages and update index
         run: |

.github/workflows/linux-gnu-cmake.yml

@@ -66,7 +66,6 @@ jobs:
               build_type: debug
               memcheck: false
 
-
     defaults:
       run:
         shell: ${{ matrix.shell }}
@@ -102,36 +101,44 @@ jobs:
           FC=${{ matrix.fcompiler }} CC=${{ matrix.ccompiler }} cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }}  ../
           make VERBOSE=1
 
+      - name: Initialize coverage counters
+        if: ${{ matrix.build_type == 'coverage' }}
+        run: |
+          sudo apt-get update -y && sudo apt-get install lcov
+          lcov --capture \
+          --initial \
+          --directory ./build/src/ \
+          --gcov=${{ matrix.gcov }}  \
+          --output-file /home/runner/work/initial.info
+
       - name: Run ctests
         run: |
-          cd build/test
-          ctest || ctest --rerun-failed --output-on-failure
+          export WORKSPACE=/home/runner/work/SELF/SELF/
+          ctest --verbose \
+                --output-on-failure \
+                --test-dir ./build/
 
       - name: Create coverage report
         if: ${{ matrix.build_type == 'coverage' }}
         run: |
-          sudo apt-get update -y && sudo apt-get install lcov
-          mkdir -p ${WORKSPACE_ROOT}/tmp/
-          lcov --no-external \
-                --zerocounters \
-                --directory /home/runner/work/SELF \
-                --exclude '*/test/*' \
-                --gcov=${{ matrix.gcov }} 
-
-          lcov --no-external \
-              --capture \
-              --gcov=${{ matrix.gcov }} \
-              --directory /home/runner/work/SELF \
-              --exclude '*/test/*' \
-              --output-file /home/runner/work/lcov.info
+          lcov --capture \
+               --directory ./build/src/ \
+               --gcov=${{ matrix.gcov }} \
+               --output-file /home/runner/work/ctest-capture.info
+
+          lcov --add-tracefile /home/runner/work/initial.info \
+               --add-tracefile /home/runner/work/ctest-capture.info \
+               --gcov=${{ matrix.gcov }}  \
+               --output-file /home/runner/work/coverage.info
+
 
       - name: codecov
         if: ${{ matrix.build_type == 'coverage' }}
         uses: codecov/codecov-action@v3
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
-          files: /home/runner/work/lcov.info
+          files: /home/runner/work/coverage.info
           flags: github-serial-ctests
 
       - name: Run memory checks with Valgrind (only Linux and GNU compilers)

.github/workflows/linux-gnu-multithreaded-cmake.yml

@@ -0,0 +1,140 @@
+name: linux-gnu-multithreaded-cmake
+
+on:
+  push:
+    branches:
+      - master
+      - main
+    paths-ignore:
+      - 'AUTHORS.md'
+      - 'LICENSE.md'
+      - 'README.md'
+  pull_request:
+    paths-ignore:
+      - 'AUTHORS.md'
+      - 'LICENSE.md'
+      - 'README.md'
+
+jobs:
+  linux-tests:
+    timeout-minutes: 30
+    if: "!contains(github.event.head_commit.message, 'skip ci')"
+    name: ${{ matrix.os }} - ${{ matrix.fcompiler }} - ${{ matrix.build_type }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+            # Linux
+            - os: ubuntu-22.04
+              fcompiler: gfortran-12
+              ccompiler: gcc-12
+              gcov: gcov-12
+              spack_compiler: gcc@=12.3.0
+              spack_os: "ubuntu:22.04"
+              shell: bash
+              build_type: release
+              memcheck: false
+
+            - os: ubuntu-22.04
+              fcompiler: gfortran-9
+              ccompiler: gcc-9
+              gcov: gcov-9
+              spack_compiler: gcc@=9.5.0
+              spack_os: "ubuntu:22.04"
+              shell: bash
+              build_type: release
+              memcheck: false
+
+            - os: ubuntu-22.04
+              fcompiler: gfortran-10
+              ccompiler: gcc-10
+              gcov: gcov-10
+              spack_compiler: gcc@=10.5.0
+              spack_os: "ubuntu:22.04"
+              shell: bash
+              build_type: release
+              memcheck: false
+
+            - os: ubuntu-22.04
+              fcompiler: gfortran-11
+              ccompiler: gcc-11
+              gcov: gcov-11
+              spack_compiler: gcc@=11.4.0
+              spack_os: "ubuntu:22.04"
+              shell: bash
+              build_type: release
+              memcheck: false
+
+    defaults:
+      run:
+        shell: ${{ matrix.shell }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Show version information
+        run: |
+          ${{ matrix.fcompiler }} --version
+          ${{ matrix.ccompiler }} --version
+
+      - name: Set up Spack
+        uses: spack/setup-spack@v2
+
+      - name: Install dependencies in spack environment
+        run: |
+          spack -e ./share/spack-env/ mirror set --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache
+          spack -e ./share/spack-env/ compiler find
+          spack -e ./share/spack-env/ external find --not-buildable
+          spack -e ./share/spack-env/ config add packages:feq-parse:require:["'%${{ matrix.spack_compiler }}'"]
+          spack -e ./share/spack-env/ config add packages:hdf5:require:["'%${{ matrix.spack_compiler }}'"]
+          spack -e ./share/spack-env/ config add packages:openmpi:require:["'%${{ matrix.spack_compiler }}'"]
+          spack -e ./share/spack-env/ concretize -f
+          spack -e ./share/spack-env/ install --no-check-signature
+
+      - name: Build with Cmake
+        shell: spack-bash {0}
+        run: |
+          spack env activate ./share/spack-env/
+          mkdir build
+          cd build
+          FC=${{ matrix.fcompiler }} CC=${{ matrix.ccompiler }} \
+          cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }}  \
+                -DSELF_ENABLE_MULTITHREADING=ON \
+                -DSELF_MULTITHREADING_NTHREADS=4 \
+                ../
+          make VERBOSE=1
+
+      - name: Run ctests
+        run: |
+          export WORKSPACE=/home/runner/work/SELF/SELF/
+          ctest --verbose \
+                --output-on-failure \
+                --test-dir ./build/
+
+      - name: codecov
+        if: ${{ matrix.build_type == 'coverage' }}
+        uses: codecov/codecov-action@v3
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        with:
+          files: /home/runner/work/lcov.info
+          flags: github-serial-ctests
+
+      - name: Run memory checks with Valgrind (only Linux and GNU compilers)
+        if: ${{ matrix.memcheck }}
+        run: |
+          sudo apt-get install -y valgrind
+          for f in $(find ./build/test/ -executable -type f)
+          do
+                  echo $f
+                  valgrind --undef-value-errors=no --error-exitcode=1 -s $f -A
+          done
+
+      - name: Push packages and update index
+        run: |
+          spack -e ./share/spack-env/ mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache
+          spack -e ./share/spack-env/ buildcache push --base-image ${{ matrix.spack_os }} --update-index local-buildcache
+        if: ${{ !cancelled() }}
+
+

.github/workflows/linux-nvfortran-cmake.yaml

@@ -81,15 +81,17 @@ jobs:
           spack env activate ./share/spack-env/
           mkdir build
           cd build
-          FC=${{ env.FC }} CC=${{ env.CC }} cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DOMP_TARGET=${{ matrix.omp_target }}  ../
+          FC=${{ env.FC }} CC=${{ env.CC }} cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }}  ../
           make VERBOSE=1
 
       - name: Run ctests
         run: |
           export OMP_NUM_THREADS=4
           export OMP_TARGET_OFFLOAD=DISABLED
-          cd build/test
-          ctest || ctest --rerun-failed --output-on-failure
+          export WORKSPACE=/home/runner/work/SELF/SELF/
+          ctest --verbose \
+                --output-on-failure \
+                --test-dir ./build/
 
       - name: Push packages and update index
         run: |

.gitignore

@@ -41,3 +41,4 @@ build/
 lcov.info
 .vscode/
 util/
+*.info

CMakeLists.txt

@@ -26,23 +26,17 @@
 cmake_minimum_required(VERSION 3.21)
 cmake_policy(VERSION 3.21...3.27)
 
-# Defaults to "none", which disables OpenMP builds
-# Override with -DOMP_TARGET
-# - Set to "none" to disable OpenMP
-# - Set to "multicore" for multicore cpu
-# - GPU offloading via openmp not supported (not by our own choice)
-option(SELF_ENABLE_OPENMP "Option to enable OpenMP multicore target. Currently only tested with nvfortran (Default Off)"  OFF)
+option(SELF_ENABLE_MULTITHREADING "Option to enable CPU multithreading for `do concurrent` loop blocks."  OFF)
 option(SELF_ENABLE_TESTING "Option to enable build of tests. (Default On)"  ON)
 option(SELF_ENABLE_GPU "Option to enable GPU backend. Requires either CUDA or HIP. (Default Off)"  OFF)
 option(SELF_ENABLE_APU "Option to enable APU backend. Requires either CUDA or HIP. (Default Off)"  OFF)
 option(SELF_ENABLE_MPI "Option to enable MPI for distributed memory parallelism (domain decomposition). (Default Off)"  OFF)
 option(SELF_ENABLE_DOUBLE_PRECISION "Option to enable double precision for floating point arithmetic. (Default On)"  ON)
 
-if(SELF_ENABLE_OPENMP)
-    set(OMP_TARGET "multicore" CACHE STRING "Target architecture for openmp.")
+if(SELF_ENABLE_MULTITHREADING)
+    set(SELF_MULITHREADING_NTHREADS "4" CACHE STRING "Number of threads to use for `do concurrent` loop blocks. This option is only used with GNU compilers. Other compilers use OMP_NUM_THREADS environment variable at runtime.")
 endif()
 
-
 if(NOT DEFINED ROCM_PATH)
     if(NOT DEFINED ENV{ROCM_PATH})
         set(ROCM_PATH "/opt/rocm/" CACHE PATH "Path to which ROCm has been installed")
@@ -51,8 +45,6 @@ if(NOT DEFINED ROCM_PATH)
     endif()
 endif()
 
-#SET(CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIS_D}/cmake/modules/")
-include("${CMAKE_CURRENT_LIST_DIR}/cmake/modules/OpenMPTargetOffloadFlags.cmake")
 # C Language is needed in order to verify Fortran compiler is C-interoperable
 # CXX language is needed to properly find "hip" package
 project(self VERSION 1.0.0
@@ -114,29 +106,25 @@ elseif( "${CMAKE_Fortran_COMPILER_ID}" STREQUAL "NVHPC" )
 endif()
 
 # Check for openmp support if offload target is provided
-if( SELF_ENABLE_OPENMP )
-    find_package(OpenMP COMPONENTS Fortran REQUIRED)
-    if( ${OpenMP_Fortran_FOUND} )
-        if( "${OMP_TARGET}" STREQUAL "multicore" )
-            get_offload_flags(RESULT OFFLOAD_FLAGS)
-        else()
-            get_offload_flags(ARCH ${OMP_TARGET} RESULT OFFLOAD_FLAGS)
-        endif()
-        set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${OFFLOAD_FLAGS}" )
-        set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} ${OFFLOAD_FLAGS}" )
-        set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} ${OFFLOAD_FLAGS}")
-        set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} ${OFFLOAD_FLAGS}")
-        set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} ${OFFLOAD_FLAGS}" )
-        if( "${CMAKE_Fortran_COMPILER_ID}" STREQUAL "NVHPC" )
-            set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -Minfo=mp" )
-            set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -Minfo=mp" )
-            set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -Minfo=mp")
-            set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -Minfo=mp")
-            set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -Minfo=mp" )
-        endif()
-    else()
-        message( WARNING "OpenMP target set to ${OMP_TARGET} but OpenMP_Fortran not found. GPU Offloading disabled.")
+if( SELF_ENABLE_MULTITHREADING )
+    if( "${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU" )
+        set( OFFLOAD_FLAGS "-ftree-parallelize-loops=${SELF_MULTITHREADING_NTHREADS} -fopt-info-loop" )
+    elseif( "${CMAKE_Fortran_COMPILER_ID}" STREQUAL "Intel" )
+        set( OFFLOAD_FLAGS "-parallel -qopt-report -qopt-report-phase=par" )
+    elseif( "${CMAKE_Fortran_COMPILER_ID}" STREQUAL "IntelLLVM" )
+        set( OFFLOAD_FLAGS "-qopenmp -qopt-report" )
+    elseif( "${CMAKE_Fortran_COMPILER_ID}" STREQUAL "Flang" )
+        set( OFFLOAD_FLAGS "-fopenmp" )
+    elseif( "${CMAKE_Fortran_COMPILER_ID}" STREQUAL "NVHPC" )
+        set( OFFLOAD_FLAGS "-stdpar=multicore -Minfo=stdpar,accel" )
     endif()
+
+    set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${OFFLOAD_FLAGS}" )
+    set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} ${OFFLOAD_FLAGS}" )
+    set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} ${OFFLOAD_FLAGS}")
+    set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} ${OFFLOAD_FLAGS}")
+    set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} ${OFFLOAD_FLAGS}" )
+
 endif()