From 1f1d2d189c85b99a7b734f8927cc50fa5fa6f1db Mon Sep 17 00:00:00 2001 From: kvrigor Date: Fri, 9 Oct 2026 14:43:17 +0200 Subject: [PATCH 01/17] Added NVHPC support to Parflow and OASIS --- cmake/BuildOASIS3MCT.cmake | 2 ++ cmake/BuildParFlow.cmake | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cmake/BuildOASIS3MCT.cmake b/cmake/BuildOASIS3MCT.cmake index c66840fd..8744249c 100644 --- a/cmake/BuildOASIS3MCT.cmake +++ b/cmake/BuildOASIS3MCT.cmake @@ -44,6 +44,8 @@ if(CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") file(APPEND ${OASIS_MAKE_INC} "FCBASEFLAGS = ${OPTIM} -I. -ffree-line-length-none -fallow-argument-mismatch ${OpenMP_Fortran_FLAGS}\n") elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "Intel" OR CMAKE_Fortran_COMPILER_ID STREQUAL "IntelLLVM") file(APPEND ${OASIS_MAKE_INC} "FCBASEFLAGS = ${OPTIM} -I. -xCORE-AVX2 -assume byterecl -mt_mpi ${OpenMP_Fortran_FLAGS}\n") +elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") + file(APPEND ${OASIS_MAKE_INC} "FCBASEFLAGS = ${OPTIM} -I. -fPIC ${OpenMP_Fortran_FLAGS}\n") else() message(FATAL_ERROR "Fortran compiler '${CMAKE_Fortran_COMPILER_ID}' is not supported.") endif() diff --git a/cmake/BuildParFlow.cmake b/cmake/BuildParFlow.cmake index 81711453..8da11511 100644 --- a/cmake/BuildParFlow.cmake +++ b/cmake/BuildParFlow.cmake @@ -51,7 +51,7 @@ if(CMAKE_C_COMPILER_ID STREQUAL "GNU") string(APPEND PF_CFLAGS " -Wno-maybe-uninitialized") endif() set(PF_FFLAGS "-ffree-line-length-none -ffixed-line-length-none") -elseif(CMAKE_C_COMPILER_ID STREQUAL "Intel" OR CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM") +elseif(CMAKE_C_COMPILER_ID STREQUAL "Intel" OR CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_C_COMPILER_ID STREQUAL "NVHPC") set(PF_CFLAGS "-Wall -Werror -Wno-unused-function -Wno-unused-variable") else() message(FATAL_ERROR "C compiler '${CMAKE_C_COMPILER_ID}' is not supported.") From 33f59978163fb39c11eff002e4cf298c314f85d5 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Fri, 9 Oct 2026 14:43:17 +0200 Subject: [PATCH 02/17] Removed Stages 2023 & 2024 --- env/jsc.2023.intel.psmpi | 34 ------------------------------- env/jsc.2024.intel.psmpi | 44 ---------------------------------------- 2 files changed, 78 deletions(-) delete mode 100644 env/jsc.2023.intel.psmpi delete mode 100644 env/jsc.2024.intel.psmpi diff --git a/env/jsc.2023.intel.psmpi b/env/jsc.2023.intel.psmpi deleted file mode 100644 index 3b9ac71e..00000000 --- a/env/jsc.2023.intel.psmpi +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -# Load modules -module --force purge -module use $OTHERSTAGES -module load Stages/2023 -module load Intel/2022.1.0 -module load ParaStationMPI/5.7.0-1 -# -module load Hypre/2.27.0-cpu -module load Silo/4.11 -module load Tcl/8.6.12 -# -module load ecCodes/2.27.0 -module load HDF5/1.12.2-serial -module load netCDF/4.9.0 -module load netCDF-Fortran/4.6.0 -module load PnetCDF/1.12.3 -module load cURL/7.83.0 -module load Szip/.2.1.1 -module load Python/3.10.4 -module load NCO/5.1.3 -module load CMake/3.23.1 -module load git/2.36.0-nodocs - -module load pkgconf/.1.8.0 - -module li - -# Set default compilers -export CC=mpicc -export FC=mpif90 -export CXX=mpicxx -export MPI_HOME=$EBROOTPSMPI diff --git a/env/jsc.2024.intel.psmpi b/env/jsc.2024.intel.psmpi deleted file mode 100644 index 2908f650..00000000 --- a/env/jsc.2024.intel.psmpi +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -# Load modules -module --force purge -module use $OTHERSTAGES -module load Stages/2024 -module load Intel/2023.2.1 -module load ParaStationMPI/5.9.2-1 -#module load IntelMPI/2021.10.0 - -# -module load Hypre/2.31.0-cpu -module load Silo/4.11.1 -module load Tcl/8.6.13 -# -module load ecCodes/2.31.0 -module load HDF5/1.14.2 -module load netCDF/4.9.2 -module load netCDF-Fortran/4.6.1 -module load PnetCDF/1.12.3 -module load cURL/8.0.1 -module load Szip/.2.1.1 -module load Python/3.11.3 -module load NCO/5.1.8 -module load CMake/3.26.3 -module load git/2.41.0-nodocs - -# Set default compilers -export CC=mpicc -export FC=mpif90 -export CXX=mpicxx -export MPI_HOME=$EBROOTPSMPI - -# Display compiler settings -module li -echo "====================================== COMPILER SETTINGS ======================================" -echo " Machine: ${SYSTEMNAME} on Stages/$STAGE" -echo " MPI lib: $(mpichversion | head -n 1 | tr -d =)" -echo " C: $($CC --version | head -n 1)" -echo " C++: $($CXX --version | head -n 1)" -echo " Fortran: $($FC --version | head -n 1)" -echo "===============================================================================================" -echo "" - From 5755aa8a29cde9f3b0b61e903a3cc4ac7e10bfca Mon Sep 17 00:00:00 2001 From: kvrigor Date: Fri, 9 Oct 2026 14:43:17 +0200 Subject: [PATCH 03/17] GNU+PSMPI env file --- env/jsc.2026.gnu.psmpi | 78 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 env/jsc.2026.gnu.psmpi diff --git a/env/jsc.2026.gnu.psmpi b/env/jsc.2026.gnu.psmpi new file mode 100644 index 00000000..e80a5d8a --- /dev/null +++ b/env/jsc.2026.gnu.psmpi @@ -0,0 +1,78 @@ +# --------------------------------------------------------------------------------------- +# Loads GNU+OpenMPI build environment for TSMP2. +# This environment is tailored for JURECA [1], JUWELS [2], and JUPITER [3] supercomputers. +# +# [1] https://apps.fz-juelich.de/jsc/software/jureca/index.xhtml +# [2] https://apps.fz-juelich.de/jsc/software/juwels/index.xhtml +# [3] https://apps.fz-juelich.de/jsc/hps/jupiter/index.html +# Usage: source jsc.2026.gnu.openmpi +# --------------------------------------------------------------------------------------- + +# Load Stages/2026 +module --force purge +module use $OTHERSTAGES +module load Stages/2026 + +# Primary compiler toolchain +module load GCC +module load ParaStationMPI + +# Basic scripting and build tools +module load Python +module load CMake +module load git + +# Storage libraries +module load HDF5 +module load netCDF +module load netCDF-Fortran +module load PnetCDF + +# ParFlow additional libraries +module load Umpire +module load SUNDIALS +module load Tcl +if [[ "$1" == "--parflowgpu" ]]; then + module load CUDA + module load UCX-settings/RC-CUDA + module load Hypre/3.0.0 + + # TODO: Verify these values + if [[ $SYSTEMNAME == "jupiter" || $SYSTEMNAME == "jureca_gh" ]]; then + export CUDAARCHS="90" + else + export CUDAARCHS="80" + fi + export CMAKE_CUDA_RUNTIME_LIBRARY="Shared" +else + module load Hypre/3.0.0-cpu +fi + +# ICON additional libraries +module load ecCodes + +# Set default MPI compilers +export CC=mpicc +export FC=mpif90 +export CXX=mpicxx + +# Variables read by TSMP2 cmake scripts +export MPI_HOME=$EBROOTPSMPI +export UMPIRE_ROOT=$EBROOTUMPIRE +export SUNDIALS_ROOT=$EBROOTSUNDIALS + +# Display compiler settings +module list +echo "=============== COMPILER SETTINGS ===============" +echo " Machine: ${SYSTEMNAME} on Stages/$STAGE" +echo " MPI lib: ParaStationMPI v$EBVERSIONPSMPI" +echo " C: $($CC --version | head -n 1)" +echo " C++: $($CXX --version | head -n 1)" +echo " Fortran: $($FC --version | head -n 1)" +if [[ "$1" == "--parflowgpu" ]]; then + echo " nvcc: $(nvcc --version | tail -n 1 | cut -d" " -f2)" + echo " CUDAARCHS: $CUDAARCHS" +fi +echo "==================================================" +echo "" + From e91f6408b4b129bc6d2282c4003caa6b6a635739 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Fri, 9 Oct 2026 14:43:17 +0200 Subject: [PATCH 04/17] BuildOASIS: clean first before building libraries --- cmake/BuildOASIS3MCT.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/BuildOASIS3MCT.cmake b/cmake/BuildOASIS3MCT.cmake index 8744249c..52deb669 100644 --- a/cmake/BuildOASIS3MCT.cmake +++ b/cmake/BuildOASIS3MCT.cmake @@ -70,7 +70,7 @@ ExternalProject_Add(OASIS3_MCT SOURCE_DIR ${OASIS_SRC} BUILD_IN_SOURCE FALSE CONFIGURE_COMMAND "" - BUILD_COMMAND make -f ${OASIS_SRC}/util/make_dir/TopMakefileOasis3 static-libs -C ${OASIS_BLD_DIR} + BUILD_COMMAND make -f ${OASIS_SRC}/util/make_dir/TopMakefileOasis3 realclean static-libs -C ${OASIS_BLD_DIR} INSTALL_COMMAND "" ) From 20fc48d03597e736bf4bf810caa8ee3b27f68262 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Tue, 13 Oct 2026 09:47:38 +0200 Subject: [PATCH 05/17] New env: jsc.2026.intel.psmpi --- env/jsc.2026.intel.psmpi | 82 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 env/jsc.2026.intel.psmpi diff --git a/env/jsc.2026.intel.psmpi b/env/jsc.2026.intel.psmpi new file mode 100644 index 00000000..33831ef3 --- /dev/null +++ b/env/jsc.2026.intel.psmpi @@ -0,0 +1,82 @@ +# -------------------------------------------------------------------------- +# Loads IntelLLVM+ParaStationMPI build environment for TSMP2. +# This environment is tailored for JURECA [1] and JUWELS [2] supercomputers. +# +# [1] https://apps.fz-juelich.de/jsc/software/jureca/index.xhtml +# [2] https://apps.fz-juelich.de/jsc/software/juwels/index.xhtml +# +# Usage: source jsc.2026.intel.psmpi +# -------------------------------------------------------------------------- + +# Load Stages/2026 +module --force purge +module use $OTHERSTAGES +module load Stages/2026 + +# Primary compiler toolchain +if [[ "$1" == "--parflowgpu" ]]; then + # TODO: Use a generic GPU switch instead of --parflowgpu + module load CUDA +fi +module load Intel +module load ParaStationMPI + +# Basic scripting and build tools +module load Python +module load CMake +module load git + +# Storage libraries +module load HDF5 +module load netCDF +module load netCDF-Fortran +module load PnetCDF + +# ParFlow additional libraries +if [[ "$1" == "--parflowgpu" ]]; then + module load CUDA + module load UCX-settings/RC-CUDA + module load Hypre/3.0.0 + module load Umpire + + # TODO: Verify these values + export CUDAARCHS="80" + export CMAKE_CUDA_RUNTIME_LIBRARY="Shared" +else + module load Hypre/3.0.0-cpu + module load Umpire +fi +module load Tcl + +# ICON additional libraries +module load ecCodes + +# Set default MPI compilers +export OMPI_CC=icx +export OMPI_CXX=icpx +export OMPI_FC=ifx +export CC=mpicc +export FC=mpif90 +export CXX=mpicxx +export MPI_HOME=$EBROOTPSMPI +export TSMP2_PARALLEL_JOBS=16 + +# Display compiler settings +module list +echo "====================================== COMPILER SETTINGS ======================================" +echo " Machine: ${SYSTEMNAME} on Stages/$STAGE" +echo " MPI lib: $(mpichversion | head -n 1 | tr -d =)" +echo " C: $($CC --version | head -n 1)" +echo " C++: $($CXX --version | head -n 1)" +echo " Fortran: $($FC --version | head -n 1)" +if [[ "$1" == "--parflowgpu" ]]; then + echo " nvcc: $(nvcc --version | tail -n 1 | cut -d" " -f2)" + echo " CUDAARCHS: $CUDAARCHS" +fi +echo "===============================================================================================" +echo "" +if [[ "$1" == "--parflowgpu" ]]; then + echo "WARNING: ParFlowGPU doesn't work on Intel toolchain due to missing Intel-built Hypre-CUDA library." + echo " Load the 'jsc.2025.gnu.openmpi' env file instead to build ParFlowGPU." +fi +echo "" From 27552186e13d4bef34f949838fd02114343dae4b Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 14 Oct 2026 10:51:59 +0200 Subject: [PATCH 06/17] OASIS3-MCT Stages/2026 NVHPC fixes - Replaced "include " with "use mpi". https://icg4geo.icg.kfa-juelich.de/ExternalReposPublic/oasis3-mct/-/commit/14503d81f877544e411fd5b2e94ec0596f58a0fd - Fixed incorrect data type passed to MPI_GATHER https://icg4geo.icg.kfa-juelich.de/ExternalReposPublic/oasis3-mct/-/commit/7c4fc975baecee2ab0ec0b71a07e4e241a8d34e6 --- cmake/BuildOASIS3MCT.cmake | 15 ++++++++++----- models/oasis3-mct | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/cmake/BuildOASIS3MCT.cmake b/cmake/BuildOASIS3MCT.cmake index 52deb669..b71edce2 100644 --- a/cmake/BuildOASIS3MCT.cmake +++ b/cmake/BuildOASIS3MCT.cmake @@ -13,6 +13,15 @@ else() message(WARNING "CMAKE_BUILD_TYPE='${CMAKE_BUILD_TYPE}' is not supported by OASIS3-MCT") endif() +set(CPPDEF_EXTRAS "") +if(${PDAF}) + string(APPEND CPPDEF_EXTRAS " -DUSE_PDAF") +endif() +if(CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") + # NVHPC doesn't support quad precision. https://forums.developer.nvidia.com/t/real-16-implementation/129969 + string(APPEND CPPDEF_EXTRAS " -D__NO_16BYTE_REALS") +endif() + set(OASIS_BLD_DIR ${CMAKE_CURRENT_BINARY_DIR}/OASIS3_MCT/bld) set(OASIS_MAKE_INC ${OASIS_BLD_DIR}/make.inc) set(OASIS_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}/OASIS3-MCT) @@ -35,11 +44,7 @@ file(APPEND ${OASIS_MAKE_INC} "AR = ar\n") file(APPEND ${OASIS_MAKE_INC} "ARFLAGS = -ruv\n") file(APPEND ${OASIS_MAKE_INC} "DYNOPT = -fPIC\n") file(APPEND ${OASIS_MAKE_INC} "LDDYNOPT = -shared\n") -if(${PDAF}) - file(APPEND ${OASIS_MAKE_INC} "CPPDEF = -Duse_netCDF -Duse_comm_$(CHAN) -D__VERBOSE -DTREAT_OVERLAY -DUSE_PDAF\n") -else() - file(APPEND ${OASIS_MAKE_INC} "CPPDEF = -Duse_netCDF -Duse_comm_$(CHAN) -D__VERBOSE -DTREAT_OVERLAY\n") -endif() +file(APPEND ${OASIS_MAKE_INC} "CPPDEF = -Duse_netCDF -Duse_comm_$(CHAN) -D__VERBOSE -DTREAT_OVERLAY ${CPPDEF_EXTRAS}\n") if(CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") file(APPEND ${OASIS_MAKE_INC} "FCBASEFLAGS = ${OPTIM} -I. -ffree-line-length-none -fallow-argument-mismatch ${OpenMP_Fortran_FLAGS}\n") elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "Intel" OR CMAKE_Fortran_COMPILER_ID STREQUAL "IntelLLVM") diff --git a/models/oasis3-mct b/models/oasis3-mct index 5253349d..7c4fc975 160000 --- a/models/oasis3-mct +++ b/models/oasis3-mct @@ -1 +1 @@ -Subproject commit 5253349d4ce15259fcd76e0443495c1ddb7788bb +Subproject commit 7c4fc975baecee2ab0ec0b71a07e4e241a8d34e6 From e2ee7cbfe001fea36fa6a9fc01b8f4a17a237604 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 14 Oct 2026 13:40:04 +0200 Subject: [PATCH 07/17] Deprecated -- switches; passed --icongpu options to env file --- build_tsmp2.sh | 18 ++++++++++-------- env/jsc.2026.gnu.openmpi | 4 ++-- env/jsc.2026.nvhpc.openmpi | 2 ++ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/build_tsmp2.sh b/build_tsmp2.sh index 9d07bc3c..d1326f45 100755 --- a/build_tsmp2.sh +++ b/build_tsmp2.sh @@ -123,14 +123,14 @@ while [[ "$#" -gt 0 ]]; do -q|--quiet) quiet=y;; -v|--verbose) verbose_makefile=y;; --version) echo "$0 version 0.2.0"; exit 0;; - --icon|icon) icon=y iconCPU=y iconCMakeModelID="ICON";; + icon) icon=y iconCPU=y iconCMakeModelID="ICON";; icongpu) icon=y iconGPU=y iconCMakeModelID="ICONGPU";; - --eclm|eclm) eclm=y;; - --parflow|parflow) parflow=y parflowCPU=y parflowCMakeModelID="ParFlow";; - --parflowgpu|parflowgpu) parflow=y parflowGPU=y parflowCMakeModelID="ParFlowGPU";; - --pdaf|pdaf) pdaf=y;; - --cosmo|cosmo) cosmo=y;; - --clm35|--clm3.5|clm3.5) clm35=y;; + eclm) eclm=y;; + parflow) parflow=y parflowCPU=y parflowCMakeModelID="ParFlow";; + parflowgpu) parflow=y parflowGPU=y parflowCMakeModelID="ParFlowGPU";; + pdaf) pdaf=y;; + cosmo) cosmo=y;; + clm3.5) clm35=y;; --no_update) update_compsrc=n;; --clean_first) clean_first=y;; --icon_src) icon_src="$2"; shift ;; @@ -233,9 +233,11 @@ fi if [[ -n "${env}" ]]; then message "Sourcing environment..." - # TODO: Fix this GPU thing on another PR + # TODO: Fix this messy GPU env parameterization on another PR if [[ "$parflowGPU" == "y" ]];then source "${env}" --parflowgpu + elif [[ "$iconGPU" == "y" ]];then + source "${env}" --icongpu else source "${env}" fi diff --git a/env/jsc.2026.gnu.openmpi b/env/jsc.2026.gnu.openmpi index d006c930..fb3f55f0 100644 --- a/env/jsc.2026.gnu.openmpi +++ b/env/jsc.2026.gnu.openmpi @@ -32,7 +32,7 @@ module load PnetCDF module load Umpire module load SUNDIALS module load Tcl -if [[ "$1" == "--parflowgpu" ]]; then +if [[ "$1" == "--parflowgpu" || "$1" == "--icongpu" ]]; then module load CUDA module load UCX-settings/RC-CUDA module load Hypre/3.0.0 @@ -69,7 +69,7 @@ echo " MPI lib: OpenMPI v$EBVERSIONOPENMPI" echo " C: $($CC --version | head -n 1)" echo " C++: $($CXX --version | head -n 1)" echo " Fortran: $($FC --version | head -n 1)" -if [[ "$1" == "--parflowgpu" ]]; then +if [[ "$1" == "--parflowgpu" || "$1" == "--icongpu" ]]; then echo " nvcc: $(nvcc --version | tail -n 1 | cut -d" " -f2)" echo " CUDAARCHS: $CUDAARCHS" fi diff --git a/env/jsc.2026.nvhpc.openmpi b/env/jsc.2026.nvhpc.openmpi index 81961028..e969a795 100644 --- a/env/jsc.2026.nvhpc.openmpi +++ b/env/jsc.2026.nvhpc.openmpi @@ -62,5 +62,7 @@ echo " MPI lib: OpenMPI v$EBVERSIONOPENMPI" echo " C: $($CC --version | head -n 2 | tail -n 1)" echo " C++: $($CXX --version | head -n 2 | tail -n 1)" echo " Fortran: $($FC --version | head -n 2 | tail -n 1)" +echo " nvcc: $(nvcc --version | tail -n 1 | cut -d" " -f2)" +echo " CUDAARCHS: $CUDAARCHS" echo "======================================================================" From 2a567ecf6dab2fc25ea280fd405540c666ed39c1 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Thu, 15 Oct 2026 14:50:53 +0200 Subject: [PATCH 08/17] eCLM-ICONGPU should build OASIS first. --- cmake/Utils.cmake | 2 +- env/jsc.2026.nvhpc.openmpi | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/Utils.cmake b/cmake/Utils.cmake index 15d11b93..adb37a2f 100644 --- a/cmake/Utils.cmake +++ b/cmake/Utils.cmake @@ -6,7 +6,7 @@ endif() # Count how many component models shoul be build, and if >=2 compile with OASIS function(check_build_oasis BUILD_OASIS) # Combine all options into a list - list(APPEND COMPONENT_MODELS ICON COSMO eCLM CLM3.5 ParFlow ParFlowGPU) + list(APPEND COMPONENT_MODELS ICON ICONGPU COSMO eCLM CLM3.5 ParFlow ParFlowGPU) # Count enabled models set(MODELCOUNT 0) foreach(comp_model IN LISTS COMPONENT_MODELS) diff --git a/env/jsc.2026.nvhpc.openmpi b/env/jsc.2026.nvhpc.openmpi index e969a795..197b2f86 100644 --- a/env/jsc.2026.nvhpc.openmpi +++ b/env/jsc.2026.nvhpc.openmpi @@ -49,7 +49,7 @@ module load ecCodes export CC=mpicc export FC=mpifort export CXX=nvc++ -export CUDACXX=nvcc +export CUDACXX=$(which nvcc) export MPI_HOME=$EBROOTOPENMPI export UMPIRE_ROOT=$EBROOTUMPIRE export SUNDIALS_ROOT=$EBROOTSUNDIALS From e0773bff953332c0ee9f634d051e20fc37935b0f Mon Sep 17 00:00:00 2001 From: kvrigor Date: Tue, 20 Oct 2026 15:15:07 +0200 Subject: [PATCH 09/17] ParflowGPU NVHPC build hacks --- CMakeLists.txt | 2 +- cmake/BuildParFlow.cmake | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c6ec4ab3..5b8b9777 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -60,7 +60,7 @@ if (${ICON} OR ${ICONGPU}) set(ICON_SRC "${CMAKE_SOURCE_DIR}/models/icon") endif() include(BuildICON) - list(APPEND COMPONENT_MODELS "ICON") + list(APPEND COMPONENT_MODELS ${ICON_ID}) if ("${MODEL_ID}" STREQUAL "") set(MODEL_ID "${ICON_ID}") else() diff --git a/cmake/BuildParFlow.cmake b/cmake/BuildParFlow.cmake index ef90156c..8bcd0d98 100644 --- a/cmake/BuildParFlow.cmake +++ b/cmake/BuildParFlow.cmake @@ -51,8 +51,11 @@ if(CMAKE_C_COMPILER_ID STREQUAL "GNU") string(APPEND PF_CFLAGS " -Wno-maybe-uninitialized") endif() set(PF_FFLAGS "-ffree-line-length-none -ffixed-line-length-none") -elseif(CMAKE_C_COMPILER_ID STREQUAL "Intel" OR CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_C_COMPILER_ID STREQUAL "NVHPC") +elseif(CMAKE_C_COMPILER_ID STREQUAL "Intel" OR CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM") set(PF_CFLAGS "-Wall -Werror -Wno-unused-function -Wno-unused-variable") +elseif(CMAKE_C_COMPILER_ID STREQUAL "NVHPC") + set(PF_CFLAGS "-Werror -Wno-unused --diag_suppress=unsigned_compare_with_negative,vla_treated_as_fixed_size_array,code_is_unreachable") + #set(PF_CFLAGS "-Werror -Wno-unused") else() message(FATAL_ERROR "C compiler '${CMAKE_C_COMPILER_ID}' is not supported.") endif() @@ -67,7 +70,13 @@ endif() # Enable Umpire if exists if (DEFINED ENV{UMPIRE_ROOT} AND NOT ${PDAF}) - set(PF_UMPIRE_FLAG "-DUMPIRE_ROOT=$ENV{UMPIRE_ROOT}") + list(APPEND PF_UMPIRE_FLAG -DUMPIRE_ROOT=$ENV{UMPIRE_ROOT}) + if(CMAKE_C_COMPILER_ID STREQUAL "NVHPC") + # Umpire leaks wrong OpenMP compile/linker flags when on NVHPC toolchain. + # See https://github.com/HPSCTerrSys/TSMP2/issues/148 + list(APPEND PF_UMPIRE_FLAG -DBLT_OPENMP_COMPILE_FLAGS=-fopenmp + -DBLT_OPENMP_LINK_FLAGS=-fopenmp) + endif() endif() # Enable SUNDIALS if exists From 381a2716a1690fa3eed25c3f826056300fb080aa Mon Sep 17 00:00:00 2001 From: kvrigor Date: Tue, 20 Oct 2026 16:36:51 +0200 Subject: [PATCH 10/17] ICON build hacks --- cmake/BuildICON.cmake | 11 ++++++++++- env/jsc.2026.nvhpc.openmpi | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/cmake/BuildICON.cmake b/cmake/BuildICON.cmake index adea7ba3..b7f09c56 100644 --- a/cmake/BuildICON.cmake +++ b/cmake/BuildICON.cmake @@ -112,7 +112,16 @@ endif() # GPU-specific options if(${ICONGPU}) - list(APPEND ICON_LIBS "-c++libs -nvmalloc -cuda") + # TODO: below flags only work for NVHPC + #find_package(OpenACC REQUIRED) + #string(APPEND ICON_FCFLAGS " ${OpenACC_Fortran_FLAGS}") + #message(STATUS "OpenACC_Fortran_FLAGS=${OpenACC_Fortran_FLAGS}") + + if(CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") + list(APPEND ICON_LIBS "-lcudart") + elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") + list(APPEND ICON_LIBS "-c++libs -nvmalloc -cuda") + endif() list(APPEND EXTRA_CONFIG_ARGS --enable-gpu=openacc --enable-mpi-gpu --enable-cuda-graphs --enable-pgi-inlib) else() list(APPEND EXTRA_CONFIG_ARGS --enable-openmp) diff --git a/env/jsc.2026.nvhpc.openmpi b/env/jsc.2026.nvhpc.openmpi index 0dec52ba..1b40822a 100644 --- a/env/jsc.2026.nvhpc.openmpi +++ b/env/jsc.2026.nvhpc.openmpi @@ -32,7 +32,7 @@ module load CUDA module load UCX-settings/RC-CUDA module load Hypre module load Umpire -module load SUNDIALS +#module load SUNDIALS # TODO: Verify these values if [[ $SYSTEMNAME == "jupiter" ]]; then From 52445f9e0c2b622d2c41e3bd744a185d44e4eb47 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Tue, 20 Oct 2026 17:17:29 +0200 Subject: [PATCH 11/17] Changed eCLM to 'dev-iconeclm-exchfix' --- .gitmodules | 2 +- models/eCLM | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 2725566e..9ac451da 100644 --- a/.gitmodules +++ b/.gitmodules @@ -17,7 +17,7 @@ [submodule "models/eCLM"] path = models/eCLM url = https://github.com/HPSCTerrSys/eCLM.git - branch = master-parallel-build + branch = dev-iconeclm-exchfix [submodule "models/oasis3-mct"] path = models/oasis3-mct url = https://icg4geo.icg.kfa-juelich.de/ExternalReposPublic/oasis3-mct diff --git a/models/eCLM b/models/eCLM index 8d9ca1ea..996b5f6c 160000 --- a/models/eCLM +++ b/models/eCLM @@ -1 +1 @@ -Subproject commit 8d9ca1ea7a3418335527568417a889ef3f259634 +Subproject commit 996b5f6cf191b62bd310cc4ee4e0dfa6613c7bdf From f5d182f2f05eb5fba6b5f137f5017347f26b415c Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 21 Oct 2026 09:27:58 +0200 Subject: [PATCH 12/17] ICONGPU should also build eCLM in coupling mode --- cmake/BuildeCLM.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/BuildeCLM.cmake b/cmake/BuildeCLM.cmake index d11854af..941aa593 100644 --- a/cmake/BuildeCLM.cmake +++ b/cmake/BuildeCLM.cmake @@ -1,9 +1,9 @@ -if(${ICON} OR ${ParFlow} OR ${ParFlowGPU}) +if(${ICON} OR ${ICONGPU} OR ${ParFlow} OR ${ParFlowGPU}) list(APPEND COUP_OAS_FLAGS -DUSE_OASIS=True) if(${ParFlow} OR ${ParFlowGPU}) list(APPEND COUP_OAS_FLAGS -DCOUP_OAS_PFL=True) endif() - if(${ICON}) + if(${ICON} OR ${ICONGPU}) list(APPEND COUP_OAS_FLAGS -DCOUP_OAS_ICON=True) endif() else() From a3a0c0898014ea5a1931da78e7d68bcf832739bb Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 21 Oct 2026 10:25:38 +0200 Subject: [PATCH 13/17] eCLM: updated dev-iconeclm-exchfix --- models/eCLM | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/eCLM b/models/eCLM index 996b5f6c..d4d595fa 160000 --- a/models/eCLM +++ b/models/eCLM @@ -1 +1 @@ -Subproject commit 996b5f6cf191b62bd310cc4ee4e0dfa6613c7bdf +Subproject commit d4d595fac9b921dc584549795452493f0f9fa614 From d319165b6a292c15c892be0f42a7a54752562569 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 21 Oct 2026 12:02:24 +0200 Subject: [PATCH 14/17] use loascplscheme_exchcoef only with ICON --- models/eCLM | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/eCLM b/models/eCLM index d4d595fa..57115693 160000 --- a/models/eCLM +++ b/models/eCLM @@ -1 +1 @@ -Subproject commit d4d595fac9b921dc584549795452493f0f9fa614 +Subproject commit 57115693bd0818edec6ef1241aed5b3668bbaaa1 From 8eccd1d83b1267cfbd182e3067a95c002aa9c25a Mon Sep 17 00:00:00 2001 From: Johannes Keller Date: Wed, 21 Oct 2026 12:30:44 +0200 Subject: [PATCH 15/17] BuildParFlow: Error message enforcing ParFlow on GPUs for NVHPC --- cmake/BuildParFlow.cmake | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmake/BuildParFlow.cmake b/cmake/BuildParFlow.cmake index 8bcd0d98..b154700e 100644 --- a/cmake/BuildParFlow.cmake +++ b/cmake/BuildParFlow.cmake @@ -56,6 +56,11 @@ elseif(CMAKE_C_COMPILER_ID STREQUAL "Intel" OR CMAKE_C_COMPILER_ID STREQUAL "Int elseif(CMAKE_C_COMPILER_ID STREQUAL "NVHPC") set(PF_CFLAGS "-Werror -Wno-unused --diag_suppress=unsigned_compare_with_negative,vla_treated_as_fixed_size_array,code_is_unreachable") #set(PF_CFLAGS "-Werror -Wno-unused") + if(NOT ${ParFlowGPU}) + message(FATAL_ERROR "BuildParFlow: NVHPC compiler detected but ParFlowGPU is not enabled. " + "NVHPC should for now only be used for GPU-accelerated ParFlow builds (parflowGPU). " + "Use a GNU or Intel compiler for CPU-only ParFlow, or enable parflowGPU.") + endif() else() message(FATAL_ERROR "C compiler '${CMAKE_C_COMPILER_ID}' is not supported.") endif() From ba9de00fd72e4202728ecdfe781331f35a026226 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 21 Oct 2026 14:23:51 +0200 Subject: [PATCH 16/17] Ensure OMPI_* vars are unset --- env/jsc.2026.gnu.openmpi | 4 ++++ env/jsc.2026.nvhpc.openmpi | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/env/jsc.2026.gnu.openmpi b/env/jsc.2026.gnu.openmpi index 63312a4f..b2b9ace1 100644 --- a/env/jsc.2026.gnu.openmpi +++ b/env/jsc.2026.gnu.openmpi @@ -56,6 +56,10 @@ export CC=mpicc export FC=mpif90 export CXX=mpicxx +# OMPI_* variables affect OpenMPI backend compilers. They should be +# unset so that the correct default backend compilers are selected. +unset OMPI_CC OMPI_FC OMPI_CXX + # Variables read by TSMP2 cmake scripts export MPI_HOME=$EBROOTOPENMPI export UMPIRE_ROOT=$EBROOTUMPIRE diff --git a/env/jsc.2026.nvhpc.openmpi b/env/jsc.2026.nvhpc.openmpi index 1b40822a..1ba81513 100644 --- a/env/jsc.2026.nvhpc.openmpi +++ b/env/jsc.2026.nvhpc.openmpi @@ -48,7 +48,13 @@ module load ecCodes # Set default MPI compilers export CC=mpicc export FC=mpifort -export CXX=nvc++ +export CXX=mpicxx + +# OMPI_* variables affect OpenMPI backend compilers. They should be +# unset so that the correct default backend compilers are selected. +unset OMPI_CC OMPI_FC OMPI_CXX + +# Variables read by TSMP2 cmake scripts export CUDACXX=$(which nvcc) export MPI_HOME=$EBROOTOPENMPI export UMPIRE_ROOT=$EBROOTUMPIRE From 4d80f4567fae8fcfa5d64a9bfc697e5d23295fb7 Mon Sep 17 00:00:00 2001 From: Johannes Keller <16795031+jjokella@users.noreply.github.com> Date: Wed, 21 Oct 2026 15:29:45 +0200 Subject: [PATCH 17/17] Enable PDAF builds on JUPITER with NVHPC compilers (#150) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Pure PDAF - **`BuildPDAF.cmake`**: Add NVHPC support - **`BuildPDAFMODEL.cmake`**: Make `-lgfortran` conditional on GNU compiler. Add GPU libraries for `ParFlowGPU` builds. - **`BuildParFlow.cmake`**: Umpire is passed to ParFlow for `ParFlowGPU+PDAF` builds - **`models/parflow_pdaf`**: Update submodule `v3.14.1-pdaf` → `v3.15.0-pdaf`, necessary update for GPU support ## OASIS (affects all coupled simulations) - **`BuildOASIS3MCT.cmake`**: Add `make realclean` as configure step --- .gitmodules | 2 +- cmake/BuildOASIS3MCT.cmake | 2 +- cmake/BuildPDAF.cmake | 39 ++++++++++++++++++++++++++++++++++++++ cmake/BuildPDAFMODEL.cmake | 27 +++++++++++++++++++++++++- cmake/BuildParFlow.cmake | 2 +- models/parflow_pdaf | 2 +- 6 files changed, 69 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index 9ac451da..055565e4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,7 +9,7 @@ [submodule "models/parflow_pdaf"] path = models/parflow_pdaf url = https://github.com/HPSCTerrSys/parflow - branch = v3.14.1-pdaf + branch = v3.15.0-pdaf [submodule "models/pdaf"] path = models/pdaf url = https://github.com/HPSCTerrSys/pdaf.git diff --git a/cmake/BuildOASIS3MCT.cmake b/cmake/BuildOASIS3MCT.cmake index 6d438904..eee469ee 100644 --- a/cmake/BuildOASIS3MCT.cmake +++ b/cmake/BuildOASIS3MCT.cmake @@ -74,7 +74,7 @@ ExternalProject_Add(OASIS3_MCT PREFIX OASIS3_MCT SOURCE_DIR ${OASIS_SRC} BUILD_IN_SOURCE FALSE - CONFIGURE_COMMAND "" + CONFIGURE_COMMAND make -f ${OASIS_SRC}/util/make_dir/TopMakefileOasis3 realclean -C ${OASIS_BLD_DIR} BUILD_COMMAND make -j ${PARALLEL_JOBS} -f ${OASIS_SRC}/util/make_dir/TopMakefileOasis3 static-libs -C ${OASIS_BLD_DIR} INSTALL_COMMAND "" ) diff --git a/cmake/BuildPDAF.cmake b/cmake/BuildPDAF.cmake index b89685c9..2c796e5b 100644 --- a/cmake/BuildPDAF.cmake +++ b/cmake/BuildPDAF.cmake @@ -141,6 +141,24 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") list(APPEND PDAF_FOPT "-fallow-argument-mismatch") list(APPEND PDAF_FOPT "-fcommon") +elseif (CMAKE_CXX_COMPILER_ID STREQUAL "NVHPC") + + # using NVIDIA HPC SDK + if (CMAKE_BUILD_TYPE STREQUAL "RELEASE") + list(APPEND PDAF_FOPT "-O2") + elseif (CMAKE_BUILD_TYPE STREQUAL "DEBUG") + list(APPEND PDAF_FOPT "-O0") + list(APPEND PDAF_FOPT "-g") + list(APPEND PDAF_FOPT "-traceback") + else() + message(FATAL_ERROR "Unsupported CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") + endif() + + list(APPEND PDAF_FOPT "-Msave") + list(APPEND PDAF_FOPT "-Mlarge_arrays") + list(APPEND PDAF_FOPT "-Mbyteswapio") + list(APPEND PDAF_FOPT "-Mfreeform") + else() message(FATAL_ERROR "Unsupported CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}") endif() @@ -189,6 +207,19 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") list(APPEND PDAF_COPT "-mcmodel=large") list(APPEND PDAF_COPT "-fcommon") +elseif (CMAKE_CXX_COMPILER_ID STREQUAL "NVHPC") + + # using NVIDIA HPC SDK + if (CMAKE_BUILD_TYPE STREQUAL "RELEASE") + list(APPEND PDAF_COPT "-O2") + elseif (CMAKE_BUILD_TYPE STREQUAL "DEBUG") + list(APPEND PDAF_COPT "-O0") + list(APPEND PDAF_COPT "-g") + list(APPEND PDAF_COPT "-traceback") + else() + message(FATAL_ERROR "Unsupported CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") + endif() + else() message(FATAL_ERROR "Unsupported CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}") endif() @@ -207,6 +238,10 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") list(APPEND PDAF_DOUBLEPRECISION "-fdefault-real-8") +elseif (CMAKE_CXX_COMPILER_ID STREQUAL "NVHPC") + + list(APPEND PDAF_DOUBLEPRECISION "-r8") + else() message(FATAL_ERROR "Unsupported CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}") endif() @@ -247,6 +282,10 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") list(APPEND PDAF_MODULEOPT "-J") +elseif (CMAKE_CXX_COMPILER_ID STREQUAL "NVHPC") + + list(APPEND PDAF_MODULEOPT "-module") + else() message(FATAL_ERROR "Unsupported CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}") endif() diff --git a/cmake/BuildPDAFMODEL.cmake b/cmake/BuildPDAFMODEL.cmake index 3e7608ae..cf715674 100644 --- a/cmake/BuildPDAFMODEL.cmake +++ b/cmake/BuildPDAFMODEL.cmake @@ -97,7 +97,10 @@ endif() # ParFlow libraries if(DEFINED PARFLOW_SRC) - list(APPEND PDAF_LIBS "-L${CMAKE_INSTALL_PREFIX}/lib -lpfsimulator -lamps -lpfkinsol -lgfortran -lcjson") + list(APPEND PDAF_LIBS "-L${CMAKE_INSTALL_PREFIX}/lib -lpfsimulator -lamps -lpfkinsol -lcjson") + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + list(APPEND PDAF_LIBS "-lgfortran") + endif() # GPU # list(APPEND PDAF_LIBS "-L${CMAKE_INSTALL_PREFIX}/rmm/lib -lstdc++ -lcudart -lrmm -lnvToolsExt") @@ -117,6 +120,28 @@ if(DEFINED PARFLOW_SRC) # Unconditional loading of stdc++ list(APPEND PDAF_LIBS "-lstdc++") + # GPU: libpfsimulator and libHYPRE (nvofbf toolchain) contain GPU objects + # requiring CUDA and Umpire libraries at final link of tsmp-pdaf. + if(${ParFlowGPU}) + list(APPEND PDAF_LIBS "-lcudart") + # CUDA math libraries needed by GPU-enabled HYPRE (nvofbf toolchain) + list(APPEND PDAF_LIBS "-lcurand") + list(APPEND PDAF_LIBS "-lcublas") + list(APPEND PDAF_LIBS "-lcusparse") + # Umpire memory manager needed by libamps and libHYPRE (nvofbf toolchain) + find_library(UMPIRE_LIB NAMES umpire + HINTS + ${UMPIRE_ROOT}/lib + $ENV{UMPIRE_ROOT}/lib + $ENV{EBROOTUMPIRE}/lib + ) + if(UMPIRE_LIB) + list(APPEND PDAF_LIBS "${UMPIRE_LIB}") + else() + message(WARNING "BuildPDAFMODEL: Umpire library not found. Set UMPIRE_ROOT or EBROOTUMPIRE if needed.") + endif() + endif() + # SLURM library if enabled (typically on HPC-system) if(ENABLE_SLURM) list(APPEND PDAF_LIBS "-lslurm") diff --git a/cmake/BuildParFlow.cmake b/cmake/BuildParFlow.cmake index b154700e..8dfa3ecb 100644 --- a/cmake/BuildParFlow.cmake +++ b/cmake/BuildParFlow.cmake @@ -74,7 +74,7 @@ else() endif() # Enable Umpire if exists -if (DEFINED ENV{UMPIRE_ROOT} AND NOT ${PDAF}) +if (DEFINED ENV{UMPIRE_ROOT} AND (NOT ${PDAF} OR ${ParFlowGPU})) list(APPEND PF_UMPIRE_FLAG -DUMPIRE_ROOT=$ENV{UMPIRE_ROOT}) if(CMAKE_C_COMPILER_ID STREQUAL "NVHPC") # Umpire leaks wrong OpenMP compile/linker flags when on NVHPC toolchain. diff --git a/models/parflow_pdaf b/models/parflow_pdaf index fa51ec62..f8e18e1b 160000 --- a/models/parflow_pdaf +++ b/models/parflow_pdaf @@ -1 +1 @@ -Subproject commit fa51ec6265e1d95b64c423c3dc4cb2b998573e15 +Subproject commit f8e18e1be291031d18020fef78a46cadb7753bb1