kvrigor · Oct 9, 2026 · Oct 9, 2026 · Oct 9, 2026 · Oct 9, 2026 · Oct 13, 2026
diff --git a/.gitmodules b/.gitmodules
@@ -9,15 +9,15 @@
 [submodule "models/parflow_pdaf"]
 	path = models/parflow_pdaf
 	url = https://github.com/HPSCTerrSys/parflow
-	branch = v3.14.1-pdaf
+	branch = v3.15.0-pdaf
 [submodule "models/pdaf"]
 	path = models/pdaf
 	url = https://github.com/HPSCTerrSys/pdaf.git
 	branch = tsmp-pdaf-patched-parallel-build
 [submodule "models/eCLM"]
 	path = models/eCLM
 	url = https://github.com/HPSCTerrSys/eCLM.git
-	branch = master-parallel-build
+	branch = dev-iconeclm-exchfix
 [submodule "models/oasis3-mct"]
         path = models/oasis3-mct
         url = https://icg4geo.icg.kfa-juelich.de/ExternalReposPublic/oasis3-mct

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -60,7 +60,7 @@ if (${ICON} OR ${ICONGPU})
     set(ICON_SRC "${CMAKE_SOURCE_DIR}/models/icon")
   endif()
   include(BuildICON)
-  list(APPEND COMPONENT_MODELS "ICON")
+  list(APPEND COMPONENT_MODELS ${ICON_ID})
   if ("${MODEL_ID}" STREQUAL "")
       set(MODEL_ID "${ICON_ID}")
   else()

diff --git a/build_tsmp2.sh b/build_tsmp2.sh
@@ -123,14 +123,14 @@ while [[ "$#" -gt 0 ]]; do
     -q|--quiet) quiet=y;;
     -v|--verbose) verbose_makefile=y;;
     --version) echo "$0 version 0.2.0"; exit 0;;
-    --icon|icon) icon=y iconCPU=y iconCMakeModelID="ICON";;
+    icon) icon=y iconCPU=y iconCMakeModelID="ICON";;
     icongpu) icon=y iconGPU=y iconCMakeModelID="ICONGPU";;
-    --eclm|eclm) eclm=y;;
-    --parflow|parflow) parflow=y parflowCPU=y parflowCMakeModelID="ParFlow";;
-    --parflowgpu|parflowgpu) parflow=y parflowGPU=y parflowCMakeModelID="ParFlowGPU";;
-    --pdaf|pdaf) pdaf=y;;
-    --cosmo|cosmo) cosmo=y;;
-    --clm35|--clm3.5|clm3.5) clm35=y;;
+    eclm) eclm=y;;
+    parflow) parflow=y parflowCPU=y parflowCMakeModelID="ParFlow";;
+    parflowgpu) parflow=y parflowGPU=y parflowCMakeModelID="ParFlowGPU";;
+    pdaf) pdaf=y;;
+    cosmo) cosmo=y;;
+    clm3.5) clm35=y;;
     --no_update) update_compsrc=n;;
     --clean_first) clean_first=y;;
     --icon_src) icon_src="$2"; shift ;;
@@ -234,9 +234,11 @@ fi
 if [[ -n "${env}" ]]; then
   message "Sourcing environment..."
 
-  # TODO: Fix this GPU thing on another PR
+  # TODO: Fix this messy GPU env parameterization on another PR
   if [[ "$parflowGPU" == "y" ]];then
     source "${env}" --parflowgpu
+  elif [[ "$iconGPU" == "y" ]];then
+    source "${env}" --icongpu
   else
     source "${env}"
   fi

diff --git a/cmake/BuildICON.cmake b/cmake/BuildICON.cmake
@@ -112,7 +112,16 @@ endif()
 
 # GPU-specific options
 if(${ICONGPU})
-  list(APPEND ICON_LIBS "-c++libs -nvmalloc -cuda")
+  # TODO: below flags only work for NVHPC
+  #find_package(OpenACC REQUIRED)
+  #string(APPEND ICON_FCFLAGS " ${OpenACC_Fortran_FLAGS}")
+  #message(STATUS "OpenACC_Fortran_FLAGS=${OpenACC_Fortran_FLAGS}")
+
+  if(CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
+    list(APPEND ICON_LIBS "-lcudart")
+  elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC")
+    list(APPEND ICON_LIBS "-c++libs -nvmalloc -cuda")
+  endif()
   list(APPEND EXTRA_CONFIG_ARGS --enable-gpu=openacc --enable-mpi-gpu --enable-cuda-graphs --enable-pgi-inlib)
 else()
   list(APPEND EXTRA_CONFIG_ARGS --enable-openmp)

diff --git a/cmake/BuildOASIS3MCT.cmake b/cmake/BuildOASIS3MCT.cmake
@@ -13,6 +13,15 @@ else()
   message(WARNING "CMAKE_BUILD_TYPE='${CMAKE_BUILD_TYPE}' is not supported by OASIS3-MCT")
 endif()
 
+set(CPPDEF_EXTRAS "")
+if(${PDAF})
+  string(APPEND CPPDEF_EXTRAS " -DUSE_PDAF")
+endif()
+if(CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC")
+  # NVHPC doesn't support quad precision. https://forums.developer.nvidia.com/t/real-16-implementation/129969
+  string(APPEND CPPDEF_EXTRAS " -D__NO_16BYTE_REALS")
+endif()
+
 set(OASIS_BLD_DIR ${CMAKE_CURRENT_BINARY_DIR}/OASIS3_MCT/bld)
 set(OASIS_MAKE_INC ${OASIS_BLD_DIR}/make.inc)
 set(OASIS_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}/OASIS3-MCT)
@@ -35,15 +44,13 @@ file(APPEND  ${OASIS_MAKE_INC} "AR              = ar\n")
 file(APPEND  ${OASIS_MAKE_INC} "ARFLAGS         = -ruv\n")
 file(APPEND  ${OASIS_MAKE_INC} "DYNOPT          = -fPIC\n")
 file(APPEND  ${OASIS_MAKE_INC} "LDDYNOPT        = -shared\n")
-if(${PDAF})
-  file(APPEND  ${OASIS_MAKE_INC} "CPPDEF          = -Duse_netCDF -Duse_comm_$(CHAN) -D__VERBOSE  -DTREAT_OVERLAY -DUSE_PDAF\n")
-else()
-  file(APPEND  ${OASIS_MAKE_INC} "CPPDEF          = -Duse_netCDF -Duse_comm_$(CHAN) -D__VERBOSE  -DTREAT_OVERLAY\n")
-endif()
+file(APPEND  ${OASIS_MAKE_INC} "CPPDEF          = -Duse_netCDF -Duse_comm_$(CHAN) -D__VERBOSE -DTREAT_OVERLAY ${CPPDEF_EXTRAS}\n")
 if(CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
   file(APPEND  ${OASIS_MAKE_INC} "FCBASEFLAGS     = ${OPTIM} -I. -ffree-line-length-none -fallow-argument-mismatch ${OpenMP_Fortran_FLAGS}\n")
 elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "Intel" OR CMAKE_Fortran_COMPILER_ID STREQUAL "IntelLLVM")
   file(APPEND  ${OASIS_MAKE_INC} "FCBASEFLAGS     = ${OPTIM} -I. -xCORE-AVX2 -assume byterecl -mt_mpi ${OpenMP_Fortran_FLAGS}\n")
+elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC")
+  file(APPEND  ${OASIS_MAKE_INC} "FCBASEFLAGS     = ${OPTIM} -I. -fPIC ${OpenMP_Fortran_FLAGS}\n")
 else()
   message(FATAL_ERROR "Fortran compiler '${CMAKE_Fortran_COMPILER_ID}' is not supported.")
 endif()
@@ -67,7 +74,7 @@ ExternalProject_Add(OASIS3_MCT
   PREFIX            OASIS3_MCT
   SOURCE_DIR        ${OASIS_SRC}
   BUILD_IN_SOURCE   FALSE
-  CONFIGURE_COMMAND ""
+  CONFIGURE_COMMAND make -f ${OASIS_SRC}/util/make_dir/TopMakefileOasis3 realclean -C ${OASIS_BLD_DIR}
   BUILD_COMMAND     make -j ${PARALLEL_JOBS} -f ${OASIS_SRC}/util/make_dir/TopMakefileOasis3 static-libs -C ${OASIS_BLD_DIR}
   INSTALL_COMMAND   ""
 )

diff --git a/cmake/BuildPDAF.cmake b/cmake/BuildPDAF.cmake
@@ -141,6 +141,24 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
   list(APPEND PDAF_FOPT "-fallow-argument-mismatch")
   list(APPEND PDAF_FOPT "-fcommon")
 
+elseif (CMAKE_CXX_COMPILER_ID STREQUAL "NVHPC")
+
+  # using NVIDIA HPC SDK
+  if (CMAKE_BUILD_TYPE STREQUAL "RELEASE")
+    list(APPEND PDAF_FOPT "-O2")
+  elseif (CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+    list(APPEND PDAF_FOPT "-O0")
+    list(APPEND PDAF_FOPT "-g")
+    list(APPEND PDAF_FOPT "-traceback")
+  else()
+    message(FATAL_ERROR "Unsupported CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
+  endif()
+
+  list(APPEND PDAF_FOPT "-Msave")
+  list(APPEND PDAF_FOPT "-Mlarge_arrays")
+  list(APPEND PDAF_FOPT "-Mbyteswapio")
+  list(APPEND PDAF_FOPT "-Mfreeform")
+
 else()
   message(FATAL_ERROR "Unsupported CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}")
 endif()
@@ -189,6 +207,19 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
   list(APPEND PDAF_COPT "-mcmodel=large")
   list(APPEND PDAF_COPT "-fcommon")
 
+elseif (CMAKE_CXX_COMPILER_ID STREQUAL "NVHPC")
+
+  # using NVIDIA HPC SDK
+  if (CMAKE_BUILD_TYPE STREQUAL "RELEASE")
+    list(APPEND PDAF_COPT "-O2")
+  elseif (CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+    list(APPEND PDAF_COPT "-O0")
+    list(APPEND PDAF_COPT "-g")
+    list(APPEND PDAF_COPT "-traceback")
+  else()
+    message(FATAL_ERROR "Unsupported CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
+  endif()
+
 else()
   message(FATAL_ERROR "Unsupported CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}")
 endif()
@@ -207,6 +238,10 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
 
   list(APPEND PDAF_DOUBLEPRECISION "-fdefault-real-8")
 
+elseif (CMAKE_CXX_COMPILER_ID STREQUAL "NVHPC")
+
+  list(APPEND PDAF_DOUBLEPRECISION "-r8")
+
 else()
   message(FATAL_ERROR "Unsupported CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}")
 endif()
@@ -247,6 +282,10 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
 
   list(APPEND PDAF_MODULEOPT "-J")
 
+elseif (CMAKE_CXX_COMPILER_ID STREQUAL "NVHPC")
+
+  list(APPEND PDAF_MODULEOPT "-module")
+
 else()
   message(FATAL_ERROR "Unsupported CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}")
 endif()

diff --git a/cmake/BuildPDAFMODEL.cmake b/cmake/BuildPDAFMODEL.cmake
@@ -97,7 +97,10 @@ endif()
 
 # ParFlow libraries
 if(DEFINED PARFLOW_SRC)
-  list(APPEND PDAF_LIBS "-L${CMAKE_INSTALL_PREFIX}/lib -lpfsimulator -lamps -lpfkinsol -lgfortran -lcjson")
+  list(APPEND PDAF_LIBS "-L${CMAKE_INSTALL_PREFIX}/lib -lpfsimulator -lamps -lpfkinsol -lcjson")
+  if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    list(APPEND PDAF_LIBS "-lgfortran")
+  endif()
   # GPU
   # list(APPEND PDAF_LIBS "-L${CMAKE_INSTALL_PREFIX}/rmm/lib -lstdc++ -lcudart -lrmm -lnvToolsExt")
 
@@ -117,6 +120,28 @@ if(DEFINED PARFLOW_SRC)
   # Unconditional loading of stdc++
   list(APPEND PDAF_LIBS "-lstdc++")
 
+  # GPU: libpfsimulator and libHYPRE (nvofbf toolchain) contain GPU objects
+  # requiring CUDA and Umpire libraries at final link of tsmp-pdaf.
+  if(${ParFlowGPU})
+    list(APPEND PDAF_LIBS "-lcudart")
+    # CUDA math libraries needed by GPU-enabled HYPRE (nvofbf toolchain)
+    list(APPEND PDAF_LIBS "-lcurand")
+    list(APPEND PDAF_LIBS "-lcublas")
+    list(APPEND PDAF_LIBS "-lcusparse")
+    # Umpire memory manager needed by libamps and libHYPRE (nvofbf toolchain)
+    find_library(UMPIRE_LIB NAMES umpire
+      HINTS
+        ${UMPIRE_ROOT}/lib
+        $ENV{UMPIRE_ROOT}/lib
+        $ENV{EBROOTUMPIRE}/lib
+    )
+    if(UMPIRE_LIB)
+      list(APPEND PDAF_LIBS "${UMPIRE_LIB}")
+    else()
+      message(WARNING "BuildPDAFMODEL: Umpire library not found. Set UMPIRE_ROOT or EBROOTUMPIRE if needed.")
+    endif()
+  endif()
+
   # SLURM library if enabled (typically on HPC-system)
   if(ENABLE_SLURM)
     list(APPEND PDAF_LIBS "-lslurm")

diff --git a/cmake/BuildParFlow.cmake b/cmake/BuildParFlow.cmake
@@ -53,6 +53,14 @@ if(CMAKE_C_COMPILER_ID STREQUAL "GNU")
     set(PF_FFLAGS "-ffree-line-length-none -ffixed-line-length-none")
 elseif(CMAKE_C_COMPILER_ID STREQUAL "Intel" OR CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM")
     set(PF_CFLAGS "-Wall -Werror -Wno-unused-function -Wno-unused-variable")
+elseif(CMAKE_C_COMPILER_ID STREQUAL "NVHPC")
+    set(PF_CFLAGS "-Werror -Wno-unused --diag_suppress=unsigned_compare_with_negative,vla_treated_as_fixed_size_array,code_is_unreachable")
+    #set(PF_CFLAGS "-Werror -Wno-unused")
+    if(NOT ${ParFlowGPU})
+      message(FATAL_ERROR "BuildParFlow: NVHPC compiler detected but ParFlowGPU is not enabled. "
+        "NVHPC should for now  only be used for GPU-accelerated ParFlow builds (parflowGPU). "
+        "Use a GNU or Intel compiler for CPU-only ParFlow, or enable parflowGPU.")
+    endif()
 else()
   message(FATAL_ERROR "C compiler '${CMAKE_C_COMPILER_ID}' is not supported.")
 endif()
@@ -66,8 +74,14 @@ else()
 endif()
 
 # Enable Umpire if exists
-if (DEFINED ENV{UMPIRE_ROOT} AND NOT ${PDAF})
-  set(PF_UMPIRE_FLAG "-DUMPIRE_ROOT=$ENV{UMPIRE_ROOT}")
+if (DEFINED ENV{UMPIRE_ROOT} AND (NOT ${PDAF} OR ${ParFlowGPU}))
+  list(APPEND PF_UMPIRE_FLAG -DUMPIRE_ROOT=$ENV{UMPIRE_ROOT})
+  if(CMAKE_C_COMPILER_ID STREQUAL "NVHPC")
+    # Umpire leaks wrong OpenMP compile/linker flags when on NVHPC toolchain.
+    # See https://github.com/HPSCTerrSys/TSMP2/issues/148
+    list(APPEND PF_UMPIRE_FLAG -DBLT_OPENMP_COMPILE_FLAGS=-fopenmp
+                               -DBLT_OPENMP_LINK_FLAGS=-fopenmp)
+  endif()
 endif()
 
 # Enable SUNDIALS if exists

diff --git a/cmake/BuildeCLM.cmake b/cmake/BuildeCLM.cmake
@@ -1,9 +1,9 @@
-if(${ICON} OR ${ParFlow} OR ${ParFlowGPU})
+if(${ICON} OR ${ICONGPU} OR ${ParFlow} OR ${ParFlowGPU})
     list(APPEND COUP_OAS_FLAGS -DUSE_OASIS=True)
     if(${ParFlow} OR ${ParFlowGPU})
         list(APPEND COUP_OAS_FLAGS  -DCOUP_OAS_PFL=True)
     endif()
-    if(${ICON})
+    if(${ICON} OR ${ICONGPU})
         list(APPEND COUP_OAS_FLAGS  -DCOUP_OAS_ICON=True)
     endif()
 else()

diff --git a/cmake/Utils.cmake b/cmake/Utils.cmake
@@ -6,7 +6,7 @@ endif()
 # Count how many component models shoul be build, and if >=2 compile with OASIS
 function(check_build_oasis BUILD_OASIS)
    # Combine all options into a list
-   list(APPEND COMPONENT_MODELS ICON COSMO eCLM CLM3.5 ParFlow ParFlowGPU)
+   list(APPEND COMPONENT_MODELS ICON ICONGPU COSMO eCLM CLM3.5 ParFlow ParFlowGPU)
    # Count enabled models
    set(MODELCOUNT 0)
    foreach(comp_model IN LISTS COMPONENT_MODELS)

diff --git a/env/jsc.2023.intel.psmpi b/env/jsc.2023.intel.psmpi
diff --git a/env/jsc.2024.intel.psmpi b/env/jsc.2024.intel.psmpi
diff --git a/env/jsc.2026.gnu.openmpi b/env/jsc.2026.gnu.openmpi
@@ -32,7 +32,7 @@ module load PnetCDF
 module load Umpire
 module load SUNDIALS
 module load Tcl
-if [[ "$1" == "--parflowgpu" ]]; then
+if [[ "$1" == "--parflowgpu" || "$1" == "--icongpu" ]]; then
   module load CUDA
   module load UCX-settings/RC-CUDA
   module load Hypre/3.0.0
@@ -56,6 +56,10 @@ export CC=mpicc
 export FC=mpif90
 export CXX=mpicxx
 
+# OMPI_* variables affect OpenMPI backend compilers. They should be
+# unset so that the correct default backend compilers are selected.
+unset OMPI_CC OMPI_FC OMPI_CXX
+
 # Variables read by TSMP2 cmake scripts
 export MPI_HOME=$EBROOTOPENMPI
 export UMPIRE_ROOT=$EBROOTUMPIRE
@@ -70,7 +74,7 @@ echo "   MPI lib: OpenMPI v$EBVERSIONOPENMPI"
 echo "         C: $($CC --version | head -n 1)"
 echo "       C++: $($CXX --version | head -n 1)"
 echo "   Fortran: $($FC --version | head -n 1)"
-if [[ "$1" == "--parflowgpu" ]]; then
+if [[ "$1" == "--parflowgpu" || "$1" == "--icongpu" ]]; then
   echo "      nvcc: $(nvcc --version | tail -n 1 | cut -d" " -f2)"
   echo " CUDAARCHS: $CUDAARCHS"
 fi