diff --git a/.gitmodules b/.gitmodules index 2725566e..6cd53250 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,7 +9,7 @@ [submodule "models/parflow_pdaf"] path = models/parflow_pdaf url = https://github.com/HPSCTerrSys/parflow - branch = v3.14.1-pdaf + branch = v3.15.0-pdaf [submodule "models/pdaf"] path = models/pdaf url = https://github.com/HPSCTerrSys/pdaf.git @@ -17,11 +17,11 @@ [submodule "models/eCLM"] path = models/eCLM url = https://github.com/HPSCTerrSys/eCLM.git - branch = master-parallel-build + branch = dev-iconeclm-exchfix [submodule "models/oasis3-mct"] path = models/oasis3-mct url = https://icg4geo.icg.kfa-juelich.de/ExternalReposPublic/oasis3-mct - branch = v5.0_patched + branch = tsmp-patches-v0.2 [submodule "models/CLM3.5"] path = models/CLM3.5 url = https://github.com/HPSCTerrSys/CLM3.5.git diff --git a/CMakeLists.txt b/CMakeLists.txt index c6ec4ab3..5b8b9777 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -60,7 +60,7 @@ if (${ICON} OR ${ICONGPU}) set(ICON_SRC "${CMAKE_SOURCE_DIR}/models/icon") endif() include(BuildICON) - list(APPEND COMPONENT_MODELS "ICON") + list(APPEND COMPONENT_MODELS ${ICON_ID}) if ("${MODEL_ID}" STREQUAL "") set(MODEL_ID "${ICON_ID}") else() diff --git a/build_tsmp2.sh b/build_tsmp2.sh index 5ae47fba..3698f767 100755 --- a/build_tsmp2.sh +++ b/build_tsmp2.sh @@ -123,14 +123,14 @@ while [[ "$#" -gt 0 ]]; do -q|--quiet) quiet=y;; -v|--verbose) verbose_makefile=y;; --version) echo "$0 version 0.2.0"; exit 0;; - --icon|icon) icon=y iconCPU=y iconCMakeModelID="ICON";; + icon) icon=y iconCPU=y iconCMakeModelID="ICON";; icongpu) icon=y iconGPU=y iconCMakeModelID="ICONGPU";; - --eclm|eclm) eclm=y;; - --parflow|parflow) parflow=y parflowCPU=y parflowCMakeModelID="ParFlow";; - --parflowgpu|parflowgpu) parflow=y parflowGPU=y parflowCMakeModelID="ParFlowGPU";; - --pdaf|pdaf) pdaf=y;; - --cosmo|cosmo) cosmo=y;; - --clm35|--clm3.5|clm3.5) clm35=y;; + eclm) eclm=y;; + parflow) parflow=y parflowCPU=y parflowCMakeModelID="ParFlow";; + parflowgpu) parflow=y parflowGPU=y parflowCMakeModelID="ParFlowGPU";; + pdaf) pdaf=y;; + cosmo) cosmo=y;; + clm3.5) clm35=y;; --no_update) update_compsrc=n;; --clean_first) clean_first=y;; --icon_src) icon_src="$2"; shift ;; @@ -234,9 +234,11 @@ fi if [[ -n "${env}" ]]; then message "Sourcing environment..." - # TODO: Fix this GPU thing on another PR + # TODO: Fix this messy GPU env parameterization on another PR if [[ "$parflowGPU" == "y" ]];then source "${env}" --parflowgpu + elif [[ "$iconGPU" == "y" ]];then + source "${env}" --icongpu else source "${env}" fi diff --git a/cmake/BuildICON.cmake b/cmake/BuildICON.cmake index adea7ba3..c635a6f6 100644 --- a/cmake/BuildICON.cmake +++ b/cmake/BuildICON.cmake @@ -112,7 +112,9 @@ endif() # GPU-specific options if(${ICONGPU}) - list(APPEND ICON_LIBS "-c++libs -nvmalloc -cuda") + if(CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") + list(APPEND ICON_LIBS "-c++libs -nvmalloc -cuda") + endif() list(APPEND EXTRA_CONFIG_ARGS --enable-gpu=openacc --enable-mpi-gpu --enable-cuda-graphs --enable-pgi-inlib) else() list(APPEND EXTRA_CONFIG_ARGS --enable-openmp) diff --git a/cmake/BuildOASIS3MCT.cmake b/cmake/BuildOASIS3MCT.cmake index 0dcbfabd..eee469ee 100644 --- a/cmake/BuildOASIS3MCT.cmake +++ b/cmake/BuildOASIS3MCT.cmake @@ -13,6 +13,15 @@ else() message(WARNING "CMAKE_BUILD_TYPE='${CMAKE_BUILD_TYPE}' is not supported by OASIS3-MCT") endif() +set(CPPDEF_EXTRAS "") +if(${PDAF}) + string(APPEND CPPDEF_EXTRAS " -DUSE_PDAF") +endif() +if(CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") + # NVHPC doesn't support quad precision. https://forums.developer.nvidia.com/t/real-16-implementation/129969 + string(APPEND CPPDEF_EXTRAS " -D__NO_16BYTE_REALS") +endif() + set(OASIS_BLD_DIR ${CMAKE_CURRENT_BINARY_DIR}/OASIS3_MCT/bld) set(OASIS_MAKE_INC ${OASIS_BLD_DIR}/make.inc) set(OASIS_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}/OASIS3-MCT) @@ -35,15 +44,13 @@ file(APPEND ${OASIS_MAKE_INC} "AR = ar\n") file(APPEND ${OASIS_MAKE_INC} "ARFLAGS = -ruv\n") file(APPEND ${OASIS_MAKE_INC} "DYNOPT = -fPIC\n") file(APPEND ${OASIS_MAKE_INC} "LDDYNOPT = -shared\n") -if(${PDAF}) - file(APPEND ${OASIS_MAKE_INC} "CPPDEF = -Duse_netCDF -Duse_comm_$(CHAN) -D__VERBOSE -DTREAT_OVERLAY -DUSE_PDAF\n") -else() - file(APPEND ${OASIS_MAKE_INC} "CPPDEF = -Duse_netCDF -Duse_comm_$(CHAN) -D__VERBOSE -DTREAT_OVERLAY\n") -endif() +file(APPEND ${OASIS_MAKE_INC} "CPPDEF = -Duse_netCDF -Duse_comm_$(CHAN) -D__VERBOSE -DTREAT_OVERLAY ${CPPDEF_EXTRAS}\n") if(CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") file(APPEND ${OASIS_MAKE_INC} "FCBASEFLAGS = ${OPTIM} -I. -ffree-line-length-none -fallow-argument-mismatch ${OpenMP_Fortran_FLAGS}\n") elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "Intel" OR CMAKE_Fortran_COMPILER_ID STREQUAL "IntelLLVM") file(APPEND ${OASIS_MAKE_INC} "FCBASEFLAGS = ${OPTIM} -I. -xCORE-AVX2 -assume byterecl -mt_mpi ${OpenMP_Fortran_FLAGS}\n") +elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") + file(APPEND ${OASIS_MAKE_INC} "FCBASEFLAGS = ${OPTIM} -I. -fPIC ${OpenMP_Fortran_FLAGS}\n") else() message(FATAL_ERROR "Fortran compiler '${CMAKE_Fortran_COMPILER_ID}' is not supported.") endif() @@ -67,7 +74,7 @@ ExternalProject_Add(OASIS3_MCT PREFIX OASIS3_MCT SOURCE_DIR ${OASIS_SRC} BUILD_IN_SOURCE FALSE - CONFIGURE_COMMAND "" + CONFIGURE_COMMAND make -f ${OASIS_SRC}/util/make_dir/TopMakefileOasis3 realclean -C ${OASIS_BLD_DIR} BUILD_COMMAND make -j ${PARALLEL_JOBS} -f ${OASIS_SRC}/util/make_dir/TopMakefileOasis3 static-libs -C ${OASIS_BLD_DIR} INSTALL_COMMAND "" ) diff --git a/cmake/BuildPDAF.cmake b/cmake/BuildPDAF.cmake index b89685c9..2c796e5b 100644 --- a/cmake/BuildPDAF.cmake +++ b/cmake/BuildPDAF.cmake @@ -141,6 +141,24 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") list(APPEND PDAF_FOPT "-fallow-argument-mismatch") list(APPEND PDAF_FOPT "-fcommon") +elseif (CMAKE_CXX_COMPILER_ID STREQUAL "NVHPC") + + # using NVIDIA HPC SDK + if (CMAKE_BUILD_TYPE STREQUAL "RELEASE") + list(APPEND PDAF_FOPT "-O2") + elseif (CMAKE_BUILD_TYPE STREQUAL "DEBUG") + list(APPEND PDAF_FOPT "-O0") + list(APPEND PDAF_FOPT "-g") + list(APPEND PDAF_FOPT "-traceback") + else() + message(FATAL_ERROR "Unsupported CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") + endif() + + list(APPEND PDAF_FOPT "-Msave") + list(APPEND PDAF_FOPT "-Mlarge_arrays") + list(APPEND PDAF_FOPT "-Mbyteswapio") + list(APPEND PDAF_FOPT "-Mfreeform") + else() message(FATAL_ERROR "Unsupported CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}") endif() @@ -189,6 +207,19 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") list(APPEND PDAF_COPT "-mcmodel=large") list(APPEND PDAF_COPT "-fcommon") +elseif (CMAKE_CXX_COMPILER_ID STREQUAL "NVHPC") + + # using NVIDIA HPC SDK + if (CMAKE_BUILD_TYPE STREQUAL "RELEASE") + list(APPEND PDAF_COPT "-O2") + elseif (CMAKE_BUILD_TYPE STREQUAL "DEBUG") + list(APPEND PDAF_COPT "-O0") + list(APPEND PDAF_COPT "-g") + list(APPEND PDAF_COPT "-traceback") + else() + message(FATAL_ERROR "Unsupported CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") + endif() + else() message(FATAL_ERROR "Unsupported CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}") endif() @@ -207,6 +238,10 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") list(APPEND PDAF_DOUBLEPRECISION "-fdefault-real-8") +elseif (CMAKE_CXX_COMPILER_ID STREQUAL "NVHPC") + + list(APPEND PDAF_DOUBLEPRECISION "-r8") + else() message(FATAL_ERROR "Unsupported CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}") endif() @@ -247,6 +282,10 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") list(APPEND PDAF_MODULEOPT "-J") +elseif (CMAKE_CXX_COMPILER_ID STREQUAL "NVHPC") + + list(APPEND PDAF_MODULEOPT "-module") + else() message(FATAL_ERROR "Unsupported CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}") endif() diff --git a/cmake/BuildPDAFMODEL.cmake b/cmake/BuildPDAFMODEL.cmake index 3e7608ae..cf715674 100644 --- a/cmake/BuildPDAFMODEL.cmake +++ b/cmake/BuildPDAFMODEL.cmake @@ -97,7 +97,10 @@ endif() # ParFlow libraries if(DEFINED PARFLOW_SRC) - list(APPEND PDAF_LIBS "-L${CMAKE_INSTALL_PREFIX}/lib -lpfsimulator -lamps -lpfkinsol -lgfortran -lcjson") + list(APPEND PDAF_LIBS "-L${CMAKE_INSTALL_PREFIX}/lib -lpfsimulator -lamps -lpfkinsol -lcjson") + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + list(APPEND PDAF_LIBS "-lgfortran") + endif() # GPU # list(APPEND PDAF_LIBS "-L${CMAKE_INSTALL_PREFIX}/rmm/lib -lstdc++ -lcudart -lrmm -lnvToolsExt") @@ -117,6 +120,28 @@ if(DEFINED PARFLOW_SRC) # Unconditional loading of stdc++ list(APPEND PDAF_LIBS "-lstdc++") + # GPU: libpfsimulator and libHYPRE (nvofbf toolchain) contain GPU objects + # requiring CUDA and Umpire libraries at final link of tsmp-pdaf. + if(${ParFlowGPU}) + list(APPEND PDAF_LIBS "-lcudart") + # CUDA math libraries needed by GPU-enabled HYPRE (nvofbf toolchain) + list(APPEND PDAF_LIBS "-lcurand") + list(APPEND PDAF_LIBS "-lcublas") + list(APPEND PDAF_LIBS "-lcusparse") + # Umpire memory manager needed by libamps and libHYPRE (nvofbf toolchain) + find_library(UMPIRE_LIB NAMES umpire + HINTS + ${UMPIRE_ROOT}/lib + $ENV{UMPIRE_ROOT}/lib + $ENV{EBROOTUMPIRE}/lib + ) + if(UMPIRE_LIB) + list(APPEND PDAF_LIBS "${UMPIRE_LIB}") + else() + message(WARNING "BuildPDAFMODEL: Umpire library not found. Set UMPIRE_ROOT or EBROOTUMPIRE if needed.") + endif() + endif() + # SLURM library if enabled (typically on HPC-system) if(ENABLE_SLURM) list(APPEND PDAF_LIBS "-lslurm") diff --git a/cmake/BuildParFlow.cmake b/cmake/BuildParFlow.cmake index 4f5531d5..546480b9 100644 --- a/cmake/BuildParFlow.cmake +++ b/cmake/BuildParFlow.cmake @@ -53,6 +53,13 @@ if(CMAKE_C_COMPILER_ID STREQUAL "GNU") set(PF_FFLAGS "-ffree-line-length-none -ffixed-line-length-none") elseif(CMAKE_C_COMPILER_ID STREQUAL "Intel" OR CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM") set(PF_CFLAGS "-Wall -Werror -Wno-unused-function -Wno-unused-variable") +elseif(CMAKE_C_COMPILER_ID STREQUAL "NVHPC") + set(PF_CFLAGS "-Werror -Wno-unused --diag_suppress=unsigned_compare_with_negative,vla_treated_as_fixed_size_array,code_is_unreachable") + if(NOT ${ParFlowGPU}) + message(FATAL_ERROR "BuildParFlow: NVHPC compiler detected but ParFlowGPU is not enabled. " + "NVHPC should for now only be used for GPU-accelerated ParFlow builds (parflowGPU). " + "Use a GNU or Intel compiler for CPU-only ParFlow, or enable parflowGPU.") + endif() else() message(FATAL_ERROR "C compiler '${CMAKE_C_COMPILER_ID}' is not supported.") endif() @@ -66,8 +73,14 @@ else() endif() # Enable Umpire if exists -if (DEFINED ENV{UMPIRE_ROOT} AND NOT ${PDAF}) - set(PF_UMPIRE_FLAG "-DUMPIRE_ROOT=$ENV{UMPIRE_ROOT}") +if (DEFINED ENV{UMPIRE_ROOT} AND (NOT ${PDAF} OR ${ParFlowGPU})) + list(APPEND PF_UMPIRE_FLAG -DUMPIRE_ROOT=$ENV{UMPIRE_ROOT}) + if(CMAKE_C_COMPILER_ID STREQUAL "NVHPC") + # Umpire leaks wrong OpenMP compile/linker flags when on NVHPC toolchain. + # See https://github.com/HPSCTerrSys/TSMP2/issues/148 + list(APPEND PF_UMPIRE_FLAG -DBLT_OPENMP_COMPILE_FLAGS=-fopenmp + -DBLT_OPENMP_LINK_FLAGS=-fopenmp) + endif() endif() # Enable SUNDIALS if exists diff --git a/cmake/BuildeCLM.cmake b/cmake/BuildeCLM.cmake index d11854af..941aa593 100644 --- a/cmake/BuildeCLM.cmake +++ b/cmake/BuildeCLM.cmake @@ -1,9 +1,9 @@ -if(${ICON} OR ${ParFlow} OR ${ParFlowGPU}) +if(${ICON} OR ${ICONGPU} OR ${ParFlow} OR ${ParFlowGPU}) list(APPEND COUP_OAS_FLAGS -DUSE_OASIS=True) if(${ParFlow} OR ${ParFlowGPU}) list(APPEND COUP_OAS_FLAGS -DCOUP_OAS_PFL=True) endif() - if(${ICON}) + if(${ICON} OR ${ICONGPU}) list(APPEND COUP_OAS_FLAGS -DCOUP_OAS_ICON=True) endif() else() diff --git a/cmake/Utils.cmake b/cmake/Utils.cmake index 15d11b93..adb37a2f 100644 --- a/cmake/Utils.cmake +++ b/cmake/Utils.cmake @@ -6,7 +6,7 @@ endif() # Count how many component models shoul be build, and if >=2 compile with OASIS function(check_build_oasis BUILD_OASIS) # Combine all options into a list - list(APPEND COMPONENT_MODELS ICON COSMO eCLM CLM3.5 ParFlow ParFlowGPU) + list(APPEND COMPONENT_MODELS ICON ICONGPU COSMO eCLM CLM3.5 ParFlow ParFlowGPU) # Count enabled models set(MODELCOUNT 0) foreach(comp_model IN LISTS COMPONENT_MODELS) diff --git a/env/jsc.2023.intel.psmpi b/env/jsc.2023.intel.psmpi deleted file mode 100644 index 3b9ac71e..00000000 --- a/env/jsc.2023.intel.psmpi +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -# Load modules -module --force purge -module use $OTHERSTAGES -module load Stages/2023 -module load Intel/2022.1.0 -module load ParaStationMPI/5.7.0-1 -# -module load Hypre/2.27.0-cpu -module load Silo/4.11 -module load Tcl/8.6.12 -# -module load ecCodes/2.27.0 -module load HDF5/1.12.2-serial -module load netCDF/4.9.0 -module load netCDF-Fortran/4.6.0 -module load PnetCDF/1.12.3 -module load cURL/7.83.0 -module load Szip/.2.1.1 -module load Python/3.10.4 -module load NCO/5.1.3 -module load CMake/3.23.1 -module load git/2.36.0-nodocs - -module load pkgconf/.1.8.0 - -module li - -# Set default compilers -export CC=mpicc -export FC=mpif90 -export CXX=mpicxx -export MPI_HOME=$EBROOTPSMPI diff --git a/env/jsc.2024.intel.psmpi b/env/jsc.2024.intel.psmpi deleted file mode 100644 index 2908f650..00000000 --- a/env/jsc.2024.intel.psmpi +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -# Load modules -module --force purge -module use $OTHERSTAGES -module load Stages/2024 -module load Intel/2023.2.1 -module load ParaStationMPI/5.9.2-1 -#module load IntelMPI/2021.10.0 - -# -module load Hypre/2.31.0-cpu -module load Silo/4.11.1 -module load Tcl/8.6.13 -# -module load ecCodes/2.31.0 -module load HDF5/1.14.2 -module load netCDF/4.9.2 -module load netCDF-Fortran/4.6.1 -module load PnetCDF/1.12.3 -module load cURL/8.0.1 -module load Szip/.2.1.1 -module load Python/3.11.3 -module load NCO/5.1.8 -module load CMake/3.26.3 -module load git/2.41.0-nodocs - -# Set default compilers -export CC=mpicc -export FC=mpif90 -export CXX=mpicxx -export MPI_HOME=$EBROOTPSMPI - -# Display compiler settings -module li -echo "====================================== COMPILER SETTINGS ======================================" -echo " Machine: ${SYSTEMNAME} on Stages/$STAGE" -echo " MPI lib: $(mpichversion | head -n 1 | tr -d =)" -echo " C: $($CC --version | head -n 1)" -echo " C++: $($CXX --version | head -n 1)" -echo " Fortran: $($FC --version | head -n 1)" -echo "===============================================================================================" -echo "" - diff --git a/env/jsc.2026.gnu.openmpi b/env/jsc.2026.gnu.openmpi index c09c1713..b2b9ace1 100644 --- a/env/jsc.2026.gnu.openmpi +++ b/env/jsc.2026.gnu.openmpi @@ -32,7 +32,7 @@ module load PnetCDF module load Umpire module load SUNDIALS module load Tcl -if [[ "$1" == "--parflowgpu" ]]; then +if [[ "$1" == "--parflowgpu" || "$1" == "--icongpu" ]]; then module load CUDA module load UCX-settings/RC-CUDA module load Hypre/3.0.0 @@ -56,6 +56,10 @@ export CC=mpicc export FC=mpif90 export CXX=mpicxx +# OMPI_* variables affect OpenMPI backend compilers. They should be +# unset so that the correct default backend compilers are selected. +unset OMPI_CC OMPI_FC OMPI_CXX + # Variables read by TSMP2 cmake scripts export MPI_HOME=$EBROOTOPENMPI export UMPIRE_ROOT=$EBROOTUMPIRE @@ -70,7 +74,7 @@ echo " MPI lib: OpenMPI v$EBVERSIONOPENMPI" echo " C: $($CC --version | head -n 1)" echo " C++: $($CXX --version | head -n 1)" echo " Fortran: $($FC --version | head -n 1)" -if [[ "$1" == "--parflowgpu" ]]; then +if [[ "$1" == "--parflowgpu" || "$1" == "--icongpu" ]]; then echo " nvcc: $(nvcc --version | tail -n 1 | cut -d" " -f2)" echo " CUDAARCHS: $CUDAARCHS" fi diff --git a/env/jsc.2026.gnu.psmpi b/env/jsc.2026.gnu.psmpi new file mode 100644 index 00000000..e80a5d8a --- /dev/null +++ b/env/jsc.2026.gnu.psmpi @@ -0,0 +1,78 @@ +# --------------------------------------------------------------------------------------- +# Loads GNU+OpenMPI build environment for TSMP2. +# This environment is tailored for JURECA [1], JUWELS [2], and JUPITER [3] supercomputers. +# +# [1] https://apps.fz-juelich.de/jsc/software/jureca/index.xhtml +# [2] https://apps.fz-juelich.de/jsc/software/juwels/index.xhtml +# [3] https://apps.fz-juelich.de/jsc/hps/jupiter/index.html +# Usage: source jsc.2026.gnu.openmpi +# --------------------------------------------------------------------------------------- + +# Load Stages/2026 +module --force purge +module use $OTHERSTAGES +module load Stages/2026 + +# Primary compiler toolchain +module load GCC +module load ParaStationMPI + +# Basic scripting and build tools +module load Python +module load CMake +module load git + +# Storage libraries +module load HDF5 +module load netCDF +module load netCDF-Fortran +module load PnetCDF + +# ParFlow additional libraries +module load Umpire +module load SUNDIALS +module load Tcl +if [[ "$1" == "--parflowgpu" ]]; then + module load CUDA + module load UCX-settings/RC-CUDA + module load Hypre/3.0.0 + + # TODO: Verify these values + if [[ $SYSTEMNAME == "jupiter" || $SYSTEMNAME == "jureca_gh" ]]; then + export CUDAARCHS="90" + else + export CUDAARCHS="80" + fi + export CMAKE_CUDA_RUNTIME_LIBRARY="Shared" +else + module load Hypre/3.0.0-cpu +fi + +# ICON additional libraries +module load ecCodes + +# Set default MPI compilers +export CC=mpicc +export FC=mpif90 +export CXX=mpicxx + +# Variables read by TSMP2 cmake scripts +export MPI_HOME=$EBROOTPSMPI +export UMPIRE_ROOT=$EBROOTUMPIRE +export SUNDIALS_ROOT=$EBROOTSUNDIALS + +# Display compiler settings +module list +echo "=============== COMPILER SETTINGS ===============" +echo " Machine: ${SYSTEMNAME} on Stages/$STAGE" +echo " MPI lib: ParaStationMPI v$EBVERSIONPSMPI" +echo " C: $($CC --version | head -n 1)" +echo " C++: $($CXX --version | head -n 1)" +echo " Fortran: $($FC --version | head -n 1)" +if [[ "$1" == "--parflowgpu" ]]; then + echo " nvcc: $(nvcc --version | tail -n 1 | cut -d" " -f2)" + echo " CUDAARCHS: $CUDAARCHS" +fi +echo "==================================================" +echo "" + diff --git a/env/jsc.2026.intel.psmpi b/env/jsc.2026.intel.psmpi new file mode 100644 index 00000000..33831ef3 --- /dev/null +++ b/env/jsc.2026.intel.psmpi @@ -0,0 +1,82 @@ +# -------------------------------------------------------------------------- +# Loads IntelLLVM+ParaStationMPI build environment for TSMP2. +# This environment is tailored for JURECA [1] and JUWELS [2] supercomputers. +# +# [1] https://apps.fz-juelich.de/jsc/software/jureca/index.xhtml +# [2] https://apps.fz-juelich.de/jsc/software/juwels/index.xhtml +# +# Usage: source jsc.2026.intel.psmpi +# -------------------------------------------------------------------------- + +# Load Stages/2026 +module --force purge +module use $OTHERSTAGES +module load Stages/2026 + +# Primary compiler toolchain +if [[ "$1" == "--parflowgpu" ]]; then + # TODO: Use a generic GPU switch instead of --parflowgpu + module load CUDA +fi +module load Intel +module load ParaStationMPI + +# Basic scripting and build tools +module load Python +module load CMake +module load git + +# Storage libraries +module load HDF5 +module load netCDF +module load netCDF-Fortran +module load PnetCDF + +# ParFlow additional libraries +if [[ "$1" == "--parflowgpu" ]]; then + module load CUDA + module load UCX-settings/RC-CUDA + module load Hypre/3.0.0 + module load Umpire + + # TODO: Verify these values + export CUDAARCHS="80" + export CMAKE_CUDA_RUNTIME_LIBRARY="Shared" +else + module load Hypre/3.0.0-cpu + module load Umpire +fi +module load Tcl + +# ICON additional libraries +module load ecCodes + +# Set default MPI compilers +export OMPI_CC=icx +export OMPI_CXX=icpx +export OMPI_FC=ifx +export CC=mpicc +export FC=mpif90 +export CXX=mpicxx +export MPI_HOME=$EBROOTPSMPI +export TSMP2_PARALLEL_JOBS=16 + +# Display compiler settings +module list +echo "====================================== COMPILER SETTINGS ======================================" +echo " Machine: ${SYSTEMNAME} on Stages/$STAGE" +echo " MPI lib: $(mpichversion | head -n 1 | tr -d =)" +echo " C: $($CC --version | head -n 1)" +echo " C++: $($CXX --version | head -n 1)" +echo " Fortran: $($FC --version | head -n 1)" +if [[ "$1" == "--parflowgpu" ]]; then + echo " nvcc: $(nvcc --version | tail -n 1 | cut -d" " -f2)" + echo " CUDAARCHS: $CUDAARCHS" +fi +echo "===============================================================================================" +echo "" +if [[ "$1" == "--parflowgpu" ]]; then + echo "WARNING: ParFlowGPU doesn't work on Intel toolchain due to missing Intel-built Hypre-CUDA library." + echo " Load the 'jsc.2025.gnu.openmpi' env file instead to build ParFlowGPU." +fi +echo "" diff --git a/env/jsc.2026.nvhpc.openmpi b/env/jsc.2026.nvhpc.openmpi index 0606c818..31c28044 100644 --- a/env/jsc.2026.nvhpc.openmpi +++ b/env/jsc.2026.nvhpc.openmpi @@ -32,7 +32,11 @@ module load CUDA module load UCX-settings/RC-CUDA module load Hypre module load Umpire -module load SUNDIALS +# +# TODO: Make Parflow+SUNDIALS work on NVHPC. +# Currently Parflow+SUNDIALS produces build errors. +# module load SUNDIALS +# # TODO: Verify these values if [[ $SYSTEMNAME == "jupiter" ]]; then @@ -48,11 +52,17 @@ module load ecCodes # Set default MPI compilers export CC=mpicc export FC=mpifort -export CXX=nvc++ -export CUDACXX=nvcc +export CXX=mpicxx + +# OMPI_* variables affect OpenMPI backend compilers. They should be +# unset so that the correct default backend compilers are selected. +unset OMPI_CC OMPI_FC OMPI_CXX + +# Variables read by TSMP2 cmake scripts +export CUDACXX=$(which nvcc) export MPI_HOME=$EBROOTOPENMPI export UMPIRE_ROOT=$EBROOTUMPIRE -export SUNDIALS_ROOT=$EBROOTSUNDIALS +#export SUNDIALS_ROOT=$EBROOTSUNDIALS export TSMP2_PARALLEL_JOBS=16 # Display compiler settings @@ -63,5 +73,7 @@ echo " MPI lib: OpenMPI v$EBVERSIONOPENMPI" echo " C: $($CC --version | head -n 2 | tail -n 1)" echo " C++: $($CXX --version | head -n 2 | tail -n 1)" echo " Fortran: $($FC --version | head -n 2 | tail -n 1)" +echo " nvcc: $(nvcc --version | tail -n 1 | cut -d" " -f2)" +echo " CUDAARCHS: $CUDAARCHS" echo "======================================================================" diff --git a/models/eCLM b/models/eCLM index 8d9ca1ea..57115693 160000 --- a/models/eCLM +++ b/models/eCLM @@ -1 +1 @@ -Subproject commit 8d9ca1ea7a3418335527568417a889ef3f259634 +Subproject commit 57115693bd0818edec6ef1241aed5b3668bbaaa1 diff --git a/models/oasis3-mct b/models/oasis3-mct index 5253349d..554c09fd 160000 --- a/models/oasis3-mct +++ b/models/oasis3-mct @@ -1 +1 @@ -Subproject commit 5253349d4ce15259fcd76e0443495c1ddb7788bb +Subproject commit 554c09fd88412086d85fc778cdb52fd2ea9e5361 diff --git a/models/parflow_pdaf b/models/parflow_pdaf index fa51ec62..f8e18e1b 160000 --- a/models/parflow_pdaf +++ b/models/parflow_pdaf @@ -1 +1 @@ -Subproject commit fa51ec6265e1d95b64c423c3dc4cb2b998573e15 +Subproject commit f8e18e1be291031d18020fef78a46cadb7753bb1