Add Parallel I/O Infrastructure

CMakeLists.txt

@@ -12,15 +12,19 @@ option(CABLE_TESTS "Build CABLE tests" OFF)
 # third party libs
 if(CABLE_MPI)
     find_package(MPI REQUIRED COMPONENTS Fortran)
+    find_package(PIO COMPONENTS Fortran QUIET)
+    if(TARGET PIO::PIO_Fortran)
+        message(STATUS "Found PIO_Fortran: ${PIO_DIR}")
+    endif()
 endif()
 find_package(PkgConfig REQUIRED)
 pkg_check_modules(NETCDF REQUIRED IMPORTED_TARGET "netcdf-fortran")
 
 if(CABLE_TESTS)
     if(CMAKE_Fortran_COMPILER_ID MATCHES "Intel" AND CMAKE_Fortran_COMPILER_VERSION VERSION_LESS "2024.0")
         message(WARNING "Intel Fortran 2024.0 or higher is recommended for building tests.")
-    elseif(CMAKE_Fortran_COMPILER_ID MATCHES "GNU" AND CMAKE_Fortran_COMPILER_VERSION VERSION_LESS "13.2")
-        message(WARNING "GNU Fortran 13.2 or higher is recommended for building tests.")
+    elseif(CMAKE_Fortran_COMPILER_ID MATCHES "GNU" AND CMAKE_Fortran_COMPILER_VERSION VERSION_LESS "14.1")
+        message(WARNING "GNU Fortran 14.1 or higher is recommended for building tests.")
     endif()
     enable_testing()
     option(FORTUNO_WITH_MPI "Fortuno: whether to build the MPI interface" ${CABLE_MPI})
@@ -302,6 +306,12 @@ else()
         src/offline/spincasacnp.F90
         src/util/cable_climate_type_mod.F90
         src/util/masks_cbl.F90
+        src/util/cable_array_utils.F90
+        src/util/netcdf/cable_netcdf_decomp_util.F90
+        src/util/netcdf/cable_netcdf.F90
+        src/util/netcdf/cable_netcdf_init.F90
+        src/util/netcdf/cable_netcdf_stub_types.F90
+        src/util/netcdf/nf90/cable_netcdf_nf90.F90
     )
 
     target_link_libraries(cable_common PRIVATE PkgConfig::NETCDF)
@@ -311,6 +321,13 @@ else()
     target_link_libraries(cable_common PRIVATE MPI::MPI_Fortran)
     endif()
 
+    if(TARGET PIO::PIO_Fortran)
+        target_link_libraries(cable_common PRIVATE PIO::PIO_Fortran)
+        target_sources(cable_common PRIVATE src/util/netcdf/pio/cable_netcdf_pio.F90)
+    else()
+        target_sources(cable_common PRIVATE src/util/netcdf/pio/cable_netcdf_pio_stub.F90)
+    endif()
+
     if(CABLE_MPI)
         add_executable(
             cable-mpi
@@ -336,23 +353,31 @@ else()
     if (CABLE_TESTS)
         add_executable(
             cable-tests
+            tests/fixtures/cable_netcdf_fixtures.F90
+            tests/utils/file_utils.F90
             tests/testapp.F90
-            tests/test_example.F90
+            tests/test_cable_netcdf.F90
         )
         if(CABLE_MPI)
             target_sources(cable-tests PRIVATE tests/utils/fortuno_interface_mpi.F90)
         else()
             target_sources(cable-tests PRIVATE tests/utils/fortuno_interface_serial.F90)
         endif()
         target_link_libraries(cable-tests PRIVATE cable_common ${fortuno_libs})
+        if(NOT TARGET PIO::PIO_Fortran)
+            target_compile_definitions(cable-tests PRIVATE SKIP_PIO_TESTS)
+        endif()
         if(CABLE_MPI)
-            add_test(
-                NAME cable-tests
-                COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 1 $<TARGET_FILE:cable-tests>
+            add_test(NAME cable-tests-serial
+                COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 1 $<TARGET_FILE:cable-tests> ~parallel
+            )
+            set_tests_properties(cable-tests-serial PROPERTIES PROCESSORS 1)
+            add_test(NAME cable-tests-parallel
+                COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 4 $<TARGET_FILE:cable-tests> parallel
             )
-            set_tests_properties(cable-tests PROPERTIES PROCESSORS 1)
+            set_tests_properties(cable-tests-parallel PROPERTIES PROCESSORS 4)
         else()
-            add_test(NAME cable-tests COMMAND $<TARGET_FILE:cable-tests>)
+            add_test(NAME cable-tests-serial COMMAND $<TARGET_FILE:cable-tests> ~parallel)
         endif()
     endif()
 endif()

build.bash

@@ -116,7 +116,7 @@ if hostname -f | grep gadi.nci.org.au > /dev/null; then
             [[ -n ${mpi} ]] && module add intel-mpi/2019.5.281
             ;;
         gnu)
-            module add gcc/13.2.0
+            module add gcc/14.1.0
             compiler_lib_install_dir=GNU
             [[ -n ${mpi} ]] && module add openmpi/4.1.4
             ;;

documentation/docs/user_guide/inputs/cable_nml.md

@@ -134,6 +134,10 @@ The cable.nml file includes some settings that are common across all CABLE appli
 | cable_user%l_limit_labile        | logical            | .TRUE. .FALSE.                                             | .FALSE.                          | Limit labile in spinup.                                                                                 |
 | cable_user%NtilesThruMetFile     | logical            | .TRUE. .FALSE.                                             | .FALSE.                          | Specify Ntiles through met file.                                                                        |
 | cable_user%l_ice_consistency     | logical            | .TRUE. .FALSE.                                             | .FALSE.                          | If true, ensures consistency between soil and vegetation tiles with permanent ice. All tiles with permanent ice for soil will have ice for vegetation and vice-versa. All the parameters for these new ice tiles are updated to the ice parameters.                       |
+| pio_settings%io_tasks            | integer            | any integer greater than 0                                 | 1                                | Number of PIO I/O tasks used to perform distributed I/O operations.                                     |
+| pio_settings%base                | integer            | any integer greater than 0                                 | 1                                | The starting MPI rank index (1-based) used to assign the subset of I/O ranks from compute ranks.        |
+| pio_settings%stride              | integer            | any integer greater than 0                                 | 1                                | The stride used to assign the subset of I/O ranks from compute ranks.                                   |
+| pio_settings%rearr               | character(len=16)  | 'box' 'subset'                                             | 'box'                            | PIO rearranger to use.                                                                                  |
 
 
 <!-- markdown-link-check-disable-line --> [Walker]: https://doi.org/10.1002/ece3.1173

src/offline/cable_driver_common.F90

@@ -17,6 +17,7 @@ MODULE cable_driver_common_mod
     l_landuse,                    &
     l_laiFeedbk,                  &
     l_vcmaxFeedbk,                &
+    pio_settings,                 &
     cable_runtime
   USE cable_IO_vars_module, ONLY : &
     soilparmnew,                   &
@@ -98,7 +99,8 @@ MODULE cable_driver_common_mod
     satuParam,      &
     snmin,          &
     cable_user,     & ! additional USER switches
-    gw_params
+    gw_params,      &
+    pio_settings
 
   PUBLIC :: cable_driver_init
   PUBLIC :: cable_driver_init_gswp

src/offline/cable_mpi.F90

@@ -33,11 +33,14 @@ MODULE cable_mpi_mod
     INTEGER :: size = -1   !! Size of the communicator
   CONTAINS
     PROCEDURE :: abort => mpi_grp_abort !! Send abort signal to processes in this group
+    PROCEDURE :: split => mpi_grp_split !! Split this group into sub-groups
+    PROCEDURE :: comm_defined => mpi_grp_comm_defined !! Check if communicator is defined
   END TYPE mpi_grp_t
 
   INTERFACE mpi_grp_t
     !* Overload the default construct for mpi_grp_t
     PROCEDURE mpi_grp_constructor
+    PROCEDURE mpi_grp_constructor_legacy
   END INTERFACE mpi_grp_t
 
 CONTAINS
@@ -120,6 +123,13 @@ FUNCTION mpi_grp_constructor(comm) RESULT(mpi_grp)
 
   END FUNCTION mpi_grp_constructor
 
+  FUNCTION mpi_grp_constructor_legacy(comm) RESULT(mpi_grp)
+    !* Contructor for mpi_grp_t using the legacy communicator type.
+    INTEGER, INTENT(IN) :: comm !! MPI communicator
+    TYPE(mpi_grp_t) :: mpi_grp
+    mpi_grp = mpi_grp_constructor(MPI_Comm(comm))
+  END FUNCTION mpi_grp_constructor_legacy
+
   SUBROUTINE mpi_grp_abort(this, error_code)
     !* Class method to abort execution of an MPI group.
     CLASS(mpi_grp_t), INTENT(IN) :: this
@@ -139,6 +149,33 @@ SUBROUTINE mpi_grp_abort(this, error_code)
 
   END SUBROUTINE mpi_grp_abort
 
+  SUBROUTINE mpi_grp_split(this, color, key, new_grp)
+    !* Class method to split an MPI group.
+    CLASS(mpi_grp_t), INTENT(IN) :: this
+    INTEGER, INTENT(IN) :: color, key
+    TYPE(mpi_grp_t), INTENT(OUT) :: new_grp
+
+    TYPE(MPI_Comm) :: new_comm
+    INTEGER :: ierr
+
+    IF (this%comm /= MPI_COMM_UNDEFINED) THEN
+#ifdef __MPI__
+      CALL MPI_Comm_split(this%comm, color, key, new_comm, ierr)
+#endif
+      call mpi_check_error(ierr)
+      new_grp = mpi_grp_t(new_comm)
+    ELSE
+      new_grp = mpi_grp_t()
+    END IF
+
+  END SUBROUTINE mpi_grp_split
+
+  LOGICAL FUNCTION mpi_grp_comm_defined(this)
+    !* Class method to check if the communicator is defined.
+    CLASS(mpi_grp_t), INTENT(IN) :: this
+    mpi_grp_comm_defined = this%comm /= MPI_COMM_UNDEFINED
+  END FUNCTION mpi_grp_comm_defined
+
   SUBROUTINE mpi_check_error(ierr)
     !* Check if an MPI return code signaled an error. If so, print the
     ! corresponding message and abort the execution.

src/util/cable_array_utils.F90

@@ -0,0 +1,41 @@
+! CSIRO Open Source Software License Agreement (variation of the BSD / MIT License)
+! Copyright (c) 2015, Commonwealth Scientific and Industrial Research Organisation
+! (CSIRO) ABN 41 687 119 230.
+
+module cable_array_utils_mod
+  !! Utility procedures for working with arrays.
+  implicit none
+  private
+
+  public array_offset
+  public array_index
+
+contains
+
+  !> Calculate the memory offset corresponding to a given index in a multi-dimensional array.
+  function array_offset(index, shape) result(offset)
+    integer, intent(in) :: index(:) !! The multi-dimensional index for which to calculate the offset.
+    integer, intent(in) :: shape(:) !! The shape of the multi-dimensional array.
+    integer :: offset !! The calculated memory offset corresponding to the given index.
+    integer :: i, scale
+    offset = 1; scale = 1
+    do i = 1, size(index)
+      offset = offset + (index(i) - 1) * scale
+      scale = scale * shape(i)
+    end do
+  end function
+
+  !> Calculate the index corresponding to a given memory offset in a multi-dimensional array.
+  subroutine array_index(offset_in, shape, index)
+    integer, intent(in) :: offset_in !! The memory offset for which to calculate the multi-dimensional index.
+    integer, intent(in) :: shape(:) !! The shape of the multi-dimensional array.
+    integer, intent(inout) :: index(:) !! The calculated multi-dimensional index corresponding to the given offset.
+    integer :: i, offset
+    offset = offset_in
+    do i = 1, size(shape)
+      index(i) = mod(offset - 1, shape(i)) + 1
+      offset = (offset - 1) / shape(i) + 1
+    end do
+  end subroutine
+
+end module cable_array_utils_mod

src/util/cable_common.F90

@@ -194,6 +194,15 @@ MODULE cable_common_module
 
   TYPE(gw_parameters_type), SAVE :: gw_params
 
+  TYPE pio_settings_type
+    INTEGER :: io_tasks = 1
+    INTEGER :: stride = 1
+    INTEGER :: base = 1
+    CHARACTER(16) :: rearr = "box"
+  END TYPE pio_settings_type
+
+  TYPE(pio_settings_type) :: pio_settings
+
 ! psi_c and psi_o below inserted by rk4417 - phase2 
   REAL, DIMENSION(17),SAVE :: psi_c = (/-2550000.0,-2550000.0,-2550000.0, &  
                                   -2240000.0,-4280000.0,-2750000.0,-2750000.0,&