From 2e36735ddd74e4ba54d13984418a709e224cb5c8 Mon Sep 17 00:00:00 2001
From: Alisa Galishnikova <alisag@login11.frontier.olcf.ornl.gov>
Date: Tue, 31 Mar 2026 15:22:16 -0400
Subject: [PATCH 1/2] A-phi works correctly with MPI on frontier

---
 src/framework/domain/metadomain_io.cpp | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/framework/domain/metadomain_io.cpp b/src/framework/domain/metadomain_io.cpp
index b9f351a0..0d59d97e 100644
--- a/src/framework/domain/metadomain_io.cpp
+++ b/src/framework/domain/metadomain_io.cpp
@@ -284,7 +284,9 @@ namespace ntt {
                               std::make_pair(N_GHOSTS, N_GHOSTS + nx1),
                               N_GHOSTS + nx2 - 1,
                               buff_idx));
-            MPI_Send(aphi_r.data(),
+            auto aphi_r_h = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace {},
+                                                                 aphi_r);
+            MPI_Send(aphi_r_h.data(),
                      nx1,
                      mpi::get_type<real_t>(),
                      rank_recv,
@@ -293,13 +295,16 @@ namespace ntt {
           } else if (static_cast<unsigned int>(local_domain->mpi_rank()) ==
                      rank_recv) {
             array_t<real_t*> aphi_r { "Aphi_r", nx1 };
-            MPI_Recv(aphi_r.data(),
+            auto             aphi_r_h = Kokkos::create_mirror_view(Kokkos::HostSpace {},
+                                                                    aphi_r);
+            MPI_Recv(aphi_r_h.data(),
                      nx1,
                      mpi::get_type<real_t>(),
                      rank_send,
                      0,
                      MPI_COMM_WORLD,
                      MPI_STATUS_IGNORE);
+            Kokkos::deep_copy(aphi_r, aphi_r_h);
             ExtractVectorPotential<S, M>(buffer, aphi_r, buff_idx, local_domain->mesh);
           }
         }

From c08982f04c2d0e98a94ffd2fa4642e97832fdbfc Mon Sep 17 00:00:00 2001
From: Alisa Galishnikova <alisag@login04.frontier.olcf.ornl.gov>
Date: Mon, 6 Apr 2026 14:30:28 -0400
Subject: [PATCH 2/2] ifdef for gpu aware mpi on

---
 src/framework/domain/metadomain_io.cpp | 41 +++++++++++++++-----------
 1 file changed, 23 insertions(+), 18 deletions(-)

diff --git a/src/framework/domain/metadomain_io.cpp b/src/framework/domain/metadomain_io.cpp
index 0d59d97e..2fc8c7b8 100644
--- a/src/framework/domain/metadomain_io.cpp
+++ b/src/framework/domain/metadomain_io.cpp
@@ -284,19 +284,35 @@ namespace ntt {
                               std::make_pair(N_GHOSTS, N_GHOSTS + nx1),
                               N_GHOSTS + nx2 - 1,
                               buff_idx));
-            auto aphi_r_h = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace {},
-                                                                 aphi_r);
+#if !defined(DEVICE_ENABLED) || defined(GPU_AWARE_MPI)
+            MPI_Send(aphi_r.data(),
+                     nx1,
+                     mpi::get_type<real_t>(),
+                     rank_recv,
+                     0,
+                     MPI_COMM_WORLD);
+#else
+            auto aphi_r_h = Kokkos::create_mirror_view(aphi_r);
+            Kokkos::deep_copy(aphi_r_h, aphi_r);
             MPI_Send(aphi_r_h.data(),
                      nx1,
                      mpi::get_type<real_t>(),
                      rank_recv,
                      0,
                      MPI_COMM_WORLD);
-          } else if (static_cast<unsigned int>(local_domain->mpi_rank()) ==
-                     rank_recv) {
+#endif
+          } else if (local_domain->mpi_rank() == rank_recv) {
             array_t<real_t*> aphi_r { "Aphi_r", nx1 };
-            auto             aphi_r_h = Kokkos::create_mirror_view(Kokkos::HostSpace {},
-                                                                    aphi_r);
+#if !defined(DEVICE_ENABLED) || defined(GPU_AWARE_MPI)
+            MPI_Recv(aphi_r.data(),
+                     nx1,
+                     mpi::get_type<real_t>(),
+                     rank_send,
+                     0,
+                     MPI_COMM_WORLD,
+                     MPI_STATUS_IGNORE);
+#else
+            auto aphi_r_h = Kokkos::create_mirror_view(aphi_r);
             MPI_Recv(aphi_r_h.data(),
                      nx1,
                      mpi::get_type<real_t>(),
@@ -305,6 +321,7 @@ namespace ntt {
                      MPI_COMM_WORLD,
                      MPI_STATUS_IGNORE);
             Kokkos::deep_copy(aphi_r, aphi_r_h);
+#endif
             ExtractVectorPotential<S, M>(buffer, aphi_r, buff_idx, local_domain->mesh);
           }
         }
@@ -487,18 +504,6 @@ namespace ntt {
                                                  {},
                                                  local_domain->fields.bckp,
                                                  c);
-            } else if (fld.id() == FldsID::V) {
-              if constexpr (S != SimEngine::GRPIC) {
-                ComputeMoments<S, M, FldsID::V>(params,
-                                                local_domain->mesh,
-                                                local_domain->species,
-                                                fld.species,
-                                                fld.comp[0],
-                                                local_domain->fields.bckp,
-                                                c);
-              } else {
-                raise::Error("Bulk velocity not supported for GRPIC", HERE);
-              }
             } else {
               raise::Error("Wrong moment requested for output", HERE);
             }