SpECTRE Documentation Coverage Report
Current view: top level - Evolution/DiscontinuousGalerkin - InboxTags.hpp Hit Total Coverage
Commit: 9a905b0737f373631c1b8e8389b8f26e67fa5313 Lines: 3 15 20.0 %
Date: 2024-03-28 09:03:18
Legend: Lines: hit not hit

          Line data    Source code
       1           0 : // Distributed under the MIT License.
       2             : // See LICENSE.txt for details.
       3             : 
       4             : #pragma once
       5             : 
       6             : #include <boost/functional/hash.hpp>
       7             : #include <cstddef>
       8             : #include <iomanip>
       9             : #include <map>
      10             : #include <memory>
      11             : #include <optional>
      12             : #include <sstream>
      13             : #include <string>
      14             : #include <tuple>
      15             : #include <type_traits>
      16             : #include <utility>
      17             : 
      18             : #include "Domain/Structure/Direction.hpp"
      19             : #include "Domain/Structure/DirectionalId.hpp"
      20             : #include "Domain/Structure/DirectionalIdMap.hpp"
      21             : #include "Domain/Structure/ElementId.hpp"
      22             : #include "Evolution/DiscontinuousGalerkin/Messages/BoundaryMessage.hpp"
      23             : #include "NumericalAlgorithms/Spectral/Mesh.hpp"
      24             : #include "Parallel/InboxInserters.hpp"
      25             : #include "Time/TimeStepId.hpp"
      26             : #include "Utilities/TMPL.hpp"
      27             : 
      28             : /// \cond
      29             : class DataVector;
      30             : /// \endcond
      31             : 
      32           1 : namespace evolution::dg::Tags {
      33             : /*!
      34             :  * \brief The inbox tag for boundary correction communication and DG-subcell
      35             :  * ghost zone cells.
      36             :  *
      37             :  * The stored data consists of the following (in argument order of the tuple):
      38             :  *
      39             :  * 1. the mesh of the ghost cell data we received. This allows eliding
      40             :  *    projection when all neighboring elements are doing DG.
      41             :  * 2. the mesh of the neighboring element's face (not the mortar mesh!)
      42             :  * 3. the variables at the ghost zone cells for finite difference/volume
      43             :  *    reconstruction
      44             :  * 4. the data on the mortar needed for computing the boundary corrections (e.g.
      45             :  *    fluxes, characteristic speeds, conserved variables)
      46             :  * 5. the TimeStepId beyond which the boundary terms are no longer valid, when
      47             :  *    using local time stepping.
      48             :  *
      49             :  * The TimeStepId is the neighboring element's next time step. When using local
      50             :  * time stepping, the neighbor's boundary data is valid up until this time,
      51             :  * which may include multiple local time steps. By receiving and storing the
      52             :  * neighbor time step, the local element knows whether or not it should remove
      53             :  * boundary data and expect new data to be sent from the neighbor.
      54             :  *
      55             :  * The ghost cell data (second element of the tuple) will be valid whenever a
      56             :  * DG-subcell scheme is being used. Whenever a DG-subcell scheme is being used,
      57             :  * elements using DG and not FD/FV always send both the ghost cells and boundary
      58             :  * correction data together. Elements using FD/FV send the ghost cells first
      59             :  * followed by the boundary correction data once the element has received all
      60             :  * neighbor ghost cell data. Note that the second send/receive only modifies the
      61             :  * flux and the TimeStepId used for the flux validity range.
      62             :  *
      63             :  * When only a DG scheme (not a DG-subcell scheme) is used the ghost cell data
      64             :  * will never be valid.
      65             :  *
      66             :  * In the DG-subcell scheme this tag is used both for communicating the ghost
      67             :  * cell data needed for the FD/FV reconstruction step and the data needed for
      68             :  * the boundary corrections.
      69             :  * - For an element using DG, both ghost cells and boundary corrections are
      70             :  *   sent using a single communication. After receiving all neighbor
      71             :  *   boundary corrections the element can finish computing the time step.
      72             :  *   The ghost cell data from neighbors is unused.
      73             :  * - For an element using FD/FV, first the ghost cells are sent. Once all
      74             :  *   neighboring ghost cell data is received, reconstruction is done and the
      75             :  *   boundary terms are computed and sent to the neighbors. After receiving all
      76             :  *   neighbor boundary corrections the element can finish computing the time
      77             :  *   step.
      78             :  * - Whether or not an extra communication is needed when an element switches
      79             :  *   from DG to FD/FV depends on how exactly the decision to switch is
      80             :  *   implemented. If the volume terms are integrated and verified to be
      81             :  *   valid before a DG element sends ghost cell and boundary data then no
      82             :  *   additional communication is needed when switching from DG to FD/FV. In this
      83             :  *   case a second check of the data that includes the boundary correction needs
      84             :  *   to be done. If the second check determines a switch from DG to FD/FV is
      85             :  *   needed, we can continue to use the DG fluxes since the evolution in the
      86             :  *   small was valid, thus avoiding an additional communication. However, to
      87             :  *   fully guarantee physical realizability a second communication or evolving
      88             :  *   the neighboring ghost cells needs to be done. We have not yet decided how
      89             :  *   to deal with the possible need for an additional communication since it
      90             :  *   also depends on whether or not we decide to couple to Voronoi instead of
      91             :  *   just Neumann neighbors.
      92             :  * - The data for the inbox tags is erased after the boundary correction is
      93             :  *   complete and the solution has been verified to be valid at the new time
      94             :  *   step. The ghost cells could be invalidated immediately after
      95             :  *   reconstruction, thus using the ghost cell data after reconstruction is
      96             :  *   complete is considered undefined behavior. That is, we make no guarantee as
      97             :  *   to whether or not it will work.
      98             :  * - The reason for minimizing the number of communications rather than having a
      99             :  *   more uniform implementation between DG and FD/FV is that it is the number
     100             :  *   of communications that adds the most overhead, not the size of each
     101             :  *   communication. Thus, one large communication is cheaper than several small
     102             :  *   communications.
     103             :  */
     104             : template <size_t Dim>
     105           1 : struct BoundaryCorrectionAndGhostCellsInbox {
     106           0 :   using stored_type =
     107             :       std::tuple<Mesh<Dim>, Mesh<Dim - 1>, std::optional<DataVector>,
     108             :                  std::optional<DataVector>, ::TimeStepId, int>;
     109             : 
     110             :  public:
     111           0 :   using temporal_id = TimeStepId;
     112           0 :   using type = std::map<TimeStepId, DirectionalIdMap<Dim, stored_type>>;
     113             : 
     114             :   template <typename Inbox, typename ReceiveDataType>
     115           0 :   static void insert_into_inbox(const gsl::not_null<Inbox*> inbox,
     116             :                                 const temporal_id& time_step_id,
     117             :                                 ReceiveDataType&& data) {
     118             :     auto& current_inbox = (*inbox)[time_step_id];
     119             :     auto& [volume_mesh_of_ghost_cell_data, face_mesh, ghost_cell_data,
     120             :            boundary_data, boundary_data_validity_range, boundary_tci_status] =
     121             :         data.second;
     122             :     (void)ghost_cell_data;
     123             : 
     124             :     if (auto it = current_inbox.find(data.first); it != current_inbox.end()) {
     125             :       auto& [current_volume_mesh_of_ghost_cell_data, current_face_mesh,
     126             :              current_ghost_cell_data, current_boundary_data,
     127             :              current_boundary_data_validity_range, current_tci_status] =
     128             :           it->second;
     129             :       (void)current_volume_mesh_of_ghost_cell_data;  // Need to use when
     130             :                                                      // optimizing subcell
     131             :       // We have already received some data at this time. Receiving data twice
     132             :       // at the same time should only occur when receiving fluxes after having
     133             :       // previously received ghost cells. We sanity check that the data we
     134             :       // already have is the ghost cells and that we have not yet received flux
     135             :       // data.
     136             :       //
     137             :       // This is used if a 2-send implementation is used (which we don't right
     138             :       // now!). We generally find that the number of communications is more
     139             :       // important than the size of each communication, and so a single
     140             :       // communication per time/sub step is preferred.
     141             :       ASSERT(current_ghost_cell_data.has_value(),
     142             :              "Have not yet received ghost cells at time step "
     143             :                  << time_step_id
     144             :                  << " but the inbox entry already exists. This is a bug in the "
     145             :                     "ordering of the actions.");
     146             :       ASSERT(not current_boundary_data.has_value(),
     147             :              "The fluxes have already been received at time step "
     148             :                  << time_step_id
     149             :                  << ". They are either being received for a second time, there "
     150             :                     "is a bug in the ordering of the actions (though a "
     151             :                     "different ASSERT should've caught that), or the incorrect "
     152             :                     "temporal ID is being sent.");
     153             : 
     154             :       ASSERT(current_face_mesh == face_mesh,
     155             :              "The mesh being received for the fluxes is different than the "
     156             :              "mesh received for the ghost cells. Mesh for fluxes: "
     157             :                  << face_mesh << " mesh for ghost cells " << current_face_mesh);
     158             :       ASSERT(current_volume_mesh_of_ghost_cell_data ==
     159             :                  volume_mesh_of_ghost_cell_data,
     160             :              "The mesh being received for the ghost cell data is different "
     161             :              "than the mesh received previously. Mesh for received when we got "
     162             :              "fluxes: "
     163             :                  << volume_mesh_of_ghost_cell_data
     164             :                  << " mesh received when we got ghost cells "
     165             :                  << current_volume_mesh_of_ghost_cell_data);
     166             : 
     167             :       // We always move here since we take ownership of the data and moves
     168             :       // implicitly decay to copies
     169             :       current_boundary_data = std::move(boundary_data);
     170             :       current_boundary_data_validity_range = boundary_data_validity_range;
     171             :       current_tci_status = boundary_tci_status;
     172             :     } else {
     173             :       // We have not received ghost cells or fluxes at this time.
     174             :       if (not current_inbox.insert(std::forward<ReceiveDataType>(data))
     175             :                   .second) {
     176             :         ERROR("Failed to insert data to receive at instance '"
     177             :               << time_step_id
     178             :               << "' with tag 'BoundaryCorrectionAndGhostCellsInbox'.\n");
     179             :       }
     180             :     }
     181             :   }
     182             : 
     183           0 :   static std::string output_inbox(const type& inbox,
     184             :                                   const size_t padding_size) {
     185             :     std::stringstream ss{};
     186             :     const std::string pad(padding_size, ' ');
     187             :     ss << std::scientific << std::setprecision(16);
     188             :     ss << pad << "BoundaryCorrectionAndGhostCellInbox:\n";
     189             : 
     190             :     for (const auto& [current_time_step_id, hash_map] : inbox) {
     191             :       ss << pad << " Current time: " << current_time_step_id << "\n";
     192             :       // We only care about the next time because that's important for deadlock
     193             :       // detection. The data itself isn't super important
     194             :       for (const auto& [key, tuple_data] : hash_map) {
     195             :         ss << pad << "  Key: " << key
     196             :            << ", next time: " << std::get<4>(tuple_data) << "\n";
     197             :       }
     198             :     }
     199             : 
     200             :     return ss.str();
     201             :   }
     202             : 
     203           0 :   void pup(PUP::er& /*p*/) {}
     204             : };
     205             : 
     206             : /*!
     207             :  * \brief The inbox tag for boundary correction communication and DG-subcell
     208             :  * ghost zone cells using a `BoundaryMessage` object
     209             :  *
     210             :  * To see what is stored within a `BoundaryMessage`, see its documentation.
     211             :  *
     212             :  * This inbox tag is very similar to `BoundaryCorrectionAndGhostCellsInbox` in
     213             :  * that it stores subcell/DG data sent from neighboring elements. To see exactly
     214             :  * when data is stored and how it's used, see the docs for
     215             :  * `BoundaryCorrectionAndGhostCellsInbox`. This inbox tag is different than
     216             :  * `BoundaryCorrectionAndGhostCellsInbox` in that it only takes a pointer to a
     217             :  * `BoundaryMessage` as an argument to `insert_into_inbox` and stores a
     218             :  * `std::unique_ptr<BoundaryMessage>` inside the inbox.
     219             :  *
     220             :  * This inbox tag is meant to be used to avoid unnecessary copies between
     221             :  * elements on the same node which share a block of memory. If two elements
     222             :  * aren't on the same node, then a copy/send is done regardless.
     223             :  *
     224             :  * \warning The `boundary_message` argument to `insert_into_inbox()` will be
     225             :  * invalid after the function is called because a `std::unique_ptr` now controls
     226             :  * the memory. Calling a method on the `boundary_message` pointer after the
     227             :  * `insert_into_inbox()` function is called can result in undefined behaviour.
     228             :  */
     229             : template <size_t Dim>
     230           1 : struct BoundaryMessageInbox {
     231           0 :   using stored_type = std::unique_ptr<BoundaryMessage<Dim>>;
     232             : 
     233             :  public:
     234           0 :   using temporal_id = TimeStepId;
     235           0 :   using type = std::map<TimeStepId, DirectionalIdMap<Dim, stored_type>>;
     236           0 :   using message_type = BoundaryMessage<Dim>;
     237             : 
     238             :   template <typename Inbox>
     239           0 :   static void insert_into_inbox(const gsl::not_null<Inbox*> inbox,
     240             :                                 BoundaryMessage<Dim>* boundary_message) {
     241             :     const auto& time_step_id = boundary_message->current_time_step_id;
     242             :     auto& current_inbox = (*inbox)[time_step_id];
     243             : 
     244             :     const auto key = DirectionalId<Dim>{boundary_message->neighbor_direction,
     245             :                                         boundary_message->element_id};
     246             : 
     247             :     if (auto it = current_inbox.find(key); it != current_inbox.end()) {
     248             :       auto& current_boundary_data = it->second;
     249             :       // We have already received some data at this time. Receiving data twice
     250             :       // at the same time should only occur when receiving fluxes after having
     251             :       // previously received ghost cells. We sanity check that the data we
     252             :       // already have is the ghost cells and that we have not yet received flux
     253             :       // data.
     254             :       //
     255             :       // This is used if a 2-send implementation is used (which we don't right
     256             :       // now!). We generally find that the number of communications is more
     257             :       // important than the size of each communication, and so a single
     258             :       // communication per time/sub step is preferred.
     259             :       ASSERT(current_boundary_data->subcell_ghost_data != nullptr,
     260             :              "Have not yet received ghost cells at time step "
     261             :                  << time_step_id
     262             :                  << " but the inbox entry already exists. This is a bug in the "
     263             :                     "ordering of the actions.");
     264             :       ASSERT(current_boundary_data->dg_flux_data == nullptr,
     265             :              "The fluxes have already been received at time step "
     266             :                  << time_step_id
     267             :                  << ". They are either being received for a second time, there "
     268             :                     "is a bug in the ordering of the actions (though a "
     269             :                     "different ASSERT should've caught that), or the incorrect "
     270             :                     "temporal ID is being sent.");
     271             :       ASSERT(boundary_message->subcell_ghost_data == nullptr,
     272             :              "Have already received ghost cells at time step "
     273             :                  << time_step_id
     274             :                  << ", but we are being sent ghost cells again. This data will "
     275             :                     "not be cleaned up while the dg flux data is being "
     276             :                     "inserted into the inbox which will cause a memory leak.");
     277             : 
     278             :       ASSERT(current_boundary_data->interface_mesh ==
     279             :                  boundary_message->interface_mesh,
     280             :              "The mesh being received for the fluxes is different than the "
     281             :              "mesh received for the ghost cells. Mesh for fluxes: "
     282             :                  << boundary_message->interface_mesh << " mesh for ghost cells "
     283             :                  << current_boundary_data->interface_mesh);
     284             :       ASSERT(current_boundary_data->volume_or_ghost_mesh ==
     285             :                  boundary_message->volume_or_ghost_mesh,
     286             :              "The mesh being received for the ghost cell data is different "
     287             :              "than the mesh received previously. Mesh for received when we got "
     288             :              "fluxes: "
     289             :                  << boundary_message->volume_or_ghost_mesh
     290             :                  << " mesh received when we got ghost cells "
     291             :                  << current_boundary_data->volume_or_ghost_mesh);
     292             : 
     293             :       // Just need to set the pointer. No need to worry about the data being
     294             :       // deleted upon destruction of boundary_message because the destructor of
     295             :       // a BoundaryMessage will only delete the pointer stored inside the
     296             :       // BoundaryMessage. It won't delete the actual data it points to. Now the
     297             :       // unique_ptr owns the data that was previously pointed to by
     298             :       // boundary_message
     299             :       current_boundary_data->dg_flux_data_size =
     300             :           boundary_message->dg_flux_data_size;
     301             :       current_boundary_data->dg_flux_data = boundary_message->dg_flux_data;
     302             :       current_boundary_data->next_time_step_id =
     303             :           boundary_message->next_time_step_id;
     304             :       current_boundary_data->tci_status = boundary_message->tci_status;
     305             :     } else {
     306             :       // We have not received ghost cells or fluxes at this time.
     307             :       // Once we insert boundary_message into the unique_ptr we cannot use
     308             :       // boundary_message anymore because it is invalidated. The unique_ptr now
     309             :       // owns the memory.
     310             :       if (not current_inbox
     311             :                   .insert(std::pair{key, std::unique_ptr<BoundaryMessage<Dim>>(
     312             :                                              boundary_message)})
     313             :                   .second) {
     314             :         ERROR("Failed to insert data to receive at instance '"
     315             :               << time_step_id << "' with tag 'BoundaryMessageInbox'.\n");
     316             :       }
     317             :     }
     318             :   }
     319             : };
     320             : }  // namespace evolution::dg::Tags

Generated by: LCOV version 1.14