Line data    Source code

       1           0 : // Distributed under the MIT License.
       2             : // See LICENSE.txt for details.
       3             : 
       4             : #pragma once
       5             : 
       6             : #include <cstddef>
       7             : #include <map>
       8             : #include <memory>
       9             : #include <string>
      10             : #include <utility>
      11             : 
      12             : #include "DataStructures/DataBox/Tag.hpp"
      13             : #include "Domain/Structure/DirectionalId.hpp"
      14             : #include "Domain/Structure/DirectionalIdMap.hpp"
      15             : #include "Evolution/DiscontinuousGalerkin/AtomicInboxBoundaryData.hpp"
      16             : #include "Evolution/DiscontinuousGalerkin/BoundaryData.hpp"
      17             : #include "Evolution/DiscontinuousGalerkin/InboxBoundaryData.hpp"
      18             : #include "Evolution/DiscontinuousGalerkin/Messages/BoundaryMessage.hpp"
      19             : #include "Time/TimeStepId.hpp"
      20             : #include "Utilities/ErrorHandling/Assert.hpp"
      21             : #include "Utilities/ErrorHandling/Error.hpp"
      22             : #include "Utilities/Gsl.hpp"
      23             : #include "Utilities/TMPL.hpp"
      24             : 
      25           1 : namespace evolution::dg::Tags {
      26             : /*!
      27             :  * \brief The inbox tag for boundary correction communication and DG-subcell
      28             :  * ghost zone cells.
      29             :  *
      30             :  * The stored data consists of the following:
      31             :  *
      32             :  * 1. the volume mesh of the element.
      33             :  * 2. the volume mesh corresponding to the ghost cell data. This allows eliding
      34             :  *    projection when all neighboring elements are doing DG.
      35             :  * 3. the mortar mesh of the data on the mortar
      36             :  * 4. the variables at the ghost zone cells for finite difference/volume
      37             :  *    reconstruction
      38             :  * 5. the data on the mortar needed for computing the boundary corrections (e.g.
      39             :  *    fluxes, characteristic speeds, conserved variables)
      40             :  * 6. the TimeStepId beyond which the boundary terms are no longer valid, when
      41             :  *    using local time stepping.
      42             :  * 7. the troublade cell indicator status using for determining halos around
      43             :  *    troubled cells.
      44             :  * 8. the integration order of the time-stepper.
      45             :  * 9. the InterpolatedBoundaryData sent by a non-conforming Element that
      46             :  *    interpolates its data to a subset of the points of the Element receiving
      47             :  *    this BoundaryData
      48             :  *
      49             :  * The TimeStepId is the neighboring element's next time step. When using local
      50             :  * time stepping, the neighbor's boundary data is valid up until this time,
      51             :  * which may include multiple local time steps. By receiving and storing the
      52             :  * neighbor time step, the local element knows whether or not it should remove
      53             :  * boundary data and expect new data to be sent from the neighbor.
      54             :  *
      55             :  * The ghost cell data will be valid whenever a DG-subcell scheme is being used.
      56             :  * Whenever a DG-subcell scheme is being used, elements using DG and not FD/FV
      57             :  * always send both the ghost cells and boundary correction data together.
      58             :  * Elements using FD/FV send the ghost cells first followed by the boundary
      59             :  * correction data once the element has received all neighbor ghost cell data.
      60             :  * Note that the second send/receive only modifies the flux and the TimeStepId
      61             :  * used for the flux validity range.
      62             :  *
      63             :  * When only a DG scheme (not a DG-subcell scheme) is used the ghost cell data
      64             :  * will never be valid.
      65             :  *
      66             :  * In the DG-subcell scheme this tag is used both for communicating the ghost
      67             :  * cell data needed for the FD/FV reconstruction step and the data needed for
      68             :  * the boundary corrections.
      69             :  * - For an element using DG, both ghost cells and boundary corrections are
      70             :  *   sent using a single communication. After receiving all neighbor
      71             :  *   boundary corrections the element can finish computing the time step.
      72             :  *   The ghost cell data from neighbors is unused.
      73             :  * - For an element using FD/FV, first the ghost cells are sent. Once all
      74             :  *   neighboring ghost cell data is received, reconstruction is done and the
      75             :  *   boundary terms are computed and sent to the neighbors. After receiving all
      76             :  *   neighbor boundary corrections the element can finish computing the time
      77             :  *   step.
      78             :  * - Whether or not an extra communication is needed when an element switches
      79             :  *   from DG to FD/FV depends on how exactly the decision to switch is
      80             :  *   implemented. If the volume terms are integrated and verified to be
      81             :  *   valid before a DG element sends ghost cell and boundary data then no
      82             :  *   additional communication is needed when switching from DG to FD/FV. In this
      83             :  *   case a second check of the data that includes the boundary correction needs
      84             :  *   to be done. If the second check determines a switch from DG to FD/FV is
      85             :  *   needed, we can continue to use the DG fluxes since the evolution in the
      86             :  *   small was valid, thus avoiding an additional communication. However, to
      87             :  *   fully guarantee physical realizability a second communication or evolving
      88             :  *   the neighboring ghost cells needs to be done. We have not yet decided how
      89             :  *   to deal with the possible need for an additional communication since it
      90             :  *   also depends on whether or not we decide to couple to Voronoi instead of
      91             :  *   just Neumann neighbors.
      92             :  * - The data for the inbox tags is erased after the boundary correction is
      93             :  *   complete and the solution has been verified to be valid at the new time
      94             :  *   step. The ghost cells could be invalidated immediately after
      95             :  *   reconstruction, thus using the ghost cell data after reconstruction is
      96             :  *   complete is considered undefined behavior. That is, we make no guarantee as
      97             :  *   to whether or not it will work.
      98             :  * - The reason for minimizing the number of communications rather than having a
      99             :  *   more uniform implementation between DG and FD/FV is that it is the number
     100             :  *   of communications that adds the most overhead, not the size of each
     101             :  *   communication. Thus, one large communication is cheaper than several small
     102             :  *   communications.
     103             :  *
     104             :  * #### DG Element Nodegroup Support
     105             :  * If you are using the `DgElementCollection` then you must set
     106             :  * `UseNodegroupDgElements` to `true`. The actions that use this tag check
     107             :  * that the parallel component and the `UseNodegroupDgElements` is consistent.
     108             :  */
     109             : template <size_t Dim, bool UseNodegroupDgElements>
     110           1 : struct BoundaryCorrectionAndGhostCellsInbox {
     111           0 :   using stored_type = evolution::dg::BoundaryData<Dim>;
     112             : 
     113             :  public:
     114           0 :   using temporal_id = TimeStepId;
     115             :   // Used by array implementation
     116           0 :   using type_map = evolution::dg::InboxBoundaryData<Dim>;
     117             : 
     118             :   // Used by nodegroup implementation
     119           0 :   using type_spsc = evolution::dg::AtomicInboxBoundaryData<Dim>;
     120             : 
     121             :   // The actual type being used.
     122           0 :   using type = tmpl::conditional_t<UseNodegroupDgElements, type_spsc, type_map>;
     123           0 :   using value_type = type;
     124             : 
     125           0 :   static bool insert_into_inbox(
     126             :       gsl::not_null<type_spsc*> inbox, const temporal_id& time_step_id,
     127             :       std::pair<DirectionalId<Dim>, evolution::dg::BoundaryData<Dim>> data);
     128             : 
     129           0 :   static bool insert_into_inbox(
     130             :       gsl::not_null<type_map*> inbox, const temporal_id& time_step_id,
     131             :       std::pair<DirectionalId<Dim>, evolution::dg::BoundaryData<Dim>> data);
     132             : 
     133           0 :   static std::string output_inbox(const type_spsc& inbox, size_t padding_size);
     134             : 
     135           0 :   static std::string output_inbox(const type_map& inbox, size_t padding_size);
     136             : };
     137             : 
     138             : /*!
     139             :  * \brief The inbox tag for boundary correction communication and DG-subcell
     140             :  * ghost zone cells using a `BoundaryMessage` object
     141             :  *
     142             :  * To see what is stored within a `BoundaryMessage`, see its documentation.
     143             :  *
     144             :  * This inbox tag is very similar to `BoundaryCorrectionAndGhostCellsInbox` in
     145             :  * that it stores subcell/DG data sent from neighboring elements. To see exactly
     146             :  * when data is stored and how it's used, see the docs for
     147             :  * `BoundaryCorrectionAndGhostCellsInbox`. This inbox tag is different than
     148             :  * `BoundaryCorrectionAndGhostCellsInbox` in that it only takes a pointer to a
     149             :  * `BoundaryMessage` as an argument to `insert_into_inbox` and stores a
     150             :  * `std::unique_ptr<BoundaryMessage>` inside the inbox.
     151             :  *
     152             :  * This inbox tag is meant to be used to avoid unnecessary copies between
     153             :  * elements on the same node which share a block of memory. If two elements
     154             :  * aren't on the same node, then a copy/send is done regardless.
     155             :  *
     156             :  * \warning The `boundary_message` argument to `insert_into_inbox()` will be
     157             :  * invalid after the function is called because a `std::unique_ptr` now controls
     158             :  * the memory. Calling a method on the `boundary_message` pointer after the
     159             :  * `insert_into_inbox()` function is called can result in undefined behaviour.
     160             :  */
     161             : template <size_t Dim>
     162           1 : struct BoundaryMessageInbox {
     163           0 :   using stored_type = std::unique_ptr<BoundaryMessage<Dim>>;
     164             : 
     165             :  public:
     166           0 :   using temporal_id = TimeStepId;
     167           0 :   using type = std::map<TimeStepId, DirectionalIdMap<Dim, stored_type>>;
     168           0 :   using message_type = BoundaryMessage<Dim>;
     169             : 
     170             :   template <typename Inbox>
     171           0 :   static bool insert_into_inbox(const gsl::not_null<Inbox*> inbox,
     172             :                                 BoundaryMessage<Dim>* boundary_message) {
     173             :     const auto& time_step_id = boundary_message->current_time_step_id;
     174             :     auto& current_inbox = (*inbox)[time_step_id];
     175             : 
     176             :     const auto key = DirectionalId<Dim>{boundary_message->neighbor_direction,
     177             :                                         boundary_message->element_id};
     178             : 
     179             :     if (auto it = current_inbox.find(key); it != current_inbox.end()) {
     180             :       auto& current_boundary_data = it->second;
     181             :       // We have already received some data at this time. Receiving data twice
     182             :       // at the same time should only occur when receiving fluxes after having
     183             :       // previously received ghost cells. We sanity check that the data we
     184             :       // already have is the ghost cells and that we have not yet received flux
     185             :       // data.
     186             :       //
     187             :       // This is used if a 2-send implementation is used (which we don't right
     188             :       // now!). We generally find that the number of communications is more
     189             :       // important than the size of each communication, and so a single
     190             :       // communication per time/sub step is preferred.
     191             :       ASSERT(current_boundary_data->subcell_ghost_data != nullptr,
     192             :              "Have not yet received ghost cells at time step "
     193             :                  << time_step_id
     194             :                  << " but the inbox entry already exists. This is a bug in the "
     195             :                     "ordering of the actions.");
     196             :       ASSERT(current_boundary_data->dg_flux_data == nullptr,
     197             :              "The fluxes have already been received at time step "
     198             :                  << time_step_id
     199             :                  << ". They are either being received for a second time, there "
     200             :                     "is a bug in the ordering of the actions (though a "
     201             :                     "different ASSERT should've caught that), or the incorrect "
     202             :                     "temporal ID is being sent.");
     203             :       ASSERT(boundary_message->subcell_ghost_data == nullptr,
     204             :              "Have already received ghost cells at time step "
     205             :                  << time_step_id
     206             :                  << ", but we are being sent ghost cells again. This data will "
     207             :                     "not be cleaned up while the dg flux data is being "
     208             :                     "inserted into the inbox which will cause a memory leak.");
     209             : 
     210             :       ASSERT(current_boundary_data->interface_mesh ==
     211             :                  boundary_message->interface_mesh,
     212             :              "The mesh being received for the fluxes is different than the "
     213             :              "mesh received for the ghost cells. Mesh for fluxes: "
     214             :                  << boundary_message->interface_mesh << " mesh for ghost cells "
     215             :                  << current_boundary_data->interface_mesh);
     216             :       ASSERT(current_boundary_data->volume_or_ghost_mesh ==
     217             :                  boundary_message->volume_or_ghost_mesh,
     218             :              "The mesh being received for the ghost cell data is different "
     219             :              "than the mesh received previously. Mesh for received when we got "
     220             :              "fluxes: "
     221             :                  << boundary_message->volume_or_ghost_mesh
     222             :                  << " mesh received when we got ghost cells "
     223             :                  << current_boundary_data->volume_or_ghost_mesh);
     224             : 
     225             :       // Just need to set the pointer. No need to worry about the data being
     226             :       // deleted upon destruction of boundary_message because the destructor of
     227             :       // a BoundaryMessage will only delete the pointer stored inside the
     228             :       // BoundaryMessage. It won't delete the actual data it points to. Now the
     229             :       // unique_ptr owns the data that was previously pointed to by
     230             :       // boundary_message
     231             :       current_boundary_data->dg_flux_data_size =
     232             :           boundary_message->dg_flux_data_size;
     233             :       current_boundary_data->dg_flux_data = boundary_message->dg_flux_data;
     234             :       current_boundary_data->next_time_step_id =
     235             :           boundary_message->next_time_step_id;
     236             :       current_boundary_data->tci_status = boundary_message->tci_status;
     237             :       current_boundary_data->integration_order =
     238             :           boundary_message->integration_order;
     239             :     } else {
     240             :       // We have not received ghost cells or fluxes at this time.
     241             :       // Once we insert boundary_message into the unique_ptr we cannot use
     242             :       // boundary_message anymore because it is invalidated. The unique_ptr now
     243             :       // owns the memory.
     244             :       if (not current_inbox
     245             :                   .insert(std::pair{key, std::unique_ptr<BoundaryMessage<Dim>>(
     246             :                                              boundary_message)})
     247             :                   .second) {
     248             :         ERROR("Failed to insert data to receive at instance '"
     249             :               << time_step_id << "' with tag 'BoundaryMessageInbox'.\n");
     250             :       }
     251             :     }
     252             :     return true;
     253             :   }
     254             : };
     255             : }  // namespace evolution::dg::Tags