Line data    Source code

       1           0 : // Distributed under the MIT License.
       2             : // See LICENSE.txt for details.
       3             : 
       4             : #pragma once
       5             : 
       6             : #include <cstddef>
       7             : #include <map>
       8             : #include <memory>
       9             : #include <string>
      10             : #include <utility>
      11             : 
      12             : #include "Domain/Structure/DirectionalId.hpp"
      13             : #include "Domain/Structure/DirectionalIdMap.hpp"
      14             : #include "Evolution/DiscontinuousGalerkin/AtomicInboxBoundaryData.hpp"
      15             : #include "Evolution/DiscontinuousGalerkin/BoundaryData.hpp"
      16             : #include "Evolution/DiscontinuousGalerkin/InboxBoundaryData.hpp"
      17             : #include "Evolution/DiscontinuousGalerkin/Messages/BoundaryMessage.hpp"
      18             : #include "Time/TimeStepId.hpp"
      19             : #include "Utilities/ErrorHandling/Error.hpp"
      20             : #include "Utilities/Gsl.hpp"
      21             : #include "Utilities/TMPL.hpp"
      22             : 
      23           1 : namespace evolution::dg::Tags {
      24             : /*!
      25             :  * \brief The inbox tag for boundary correction communication and DG-subcell
      26             :  * ghost zone cells.
      27             :  *
      28             :  * The stored data consists of the following:
      29             :  *
      30             :  * 1. the volume mesh of the element.
      31             :  * 2. the volume mesh corresponding to the ghost cell data. This allows eliding
      32             :  *    projection when all neighboring elements are doing DG.
      33             :  * 3. the mortar mesh of the data on the mortar
      34             :  * 4. the variables at the ghost zone cells for finite difference/volume
      35             :  *    reconstruction
      36             :  * 5. the data on the mortar needed for computing the boundary corrections (e.g.
      37             :  *    fluxes, characteristic speeds, conserved variables)
      38             :  * 6. the TimeStepId beyond which the boundary terms are no longer valid, when
      39             :  *    using local time stepping.
      40             :  * 7. the troublade cell indicator status using for determining halos around
      41             :  *    troubled cells.
      42             :  * 8. the integration order of the time-stepper.
      43             :  * 9. the InterpolatedBoundaryData sent by a non-conforming Element that
      44             :  *    interpolates its data to a subset of the points of the Element receiving
      45             :  *    this BoundaryData
      46             :  *
      47             :  * The TimeStepId is the neighboring element's next time step. When using local
      48             :  * time stepping, the neighbor's boundary data is valid up until this time,
      49             :  * which may include multiple local time steps. By receiving and storing the
      50             :  * neighbor time step, the local element knows whether or not it should remove
      51             :  * boundary data and expect new data to be sent from the neighbor.
      52             :  *
      53             :  * The ghost cell data will be valid whenever a DG-subcell scheme is being used.
      54             :  * Whenever a DG-subcell scheme is being used, elements using DG and not FD/FV
      55             :  * always send both the ghost cells and boundary correction data together.
      56             :  * Elements using FD/FV send the ghost cells first followed by the boundary
      57             :  * correction data once the element has received all neighbor ghost cell data.
      58             :  * Note that the second send/receive only modifies the flux and the TimeStepId
      59             :  * used for the flux validity range.
      60             :  *
      61             :  * When only a DG scheme (not a DG-subcell scheme) is used the ghost cell data
      62             :  * will never be valid.
      63             :  *
      64             :  * In the DG-subcell scheme this tag is used both for communicating the ghost
      65             :  * cell data needed for the FD/FV reconstruction step and the data needed for
      66             :  * the boundary corrections.
      67             :  * - For an element using DG, both ghost cells and boundary corrections are
      68             :  *   sent using a single communication. After receiving all neighbor
      69             :  *   boundary corrections the element can finish computing the time step.
      70             :  *   The ghost cell data from neighbors is unused.
      71             :  * - For an element using FD/FV, first the ghost cells are sent. Once all
      72             :  *   neighboring ghost cell data is received, reconstruction is done and the
      73             :  *   boundary terms are computed and sent to the neighbors. After receiving all
      74             :  *   neighbor boundary corrections the element can finish computing the time
      75             :  *   step.
      76             :  * - Whether or not an extra communication is needed when an element switches
      77             :  *   from DG to FD/FV depends on how exactly the decision to switch is
      78             :  *   implemented. If the volume terms are integrated and verified to be
      79             :  *   valid before a DG element sends ghost cell and boundary data then no
      80             :  *   additional communication is needed when switching from DG to FD/FV. In this
      81             :  *   case a second check of the data that includes the boundary correction needs
      82             :  *   to be done. If the second check determines a switch from DG to FD/FV is
      83             :  *   needed, we can continue to use the DG fluxes since the evolution in the
      84             :  *   small was valid, thus avoiding an additional communication. However, to
      85             :  *   fully guarantee physical realizability a second communication or evolving
      86             :  *   the neighboring ghost cells needs to be done. We have not yet decided how
      87             :  *   to deal with the possible need for an additional communication since it
      88             :  *   also depends on whether or not we decide to couple to Voronoi instead of
      89             :  *   just Neumann neighbors.
      90             :  * - The data for the inbox tags is erased after the boundary correction is
      91             :  *   complete and the solution has been verified to be valid at the new time
      92             :  *   step. The ghost cells could be invalidated immediately after
      93             :  *   reconstruction, thus using the ghost cell data after reconstruction is
      94             :  *   complete is considered undefined behavior. That is, we make no guarantee as
      95             :  *   to whether or not it will work.
      96             :  * - The reason for minimizing the number of communications rather than having a
      97             :  *   more uniform implementation between DG and FD/FV is that it is the number
      98             :  *   of communications that adds the most overhead, not the size of each
      99             :  *   communication. Thus, one large communication is cheaper than several small
     100             :  *   communications.
     101             :  *
     102             :  * #### DG Element Nodegroup Support
     103             :  * If you are using the `DgElementCollection` then you must set
     104             :  * `UseNodegroupDgElements` to `true`. The actions that use this tag check
     105             :  * that the parallel component and the `UseNodegroupDgElements` is consistent.
     106             :  */
     107             : template <size_t Dim, bool UseNodegroupDgElements>
     108           1 : struct BoundaryCorrectionAndGhostCellsInbox {
     109           0 :   using stored_type = evolution::dg::BoundaryData<Dim>;
     110             : 
     111             :  public:
     112           0 :   using temporal_id = TimeStepId;
     113             :   // Used by array implementation
     114           0 :   using type_map = evolution::dg::InboxBoundaryData<Dim>;
     115             : 
     116             :   // Used by nodegroup implementation
     117           0 :   using type_spsc = evolution::dg::AtomicInboxBoundaryData<Dim>;
     118             : 
     119             :   // The actual type being used.
     120           0 :   using type = tmpl::conditional_t<UseNodegroupDgElements, type_spsc, type_map>;
     121           0 :   using value_type = type;
     122             : 
     123           0 :   static bool insert_into_inbox(
     124             :       gsl::not_null<type_spsc*> inbox, const temporal_id& time_step_id,
     125             :       std::pair<DirectionalId<Dim>, evolution::dg::BoundaryData<Dim>> data);
     126             : 
     127           0 :   static bool insert_into_inbox(
     128             :       gsl::not_null<type_map*> inbox, const temporal_id& time_step_id,
     129             :       std::pair<DirectionalId<Dim>, evolution::dg::BoundaryData<Dim>> data);
     130             : 
     131           0 :   static std::string output_inbox(const type_spsc& inbox, size_t padding_size);
     132             : 
     133           0 :   static std::string output_inbox(const type_map& inbox, size_t padding_size);
     134             : };
     135             : 
     136             : /*!
     137             :  * \brief The inbox tag for boundary correction communication and DG-subcell
     138             :  * ghost zone cells using a `BoundaryMessage` object
     139             :  *
     140             :  * To see what is stored within a `BoundaryMessage`, see its documentation.
     141             :  *
     142             :  * This inbox tag is very similar to `BoundaryCorrectionAndGhostCellsInbox` in
     143             :  * that it stores subcell/DG data sent from neighboring elements. To see exactly
     144             :  * when data is stored and how it's used, see the docs for
     145             :  * `BoundaryCorrectionAndGhostCellsInbox`. This inbox tag is different than
     146             :  * `BoundaryCorrectionAndGhostCellsInbox` in that it only takes a pointer to a
     147             :  * `BoundaryMessage` as an argument to `insert_into_inbox` and stores a
     148             :  * `std::unique_ptr<BoundaryMessage>` inside the inbox.
     149             :  *
     150             :  * This inbox tag is meant to be used to avoid unnecessary copies between
     151             :  * elements on the same node which share a block of memory. If two elements
     152             :  * aren't on the same node, then a copy/send is done regardless.
     153             :  *
     154             :  * \warning The `boundary_message` argument to `insert_into_inbox()` will be
     155             :  * invalid after the function is called because a `std::unique_ptr` now controls
     156             :  * the memory. Calling a method on the `boundary_message` pointer after the
     157             :  * `insert_into_inbox()` function is called can result in undefined behaviour.
     158             :  */
     159             : template <size_t Dim>
     160           1 : struct BoundaryMessageInbox {
     161           0 :   using stored_type = std::unique_ptr<BoundaryMessage<Dim>>;
     162             : 
     163             :  public:
     164           0 :   using temporal_id = TimeStepId;
     165           0 :   using type = std::map<TimeStepId, DirectionalIdMap<Dim, stored_type>>;
     166           0 :   using message_type = BoundaryMessage<Dim>;
     167             : 
     168             :   template <typename Inbox>
     169           0 :   static bool insert_into_inbox(const gsl::not_null<Inbox*> inbox,
     170             :                                 BoundaryMessage<Dim>* boundary_message) {
     171             :     const auto& time_step_id = boundary_message->current_time_step_id;
     172             :     auto& current_inbox = (*inbox)[time_step_id];
     173             : 
     174             :     const auto key = DirectionalId<Dim>{boundary_message->neighbor_direction,
     175             :                                         boundary_message->element_id};
     176             : 
     177             :     // Once we insert boundary_message into the unique_ptr we cannot use
     178             :     // boundary_message anymore because it is invalidated. The unique_ptr now
     179             :     // owns the memory.
     180             :     if (not current_inbox
     181             :                 .insert(std::pair{key, std::unique_ptr<BoundaryMessage<Dim>>(
     182             :                                            boundary_message)})
     183             :                 .second) {
     184             :       ERROR("Failed to insert data to receive at instance '"
     185             :             << time_step_id << "' with tag 'BoundaryMessageInbox'.\n");
     186             :     }
     187             :     return true;
     188             :   }
     189             : };
     190             : }  // namespace evolution::dg::Tags