InboxTags.hpp
1 // Distributed under the MIT License.
2 // See LICENSE.txt for details.
3 
4 #pragma once
5 
6 #include <boost/functional/hash.hpp>
7 #include <cstddef>
8 #include <map>
9 #include <optional>
10 #include <tuple>
11 #include <type_traits>
12 #include <utility>
13 
14 #include "DataStructures/FixedHashMap.hpp"
17 #include "Domain/Structure/MaxNumberOfNeighbors.hpp"
19 #include "Parallel/InboxInserters.hpp"
20 #include "Time/TimeStepId.hpp"
21 #include "Utilities/TMPL.hpp"
22 
23 namespace evolution::dg::Tags {
24 /*!
25  * \brief The inbox tag for boundary correction communication and DG-subcell
26  * ghost zone cells.
27  *
28  * The stored data consists of the following (in argument order of the tuple):
29  *
30  * 1. the mesh of the neighboring element's face (not the mortar mesh!)
31  * 2. the variables at the ghost zone cells for finite difference/volume
32  * reconstruction
33  * 3. the data on the mortar needed for computing the boundary corrections (e.g.
34  * fluxes, characteristic speeds, conserved variables)
35  * 4. the TimeStepId beyond which the boundary terms are no longer valid, when
36  * using local time stepping.
37  *
38  * The TimeStepId is the neighboring element's next time step. When using local
39  * time stepping, the neighbor's boundary data is valid up until this time,
40  * which may include multiple local time steps. By receiving and storing the
41  * neighbor time step, the local element knows whether or not it should remove
42  * boundary data and expect new data to be sent from the neighbor.
43  *
44  * The ghost cell data (second element of the tuple) will be valid whenever a
45  * DG-subcell scheme is being used. Whenever a DG-subcell scheme is being used,
46  * elements using DG and not FD/FV always send both the ghost cells and boundary
47  * correction data together. Elements using FD/FV send the ghost cells first
48  * followed by the boundary correction data once the element has received all
49  * neighbor ghost cell data. Note that the second send/receive only modifies the
50  * flux and the TimeStepId used for the flux validity range.
51  *
52  * When only a DG scheme (not a DG-subcell scheme) is used the ghost cell data
53  * will never be valid.
54  *
55  * In the DG-subcell scheme this tag is used both for communicating the ghost
56  * cell data needed for the FD/FV reconstruction step and the data needed for
57  * the boundary corrections.
58  * - For an element using DG, both ghost cells and boundary corrections are
59  * sent using a single communication. After receiving all neighbor
60  * boundary corrections the element can finish computing the time step.
61  * The ghost cell data from neighbors is unused.
62  * - For an element using FD/FV, first the ghost cells are sent. Once all
63  * neighboring ghost cell data is received, reconstruction is done and the
64  * boundary terms are computed and sent to the neighbors. After receiving all
65  * neighbor boundary corrections the element can finish computing the time
66  * step.
67  * - Whether or not an extra communication is needed when an element switches
68  * from DG to FD/FV depends on how exactly the decision to switch is
69  * implemented. If the volume terms are integrated and verified to be
70  * valid before a DG element sends ghost cell and bonudary data then no
71  * additional communication is needed when switching from DG to FD/FV. In this
72  * case a second check of the data that includes the boundary correction needs
73  * to be done. If the second check determines a switch from DG to FD/FV is
74  * needed, we can continue to use the DG fluxes since the evolution in the
75  * small was valid, thus avoiding an additional communication. However, to
76  * fully guarantee physical realizability a second communication or evolving
77  * the neighboring ghost cells needs to be done. We have not yet decided how
78  * to deal with the possible need for an additional communication since it
79  * also depends on whether or not we decide to couple to Voronoi instead of
80  * just Neumann neighbors.
81  * - The data for the inbox tags is erased after the boundary correction is
82  * complete and the solution has been verified to be valid at the new time
83  * step. The ghost cells could be invalidated immediately after
84  * reconstruction, thus using the ghost cell data after reconstruction is
85  * complete is considered undefined behavior. That is, we make no guarantee as
86  * to whether or not it will work.
87  * - The reason for minimizing the number of communications rather than having a
88  * more uniform implementation between DG and FD/FV is that it is the number
89  * of communications that adds the most overhead, not the size of each
90  * communication. Thus, one large communication is cheaper than several small
91  * communications.
92  */
93 template <size_t Dim>
95  using stored_type =
98 
99  public:
100  using temporal_id = TimeStepId;
101  using type = std::map<
102  TimeStepId,
105  boost::hash<std::pair<Direction<Dim>, ElementId<Dim>>>>>;
106 
107  template <typename Inbox, typename ReceiveDataType>
108  static void insert_into_inbox(const gsl::not_null<Inbox*> inbox,
109  const temporal_id& time_step_id,
110  ReceiveDataType&& data) noexcept {
111  auto& current_inbox = (*inbox)[time_step_id];
112  auto& [mesh, ghost_cell_data, boundary_data, boundary_data_validity_range] =
113  data.second;
114  (void)ghost_cell_data;
115 
116  if (auto it = current_inbox.find(data.first); it != current_inbox.end()) {
117  auto& [current_mesh, current_ghost_cell_data, current_boundary_data,
118  current_boundary_data_validity_range] = it->second;
119  // We have already received some data at this time. Receiving data twice
120  // at the same time should only occur when receiving fluxes after having
121  // previously received ghost cells. We sanity check that the data we
122  // already have is the ghost cells and that we have not yet received flux
123  // data.
124  ASSERT(current_ghost_cell_data.has_value(),
125  "Have not yet received ghost cells at time step "
126  << time_step_id
127  << " but the inbox entry already exists. This is a bug in the "
128  "ordering of the actions.");
129  ASSERT(not current_boundary_data.has_value(),
130  "The fluxes have already been received at time step "
131  << time_step_id
132  << ". They are either being received for a second time, there "
133  "is a bug in the ordering of the actions (though a "
134  "different ASSERT should've caught that), or the incorrect "
135  "temporal ID is being sent.");
136 
137  ASSERT(current_mesh == mesh,
138  "The mesh being received for the fluxes is different than the "
139  "mesh received for the ghost cells. Mesh for fluxes: "
140  << mesh << " mesh for ghost cells " << current_mesh);
141 
142  // We always move here since we take ownership of the data and moves
143  // implicitly decay to copies
144  current_boundary_data = std::move(boundary_data);
145  current_boundary_data_validity_range = boundary_data_validity_range;
146  } else {
147  // We have not received ghost cells or fluxes at this time.
148  if (not current_inbox.insert(std::forward<ReceiveDataType>(data))
149  .second) {
150  ERROR("Failed to insert data to receive at instance '"
151  << time_step_id
152  << "' with tag 'BoundaryCorrectionAndGhostCellsInbox'.\n");
153  }
154  }
155  }
156 };
157 } // namespace evolution::dg::Tags
maximum_number_of_neighbors
constexpr size_t maximum_number_of_neighbors(const size_t dim)
Definition: MaxNumberOfNeighbors.hpp:15
utility
std::pair
tuple
ElementId< Dim >
ElementId.hpp
ERROR
#define ERROR(m)
prints an error message to the standard error stream and aborts the program.
Definition: Error.hpp:36
cstddef
map
TimeStepId
Definition: TimeStepId.hpp:25
ASSERT
#define ASSERT(a, m)
Assert that an expression should be true.
Definition: Assert.hpp:51
Mesh
Holds the number of grid points, basis, and quadrature in each direction of the computational grid.
Definition: Mesh.hpp:47
TimeStepId.hpp
evolution::dg::Tags::BoundaryCorrectionAndGhostCellsInbox
The inbox tag for boundary correction communication and DG-subcell ghost zone cells.
Definition: InboxTags.hpp:94
optional
Direction.hpp
Mesh.hpp
FixedHashMap
A hash table with a compile-time specified maximum size and ability to efficiently handle perfect has...
Definition: FixedHashMap.hpp:81
type_traits
TMPL.hpp
gsl::not_null
Require a pointer to not be a nullptr
Definition: Gsl.hpp:183