FluxCommunication.hpp
Go to the documentation of this file.
1 // Distributed under the MIT License.
2 // See LICENSE.txt for details.
3 
4 /// \file
5 /// Defines actions SendDataForFluxes and ReceiveDataForFluxes
6 
7 #pragma once
8 
9 #include <algorithm>
10 #include <cstddef>
11 #include <tuple>
12 #include <utility>
13 
17 #include "Domain/FaceNormal.hpp"
18 #include "Domain/Tags.hpp"
19 #include "ErrorHandling/Assert.hpp"
20 #include "NumericalAlgorithms/DiscontinuousGalerkin/Actions/InterfaceActionHelpers.hpp"
21 #include "NumericalAlgorithms/DiscontinuousGalerkin/FluxCommunicationTypes.hpp"
22 #include "NumericalAlgorithms/DiscontinuousGalerkin/MortarHelpers.hpp"
23 #include "NumericalAlgorithms/DiscontinuousGalerkin/Tags.hpp"
25 #include "Parallel/Invoke.hpp"
26 #include "Utilities/Gsl.hpp"
27 #include "Utilities/TMPL.hpp"
29 
30 /// \cond
31 namespace Tags {
32 template <typename Tag>
33 struct Magnitude;
34 template <typename Tag>
35 struct Next;
36 } // namespace Tags
37 // IWYU pragma: no_forward_declare db::DataBox
38 /// \endcond
39 
40 namespace dg {
41 namespace Actions {
42 /// \ingroup ActionsGroup
43 /// \ingroup DiscontinuousGalerkinGroup
44 /// \brief Receive boundary data needed for fluxes from neighbors.
45 ///
46 /// Uses:
47 /// - DataBox:
48 /// - Metavariables::temporal_id
49 /// - Tags::Next<Metavariables::temporal_id>
50 /// DataBox changes:
51 /// - Adds: nothing
52 /// - Removes: nothing
53 /// - Modifies:
54 /// - Tags::Mortars<Tags::Next<Metavariables::temporal_id>, volume_dim>
55 /// - Tags::VariablesBoundaryData
56 ///
57 /// \see SendDataForFluxes
58 template <typename Metavariables>
60  using const_global_cache_tags =
61  tmpl::list<typename Metavariables::normal_dot_numerical_flux>;
62 
63  private:
65 
66  public:
67  using inbox_tags = tmpl::list<typename flux_comm_types::FluxesTag>;
68 
69  template <typename DbTags, typename... InboxTags, typename ArrayIndex,
70  typename ActionList, typename ParallelComponent>
74  const ArrayIndex& /*array_index*/, const ActionList /*meta*/,
75  const ParallelComponent* const /*meta*/) noexcept {
76  constexpr size_t volume_dim = Metavariables::system::volume_dim;
77  using temporal_id_tag = typename Metavariables::temporal_id;
78  using neighbor_temporal_id_tag =
80  db::mutate<Tags::VariablesBoundaryData, neighbor_temporal_id_tag>(
81  make_not_null(&box),
82  [&inboxes](const gsl::not_null<
84  mortar_data,
86  neighbor_next_temporal_ids,
88  local_next_temporal_id) noexcept {
89  auto& inbox =
90  tuples::get<typename flux_comm_types::FluxesTag>(inboxes);
91  for (auto received_data = inbox.begin();
92  received_data != inbox.end() and
93  received_data->first < local_next_temporal_id;
94  received_data = inbox.erase(received_data)) {
95  const auto& receive_temporal_id = received_data->first;
96  for (auto& received_mortar_data : received_data->second) {
97  const auto mortar_id = received_mortar_data.first;
98  ASSERT(neighbor_next_temporal_ids->at(mortar_id) ==
99  receive_temporal_id,
100  "Expected data at "
101  << neighbor_next_temporal_ids->at(mortar_id)
102  << " but received at " << receive_temporal_id);
103  neighbor_next_temporal_ids->at(mortar_id) =
104  received_mortar_data.second.first;
105  mortar_data->at(mortar_id).remote_insert(
106  receive_temporal_id,
107  std::move(received_mortar_data.second.second));
108  }
109  }
110 
111  // The apparently pointless lambda wrapping this check
112  // prevents gcc-7.3.0 from segfaulting.
113  ASSERT(([
114  &neighbor_next_temporal_ids, &local_next_temporal_id
115  ]() noexcept {
116  return std::all_of(
117  neighbor_next_temporal_ids->begin(),
118  neighbor_next_temporal_ids->end(),
119  [&local_next_temporal_id](const auto& next) noexcept {
120  return next.first.second ==
121  ElementId<
122  volume_dim>::external_boundary_id() or
123  next.second >= local_next_temporal_id;
124  });
125  }()),
126  "apply called before all data received");
127  ASSERT(
128  inbox.empty() or (inbox.size() == 1 and
129  inbox.begin()->first == local_next_temporal_id),
130  "Shouldn't have received data that depended upon the step being "
131  "taken: Received data at " << inbox.begin()->first
132  << " while stepping to " << local_next_temporal_id);
133  },
134  db::get<Tags::Next<temporal_id_tag>>(box));
135 
136  return std::forward_as_tuple(std::move(box));
137  }
138 
139  template <typename DbTags, typename... InboxTags, typename ArrayIndex>
140  static bool is_ready(
141  const db::DataBox<DbTags>& box,
142  const tuples::TaggedTuple<InboxTags...>& inboxes,
144  const ArrayIndex& /*array_index*/) noexcept {
145  constexpr size_t volume_dim = Metavariables::system::volume_dim;
146  using temporal_id = typename Metavariables::temporal_id;
147 
148  const auto& inbox =
149  tuples::get<typename flux_comm_types::FluxesTag>(inboxes);
150  const auto& local_next_temporal_id = db::get<Tags::Next<temporal_id>>(box);
151  const auto& mortars_next_temporal_id =
152  db::get<Tags::Mortars<Tags::Next<temporal_id>, volume_dim>>(box);
153  for (const auto& mortar_id_next_temporal_id : mortars_next_temporal_id) {
154  const auto& mortar_id = mortar_id_next_temporal_id.first;
155  // If on an external boundary
156  if (mortar_id.second == ElementId<volume_dim>::external_boundary_id()) {
157  continue;
158  }
159  auto next_temporal_id = mortar_id_next_temporal_id.second;
160  while (next_temporal_id < local_next_temporal_id) {
161  const auto temporal_received = inbox.find(next_temporal_id);
162  if (temporal_received == inbox.end()) {
163  return false;
164  }
165  const auto mortar_received = temporal_received->second.find(mortar_id);
166  if (mortar_received == temporal_received->second.end()) {
167  return false;
168  }
169  next_temporal_id = mortar_received->second.first;
170  }
171  }
172  return true;
173  }
174 };
175 
176 /// \ingroup ActionsGroup
177 /// \ingroup DiscontinuousGalerkinGroup
178 /// \brief Send local boundary data needed for fluxes to neighbors.
179 ///
180 /// With:
181 /// - `Interface<Tag> =
182 /// Tags::Interface<Tags::InternalDirections<volume_dim>, Tag>`
183 ///
184 /// Uses:
185 /// - ConstGlobalCache: Metavariables::normal_dot_numerical_flux
186 /// - DataBox:
187 /// - Tags::Element<volume_dim>
188 /// - Interface<Tags listed in
189 /// Metavariables::normal_dot_numerical_flux::type::argument_tags>
190 /// - Interface<Tags::Mesh<volume_dim - 1>>
191 /// - Interface<Tags::Magnitude<Tags::UnnormalizedFaceNormal<volume_dim>>>,
192 /// - Metavariables::temporal_id
193 /// - Tags::Mortars<Tags::Mesh<volume_dim - 1>, volume_dim>
194 /// - Tags::Mortars<Tags::MortarSize<volume_dim - 1>, volume_dim>
195 /// - Tags::Next<Metavariables::temporal_id>
196 ///
197 /// DataBox changes:
198 /// - Adds: nothing
199 /// - Removes: nothing
200 /// - Modifies: Tags::VariablesBoundaryData
201 ///
202 /// \see ReceiveDataForFluxes
203 template <typename Metavariables>
205  using const_global_cache_tags =
206  tmpl::list<typename Metavariables::normal_dot_numerical_flux>;
207 
208  template <typename DbTags, typename... InboxTags, typename ArrayIndex,
209  typename ActionList, typename ParallelComponent>
213  const ArrayIndex& /*array_index*/, const ActionList /*meta*/,
214  const ParallelComponent* const /*meta*/) noexcept {
215  using system = typename Metavariables::system;
216  constexpr size_t volume_dim = system::volume_dim;
217 
218  using flux_comm_types = FluxCommunicationTypes<Metavariables>;
219 
220  using interface_normal_dot_fluxes_tag =
222  typename flux_comm_types::normal_dot_fluxes_tag>;
223 
224  const auto& normal_dot_numerical_flux_computer =
225  get<typename Metavariables::normal_dot_numerical_flux>(cache);
226 
227  auto& receiver_proxy =
228  Parallel::get_parallel_component<ParallelComponent>(cache);
229 
230  const auto& element = db::get<Tags::Element<volume_dim>>(box);
231  const auto& temporal_id = db::get<typename Metavariables::temporal_id>(box);
232  const auto& next_temporal_id =
233  db::get<Tags::Next<typename Metavariables::temporal_id>>(box);
234 
235  for (const auto& direction_neighbors : element.neighbors()) {
236  const auto& direction = direction_neighbors.first;
237  const size_t dimension = direction.dimension();
238  const auto& neighbors_in_direction = direction_neighbors.second;
239  const auto& orientation = neighbors_in_direction.orientation();
240  const auto& boundary_mesh =
241  db::get<Tags::Interface<Tags::InternalDirections<volume_dim>,
242  Tags::Mesh<volume_dim - 1>>>(box)
243  .at(direction);
244 
245  // We compute the parts of the numerical flux that only depend on data
246  // from this side of the mortar now, then package it into a Variables.
247  // We store one copy of the Variables and send another, since we need
248  // the data on both sides of the mortar.
249 
250  const auto packaged_data = DgActions_detail::compute_packaged_data(
251  box, direction, normal_dot_numerical_flux_computer,
253 
254  const auto direction_from_neighbor = orientation(direction.opposite());
255 
256  for (const auto& neighbor : neighbors_in_direction) {
257  const auto mortar_id = std::make_pair(direction, neighbor);
258  const auto& mortar_mesh =
259  db::get<Tags::Mortars<Tags::Mesh<volume_dim - 1>, volume_dim>>(box)
260  .at(mortar_id);
261  const auto& mortar_size = db::get<
262  Tags::Mortars<Tags::MortarSize<volume_dim - 1>, volume_dim>>(box)
263  .at(mortar_id);
264 
265  auto projected_packaged_data = project_to_mortar(
266  packaged_data, boundary_mesh, mortar_mesh, mortar_size);
267 
268  typename flux_comm_types::LocalData local_data{};
269  local_data.magnitude_of_face_normal = db::get<Tags::Interface<
272  .at(direction);
273 
274  local_data.mortar_data.initialize(mortar_mesh.number_of_grid_points());
275  local_data.mortar_data.assign_subset(projected_packaged_data);
276  if (tmpl::size<
277  typename flux_comm_types::LocalMortarData::tags_list>::value !=
278  tmpl::size<
279  typename flux_comm_types::PackagedData::tags_list>::value) {
280  // The local fluxes were not (all) included in the packaged
281  // data, so we need to add them to the mortar data
282  // explicitly.
283  const auto& normal_dot_fluxes =
284  db::get<interface_normal_dot_fluxes_tag>(box).at(direction);
285  local_data.mortar_data.assign_subset(
286  boundary_mesh == mortar_mesh
287  ? normal_dot_fluxes
288  : project_to_mortar(normal_dot_fluxes, boundary_mesh,
290  }
291 
292  if (not orientation.is_aligned()) {
293  projected_packaged_data = orient_variables_on_slice(
294  projected_packaged_data, mortar_mesh.extents(), dimension,
295  orientation);
296  }
297 
298  Parallel::receive_data<typename flux_comm_types::FluxesTag>(
299  receiver_proxy[neighbor], temporal_id,
300  std::make_pair(
301  std::make_pair(direction_from_neighbor, element.id()),
302  std::make_pair(next_temporal_id,
303  std::move(projected_packaged_data))));
304 
305  db::mutate<Tags::VariablesBoundaryData>(
306  make_not_null(&box),
307  [&mortar_id, &temporal_id, &local_data](
308  const gsl::not_null<
310  mortar_data) noexcept {
311  mortar_data->at(mortar_id).local_insert(temporal_id,
312  std::move(local_data));
313  });
314  } // loop over neighbors_in_direction
315  } // loop over element.neighbors()
316 
317  return std::forward_as_tuple(std::move(box));
318  }
319 };
320 } // namespace Actions
321 } // namespace dg
The set of directions to neighboring Elements.
Definition: Tags.hpp:145
Defines class tuples::TaggedTuple.
Definition: InitializeElement.hpp:63
An ElementId uniquely labels an Element. It is constructed from the BlockId of the Block to which the...
Definition: ElementId.hpp:36
Mesh< Dim > mortar_mesh(const Mesh< Dim > &face_mesh1, const Mesh< Dim > &face_mesh2) noexcept
Find a mesh for a mortar capable of representing data from either of two faces.
Definition: MortarHelpers.cpp:19
Receive boundary data needed for fluxes from neighbors.
Definition: FluxCommunication.hpp:59
Variables< Tags > project_to_mortar(const Variables< Tags > &vars, const Mesh< Dim > &face_mesh, const Mesh< Dim > &mortar_mesh, const std::array< Spectral::MortarSize, Dim > &mortar_size) noexcept
Project variables from a face to a mortar.
Definition: MortarHelpers.hpp:55
#define ASSERT(a, m)
Assert that an expression should be true.
Definition: Assert.hpp:49
constexpr auto apply(F &&f, const DataBox< BoxTags > &box, Args &&... args)
Apply the function f with argument Tags TagsList from DataBox box
Definition: DataBox.hpp:1595
Data on mortars, indexed by (Direction, ElementId) pairs.
Definition: Tags.hpp:34
Holds an IterationId that identifies a step in the linear solver algorithm.
Definition: Tags.hpp:66
An associative container that is indexed by structs.
Definition: TaggedTuple.hpp:272
Defines classes and functions used for manipulating DataBox&#39;s.
Definition: InterpolationTargetWedgeSectionTorus.hpp:24
Send local boundary data needed for fluxes to neighbors.
Definition: FluxCommunication.hpp:204
Types related to flux communication.
Definition: FluxCommunicationTypes.hpp:37
Definition: DataBoxTag.hpp:29
A Charm++ chare that caches constant data once per Charm++ node.
Definition: ConstGlobalCache.hpp:76
Size of a mortar, relative to the element face. That is, the part of the face that it covers...
Definition: Tags.hpp:46
Declares function unnormalized_face_normal.
std::array< Spectral::MortarSize, Dim - 1 > mortar_size(const ElementId< Dim > &self, const ElementId< Dim > &neighbor, const size_t dimension, const OrientationMap< Dim > &orientation) noexcept
Determine the size of the mortar (i.e., the part of the face it covers) for communicating with a neig...
Definition: MortarHelpers.cpp:36
Defines macro ASSERT.
const auto & get(const DataBox< TagList > &box) noexcept
Retrieve the item with tag Tag from the DataBox.
Definition: DataBox.hpp:1211
Variables< TagsList > orient_variables_on_slice(const Variables< TagsList > &variables_on_slice, const Index< VolumeDim - 1 > &slice_extents, const size_t sliced_dim, const OrientationMap< VolumeDim > &orientation_of_neighbor) noexcept
Orient variables to the data-storage order of a neighbor element with the given orientation.
Definition: VariablesHelpers.hpp:264
The Poisson equation formulated as a set of coupled first-order PDEs.
Definition: FirstOrderSystem.hpp:55
Wraps the template metaprogramming library used (brigand)
The computational grid of the Element in the DataBox.
Definition: Tags.hpp:75
typename DataBox_detail::item_type_impl< TagList, Tag >::type item_type
Get the type that is returned by the Tag. If it is a base tag then a TagList must be passed as a seco...
Definition: DataBoxTag.hpp:410
Defines functions and classes from the GSL.
gsl::not_null< T * > make_not_null(T *ptr) noexcept
Construct a not_null from a pointer. Often this will be done as an implicit conversion, but it may be necessary to perform the conversion explicitly when type deduction is desired.
Definition: Gsl.hpp:863
Defines tags related to domain quantities.
The magnitude of a (co)vector.
Definition: Magnitude.hpp:53
Defines helper functions for use with Variables class.
Definition: SolvePoissonProblem.hpp:38
Defines classes SimpleTag, PrefixTag, ComputeTag and several functions for retrieving tag info...
Defines class template ConstGlobalCache.
Tag which is either a SimpleTag for quantities on an interface, base tag to a compute item which acts...
Definition: Tags.hpp:217
Definition: ComputeTimeDerivative.hpp:28
Require a pointer to not be a nullptr
Definition: ConservativeFromPrimitive.hpp:12
constexpr T & at(std::array< T, N > &arr, Size index)
Retrieve a entry from a container, with checks in Debug mode that the index being retrieved is valid...
Definition: Gsl.hpp:124