CommunicateOverlapFields.hpp
Go to the documentation of this file.
1 // Distributed under the MIT License.
2 // See LICENSE.txt for details.
3 
4 /// \file
5 /// Actions for communicating data on regions that overlap with the subdomains
6 /// of other elements
7 
8 #pragma once
9 
10 #include <cstddef>
11 #include <map>
12 #include <tuple>
13 #include <utility>
14 
17 #include "Domain/Tags.hpp"
18 #include "IO/Logging/Tags.hpp"
19 #include "IO/Logging/Verbosity.hpp"
20 #include "NumericalAlgorithms/Convergence/Tags.hpp"
21 #include "NumericalAlgorithms/DiscontinuousGalerkin/HasReceivedFromAllMortars.hpp"
22 #include "Parallel/AlgorithmMetafunctions.hpp"
23 #include "Parallel/GlobalCache.hpp"
24 #include "Parallel/InboxInserters.hpp"
25 #include "Parallel/Invoke.hpp"
26 #include "Parallel/Printf.hpp"
27 #include "ParallelAlgorithms/LinearSolver/Schwarz/OverlapHelpers.hpp"
28 #include "ParallelAlgorithms/LinearSolver/Schwarz/Tags.hpp"
29 #include "Utilities/TMPL.hpp"
30 #include "Utilities/TaggedTuple.hpp"
31 
32 /// \cond
33 template <size_t Dim>
34 struct ElementId;
35 /// \endcond
36 
38 /// Actions related to the Schwarz solver
39 namespace Actions {
40 
41 namespace detail {
42 template <size_t Dim, typename OverlapFields, typename OptionsGroup>
43 struct OverlapFieldsTag
45  OverlapFieldsTag<Dim, OverlapFields, OptionsGroup>> {
46  using temporal_id = size_t;
47  using type = std::map<
48  temporal_id,
49  OverlapMap<Dim, tmpl::conditional_t<
50  (tmpl::size<OverlapFields>::value > 1),
51  tuples::tagged_tuple_from_typelist<OverlapFields>,
52  typename tmpl::front<OverlapFields>::type>>>;
53 };
54 } // namespace detail
55 
56 /*!
57  * \brief Send data on regions that overlap with other subdomains to their
58  * corresponding elements
59  *
60  * Collect the `OverlapFields` on "intruding overlaps", i.e. regions that
61  * overlap with the subdomains of other elements, and send the data to those
62  * elements. The `OverlapFields` can be tags holding either `Variables` or
63  * `Tensor`s. The `RestrictToOverlap` flag controls whether the tags are simply
64  * retrieved from the element and sent as-is (`false`) or only the data that
65  * intersect the overlap region are sent (`true`). If `RestrictToOverlap` is
66  * `false` this action can also be used to communicate non-tensor data.
67  *
68  * This actions should be followed by
69  * `LinearSolver::Schwarz::Actions::ReceiveOverlapFields` in the action list.
70  */
71 template <typename OverlapFields, typename OptionsGroup, bool RestrictToOverlap>
73 
74 /// \cond
75 template <typename... OverlapFields, typename OptionsGroup,
76  bool RestrictToOverlap>
77 struct SendOverlapFields<tmpl::list<OverlapFields...>, OptionsGroup,
78  RestrictToOverlap> {
79  using const_global_cache_tags =
80  tmpl::list<Tags::MaxOverlap<OptionsGroup>,
82 
83  template <typename DbTagsList, typename... InboxTags, typename Metavariables,
84  size_t Dim, typename ActionList, typename ParallelComponent>
85  static std::tuple<db::DataBox<DbTagsList>&&> apply(
86  db::DataBox<DbTagsList>& box,
87  const tuples::TaggedTuple<InboxTags...>& /*inboxes*/,
89  const ElementId<Dim>& element_id, const ActionList /*meta*/,
90  const ParallelComponent* const /*meta*/) noexcept {
91  const auto& element = get<domain::Tags::Element<Dim>>(box);
92 
93  // Skip communicating if the overlap is empty
95  element.number_of_neighbors() == 0)) {
96  return {std::move(box)};
97  }
98 
99  // Do some logging
100  const auto& iteration_id =
101  get<Convergence::Tags::IterationId<OptionsGroup>>(box);
103  ::Verbosity::Debug)) {
104  Parallel::printf("%s %s(%zu): Send overlap fields\n", element_id,
105  Options::name<OptionsGroup>(), iteration_id);
106  }
107 
108  // Send data on intruding overlaps to the corresponding neighbors
109  auto& receiver_proxy =
110  Parallel::get_parallel_component<ParallelComponent>(cache);
111  for (const auto& direction_and_neighbors : element.neighbors()) {
112  const auto& direction = direction_and_neighbors.first;
113  const auto& neighbors = direction_and_neighbors.second;
114  // Collect the data on intruding overlaps
115  tuples::TaggedTuple<OverlapFields...> overlap_fields{};
116  if constexpr (RestrictToOverlap) {
117  const auto& element_extents =
118  get<domain::Tags::Mesh<Dim>>(box).extents();
119  const size_t intruding_extent =
120  gsl::at(get<Tags::IntrudingExtents<Dim, OptionsGroup>>(box),
121  direction.dimension());
122  expand_pack((get<OverlapFields>(overlap_fields) =
124  db::get<OverlapFields>(box), element_extents,
125  intruding_extent, direction))...);
126  } else {
127  expand_pack((get<OverlapFields>(overlap_fields) =
128  db::get<OverlapFields>(box))...);
129  }
130  // We elide the tagged tuple in the inbox tag if only a single tag is
131  // communicated. This optimization allows moving the overlap-map into the
132  // DataBox in one piece.
133  auto& collapsed_overlap_fields = [&overlap_fields]() noexcept -> auto& {
134  if constexpr (sizeof...(OverlapFields) > 1) {
135  return overlap_fields;
136  } else {
137  return get<OverlapFields...>(overlap_fields);
138  }
139  }
140  ();
141  // Copy data to send to neighbors, but move it for the last one
142  const auto direction_from_neighbor =
143  neighbors.orientation()(direction.opposite());
144  for (auto neighbor = neighbors.begin(); neighbor != neighbors.end();
145  ++neighbor) {
146  Parallel::receive_data<detail::OverlapFieldsTag<
147  Dim, tmpl::list<OverlapFields...>, OptionsGroup>>(
148  receiver_proxy[*neighbor], iteration_id,
149  std::make_pair(
150  OverlapId<Dim>{direction_from_neighbor, element.id()},
151  (std::next(neighbor) == neighbors.end())
152  // NOLINTNEXTLINE(bugprone-use-after-move)
153  ? std::move(collapsed_overlap_fields)
154  : collapsed_overlap_fields));
155  }
156  }
157  return {std::move(box)};
158  }
159 };
160 /// \endcond
161 
162 /*!
163  * \brief Receive data from regions of this element's subdomain that overlap
164  * with other elements
165  *
166  * This action waits until overlap data from all neighboring elements has been
167  * received and then moves the data into the DataBox as
168  * `LinearSolver::Schwarz::Tags::Overlaps<OverlapFields...>`.
169  *
170  * This actions should be preceded by
171  * `LinearSolver::Schwarz::Actions::SendOverlapFields` in the action list.
172  */
173 template <size_t Dim, typename OverlapFields, typename OptionsGroup>
175 
176 /// \cond
177 template <size_t Dim, typename... OverlapFields, typename OptionsGroup>
178 struct ReceiveOverlapFields<Dim, tmpl::list<OverlapFields...>, OptionsGroup> {
179  private:
180  using overlap_fields_tag =
181  detail::OverlapFieldsTag<Dim, tmpl::list<OverlapFields...>, OptionsGroup>;
182 
183  public:
184  using const_global_cache_tags =
185  tmpl::list<Tags::MaxOverlap<OptionsGroup>,
187  using inbox_tags = tmpl::list<overlap_fields_tag>;
188 
189  template <typename DbTagsList, typename... InboxTags, typename Metavariables,
190  typename ActionList, typename ParallelComponent>
192  apply(db::DataBox<DbTagsList>& box,
194  const Parallel::GlobalCache<Metavariables>& /*cache*/,
195  const ElementId<Dim>& element_id, const ActionList /*meta*/,
196  const ParallelComponent* const /*meta*/) noexcept {
197  const auto& iteration_id =
198  get<Convergence::Tags::IterationId<OptionsGroup>>(box);
199  const auto& element = get<domain::Tags::Element<Dim>>(box);
200 
201  // Nothing to receive if overlap is empty
203  element.number_of_neighbors() == 0)) {
204  return {std::move(box), Parallel::AlgorithmExecution::Continue};
205  }
206 
207  if (not dg::has_received_from_all_mortars<overlap_fields_tag>(
208  iteration_id, element, inboxes)) {
209  return {std::move(box), Parallel::AlgorithmExecution::Retry};
210  }
211 
212  // Do some logging
214  ::Verbosity::Debug)) {
215  Parallel::printf("%s %s(%zu): Receive overlap fields\n", element_id,
216  Options::name<OptionsGroup>(), iteration_id);
217  }
218 
219  // Move received overlap data into DataBox
220  auto received_overlap_fields =
221  std::move(tuples::get<overlap_fields_tag>(inboxes)
222  .extract(iteration_id)
223  .mapped());
224  db::mutate<Tags::Overlaps<OverlapFields, Dim, OptionsGroup>...>(
225  make_not_null(&box), [&received_overlap_fields](
226  const auto... local_overlap_fields) noexcept {
227  if constexpr (sizeof...(OverlapFields) > 1) {
228  for (auto& [overlap_id, overlap_fields] : received_overlap_fields) {
229  expand_pack((*local_overlap_fields)[overlap_id] =
230  std::move(get<OverlapFields>(overlap_fields))...);
231  }
232  } else {
233  expand_pack((*local_overlap_fields =
234  std::move(received_overlap_fields))...);
235  }
236  });
237 
238  return {std::move(box), Parallel::AlgorithmExecution::Continue};
239  }
240 };
241 /// \endcond
242 
243 } // namespace Actions
244 } // namespace LinearSolver::Schwarz
expand_pack
constexpr void expand_pack(Ts &&...) noexcept
Allows zero-cost unordered expansion of a parameter.
Definition: TMPL.hpp:549
gsl::at
constexpr T & at(std::array< T, N > &arr, Size index)
Retrieve a entry from a container, with checks in Debug mode that the index being retrieved is valid.
Definition: Gsl.hpp:125
utility
Parallel::AlgorithmExecution::Retry
@ Retry
Temporarily stop executing iterable actions, but try the same action again after receiving data from ...
UNLIKELY
#define UNLIKELY(x)
Definition: Gsl.hpp:73
Parallel::GlobalCache
Definition: ElementReceiveInterpPoints.hpp:15
GlobalCache.hpp
Tags.hpp
LinearSolver::Schwarz::Actions::ReceiveOverlapFields
Receive data from regions of this element's subdomain that overlap with other elements.
Definition: CommunicateOverlapFields.hpp:174
Parallel::printf
void printf(const std::string &format, Args &&... args)
Print an atomic message to stdout with C printf usage.
Definition: Printf.hpp:103
LinearSolver::Schwarz::Tags::MaxOverlap
Number of points a subdomain can overlap with its neighbor.
Definition: Tags.hpp:63
tuple
db::get
const auto & get(const DataBox< TagList > &box) noexcept
Retrieve the item with tag Tag from the DataBox.
Definition: DataBox.hpp:791
LinearSolver::Schwarz
Items related to the Schwarz linear solver.
Definition: CommunicateOverlapFields.hpp:37
LinearSolver::Schwarz::Actions::SendOverlapFields
Send data on regions that overlap with other subdomains to their corresponding elements.
Definition: CommunicateOverlapFields.hpp:72
ElementId
An ElementId uniquely labels an Element.
Definition: ElementId.hpp:51
Printf.hpp
Parallel::AlgorithmExecution::Continue
@ Continue
Leave the algorithm termination flag in its current state.
DataBox.hpp
cstddef
logging::Tags::Verbosity
Tag for putting Verbosity in a DataBox.
Definition: Tags.hpp:33
LinearSolver::Schwarz::data_on_overlap
void data_on_overlap(const gsl::not_null< Tensor< DataType, TensorStructure... > * > restricted_tensor, const Tensor< DataType, TensorStructure... > &tensor, const Index< Dim > &volume_extents, const size_t overlap_extent, const Direction< Dim > &direction) noexcept
The part of the tensor data that lies within the overlap region.
Definition: OverlapHelpers.hpp:139
tuples::TaggedTuple
An associative container that is indexed by structs.
Definition: TaggedTuple.hpp:271
map
Parallel::AlgorithmExecution
AlgorithmExecution
The possible options for altering the current execution of the algorithm, used in the return type of ...
Definition: AlgorithmMetafunctions.hpp:31
Parallel::receive_data
void receive_data(Proxy &&proxy, typename ReceiveTag::temporal_id temporal_id, ReceiveDataType &&receive_data, const bool enable_if_disabled=false) noexcept
Send the data args... to the algorithm running on proxy, and tag the message with the identifier temp...
Definition: Invoke.hpp:32
Element.hpp
ActionTesting::cache
Parallel::GlobalCache< Metavariables > & cache(MockRuntimeSystem< Metavariables > &runner, const ArrayIndex &array_index) noexcept
Returns the GlobalCache of Component with index array_index.
Definition: MockRuntimeSystemFreeFunctions.hpp:382
make_not_null
gsl::not_null< T * > make_not_null(T *ptr) noexcept
Construct a not_null from a pointer. Often this will be done as an implicit conversion,...
Definition: Gsl.hpp:880
FixedHashMap
A hash table with a compile-time specified maximum size and ability to efficiently handle perfect has...
Definition: FixedHashMap.hpp:82
TMPL.hpp
Parallel::InboxInserters::Map
Inserter for inserting data that is received as the value_type (with non-const key_type) of a map dat...
Definition: InboxInserters.hpp:36