CommunicateOverlapFields.hpp
Go to the documentation of this file.
1 // Distributed under the MIT License.
2 // See LICENSE.txt for details.
3 
4 /// \file
5 /// Actions for communicating data on regions that overlap with the subdomains
6 /// of other elements
7 
8 #pragma once
9 
10 #include <cstddef>
11 #include <map>
12 #include <tuple>
13 #include <utility>
14 
17 #include "Domain/Tags.hpp"
18 #include "Informer/Tags.hpp"
19 #include "Informer/Verbosity.hpp"
20 #include "NumericalAlgorithms/Convergence/Tags.hpp"
21 #include "Parallel/GlobalCache.hpp"
22 #include "Parallel/InboxInserters.hpp"
23 #include "Parallel/Invoke.hpp"
24 #include "Parallel/Printf.hpp"
25 #include "ParallelAlgorithms/DiscontinuousGalerkin/HasReceivedFromAllMortars.hpp"
26 #include "ParallelAlgorithms/LinearSolver/Schwarz/OverlapHelpers.hpp"
27 #include "ParallelAlgorithms/LinearSolver/Schwarz/Tags.hpp"
28 #include "Utilities/TMPL.hpp"
29 #include "Utilities/TaggedTuple.hpp"
30 
31 /// \cond
32 template <size_t Dim>
33 struct ElementId;
34 /// \endcond
35 
37 /// Actions related to the Schwarz solver
38 namespace Actions {
39 
40 namespace detail {
41 template <size_t Dim, typename OverlapFields, typename OptionsGroup>
42 struct OverlapFieldsTag
44  OverlapFieldsTag<Dim, OverlapFields, OptionsGroup>> {
45  using temporal_id = size_t;
46  using type = std::map<
47  temporal_id,
49 };
50 } // namespace detail
51 
52 /*!
53  * \brief Send data on regions that overlap with other subdomains to their
54  * corresponding elements
55  *
56  * Collect the `OverlapFields` on "intruding overlaps", i.e. regions that
57  * overlap with the subdomains of other elements, and send the data to those
58  * elements. The `OverlapFields` can be tags holding either `Variables` or
59  * `Tensor`s. The `RestrictToOverlap` flag controls whether the tags are simply
60  * retrieved from the element and sent as-is (`false`) or only the data that
61  * intersect the overlap region are sent (`true`). If `RestrictToOverlap` is
62  * `false` this action can also be used to communicate non-tensor data.
63  *
64  * This actions should be followed by
65  * `LinearSolver::Schwarz::Actions::ReceiveOverlapFields` in the action list.
66  */
67 template <typename OverlapFields, typename OptionsGroup, bool RestrictToOverlap>
69 
70 /// \cond
71 template <typename... OverlapFields, typename OptionsGroup,
72  bool RestrictToOverlap>
73 struct SendOverlapFields<tmpl::list<OverlapFields...>, OptionsGroup,
74  RestrictToOverlap> {
75  using const_global_cache_tags =
76  tmpl::list<Tags::MaxOverlap<OptionsGroup>,
78 
79  template <typename DbTagsList, typename... InboxTags, typename Metavariables,
80  size_t Dim, typename ActionList, typename ParallelComponent>
81  static std::tuple<db::DataBox<DbTagsList>&&> apply(
82  db::DataBox<DbTagsList>& box,
83  const tuples::TaggedTuple<InboxTags...>& /*inboxes*/,
85  const ElementId<Dim>& element_id, const ActionList /*meta*/,
86  const ParallelComponent* const /*meta*/) noexcept {
87  const auto& element = get<domain::Tags::Element<Dim>>(box);
88 
89  // Skip communicating if the overlap is empty
91  element.number_of_neighbors() == 0)) {
92  return {std::move(box)};
93  }
94 
95  // Do some logging
96  const auto& iteration_id =
97  get<Convergence::Tags::IterationId<OptionsGroup>>(box);
99  ::Verbosity::Debug)) {
100  Parallel::printf("%s " + Options::name<OptionsGroup>() +
101  "(%zu): Send overlap fields\n",
102  element_id, iteration_id);
103  }
104 
105  // Send data on intruding overlaps to the corresponding neighbors
106  auto& receiver_proxy =
107  Parallel::get_parallel_component<ParallelComponent>(cache);
108  for (const auto& direction_and_neighbors : element.neighbors()) {
109  const auto& direction = direction_and_neighbors.first;
110  const auto& neighbors = direction_and_neighbors.second;
111  // Collect the data on intruding overlaps
112  tuples::TaggedTuple<OverlapFields...> overlap_fields{};
113  if constexpr (RestrictToOverlap) {
114  const auto& element_extents =
115  get<domain::Tags::Mesh<Dim>>(box).extents();
116  const size_t intruding_extent =
117  gsl::at(get<Tags::IntrudingExtents<Dim, OptionsGroup>>(box),
118  direction.dimension());
119  expand_pack((get<OverlapFields>(overlap_fields) =
121  db::get<OverlapFields>(box), element_extents,
122  intruding_extent, direction))...);
123  } else {
124  expand_pack((get<OverlapFields>(overlap_fields) =
125  db::get<OverlapFields>(box))...);
126  }
127  // Copy data to send to neighbors, but move it for the last one
128  const auto direction_from_neighbor =
129  neighbors.orientation()(direction.opposite());
130  for (auto neighbor = neighbors.begin(); neighbor != neighbors.end();
131  ++neighbor) {
132  Parallel::receive_data<detail::OverlapFieldsTag<
133  Dim, tmpl::list<OverlapFields...>, OptionsGroup>>(
134  receiver_proxy[*neighbor], iteration_id,
135  std::make_pair(
136  OverlapId<Dim>{direction_from_neighbor, element.id()},
137  (std::next(neighbor) == neighbors.end())
138  // NOLINTNEXTLINE(bugprone-use-after-move)
139  ? std::move(overlap_fields)
140  : overlap_fields));
141  }
142  }
143  return {std::move(box)};
144  }
145 };
146 /// \endcond
147 
148 /*!
149  * \brief Receive data from regions of this element's subdomain that overlap
150  * with other elements
151  *
152  * This action waits until overlap data from all neighboring elements has been
153  * received and then moves the data into the DataBox as
154  * `LinearSolver::Schwarz::Tags::Overlaps<OverlapFields...>`.
155  *
156  * This actions should be preceded by
157  * `LinearSolver::Schwarz::Actions::SendOverlapFields` in the action list.
158  */
159 template <size_t Dim, typename OverlapFields, typename OptionsGroup>
161 
162 /// \cond
163 template <size_t Dim, typename... OverlapFields, typename OptionsGroup>
164 struct ReceiveOverlapFields<Dim, tmpl::list<OverlapFields...>, OptionsGroup> {
165  private:
166  using overlap_fields_tag =
167  detail::OverlapFieldsTag<Dim, tmpl::list<OverlapFields...>, OptionsGroup>;
168 
169  public:
170  using const_global_cache_tags =
171  tmpl::list<Tags::MaxOverlap<OptionsGroup>,
173  using inbox_tags = tmpl::list<overlap_fields_tag>;
174 
175  template <typename DbTagsList, typename... InboxTags, typename Metavariables>
176  static bool is_ready(const db::DataBox<DbTagsList>& box,
177  const tuples::TaggedTuple<InboxTags...>& inboxes,
178  const Parallel::GlobalCache<Metavariables>& /*cache*/,
179  const ElementId<Dim>& /*element_id*/) noexcept {
181  return true;
182  }
183  return dg::has_received_from_all_mortars<overlap_fields_tag>(
185  get<domain::Tags::Element<Dim>>(box), inboxes);
186  }
187 
188  template <typename DbTagsList, typename... InboxTags, typename Metavariables,
189  typename ActionList, typename ParallelComponent>
190  static std::tuple<db::DataBox<DbTagsList>&&> apply(
191  db::DataBox<DbTagsList>& box, tuples::TaggedTuple<InboxTags...>& inboxes,
192  const Parallel::GlobalCache<Metavariables>& /*cache*/,
193  const ElementId<Dim>& element_id, const ActionList /*meta*/,
194  const ParallelComponent* const /*meta*/) noexcept {
195  // Nothing to receive if overlap is empty
197  get<domain::Tags::Element<Dim>>(box).number_of_neighbors() ==
198  0)) {
199  return {std::move(box)};
200  }
201 
202  // Do some logging
203  const auto& iteration_id =
204  get<Convergence::Tags::IterationId<OptionsGroup>>(box);
206  ::Verbosity::Debug)) {
207  Parallel::printf("%s " + Options::name<OptionsGroup>() +
208  "(%zu): Receive overlap fields\n",
209  element_id, iteration_id);
210  }
211 
212  // Move received overlap data into DataBox
213  auto received_overlap_fields =
214  std::move(tuples::get<overlap_fields_tag>(inboxes)
215  .extract(iteration_id)
216  .mapped());
217  db::mutate<Tags::Overlaps<OverlapFields, Dim, OptionsGroup>...>(
218  make_not_null(&box), [&received_overlap_fields](
219  const auto... local_overlap_fields) noexcept {
220  for (auto& [overlap_id, overlap_fields] : received_overlap_fields) {
221  expand_pack((*local_overlap_fields)[overlap_id] =
222  std::move(get<OverlapFields>(overlap_fields))...);
223  }
224  });
225 
226  return {std::move(box)};
227  }
228 };
229 /// \endcond
230 
231 } // namespace Actions
232 } // namespace LinearSolver::Schwarz
expand_pack
constexpr void expand_pack(Ts &&...) noexcept
Allows zero-cost unordered expansion of a parameter.
Definition: TMPL.hpp:547
gsl::at
constexpr T & at(std::array< T, N > &arr, Size index)
Retrieve a entry from a container, with checks in Debug mode that the index being retrieved is valid.
Definition: Gsl.hpp:125
utility
UNLIKELY
#define UNLIKELY(x)
Definition: Gsl.hpp:73
get
constexpr Tag::type & get(Variables< TagList > &v) noexcept
Return Tag::type pointing into the contiguous array.
Definition: Variables.hpp:638
Parallel::GlobalCache
Definition: ElementReceiveInterpPoints.hpp:15
GlobalCache.hpp
Tags.hpp
domain::Tags::Element
Definition: Tags.hpp:97
LinearSolver::Schwarz::Actions::ReceiveOverlapFields
Receive data from regions of this element's subdomain that overlap with other elements.
Definition: CommunicateOverlapFields.hpp:160
Parallel::printf
void printf(const std::string &format, Args &&... args)
Print an atomic message to stdout with C printf usage.
Definition: Printf.hpp:103
LinearSolver::Schwarz::Tags::MaxOverlap
Number of points a subdomain can overlap with its neighbor.
Definition: Tags.hpp:48
tuple
db::get
const auto & get(const DataBox< TagList > &box) noexcept
Retrieve the item with tag Tag from the DataBox.
Definition: DataBox.hpp:969
LinearSolver::Schwarz
Items related to the Schwarz linear solver.
Definition: CommunicateOverlapFields.hpp:36
LinearSolver::Schwarz::Actions::SendOverlapFields
Send data on regions that overlap with other subdomains to their corresponding elements.
Definition: CommunicateOverlapFields.hpp:68
ElementId
An ElementId uniquely labels an Element.
Definition: ElementId.hpp:49
Printf.hpp
DataBox.hpp
cstddef
logging::Tags::Verbosity
Tag for putting Verbosity in a DataBox.
Definition: Tags.hpp:33
LinearSolver::Schwarz::data_on_overlap
void data_on_overlap(const gsl::not_null< Tensor< DataType, TensorStructure... > * > restricted_tensor, const Tensor< DataType, TensorStructure... > &tensor, const Index< Dim > &volume_extents, const size_t overlap_extent, const Direction< Dim > &direction) noexcept
The part of the tensor data that lies within the overlap region.
Definition: OverlapHelpers.hpp:139
tuples::TaggedTuple
An associative container that is indexed by structs.
Definition: TaggedTuple.hpp:271
map
Element.hpp
Convergence::Tags::IterationId
Identifies a step in an iterative algorithm.
Definition: Tags.hpp:74
Parallel::receive_data
void receive_data(Proxy &&proxy, typename ReceiveTag::temporal_id temporal_id, ReceiveDataType &&receive_data, const bool enable_if_disabled=false) noexcept
Send the data args... to the algorithm running on proxy, and tag the message with the identifier temp...
Definition: Invoke.hpp:47
make_not_null
gsl::not_null< T * > make_not_null(T *ptr) noexcept
Construct a not_null from a pointer. Often this will be done as an implicit conversion,...
Definition: Gsl.hpp:880
FixedHashMap
A hash table with a compile-time specified maximum size and ability to efficiently handle perfect has...
Definition: FixedHashMap.hpp:81
TMPL.hpp
Parallel::InboxInserters::Map
Inserter for inserting data that is received as the value_type (with non-const key_type) of a map dat...
Definition: InboxInserters.hpp:36