Line data Source code
1 1 : // Distributed under the MIT License.
2 : // See LICENSE.txt for details.
3 :
4 : /// \file
5 : /// Actions for communicating data on regions that overlap with the subdomains
6 : /// of other elements
7 :
8 : #pragma once
9 :
10 : #include <cstddef>
11 : #include <map>
12 : #include <optional>
13 : #include <tuple>
14 : #include <utility>
15 :
16 : #include "DataStructures/DataBox/DataBox.hpp"
17 : #include "DataStructures/TaggedTuple.hpp"
18 : #include "Domain/Structure/Element.hpp"
19 : #include "Domain/Tags.hpp"
20 : #include "IO/Logging/Tags.hpp"
21 : #include "IO/Logging/Verbosity.hpp"
22 : #include "NumericalAlgorithms/Convergence/Tags.hpp"
23 : #include "NumericalAlgorithms/DiscontinuousGalerkin/HasReceivedFromAllMortars.hpp"
24 : #include "Parallel/AlgorithmExecution.hpp"
25 : #include "Parallel/GlobalCache.hpp"
26 : #include "Parallel/InboxInserters.hpp"
27 : #include "Parallel/Invoke.hpp"
28 : #include "Parallel/Printf/Printf.hpp"
29 : #include "ParallelAlgorithms/LinearSolver/Schwarz/OverlapHelpers.hpp"
30 : #include "ParallelAlgorithms/LinearSolver/Schwarz/Tags.hpp"
31 : #include "Utilities/PrettyType.hpp"
32 : #include "Utilities/TMPL.hpp"
33 :
34 : /// \cond
35 : template <size_t Dim>
36 : struct ElementId;
37 : /// \endcond
38 :
39 1 : namespace LinearSolver::Schwarz {
40 : /// Actions related to the Schwarz solver
41 1 : namespace Actions {
42 :
43 : namespace detail {
44 : template <size_t Dim, typename OverlapFields, typename OptionsGroup>
45 : struct OverlapFieldsTag
46 : : public Parallel::InboxInserters::Map<
47 : OverlapFieldsTag<Dim, OverlapFields, OptionsGroup>> {
48 : using temporal_id = size_t;
49 : using type = std::map<
50 : temporal_id,
51 : OverlapMap<Dim, tmpl::conditional_t<
52 : (tmpl::size<OverlapFields>::value > 1),
53 : tuples::tagged_tuple_from_typelist<OverlapFields>,
54 : typename tmpl::front<OverlapFields>::type>>>;
55 : };
56 : } // namespace detail
57 :
58 : /*!
59 : * \brief Send data on regions that overlap with other subdomains to their
60 : * corresponding elements
61 : *
62 : * Collect the `OverlapFields` on "intruding overlaps", i.e. regions that
63 : * overlap with the subdomains of other elements, and send the data to those
64 : * elements. The `OverlapFields` can be tags holding either `Variables` or
65 : * `Tensor`s. The `RestrictToOverlap` flag controls whether the tags are simply
66 : * retrieved from the element and sent as-is (`false`) or only the data that
67 : * intersect the overlap region are sent (`true`). If `RestrictToOverlap` is
68 : * `false` this action can also be used to communicate non-tensor data.
69 : *
70 : * This actions should be followed by
71 : * `LinearSolver::Schwarz::Actions::ReceiveOverlapFields` in the action list.
72 : */
73 : template <typename OverlapFields, typename OptionsGroup, bool RestrictToOverlap,
74 : typename TemporalIdTag = Convergence::Tags::IterationId<OptionsGroup>>
75 1 : struct SendOverlapFields;
76 :
77 : /// \cond
78 : template <typename... OverlapFields, typename OptionsGroup,
79 : bool RestrictToOverlap, typename TemporalIdTag>
80 : struct SendOverlapFields<tmpl::list<OverlapFields...>, OptionsGroup,
81 : RestrictToOverlap, TemporalIdTag> {
82 : using const_global_cache_tags =
83 : tmpl::list<Tags::MaxOverlap<OptionsGroup>,
84 : logging::Tags::Verbosity<OptionsGroup>>;
85 :
86 : template <typename DbTagsList, typename... InboxTags, typename Metavariables,
87 : size_t Dim, typename ActionList, typename ParallelComponent>
88 : static Parallel::iterable_action_return_t apply(
89 : db::DataBox<DbTagsList>& box,
90 : const tuples::TaggedTuple<InboxTags...>& /*inboxes*/,
91 : Parallel::GlobalCache<Metavariables>& cache,
92 : const ElementId<Dim>& element_id, const ActionList /*meta*/,
93 : const ParallelComponent* const /*meta*/) {
94 : const auto& element = get<domain::Tags::Element<Dim>>(box);
95 :
96 : // Skip communicating if the overlap is empty and we RestrictToOverlap,
97 : // because we would end up with empty tensor data.
98 : if (UNLIKELY((RestrictToOverlap and
99 : db::get<Tags::MaxOverlap<OptionsGroup>>(box) == 0) or
100 : element.number_of_neighbors() == 0)) {
101 : return {Parallel::AlgorithmExecution::Continue, std::nullopt};
102 : }
103 :
104 : // Do some logging
105 : const auto& iteration_id = get<TemporalIdTag>(box);
106 : if (UNLIKELY(get<logging::Tags::Verbosity<OptionsGroup>>(box) >=
107 : ::Verbosity::Debug)) {
108 : Parallel::printf("%s %s(%zu): Send overlap fields\n", element_id,
109 : pretty_type::name<OptionsGroup>(), iteration_id);
110 : }
111 :
112 : // Send data on intruding overlaps to the corresponding neighbors
113 : auto& receiver_proxy =
114 : Parallel::get_parallel_component<ParallelComponent>(cache);
115 : for (const auto& direction_and_neighbors : element.neighbors()) {
116 : const auto& direction = direction_and_neighbors.first;
117 : const auto& neighbors = direction_and_neighbors.second;
118 : // Collect the data on intruding overlaps
119 : tuples::TaggedTuple<OverlapFields...> overlap_fields{};
120 : if constexpr (RestrictToOverlap) {
121 : const auto& element_extents =
122 : get<domain::Tags::Mesh<Dim>>(box).extents();
123 : const size_t intruding_extent =
124 : gsl::at(get<Tags::IntrudingExtents<Dim, OptionsGroup>>(box),
125 : direction.dimension());
126 : expand_pack((get<OverlapFields>(overlap_fields) =
127 : LinearSolver::Schwarz::data_on_overlap(
128 : db::get<OverlapFields>(box), element_extents,
129 : intruding_extent, direction))...);
130 : } else {
131 : expand_pack((get<OverlapFields>(overlap_fields) =
132 : db::get<OverlapFields>(box))...);
133 : }
134 : // We elide the tagged tuple in the inbox tag if only a single tag is
135 : // communicated. This optimization allows moving the overlap-map into the
136 : // DataBox in one piece.
137 : auto& collapsed_overlap_fields = [&overlap_fields]() -> auto& {
138 : if constexpr (sizeof...(OverlapFields) > 1) {
139 : return overlap_fields;
140 : } else {
141 : return get<OverlapFields...>(overlap_fields);
142 : }
143 : }
144 : ();
145 : // Copy data to send to neighbors, but move it for the last one
146 : for (auto neighbor = neighbors.begin(); neighbor != neighbors.end();
147 : ++neighbor) {
148 : const auto direction_from_neighbor =
149 : neighbors.orientation(*neighbor)(direction.opposite());
150 : Parallel::receive_data<detail::OverlapFieldsTag<
151 : Dim, tmpl::list<OverlapFields...>, OptionsGroup>>(
152 : receiver_proxy[*neighbor], iteration_id,
153 : std::make_pair(
154 : OverlapId<Dim>{direction_from_neighbor, element.id()},
155 : (std::next(neighbor) == neighbors.end())
156 : // NOLINTNEXTLINE(bugprone-use-after-move)
157 : ? std::move(collapsed_overlap_fields)
158 : : collapsed_overlap_fields));
159 : }
160 : }
161 : return {Parallel::AlgorithmExecution::Continue, std::nullopt};
162 : }
163 : };
164 : /// \endcond
165 :
166 : /*!
167 : * \brief Receive data from regions of this element's subdomain that overlap
168 : * with other elements
169 : *
170 : * This action waits until overlap data from all neighboring elements has been
171 : * received and then moves the data into the DataBox as
172 : * `LinearSolver::Schwarz::Tags::Overlaps<OverlapFields...>`.
173 : *
174 : * This actions should be preceded by
175 : * `LinearSolver::Schwarz::Actions::SendOverlapFields` in the action list.
176 : */
177 : template <size_t Dim, typename OverlapFields, typename OptionsGroup,
178 : bool RestrictToOverlap,
179 : typename TemporalIdTag = Convergence::Tags::IterationId<OptionsGroup>>
180 1 : struct ReceiveOverlapFields;
181 :
182 : /// \cond
183 : template <size_t Dim, typename... OverlapFields, typename OptionsGroup,
184 : bool RestrictToOverlap, typename TemporalIdTag>
185 : struct ReceiveOverlapFields<Dim, tmpl::list<OverlapFields...>, OptionsGroup,
186 : RestrictToOverlap, TemporalIdTag> {
187 : private:
188 : using overlap_fields_tag =
189 : detail::OverlapFieldsTag<Dim, tmpl::list<OverlapFields...>, OptionsGroup>;
190 :
191 : public:
192 : using simple_tags =
193 : tmpl::list<Tags::Overlaps<OverlapFields, Dim, OptionsGroup>...>;
194 : using const_global_cache_tags =
195 : tmpl::list<Tags::MaxOverlap<OptionsGroup>,
196 : logging::Tags::Verbosity<OptionsGroup>>;
197 : using inbox_tags = tmpl::list<overlap_fields_tag>;
198 :
199 : template <typename DbTagsList, typename... InboxTags, typename Metavariables,
200 : typename ActionList, typename ParallelComponent>
201 : static Parallel::iterable_action_return_t apply(
202 : db::DataBox<DbTagsList>& box, tuples::TaggedTuple<InboxTags...>& inboxes,
203 : const Parallel::GlobalCache<Metavariables>& /*cache*/,
204 : const ElementId<Dim>& element_id, const ActionList /*meta*/,
205 : const ParallelComponent* const /*meta*/) {
206 : const auto& iteration_id = get<TemporalIdTag>(box);
207 : const auto& element = get<domain::Tags::Element<Dim>>(box);
208 :
209 : // Nothing to receive if nothing was sent in the action above
210 : if (UNLIKELY((RestrictToOverlap and
211 : db::get<Tags::MaxOverlap<OptionsGroup>>(box) == 0) or
212 : element.number_of_neighbors() == 0)) {
213 : return {Parallel::AlgorithmExecution::Continue, std::nullopt};
214 : }
215 :
216 : if (not dg::has_received_from_all_mortars<overlap_fields_tag>(
217 : iteration_id, element, inboxes)) {
218 : return {Parallel::AlgorithmExecution::Retry, std::nullopt};
219 : }
220 :
221 : // Do some logging
222 : if (UNLIKELY(get<logging::Tags::Verbosity<OptionsGroup>>(box) >=
223 : ::Verbosity::Debug)) {
224 : Parallel::printf("%s %s(%zu): Receive overlap fields\n", element_id,
225 : pretty_type::name<OptionsGroup>(), iteration_id);
226 : }
227 :
228 : // Move received overlap data into DataBox
229 : auto received_overlap_fields =
230 : std::move(tuples::get<overlap_fields_tag>(inboxes)
231 : .extract(iteration_id)
232 : .mapped());
233 : db::mutate<Tags::Overlaps<OverlapFields, Dim, OptionsGroup>...>(
234 : [&received_overlap_fields](const auto... local_overlap_fields) {
235 : if constexpr (sizeof...(OverlapFields) > 1) {
236 : (local_overlap_fields->clear(), ...);
237 : for (auto& [overlap_id, overlap_fields] : received_overlap_fields) {
238 : expand_pack((*local_overlap_fields)[overlap_id] =
239 : std::move(get<OverlapFields>(overlap_fields))...);
240 : }
241 : } else {
242 : expand_pack((*local_overlap_fields =
243 : std::move(received_overlap_fields))...);
244 : }
245 : },
246 : make_not_null(&box));
247 :
248 : return {Parallel::AlgorithmExecution::Continue, std::nullopt};
249 : }
250 : };
251 : /// \endcond
252 :
253 : } // namespace Actions
254 : } // namespace LinearSolver::Schwarz
|
