Line data Source code
1 0 : // Distributed under the MIT License.
2 : // See LICENSE.txt for details.
3 :
4 : #pragma once
5 :
6 : #include <array>
7 : #include <cstddef>
8 : #include <unordered_set>
9 :
10 : #include "DataStructures/DataBox/DataBox.hpp"
11 : #include "Domain/Amr/Flag.hpp"
12 : #include "Domain/Amr/Helpers.hpp"
13 : #include "Domain/Amr/NewNeighborIds.hpp"
14 : #include "Domain/Amr/Tags/Flags.hpp"
15 : #include "Domain/Amr/Tags/NeighborFlags.hpp"
16 : #include "Domain/Structure/Direction.hpp"
17 : #include "Domain/Structure/DirectionMap.hpp"
18 : #include "Domain/Structure/DirectionalIdMap.hpp"
19 : #include "Domain/Structure/Element.hpp"
20 : #include "Domain/Structure/ElementId.hpp"
21 : #include "Domain/Structure/Neighbors.hpp"
22 : #include "Domain/Tags.hpp"
23 : #include "Domain/Tags/NeighborMesh.hpp"
24 : #include "IO/Logging/Tags.hpp"
25 : #include "IO/Logging/Verbosity.hpp"
26 : #include "NumericalAlgorithms/Spectral/Mesh.hpp"
27 : #include "Parallel/ArrayCollection/IsDgElementCollection.hpp"
28 : #include "Parallel/GlobalCache.hpp"
29 : #include "Parallel/Invoke.hpp"
30 : #include "Parallel/Phase.hpp"
31 : #include "Parallel/Printf/Printf.hpp"
32 : #include "Parallel/Tags/DistributedObjectTags.hpp"
33 : #include "Parallel/Tags/Section.hpp"
34 : #include "ParallelAlgorithms/Amr/Actions/CreateChild.hpp"
35 : #include "ParallelAlgorithms/Amr/Actions/CreateParent.hpp"
36 : #include "ParallelAlgorithms/Amr/Projectors/Mesh.hpp"
37 : #include "ParallelAlgorithms/Amr/Protocols/Projector.hpp"
38 : #include "ParallelAlgorithms/Amr/Tags.hpp"
39 : #include "Utilities/Algorithm.hpp"
40 : #include "Utilities/ErrorHandling/Assert.hpp"
41 : #include "Utilities/Gsl.hpp"
42 : #include "Utilities/Literals.hpp"
43 : #include "Utilities/PrettyType.hpp"
44 : #include "Utilities/ProtocolHelpers.hpp"
45 : #include "Utilities/TMPL.hpp"
46 :
47 : namespace amr {
48 : /// \cond
49 : template <class Metavariables>
50 : struct Component;
51 : /// \endcond
52 : } // namespace amr
53 :
54 : namespace detail {
55 : template <typename ListOfProjectors>
56 : struct GetMutatedTags;
57 :
58 : template <typename... Projectors>
59 : struct GetMutatedTags<tmpl::list<Projectors...>> {
60 : using type = tmpl::remove_duplicates<
61 : tmpl::flatten<tmpl::append<typename Projectors::return_tags...>>>;
62 : };
63 : } // namespace detail
64 :
65 : namespace amr::Actions {
66 : /// \brief Adjusts the domain given the refinement criteria
67 : ///
68 : /// \details
69 : /// - Checks if an Element wants to split in any dimension; if yes, determines
70 : /// the ElementId%s of the new children Element%s and calls
71 : /// amr::Actions::CreateChild on the amr::Component, then exits the action.
72 : /// - Checks if an Element wants to join in any dimension; if yes, either calls
73 : /// amr::Actions::CreateParent on the amr::Component if this is the child
74 : /// Element that should create the parent Element or does nothing, and then
75 : /// exits the action.
76 : /// - Checks if an Element wants to increase or decrease its resolution, if yes,
77 : /// mutates the Mesh
78 : /// - Updates the Neighbors of the Element
79 : /// - Resets amr::Tags::Flag%s to amr::Flag::Undefined
80 : /// - Resets amr::Tags::NeighborInfo to an empty map
81 : /// - Mutates all return_tags of Metavariables::amr::projectors
82 1 : struct AdjustDomain {
83 : template <typename ParallelComponent, typename DbTagList,
84 : typename Metavariables>
85 0 : static void apply(db::DataBox<DbTagList>& box,
86 : Parallel::GlobalCache<Metavariables>& cache,
87 : const ElementId<Metavariables::volume_dim>& element_id) {
88 : if constexpr (Parallel::is_dg_element_collection_v<ParallelComponent>) {
89 : ERROR("Can't adjust the domain for DG elements running on nodegroups.");
90 : } else {
91 : constexpr size_t volume_dim = Metavariables::volume_dim;
92 : using amr_projectors = typename Metavariables::amr::projectors;
93 : static_assert(
94 : tmpl::all<amr_projectors,
95 : tt::assert_conforms_to<tmpl::_1,
96 : amr::protocols::Projector>>::value,
97 : "All AMR projectors must conform to 'amr::protocols::Projector'.");
98 :
99 : // To prevent bugs when new mutable items are added to a DataBox, we
100 : // require that all mutable_item_creation_tags of box are either:
101 : // - mutated by one of the projectors in Metavariables::amr::projectors
102 : // - in the list of distributed_object_tags
103 : // - or in the list of tags mutated by this action
104 : using distributed_object_tags =
105 : typename ::Parallel::Tags::distributed_object_tags<
106 : Metavariables, ElementId<volume_dim>>;
107 : using tags_mutated_by_this_action = tmpl::list<
108 : ::domain::Tags::Element<volume_dim>, ::domain::Tags::Mesh<volume_dim>,
109 : ::domain::Tags::NeighborMesh<volume_dim>, amr::Tags::Info<volume_dim>,
110 : amr::Tags::NeighborInfo<volume_dim>, amr::Tags::ParentId<volume_dim>,
111 : amr::Tags::ChildIds<volume_dim>, amr::Tags::ParentMesh<volume_dim>,
112 : Parallel::Tags::Section<ParallelComponent, amr::Tags::GridIndex>,
113 : Parallel::Tags::Section<ParallelComponent, amr::Tags::IsFinestGrid>>;
114 : using mutated_tags =
115 : tmpl::append<distributed_object_tags, tags_mutated_by_this_action,
116 : typename detail::GetMutatedTags<amr_projectors>::type>;
117 : using mutable_tags =
118 : typename db::DataBox<DbTagList>::mutable_item_creation_tags;
119 : using mutable_tags_not_mutated =
120 : tmpl::list_difference<mutable_tags, mutated_tags>;
121 : static_assert(
122 : std::is_same_v<mutable_tags_not_mutated, tmpl::list<>>,
123 : "All mutable tags in the DataBox must be explicitly mutated "
124 : "by an amr::projector. Default initialized objects can use "
125 : "amr::projector::DefaultInitialize.");
126 :
127 : const auto& my_amr_info = db::get<amr::Tags::Info<volume_dim>>(box);
128 : const auto& my_amr_flags = my_amr_info.flags;
129 : auto& element_array =
130 : Parallel::get_parallel_component<ParallelComponent>(cache);
131 : auto& amr_component =
132 : Parallel::get_parallel_component<amr::Component<Metavariables>>(
133 : cache);
134 : const auto& phase_bookmarks =
135 : Parallel::local(element_array[element_id])->phase_bookmarks();
136 : const auto& verbosity =
137 : db::get<logging::Tags::Verbosity<amr::OptionTags::AmrGroup>>(box);
138 :
139 : if (alg::all_of(my_amr_flags, [](amr::Flag flag) {
140 : return flag == amr::Flag::Undefined;
141 : })) {
142 : // AMR flags are undefined. This state can be reached when the AMR
143 : // component broadcasts the `AdjustDomain` simple action to the entire
144 : // element array, then some of those elements run the simple action and
145 : // create new child elements, and then the broadcast also arrives at
146 : // these new elements for some reason (seems like a Charm++ bug). The
147 : // AMR flags will be undefined in this case, so we just ignore the
148 : // broadcast and return early here.
149 : return;
150 : } else if (alg::any_of(my_amr_flags, [](amr::Flag flag) {
151 : return flag == amr::Flag::Split;
152 : })) {
153 : // h-refinement
154 : using ::operator<<;
155 : ASSERT(alg::count(my_amr_flags, amr::Flag::Join) == 0,
156 : "Element " << element_id
157 : << " cannot both split and join, but had AMR flags "
158 : << my_amr_flags << "\n");
159 : const size_t child_grid_index = Metavariables::amr::keep_coarse_grids
160 : ? element_id.grid_index() + 1
161 : : element_id.grid_index();
162 : auto children_ids =
163 : amr::ids_of_children(element_id, my_amr_flags, child_grid_index);
164 : if (verbosity >= Verbosity::Debug) {
165 : Parallel::printf("Splitting element %s into %zu: %s\n", element_id,
166 : children_ids.size(), children_ids);
167 : }
168 : Parallel::simple_action<CreateChild>(amr_component, element_array,
169 : element_id, children_ids, 0_st,
170 : phase_bookmarks);
171 : return;
172 : } else if (alg::any_of(my_amr_flags, [](amr::Flag flag) {
173 : return flag == amr::Flag::Join;
174 : })) {
175 : // h-coarsening
176 : if (Metavariables::amr::keep_coarse_grids) {
177 : ERROR(
178 : "When AMR keeps coarse grids then no h-coarsening during AMR is "
179 : "allowed, but element "
180 : << element_id
181 : << " requested h-coarsening. Set the 'AllowCoarsening' policy to "
182 : "false to disable coarsening.");
183 : // Note: this restriction could be relaxed if ever needed.
184 : return;
185 : }
186 : // Only one element should create the new parent
187 : if (amr::is_child_that_creates_parent(element_id, my_amr_flags)) {
188 : auto parent_id = amr::id_of_parent(element_id, my_amr_flags);
189 : const auto& element =
190 : db::get<::domain::Tags::Element<volume_dim>>(box);
191 : auto ids_to_join =
192 : amr::ids_of_joining_neighbors(element, my_amr_flags);
193 : if (verbosity >= Verbosity::Debug) {
194 : Parallel::printf("Joining %zu elements: %s -> %s\n",
195 : ids_to_join.size(), ids_to_join, parent_id);
196 : }
197 : Parallel::simple_action<CreateParent>(
198 : amr_component, element_array, std::move(parent_id), element_id,
199 : std::move(ids_to_join), phase_bookmarks);
200 : }
201 : return;
202 : }
203 :
204 : // Neither h-refinement nor h-coarsening. This element will remain.
205 :
206 : if constexpr (Metavariables::amr::keep_coarse_grids) {
207 : // Create new element that covers this one with an incremented grid
208 : // index. p-refinement will be handled by the newly created element.
209 : ElementId<volume_dim> new_element_id{element_id.block_id(),
210 : element_id.segment_ids(),
211 : element_id.grid_index() + 1};
212 : Parallel::simple_action<CreateChild>(
213 : amr_component, element_array, element_id,
214 : std::vector<ElementId<volume_dim>>{new_element_id}, 0_st,
215 : phase_bookmarks);
216 : return;
217 : }
218 :
219 : const auto old_mesh_and_element =
220 : std::make_pair(db::get<::domain::Tags::Mesh<volume_dim>>(box),
221 : db::get<::domain::Tags::Element<volume_dim>>(box));
222 : const auto& old_mesh = old_mesh_and_element.first;
223 :
224 : // Determine new neighbors and update the Element
225 : const auto& amr_info_of_neighbors =
226 : db::get<amr::Tags::NeighborInfo<volume_dim>>(box);
227 : auto [new_neighbors, new_neighbor_meshes] =
228 : neighbors_of_child(old_mesh_and_element.second, my_amr_info,
229 : amr_info_of_neighbors, element_id);
230 : ::Initialization::mutate_assign<
231 : tmpl::list<::domain::Tags::Element<volume_dim>,
232 : ::domain::Tags::NeighborMesh<volume_dim>>>(
233 : make_not_null(&box),
234 : Element<volume_dim>(element_id, std::move(new_neighbors)),
235 : std::move(new_neighbor_meshes));
236 :
237 : // Check for p-refinement
238 : if (alg::any_of(my_amr_flags, [](amr::Flag flag) {
239 : return (flag == amr::Flag::IncreaseResolution or
240 : flag == amr::Flag::DecreaseResolution);
241 : })) {
242 : db::mutate<::domain::Tags::Mesh<volume_dim>>(
243 : [&old_mesh,
244 : &my_amr_flags](const gsl::not_null<Mesh<volume_dim>*> mesh) {
245 : *mesh = amr::projectors::mesh(old_mesh, my_amr_flags);
246 : },
247 : make_not_null(&box));
248 :
249 : if (verbosity >= Verbosity::Debug) {
250 : Parallel::printf(
251 : "Increasing order of element %s: %s -> %s\n", element_id,
252 : old_mesh.extents(),
253 : db::get<::domain::Tags::Mesh<volume_dim>>(box).extents());
254 : }
255 : }
256 :
257 : // Run the projectors on all elements, even if they did no p-refinement.
258 : // This allows projectors to update mutable items that depend upon the
259 : // neighbors of the element.
260 : tmpl::for_each<amr_projectors>(
261 : [&box, &old_mesh_and_element](auto projector_v) {
262 : using projector = typename decltype(projector_v)::type;
263 : try {
264 : db::mutate_apply<projector>(make_not_null(&box),
265 : old_mesh_and_element);
266 : } catch (std::exception& e) {
267 : ERROR("Error in AMR projector '"
268 : << pretty_type::get_name<projector>() << "':\n"
269 : << e.what());
270 : }
271 : });
272 :
273 : // Reset the AMR flags
274 : db::mutate<amr::Tags::Info<volume_dim>,
275 : amr::Tags::NeighborInfo<volume_dim>>(
276 : [](const gsl::not_null<amr::Info<volume_dim>*> amr_info,
277 : const gsl::not_null<std::unordered_map<ElementId<volume_dim>,
278 : amr::Info<volume_dim>>*>
279 : local_amr_info_of_neighbors) {
280 : local_amr_info_of_neighbors->clear();
281 : for (size_t d = 0; d < volume_dim; ++d) {
282 : amr_info->flags[d] = amr::Flag::Undefined;
283 : }
284 : },
285 : make_not_null(&box));
286 : }
287 : }
288 : };
289 : } // namespace amr::Actions
|
