Line data Source code
1 0 : // Distributed under the MIT License.
2 : // See LICENSE.txt for details.
3 :
4 : #pragma once
5 :
6 : #include <array>
7 : #include <cstddef>
8 : #include <unordered_set>
9 :
10 : #include "DataStructures/DataBox/DataBox.hpp"
11 : #include "Domain/Amr/Flag.hpp"
12 : #include "Domain/Amr/Helpers.hpp"
13 : #include "Domain/Amr/NewNeighborIds.hpp"
14 : #include "Domain/Amr/Tags/Flags.hpp"
15 : #include "Domain/Amr/Tags/NeighborFlags.hpp"
16 : #include "Domain/Structure/Direction.hpp"
17 : #include "Domain/Structure/DirectionMap.hpp"
18 : #include "Domain/Structure/DirectionalIdMap.hpp"
19 : #include "Domain/Structure/Element.hpp"
20 : #include "Domain/Structure/ElementId.hpp"
21 : #include "Domain/Structure/IsValidDgMesh.hpp"
22 : #include "Domain/Structure/Neighbors.hpp"
23 : #include "Domain/Tags.hpp"
24 : #include "Domain/Tags/NeighborMesh.hpp"
25 : #include "IO/Logging/Tags.hpp"
26 : #include "IO/Logging/Verbosity.hpp"
27 : #include "NumericalAlgorithms/Spectral/Mesh.hpp"
28 : #include "Parallel/ArrayCollection/IsDgElementCollection.hpp"
29 : #include "Parallel/GlobalCache.hpp"
30 : #include "Parallel/Invoke.hpp"
31 : #include "Parallel/Phase.hpp"
32 : #include "Parallel/Printf/Printf.hpp"
33 : #include "Parallel/Tags/DistributedObjectTags.hpp"
34 : #include "Parallel/Tags/Section.hpp"
35 : #include "ParallelAlgorithms/Amr/Actions/CreateChild.hpp"
36 : #include "ParallelAlgorithms/Amr/Actions/CreateParent.hpp"
37 : #include "ParallelAlgorithms/Amr/Projectors/Mesh.hpp"
38 : #include "ParallelAlgorithms/Amr/Protocols/Projector.hpp"
39 : #include "ParallelAlgorithms/Amr/Tags.hpp"
40 : #include "Utilities/Algorithm.hpp"
41 : #include "Utilities/ErrorHandling/Assert.hpp"
42 : #include "Utilities/ErrorHandling/Error.hpp"
43 : #include "Utilities/Gsl.hpp"
44 : #include "Utilities/Literals.hpp"
45 : #include "Utilities/PrettyType.hpp"
46 : #include "Utilities/ProtocolHelpers.hpp"
47 : #include "Utilities/TMPL.hpp"
48 :
49 : namespace amr {
50 : /// \cond
51 : template <class Metavariables>
52 : struct Component;
53 : /// \endcond
54 : } // namespace amr
55 :
56 : namespace detail {
57 : template <typename ListOfProjectors>
58 : struct GetMutatedTags;
59 :
60 : template <typename... Projectors>
61 : struct GetMutatedTags<tmpl::list<Projectors...>> {
62 : using type = tmpl::remove_duplicates<
63 : tmpl::flatten<tmpl::append<typename Projectors::return_tags...>>>;
64 : };
65 : } // namespace detail
66 :
67 : namespace amr::Actions {
68 : /// \brief Adjusts the domain given the refinement criteria
69 : ///
70 : /// \details
71 : /// - Checks if an Element wants to split in any dimension; if yes, determines
72 : /// the ElementId%s of the new children Element%s and calls
73 : /// amr::Actions::CreateChild on the amr::Component, then exits the action.
74 : /// - Checks if an Element wants to join in any dimension; if yes, either calls
75 : /// amr::Actions::CreateParent on the amr::Component if this is the child
76 : /// Element that should create the parent Element or does nothing, and then
77 : /// exits the action.
78 : /// - Checks if an Element wants to increase or decrease its resolution, if yes,
79 : /// mutates the Mesh
80 : /// - Updates the Neighbors of the Element
81 : /// - Resets amr::Tags::Flag%s to amr::Flag::Undefined
82 : /// - Resets amr::Tags::NeighborInfo to an empty map
83 : /// - Mutates all return_tags of Metavariables::amr::projectors
84 1 : struct AdjustDomain {
85 : template <typename ParallelComponent, typename DbTagList,
86 : typename Metavariables>
87 0 : static void apply(db::DataBox<DbTagList>& box,
88 : Parallel::GlobalCache<Metavariables>& cache,
89 : const ElementId<Metavariables::volume_dim>& element_id) {
90 : if constexpr (Parallel::is_dg_element_collection_v<ParallelComponent>) {
91 : ERROR("Can't adjust the domain for DG elements running on nodegroups.");
92 : } else {
93 : constexpr size_t volume_dim = Metavariables::volume_dim;
94 : using amr_projectors = typename Metavariables::amr::projectors;
95 : static_assert(
96 : tmpl::all<amr_projectors,
97 : tt::assert_conforms_to<tmpl::_1,
98 : amr::protocols::Projector>>::value,
99 : "All AMR projectors must conform to 'amr::protocols::Projector'.");
100 :
101 : // To prevent bugs when new mutable items are added to a DataBox, we
102 : // require that all mutable_item_creation_tags of box are either:
103 : // - mutated by one of the projectors in Metavariables::amr::projectors
104 : // - in the list of distributed_object_tags
105 : // - or in the list of tags mutated by this action
106 : using distributed_object_tags =
107 : typename ::Parallel::Tags::distributed_object_tags<
108 : Metavariables, ElementId<volume_dim>>;
109 : using tags_mutated_by_this_action = tmpl::list<
110 : ::domain::Tags::Element<volume_dim>, ::domain::Tags::Mesh<volume_dim>,
111 : ::domain::Tags::NeighborMesh<volume_dim>, amr::Tags::Info<volume_dim>,
112 : amr::Tags::NeighborInfo<volume_dim>, amr::Tags::ParentId<volume_dim>,
113 : amr::Tags::ChildIds<volume_dim>, amr::Tags::ParentMesh<volume_dim>,
114 : Parallel::Tags::Section<ParallelComponent, amr::Tags::GridIndex>,
115 : Parallel::Tags::Section<ParallelComponent, amr::Tags::IsFinestGrid>>;
116 : using mutated_tags =
117 : tmpl::append<distributed_object_tags, tags_mutated_by_this_action,
118 : typename detail::GetMutatedTags<amr_projectors>::type>;
119 : using mutable_tags =
120 : typename db::DataBox<DbTagList>::mutable_item_creation_tags;
121 : using mutable_tags_not_mutated =
122 : tmpl::list_difference<mutable_tags, mutated_tags>;
123 : static_assert(
124 : std::is_same_v<mutable_tags_not_mutated, tmpl::list<>>,
125 : "All mutable tags in the DataBox must be explicitly mutated "
126 : "by an amr::projector. Default initialized objects can use "
127 : "amr::projector::DefaultInitialize.");
128 :
129 : const auto& my_amr_info = db::get<amr::Tags::Info<volume_dim>>(box);
130 : const auto& my_amr_flags = my_amr_info.flags;
131 : auto& element_array =
132 : Parallel::get_parallel_component<ParallelComponent>(cache);
133 : auto& amr_component =
134 : Parallel::get_parallel_component<amr::Component<Metavariables>>(
135 : cache);
136 : const auto& phase_bookmarks =
137 : Parallel::local(element_array[element_id])->phase_bookmarks();
138 : const auto& verbosity =
139 : db::get<logging::Tags::Verbosity<amr::OptionTags::AmrGroup>>(box);
140 :
141 : if (alg::all_of(my_amr_flags, [](amr::Flag flag) {
142 : return flag == amr::Flag::Undefined;
143 : })) {
144 : // AMR flags are undefined. This state can be reached when the AMR
145 : // component broadcasts the `AdjustDomain` simple action to the entire
146 : // element array, then some of those elements run the simple action and
147 : // create new child elements, and then the broadcast also arrives at
148 : // these new elements for some reason (seems like a Charm++ bug). The
149 : // AMR flags will be undefined in this case, so we just ignore the
150 : // broadcast and return early here.
151 : return;
152 : } else if (alg::any_of(my_amr_flags, [](amr::Flag flag) {
153 : return flag == amr::Flag::Split;
154 : })) {
155 : // h-refinement
156 : using ::operator<<;
157 : ASSERT(alg::count(my_amr_flags, amr::Flag::Join) == 0,
158 : "Element " << element_id
159 : << " cannot both split and join, but had AMR flags "
160 : << my_amr_flags << "\n");
161 : const size_t child_grid_index = Metavariables::amr::keep_coarse_grids
162 : ? element_id.grid_index() + 1
163 : : element_id.grid_index();
164 : auto children_ids =
165 : amr::ids_of_children(element_id, my_amr_flags, child_grid_index);
166 : if (verbosity >= Verbosity::Debug) {
167 : Parallel::printf("Splitting element %s into %zu: %s\n", element_id,
168 : children_ids.size(), children_ids);
169 : }
170 : Parallel::simple_action<CreateChild>(amr_component, element_array,
171 : element_id, children_ids, 0_st,
172 : phase_bookmarks);
173 : return;
174 : } else if (alg::any_of(my_amr_flags, [](amr::Flag flag) {
175 : return flag == amr::Flag::Join;
176 : })) {
177 : // h-coarsening
178 : if (Metavariables::amr::keep_coarse_grids) {
179 : ERROR(
180 : "When AMR keeps coarse grids then no h-coarsening during AMR is "
181 : "allowed, but element "
182 : << element_id
183 : << " requested h-coarsening. Set the 'AllowCoarsening' policy to "
184 : "false to disable coarsening.");
185 : // Note: this restriction could be relaxed if ever needed.
186 : return;
187 : }
188 : // Only one element should create the new parent
189 : if (amr::is_child_that_creates_parent(element_id, my_amr_flags)) {
190 : auto parent_id = amr::id_of_parent(element_id, my_amr_flags);
191 : const auto& element =
192 : db::get<::domain::Tags::Element<volume_dim>>(box);
193 : auto ids_to_join =
194 : amr::ids_of_joining_neighbors(element, my_amr_flags);
195 : if (verbosity >= Verbosity::Debug) {
196 : Parallel::printf("Joining %zu elements: %s -> %s\n",
197 : ids_to_join.size(), ids_to_join, parent_id);
198 : }
199 : Parallel::simple_action<CreateParent>(
200 : amr_component, element_array, std::move(parent_id), element_id,
201 : std::move(ids_to_join), phase_bookmarks);
202 : }
203 : return;
204 : }
205 :
206 : // Neither h-refinement nor h-coarsening. This element will remain.
207 :
208 : if constexpr (Metavariables::amr::keep_coarse_grids) {
209 : // Create new element that covers this one with an incremented grid
210 : // index. p-refinement will be handled by the newly created element.
211 : ElementId<volume_dim> new_element_id{element_id.block_id(),
212 : element_id.segment_ids(),
213 : element_id.grid_index() + 1};
214 : Parallel::simple_action<CreateChild>(
215 : amr_component, element_array, element_id,
216 : std::vector<ElementId<volume_dim>>{new_element_id}, 0_st,
217 : phase_bookmarks);
218 : return;
219 : }
220 :
221 : const auto old_mesh_and_element =
222 : std::make_pair(db::get<::domain::Tags::Mesh<volume_dim>>(box),
223 : db::get<::domain::Tags::Element<volume_dim>>(box));
224 : const auto& old_mesh = old_mesh_and_element.first;
225 : const auto& old_element = old_mesh_and_element.second;
226 :
227 : // Determine new neighbors and update the Element
228 : const auto& amr_info_of_neighbors =
229 : db::get<amr::Tags::NeighborInfo<volume_dim>>(box);
230 : auto [new_neighbors, new_neighbor_meshes] = neighbors_of_child(
231 : old_element, my_amr_info, amr_info_of_neighbors, element_id);
232 : ::Initialization::mutate_assign<
233 : tmpl::list<::domain::Tags::Element<volume_dim>,
234 : ::domain::Tags::NeighborMesh<volume_dim>>>(
235 : make_not_null(&box),
236 : Element<volume_dim>(element_id, std::move(new_neighbors),
237 : old_element.topologies()),
238 : std::move(new_neighbor_meshes));
239 :
240 : // Check for p-refinement
241 : if (alg::any_of(my_amr_flags, [](amr::Flag flag) {
242 : return (flag == amr::Flag::IncreaseResolution or
243 : flag == amr::Flag::DecreaseResolution);
244 : })) {
245 : db::mutate<::domain::Tags::Mesh<volume_dim>>(
246 : [&old_mesh, &old_element,
247 : &my_amr_flags](const gsl::not_null<Mesh<volume_dim>*> mesh) {
248 : *mesh = amr::projectors::p_refined_mesh(old_mesh, my_amr_flags,
249 : old_element.topologies());
250 : },
251 : make_not_null(&box));
252 :
253 : if (verbosity >= Verbosity::Debug) {
254 : Parallel::printf(
255 : "Changing order of element %s: %s -> %s\n", element_id,
256 : old_mesh.extents(),
257 : db::get<::domain::Tags::Mesh<volume_dim>>(box).extents());
258 : }
259 : }
260 :
261 : const auto& new_mesh = db::get<::domain::Tags::Mesh<volume_dim>>(box);
262 : const auto& new_element =
263 : db::get<::domain::Tags::Element<volume_dim>>(box);
264 : if (not domain::is_valid_dg_mesh(new_mesh, new_element)) {
265 : ERROR("Invalid mesh " << new_mesh << " for Element " << new_element);
266 : }
267 :
268 : // Run the projectors on all elements, even if they did no p-refinement.
269 : // This allows projectors to update mutable items that depend upon the
270 : // neighbors of the element.
271 : tmpl::for_each<amr_projectors>(
272 : [&box, &old_mesh_and_element](auto projector_v) {
273 : using projector = typename decltype(projector_v)::type;
274 : try {
275 : db::mutate_apply<projector>(make_not_null(&box),
276 : old_mesh_and_element);
277 : } catch (std::exception& e) {
278 : ERROR("Error in AMR projector '"
279 : << pretty_type::get_name<projector>() << "':\n"
280 : << e.what());
281 : }
282 : });
283 :
284 : // Reset the AMR flags
285 : db::mutate<amr::Tags::Info<volume_dim>,
286 : amr::Tags::NeighborInfo<volume_dim>>(
287 : [](const gsl::not_null<amr::Info<volume_dim>*> amr_info,
288 : const gsl::not_null<std::unordered_map<ElementId<volume_dim>,
289 : amr::Info<volume_dim>>*>
290 : local_amr_info_of_neighbors) {
291 : local_amr_info_of_neighbors->clear();
292 : for (size_t d = 0; d < volume_dim; ++d) {
293 : amr_info->flags[d] = amr::Flag::Undefined;
294 : }
295 : },
296 : make_not_null(&box));
297 : }
298 : }
299 : };
300 : } // namespace amr::Actions
|
