Line data Source code
1 0 : // Distributed under the MIT License.
2 : // See LICENSE.txt for details.
3 :
4 : #pragma once
5 :
6 : #include <array>
7 : #include <cstddef>
8 : #include <unordered_set>
9 :
10 : #include "DataStructures/DataBox/DataBox.hpp"
11 : #include "Domain/Amr/Flag.hpp"
12 : #include "Domain/Amr/Helpers.hpp"
13 : #include "Domain/Amr/NewNeighborIds.hpp"
14 : #include "Domain/Amr/Tags/Flags.hpp"
15 : #include "Domain/Amr/Tags/NeighborFlags.hpp"
16 : #include "Domain/Structure/Direction.hpp"
17 : #include "Domain/Structure/DirectionMap.hpp"
18 : #include "Domain/Structure/DirectionalIdMap.hpp"
19 : #include "Domain/Structure/Element.hpp"
20 : #include "Domain/Structure/ElementId.hpp"
21 : #include "Domain/Structure/Neighbors.hpp"
22 : #include "Domain/Tags.hpp"
23 : #include "Domain/Tags/NeighborMesh.hpp"
24 : #include "IO/Logging/Tags.hpp"
25 : #include "IO/Logging/Verbosity.hpp"
26 : #include "NumericalAlgorithms/Spectral/Mesh.hpp"
27 : #include "Parallel/GlobalCache.hpp"
28 : #include "Parallel/Invoke.hpp"
29 : #include "Parallel/Phase.hpp"
30 : #include "Parallel/Printf/Printf.hpp"
31 : #include "Parallel/Tags/DistributedObjectTags.hpp"
32 : #include "ParallelAlgorithms/Amr/Actions/CreateChild.hpp"
33 : #include "ParallelAlgorithms/Amr/Actions/CreateParent.hpp"
34 : #include "ParallelAlgorithms/Amr/Projectors/Mesh.hpp"
35 : #include "ParallelAlgorithms/Amr/Protocols/Projector.hpp"
36 : #include "ParallelAlgorithms/Amr/Tags.hpp"
37 : #include "Utilities/Algorithm.hpp"
38 : #include "Utilities/ErrorHandling/Assert.hpp"
39 : #include "Utilities/Gsl.hpp"
40 : #include "Utilities/Literals.hpp"
41 : #include "Utilities/PrettyType.hpp"
42 : #include "Utilities/ProtocolHelpers.hpp"
43 : #include "Utilities/TMPL.hpp"
44 :
45 : namespace amr {
46 : /// \cond
47 : template <class Metavariables>
48 : struct Component;
49 : /// \endcond
50 : } // namespace amr
51 :
52 : namespace detail {
53 : template <typename ListOfProjectors>
54 : struct GetMutatedTags;
55 :
56 : template <typename... Projectors>
57 : struct GetMutatedTags<tmpl::list<Projectors...>> {
58 : using type = tmpl::remove_duplicates<
59 : tmpl::flatten<tmpl::append<typename Projectors::return_tags...>>>;
60 : };
61 : } // namespace detail
62 :
63 : namespace amr::Actions {
64 : /// \brief Adjusts the domain given the refinement criteria
65 : ///
66 : /// \details
67 : /// - Checks if an Element wants to split in any dimension; if yes, determines
68 : /// the ElementId%s of the new children Element%s and calls
69 : /// amr::Actions::CreateChild on the amr::Component, then exits the action.
70 : /// - Checks if an Element wants to join in any dimension; if yes, either calls
71 : /// amr::Actions::CreateParent on the amr::Component if this is the child
72 : /// Element that should create the parent Element or does nothing, and then
73 : /// exits the action.
74 : /// - Checks if an Element wants to increase or decrease its resolution, if yes,
75 : /// mutates the Mesh
76 : /// - Updates the Neighbors of the Element
77 : /// - Resets amr::Tags::Flag%s to amr::Flag::Undefined
78 : /// - Resets amr::Tags::NeighborInfo to an empty map
79 : /// - Mutates all return_tags of Metavariables::amr::projectors
80 1 : struct AdjustDomain {
81 : template <typename ParallelComponent, typename DbTagList,
82 : typename Metavariables>
83 0 : static void apply(db::DataBox<DbTagList>& box,
84 : Parallel::GlobalCache<Metavariables>& cache,
85 : const ElementId<Metavariables::volume_dim>& element_id) {
86 : constexpr size_t volume_dim = Metavariables::volume_dim;
87 : using amr_projectors = typename Metavariables::amr::projectors;
88 : static_assert(
89 : tmpl::all<
90 : amr_projectors,
91 : tt::assert_conforms_to<tmpl::_1, amr::protocols::Projector>>::value,
92 : "All AMR projectors must conform to 'amr::protocols::Projector'.");
93 :
94 : // To prevent bugs when new mutable items are added to a DataBox, we require
95 : // that all mutable_item_creation_tags of box are either:
96 : // - mutated by one of the projectors in Metavariables::amr::projectors
97 : // - in the list of distributed_object_tags
98 : // - or in the list of tags mutated by this action
99 : using distributed_object_tags =
100 : typename ::Parallel::Tags::distributed_object_tags<
101 : Metavariables, ElementId<volume_dim>>;
102 : using tags_mutated_by_this_action = tmpl::list<
103 : ::domain::Tags::Element<volume_dim>, ::domain::Tags::Mesh<volume_dim>,
104 : ::domain::Tags::NeighborMesh<volume_dim>, amr::Tags::Info<volume_dim>,
105 : amr::Tags::NeighborInfo<volume_dim>>;
106 : using mutated_tags =
107 : tmpl::append<distributed_object_tags, tags_mutated_by_this_action,
108 : typename detail::GetMutatedTags<amr_projectors>::type>;
109 : using mutable_tags =
110 : typename db::DataBox<DbTagList>::mutable_item_creation_tags;
111 : using mutable_tags_not_mutated =
112 : tmpl::list_difference<mutable_tags, mutated_tags>;
113 : static_assert(std::is_same_v<mutable_tags_not_mutated, tmpl::list<>>,
114 : "All mutable tags in the DataBox must be explicitly mutated "
115 : "by an amr::projector. Default initialized objects can use "
116 : "amr::projector::DefaultInitialize.");
117 :
118 : const auto& my_amr_info = db::get<amr::Tags::Info<volume_dim>>(box);
119 : const auto& my_amr_flags = my_amr_info.flags;
120 : auto& element_array =
121 : Parallel::get_parallel_component<ParallelComponent>(cache);
122 : const auto& phase_bookmarks =
123 : Parallel::local(element_array[element_id])->phase_bookmarks();
124 : const auto& verbosity =
125 : db::get<logging::Tags::Verbosity<amr::OptionTags::AmrGroup>>(box);
126 :
127 : if (alg::all_of(my_amr_flags, [](amr::Flag flag) {
128 : return flag == amr::Flag::Undefined;
129 : })) {
130 : // AMR flags are undefined. This state can be reached when the AMR
131 : // component broadcasts the `AdjustDomain` simple action to the entire
132 : // element array, then some of those elements run the simple action and
133 : // create new child elements, and then the broadcast also arrives at these
134 : // new elements for some reason (seems like a Charm++ bug). The AMR flags
135 : // will be undefined in this case, so we just ignore the broadcast and
136 : // return early here.
137 : return;
138 : } else if (alg::any_of(my_amr_flags, [](amr::Flag flag) {
139 : return flag == amr::Flag::Split;
140 : })) {
141 : // h-refinement
142 : using ::operator<<;
143 : ASSERT(alg::count(my_amr_flags, amr::Flag::Join) == 0,
144 : "Element " << element_id
145 : << " cannot both split and join, but had AMR flags "
146 : << my_amr_flags << "\n");
147 : auto children_ids = amr::ids_of_children(element_id, my_amr_flags);
148 : auto& amr_component =
149 : Parallel::get_parallel_component<amr::Component<Metavariables>>(
150 : cache);
151 : if (verbosity >= Verbosity::Debug) {
152 : Parallel::printf("Splitting element %s into %zu: %s\n", element_id,
153 : children_ids.size(), children_ids);
154 : }
155 : Parallel::simple_action<CreateChild>(amr_component, element_array,
156 : element_id, children_ids, 0_st,
157 : phase_bookmarks);
158 :
159 : } else if (alg::any_of(my_amr_flags, [](amr::Flag flag) {
160 : return flag == amr::Flag::Join;
161 : })) {
162 : // h-coarsening
163 : // Only one element should create the new parent
164 : if (amr::is_child_that_creates_parent(element_id, my_amr_flags)) {
165 : auto parent_id = amr::id_of_parent(element_id, my_amr_flags);
166 : const auto& element = db::get<::domain::Tags::Element<volume_dim>>(box);
167 : auto ids_to_join = amr::ids_of_joining_neighbors(element, my_amr_flags);
168 : auto& amr_component =
169 : Parallel::get_parallel_component<amr::Component<Metavariables>>(
170 : cache);
171 : if (verbosity >= Verbosity::Debug) {
172 : Parallel::printf("Joining %zu elements: %s -> %s\n",
173 : ids_to_join.size(), ids_to_join, parent_id);
174 : }
175 : Parallel::simple_action<CreateParent>(
176 : amr_component, element_array, std::move(parent_id), element_id,
177 : std::move(ids_to_join), phase_bookmarks);
178 : }
179 :
180 : } else {
181 : // Neither h-refinement nor h-coarsening. This element will remain.
182 : const auto old_mesh_and_element =
183 : std::make_pair(db::get<::domain::Tags::Mesh<volume_dim>>(box),
184 : db::get<::domain::Tags::Element<volume_dim>>(box));
185 : const auto& old_mesh = old_mesh_and_element.first;
186 :
187 : // Determine new neighbors and update the Element
188 : { // avoid shadowing when mutating flags below
189 : using NeighborMeshType =
190 : DirectionalIdMap<volume_dim, ::Mesh<volume_dim>>;
191 : const auto& amr_info_of_neighbors =
192 : db::get<amr::Tags::NeighborInfo<volume_dim>>(box);
193 : db::mutate<::domain::Tags::Element<volume_dim>,
194 : ::domain::Tags::NeighborMesh<volume_dim>>(
195 : [&element_id, &amr_info_of_neighbors](
196 : const gsl::not_null<Element<volume_dim>*> element,
197 : const gsl::not_null<NeighborMeshType*> neighbor_meshes) {
198 : auto new_neighbors = element->neighbors();
199 : neighbor_meshes->clear();
200 : for (auto& [direction, neighbors] : new_neighbors) {
201 : const auto new_neighbor_ids_and_meshes = amr::new_neighbor_ids(
202 : element_id, direction, neighbors, amr_info_of_neighbors);
203 : std::unordered_set<ElementId<volume_dim>> new_neighbor_ids;
204 : for (const auto& [id, mesh] : new_neighbor_ids_and_meshes) {
205 : neighbor_meshes->insert({{direction, id}, mesh});
206 : new_neighbor_ids.insert(id);
207 : }
208 : neighbors.set_ids_to(new_neighbor_ids);
209 : }
210 : *element =
211 : Element<volume_dim>(element_id, std::move(new_neighbors));
212 : },
213 : make_not_null(&box));
214 : }
215 :
216 : // Check for p-refinement
217 : if (alg::any_of(my_amr_flags, [](amr::Flag flag) {
218 : return (flag == amr::Flag::IncreaseResolution or
219 : flag == amr::Flag::DecreaseResolution);
220 : })) {
221 : db::mutate<::domain::Tags::Mesh<volume_dim>>(
222 : [&old_mesh,
223 : &my_amr_flags](const gsl::not_null<Mesh<volume_dim>*> mesh) {
224 : *mesh = amr::projectors::mesh(old_mesh, my_amr_flags);
225 : },
226 : make_not_null(&box));
227 :
228 : if (verbosity >= Verbosity::Debug) {
229 : Parallel::printf(
230 : "Increasing order of element %s: %s -> %s\n", element_id,
231 : old_mesh.extents(),
232 : db::get<::domain::Tags::Mesh<volume_dim>>(box).extents());
233 : }
234 : }
235 :
236 : // Run the projectors on all elements, even if they did no h-refinement.
237 : // This allows projectors to update mutable items that depend upon the
238 : // neighbors of the element.
239 : tmpl::for_each<amr_projectors>(
240 : [&box, &old_mesh_and_element](auto projector_v) {
241 : using projector = typename decltype(projector_v)::type;
242 : try {
243 : db::mutate_apply<projector>(make_not_null(&box),
244 : old_mesh_and_element);
245 : } catch (std::exception& e) {
246 : ERROR("Error in AMR projector '"
247 : << pretty_type::get_name<projector>() << "':\n"
248 : << e.what());
249 : }
250 : });
251 :
252 : // Reset the AMR flags
253 : db::mutate<amr::Tags::Info<volume_dim>,
254 : amr::Tags::NeighborInfo<volume_dim>>(
255 : [](const gsl::not_null<amr::Info<volume_dim>*> amr_info,
256 : const gsl::not_null<std::unordered_map<ElementId<volume_dim>,
257 : amr::Info<volume_dim>>*>
258 : amr_info_of_neighbors) {
259 : amr_info_of_neighbors->clear();
260 : for (size_t d = 0; d < volume_dim; ++d) {
261 : amr_info->flags[d] = amr::Flag::Undefined;
262 : }
263 : },
264 : make_not_null(&box));
265 : }
266 : }
267 : };
268 : } // namespace amr::Actions
|
