Line data Source code
1 1 : // Distributed under the MIT License.
2 : // See LICENSE.txt for details.
3 :
4 : /// \file
5 : /// The actions in this file keep track of all element IDs during AMR and update
6 : /// the array sections that represent the grid hierarchy.
7 :
8 : #pragma once
9 :
10 : #include <algorithm>
11 : #include <charm++.h>
12 : #include <cstddef>
13 : #include <optional>
14 : #include <unordered_map>
15 : #include <unordered_set>
16 : #include <vector>
17 :
18 : #include "Domain/Structure/ElementId.hpp"
19 : #include "Parallel/AlgorithmExecution.hpp"
20 : #include "Parallel/ElementRegistration.hpp"
21 : #include "Parallel/GlobalCache.hpp"
22 : #include "Parallel/Invoke.hpp"
23 : #include "Parallel/Local.hpp"
24 : #include "Parallel/Printf/Printf.hpp"
25 : #include "Parallel/Protocols/ElementRegistrar.hpp"
26 : #include "Parallel/Section.hpp"
27 : #include "Parallel/Tags/Section.hpp"
28 : #include "ParallelAlgorithms/Amr/Protocols/Projector.hpp"
29 : #include "ParallelAlgorithms/Amr/Tags.hpp"
30 : #include "Utilities/ProtocolHelpers.hpp"
31 : #include "Utilities/TMPL.hpp"
32 :
33 : namespace amr {
34 : template <typename Metavariables>
35 : struct Component;
36 : } // namespace amr
37 :
38 : namespace amr::Actions {
39 :
40 : template <size_t Dim>
41 0 : struct InitializeElementsRegistration {
42 0 : using simple_tags = tmpl::list<Tags::AllElementIds<Dim>>;
43 0 : using compute_tags = tmpl::list<>;
44 :
45 : template <typename DbTagsList, typename... InboxTags, typename ArrayIndex,
46 : typename Metavariables, typename ActionList,
47 : typename ParallelComponent>
48 0 : static Parallel::iterable_action_return_t apply(
49 : db::DataBox<DbTagsList>& /*box*/,
50 : const tuples::TaggedTuple<InboxTags...>& /*inboxes*/,
51 : const Parallel::GlobalCache<Metavariables>& /*cache*/,
52 : const ArrayIndex& /*array_index*/, const ActionList /*meta*/,
53 : const ParallelComponent* const /*meta*/) {
54 : return {Parallel::AlgorithmExecution::Pause, std::nullopt};
55 : }
56 : };
57 :
58 0 : struct RegisterOrDeregisterElement {
59 : template <typename ParallelComponent, typename DbTagsList,
60 : typename Metavariables, typename ArrayIndex, size_t Dim>
61 0 : static void apply(db::DataBox<DbTagsList>& box,
62 : const Parallel::GlobalCache<Metavariables>& /*cache*/,
63 : const ArrayIndex& /*array_index*/,
64 : const ElementId<Dim>& element_id,
65 : const bool register_or_deregister) {
66 : db::mutate<Tags::AllElementIds<Dim>>(
67 : [&element_id, register_or_deregister](const auto all_element_ids) {
68 : auto& element_ids = (*all_element_ids)[element_id.grid_index()];
69 : if (register_or_deregister) {
70 : element_ids.insert(element_id);
71 : } else {
72 : element_ids.erase(element_id);
73 : }
74 : },
75 : make_not_null(&box));
76 : }
77 : };
78 :
79 0 : struct RegisterElement : tt::ConformsTo<Parallel::protocols::ElementRegistrar> {
80 : public: // ElementRegistrar protocol
81 : template <typename ParallelComponent, typename DbTagList,
82 : typename Metavariables, size_t Dim>
83 0 : static void perform_registration(const db::DataBox<DbTagList>& /*box*/,
84 : Parallel::GlobalCache<Metavariables>& cache,
85 : const ElementId<Dim>& element_id) {
86 : Parallel::simple_action<RegisterOrDeregisterElement>(
87 : Parallel::get_parallel_component<::amr::Component<Metavariables>>(
88 : cache),
89 : element_id, true);
90 : }
91 :
92 : template <typename ParallelComponent, typename DbTagList,
93 : typename Metavariables, size_t Dim>
94 0 : static void perform_deregistration(
95 : const db::DataBox<DbTagList>& /*box*/,
96 : Parallel::GlobalCache<Metavariables>& cache,
97 : const ElementId<Dim>& element_id) {
98 : Parallel::simple_action<RegisterOrDeregisterElement>(
99 : Parallel::get_parallel_component<::amr::Component<Metavariables>>(
100 : cache),
101 : element_id, false);
102 : }
103 :
104 : public: // Iterable action
105 : template <typename DbTagList, typename... InboxTags, typename Metavariables,
106 : typename ActionList, typename ParallelComponent, size_t Dim>
107 0 : static Parallel::iterable_action_return_t apply(
108 : db::DataBox<DbTagList>& box,
109 : const tuples::TaggedTuple<InboxTags...>& /*inboxes*/,
110 : Parallel::GlobalCache<Metavariables>& cache,
111 : const ElementId<Dim>& element_id, const ActionList /*meta*/,
112 : const ParallelComponent* const /*meta*/) {
113 : perform_registration<ParallelComponent>(box, cache, element_id);
114 : return {Parallel::AlgorithmExecution::Continue, std::nullopt};
115 : }
116 : };
117 :
118 0 : struct UpdateSectionsOnElement {
119 : template <typename ParallelComponent, typename DbTagsList,
120 : typename Metavariables, size_t Dim>
121 0 : static void apply(
122 : db::DataBox<DbTagsList>& box,
123 : const Parallel::GlobalCache<Metavariables>& /*cache*/,
124 : const ElementId<Dim>& element_id, const size_t grid_index,
125 : Parallel::Section<ParallelComponent, Tags::GridIndex> grid_index_section,
126 : std::optional<Parallel::Section<ParallelComponent, Tags::IsFinestGrid>>
127 : finest_grid_section) {
128 : if (grid_index != element_id.grid_index()) {
129 : // Discard broadcast to elements that are not part of the section. This
130 : // happens because we broadcast to all elements, not just to the section.
131 : // Broadcasting to the section fails with a segfault for some reason.
132 : return;
133 : }
134 : db::mutate<Parallel::Tags::Section<ParallelComponent, Tags::GridIndex>,
135 : Parallel::Tags::Section<ParallelComponent, Tags::IsFinestGrid>>(
136 : [&grid_index_section, &finest_grid_section](
137 : const auto stored_grid_index_section,
138 : const auto stored_finest_grid_section) {
139 : // Only update the grid index section if we don't have one already
140 : // because the elements in the old grid didn't change. This avoids a
141 : // bug(?) with Charm++ where section reductions don't work with the
142 : // new section. It's possible that this issue is with (not) updating
143 : // the section cookie, but it's not clear how to do that because a
144 : // multicast message is needed for that and we can't even do a
145 : // broadcast to the section without a segfault.
146 : if (not stored_grid_index_section->has_value()) {
147 : *stored_grid_index_section = std::move(grid_index_section);
148 : }
149 : *stored_finest_grid_section = std::move(finest_grid_section);
150 : },
151 : make_not_null(&box));
152 : }
153 : };
154 :
155 0 : struct DestroyGrid {
156 : template <typename ParallelComponent, typename DbTagsList,
157 : typename Metavariables, size_t Dim>
158 0 : static void apply(db::DataBox<DbTagsList>& box,
159 : Parallel::GlobalCache<Metavariables>& cache,
160 : const ElementId<Dim>& element_id, const size_t grid_index) {
161 : if (grid_index == element_id.grid_index()) {
162 : // Destroy the element
163 : Parallel::deregister_element<ParallelComponent>(box, cache, element_id);
164 : auto& array_proxy =
165 : Parallel::get_parallel_component<ParallelComponent>(cache);
166 : array_proxy[element_id].ckDestroy();
167 : return;
168 : }
169 : // Unregister the parent ID if it was destroyed
170 : const auto& parent_id = db::get<amr::Tags::ParentId<Dim>>(box);
171 : if (parent_id.has_value() and parent_id->grid_index() == grid_index) {
172 : db::mutate<amr::Tags::ParentId<Dim>, amr::Tags::ParentMesh<Dim>>(
173 : [](const gsl::not_null<std::optional<ElementId<Dim>>*>
174 : stored_parent_id,
175 : const gsl::not_null<std::optional<Mesh<Dim>>*>
176 : stored_parent_mesh) {
177 : *stored_parent_id = std::nullopt;
178 : *stored_parent_mesh = std::nullopt;
179 : },
180 : make_not_null(&box));
181 : }
182 : }
183 : };
184 :
185 : template <typename ElementArray>
186 0 : struct UpdateSections {
187 0 : using const_global_cache_tags = tmpl::list<amr::Tags::MaxCoarseLevels>;
188 :
189 : template <typename DbTagList, typename... InboxTags, typename Metavariables,
190 : typename ArrayIndex, typename ActionList,
191 : typename ParallelComponent>
192 0 : static Parallel::iterable_action_return_t apply(
193 : db::DataBox<DbTagList>& box,
194 : const tuples::TaggedTuple<InboxTags...>& /*inboxes*/,
195 : Parallel::GlobalCache<Metavariables>& cache,
196 : const ArrayIndex& /*array_index*/, const ActionList /*meta*/,
197 : const ParallelComponent* const /*meta*/) {
198 : static constexpr size_t Dim = Metavariables::volume_dim;
199 : const auto& all_element_ids = db::get<Tags::AllElementIds<Dim>>(box);
200 : auto& element_array = Parallel::get_parallel_component<ElementArray>(cache);
201 : const size_t finest_grid_index = std::prev(all_element_ids.end())->first;
202 : const std::optional<size_t> max_coarse_levels =
203 : db::get<amr::Tags::MaxCoarseLevels>(box);
204 : for (const auto& [grid_index, element_ids] : all_element_ids) {
205 : if (max_coarse_levels.has_value() and
206 : finest_grid_index - grid_index > max_coarse_levels.value()) {
207 : // Delete grids that are coarser than the maximum allowed level
208 : Parallel::simple_action<DestroyGrid>(element_array, grid_index);
209 : continue;
210 : }
211 : std::vector<CkArrayIndex> array_indices(element_ids.size());
212 : std::transform(
213 : element_ids.begin(), element_ids.end(), array_indices.begin(),
214 : [](const ElementId<Dim>& local_element_id) {
215 : return Parallel::ArrayIndex<ElementId<Dim>>(local_element_id);
216 : });
217 : using GridIndexSection = Parallel::Section<ElementArray, Tags::GridIndex>;
218 : GridIndexSection grid_index_section{
219 : grid_index, GridIndexSection::cproxy_section::ckNew(
220 : element_array.ckGetArrayID(), array_indices.data(),
221 : array_indices.size())};
222 : using FinestGridSection =
223 : Parallel::Section<ElementArray, Tags::IsFinestGrid>;
224 : const std::optional<FinestGridSection> finest_grid_section =
225 : grid_index == finest_grid_index
226 : ? std::make_optional(FinestGridSection{
227 : true, FinestGridSection::cproxy_section::ckNew(
228 : element_array.ckGetArrayID(),
229 : array_indices.data(), array_indices.size())})
230 : : std::nullopt;
231 : // Send new sections to all elements. Broadcasting to the section fails
232 : // with a segfault for some reason.
233 : Parallel::simple_action<UpdateSectionsOnElement>(
234 : element_array, grid_index, grid_index_section, finest_grid_section);
235 : }
236 : return {Parallel::AlgorithmExecution::Pause, std::nullopt};
237 : }
238 : };
239 :
240 : } // namespace amr::Actions
|
