Line data Source code
1 0 : // Distributed under the MIT License.
2 : // See LICENSE.txt for details.
3 :
4 : #pragma once
5 :
6 : #include <cstddef>
7 : #include <optional>
8 : #include <tuple>
9 : #include <unordered_map>
10 : #include <unordered_set>
11 : #include <utility>
12 : #include <vector>
13 :
14 : #include "DataStructures/DataBox/DataBox.hpp"
15 : #include "DataStructures/TaggedTuple.hpp"
16 : #include "Domain/Block.hpp"
17 : #include "Domain/Creators/Tags/Domain.hpp"
18 : #include "Domain/Creators/Tags/InitialExtents.hpp"
19 : #include "Domain/Creators/Tags/InitialRefinementLevels.hpp"
20 : #include "Domain/Domain.hpp"
21 : #include "Domain/Structure/ElementId.hpp"
22 : #include "Domain/Tags/ElementDistribution.hpp"
23 : #include "Evolution/DiscontinuousGalerkin/Initialization/QuadratureTag.hpp"
24 : #include "Evolution/DiscontinuousGalerkin/SubcellElementDistribution.hpp"
25 : #include "NumericalAlgorithms/Spectral/Basis.hpp"
26 : #include "NumericalAlgorithms/Spectral/Quadrature.hpp"
27 : #include "Parallel/AlgorithmExecution.hpp"
28 : #include "Parallel/ArrayCollection/CreateElementsUsingDistribution.hpp"
29 : #include "Parallel/ArrayCollection/SpawnInitializeElementsInCollection.hpp"
30 : #include "Parallel/ArrayCollection/Tags/ElementCollection.hpp"
31 : #include "Parallel/ArrayCollection/Tags/ElementLocations.hpp"
32 : #include "Parallel/ArrayCollection/Tags/NumberOfElementsTerminated.hpp"
33 : #include "Parallel/GlobalCache.hpp"
34 : #include "Parallel/Info.hpp"
35 : #include "Parallel/Local.hpp"
36 : #include "Parallel/NodeLock.hpp"
37 : #include "Parallel/Reduction.hpp"
38 : #include "Utilities/ErrorHandling/Assert.hpp"
39 : #include "Utilities/ErrorHandling/Error.hpp"
40 : #include "Utilities/Functional.hpp"
41 : #include "Utilities/Gsl.hpp"
42 :
43 0 : namespace Parallel::Actions {
44 : /*!
45 : * \brief Creates the `DgElementArrayMember`s on the (node)group component.
46 : *
47 : * First the distribution of elements is computed using
48 : * `Parallel::create_elements_using_distribution()`, they are inserted on
49 : * each (node)group element. A reduction is done over the (node)group before
50 : * initializing the `DgElementArrayMember`s themselves, since they are allowed
51 : * to communicate with each other during their initialization. The reduction
52 : * target is `Parallel::Actions::SpawnInitializeElementsInCollection`.
53 : *
54 : * Uses:
55 : * - DataBox:
56 : * - `domain::Tags::Domain<Dim>`
57 : * - `domain::Tags::InitialRefinementLevels<Dim>`
58 : * - `domain::Tags::InitialExtents<Dim>`
59 : * - `evolution::dg::Tags::Quadrature`
60 : * - `domain::Tags::ElementDistribution`
61 : *
62 : * DataBox changes:
63 : * - Adds:
64 : * - `Parallel::Tags::ElementCollection`
65 : * - `Parallel::Tags::ElementLocations<Dim>`
66 : * - `Parallel::Tags::NumberOfElementsTerminated`
67 : * - Removes: nothing
68 : * - Modifies:
69 : * - `Parallel::Tags::ElementCollection`
70 : * - `Parallel::Tags::ElementLocations<Dim>`
71 : * - `Parallel::Tags::NumberOfElementsTerminated`
72 : */
73 : template <size_t Dim, class Metavariables, class PhaseDepActionList,
74 : typename SimpleTagsFromOptions>
75 1 : struct CreateElementCollection {
76 0 : using simple_tags = tmpl::list<
77 : Parallel::Tags::ElementCollection<Dim, Metavariables, PhaseDepActionList,
78 : SimpleTagsFromOptions>,
79 : Parallel::Tags::ElementLocations<Dim>, Tags::NumberOfElementsTerminated>;
80 0 : using compute_tags = tmpl::list<>;
81 0 : using const_global_cache_tags =
82 : ::evolution::dg::element_distribution_cache_tags<Metavariables, Dim>;
83 :
84 0 : using return_tag_list = tmpl::append<simple_tags, compute_tags>;
85 :
86 : template <typename DbTagsList, typename... InboxTags, typename ArrayIndex,
87 : typename ActionList, typename ParallelComponent>
88 0 : static Parallel::iterable_action_return_t apply(
89 : db::DataBox<DbTagsList>& box,
90 : const tuples::TaggedTuple<InboxTags...>& /*inboxes*/,
91 : Parallel::GlobalCache<Metavariables>& local_cache,
92 : const ArrayIndex& /*array_index*/, const ActionList /*meta*/,
93 : const ParallelComponent* const /*meta*/) {
94 : const std::unordered_set<size_t> procs_to_ignore{};
95 :
96 : const auto& domain = Parallel::get<domain::Tags::Domain<Dim>>(local_cache);
97 : const auto& initial_refinement_levels =
98 : get<domain::Tags::InitialRefinementLevels<Dim>>(box);
99 : const auto& initial_extents = get<domain::Tags::InitialExtents<Dim>>(box);
100 : const auto i1_basis = Spectral::Basis::Legendre;
101 : const auto& i1_quadrature = get<evolution::dg::Tags::Quadrature>(box);
102 : const std::optional<domain::ElementWeight>& element_weight =
103 : Parallel::get<domain::Tags::ElementDistribution>(local_cache);
104 :
105 : const size_t number_of_procs =
106 : Parallel::number_of_procs<size_t>(local_cache);
107 : const size_t number_of_nodes =
108 : Parallel::number_of_nodes<size_t>(local_cache);
109 : const size_t num_of_procs_to_use = number_of_procs - procs_to_ignore.size();
110 :
111 : const auto& blocks = domain.blocks();
112 :
113 : const auto weighting_extents_override =
114 : ::evolution::dg::weighting_extents_override_from_cache(local_cache,
115 : initial_extents);
116 :
117 : const size_t total_num_elements = [&blocks, &initial_refinement_levels]() {
118 : size_t result = 0;
119 : for (const auto& block : blocks) {
120 : const auto& initial_ref_levs = initial_refinement_levels[block.id()];
121 : for (const size_t ref_lev : initial_ref_levs) {
122 : result += two_to_the(ref_lev);
123 : }
124 : }
125 : return result;
126 : }();
127 : std::vector<std::pair<ElementId<Dim>, size_t>> my_elements_and_cores{};
128 : my_elements_and_cores.reserve(total_num_elements / number_of_nodes + 1);
129 : std::unordered_map<ElementId<Dim>, size_t> node_of_elements{};
130 : const size_t my_node = Parallel::my_node<size_t>(local_cache);
131 :
132 : Parallel::create_elements_using_distribution(
133 : [&my_elements_and_cores, my_node, &node_of_elements](
134 : const ElementId<Dim>& element_id, const size_t target_proc,
135 : const size_t target_node) {
136 : node_of_elements.insert(std::pair{element_id, target_node});
137 : if (target_node == my_node) {
138 : my_elements_and_cores.push_back(std::pair{element_id, target_proc});
139 : }
140 : },
141 : element_weight, blocks, initial_extents, initial_refinement_levels,
142 : i1_basis, i1_quadrature,
143 : // The below arguments control how the elements are mapped to the
144 : // hardware.
145 : procs_to_ignore, number_of_procs, number_of_nodes, num_of_procs_to_use,
146 : local_cache, my_node == 0, weighting_extents_override);
147 :
148 : tuples::tagged_tuple_from_typelist<SimpleTagsFromOptions>
149 : initialization_items = db::copy_items<SimpleTagsFromOptions>(box);
150 :
151 : const gsl::not_null<Parallel::NodeLock*> node_lock = make_not_null(
152 : &Parallel::local_branch(
153 : Parallel::get_parallel_component<ParallelComponent>(local_cache))
154 : ->get_node_lock());
155 : db::mutate<Tags::ElementLocations<Dim>,
156 : Tags::ElementCollection<Dim, Metavariables, PhaseDepActionList,
157 : SimpleTagsFromOptions>,
158 : Tags::NumberOfElementsTerminated>(
159 : [&local_cache, &initialization_items, &my_elements_and_cores,
160 : &node_of_elements](
161 : const auto element_locations_ptr, const auto collection_ptr,
162 : const gsl::not_null<size_t*> number_of_elements_terminated) {
163 : *number_of_elements_terminated = 0;
164 : const auto serialized_initialization_items =
165 : serialize(initialization_items);
166 : *element_locations_ptr = std::move(node_of_elements);
167 : for (const auto& element_id_and_core : my_elements_and_cores) {
168 : const auto& element_id = element_id_and_core.first;
169 : const auto core = element_id_and_core.second;
170 : if (not collection_ptr
171 : ->emplace(
172 : std::piecewise_construct,
173 : std::forward_as_tuple(element_id),
174 : std::forward_as_tuple(
175 : local_cache.get_this_proxy(),
176 : deserialize<tuples::tagged_tuple_from_typelist<
177 : SimpleTagsFromOptions>>(
178 : serialized_initialization_items.data()),
179 : element_id))
180 : .second) {
181 : ERROR("Failed to insert element with ID: " << element_id);
182 : }
183 : if (collection_ptr->at(element_id).get_terminate() == true) {
184 : ++(*number_of_elements_terminated);
185 : } else {
186 : ERROR("Inserted element with ID "
187 : << element_id
188 : << " was not initialized in a terminated state. This is a "
189 : "bug.");
190 : }
191 : collection_ptr->at(element_id).set_core(core);
192 : }
193 : if (*number_of_elements_terminated != collection_ptr->size()) {
194 : ERROR(
195 : "The number of elements inserted must match the number of "
196 : "elements set to terminate since the default state of an "
197 : "inserted element is terminated. This is a bug.");
198 : }
199 : },
200 : make_not_null(&box));
201 :
202 : Parallel::contribute_to_reduction<
203 : Parallel::Actions::SpawnInitializeElementsInCollection>(
204 : Parallel::ReductionData<
205 : Parallel::ReductionDatum<int, funcl::AssertEqual<>>>{0},
206 : Parallel::get_parallel_component<ParallelComponent>(
207 : local_cache)[my_node],
208 : Parallel::get_parallel_component<ParallelComponent>(local_cache));
209 :
210 : return {Parallel::AlgorithmExecution::Continue, std::nullopt};
211 : }
212 : };
213 : } // namespace Parallel::Actions
|
