Line data Source code
1 0 : // Distributed under the MIT License.
2 : // See LICENSE.txt for details.
3 :
4 : #pragma once
5 :
6 : #include <cstddef>
7 : #include <optional>
8 : #include <tuple>
9 : #include <unordered_set>
10 : #include <utility>
11 : #include <vector>
12 :
13 : #include "DataStructures/DataBox/DataBox.hpp"
14 : #include "Domain/Block.hpp"
15 : #include "Domain/Creators/Tags/Domain.hpp"
16 : #include "Domain/Creators/Tags/InitialExtents.hpp"
17 : #include "Domain/Creators/Tags/InitialRefinementLevels.hpp"
18 : #include "Domain/Domain.hpp"
19 : #include "Domain/Structure/ElementId.hpp"
20 : #include "Domain/Tags/ElementDistribution.hpp"
21 : #include "Evolution/DiscontinuousGalerkin/Initialization/QuadratureTag.hpp"
22 : #include "NumericalAlgorithms/Spectral/Basis.hpp"
23 : #include "NumericalAlgorithms/Spectral/Quadrature.hpp"
24 : #include "Parallel/AlgorithmExecution.hpp"
25 : #include "Parallel/ArrayCollection/CreateElementsUsingDistribution.hpp"
26 : #include "Parallel/ArrayCollection/SpawnInitializeElementsInCollection.hpp"
27 : #include "Parallel/ArrayCollection/Tags/ElementCollection.hpp"
28 : #include "Parallel/ArrayCollection/Tags/ElementLocations.hpp"
29 : #include "Parallel/ArrayCollection/Tags/NumberOfElementsTerminated.hpp"
30 : #include "Parallel/GlobalCache.hpp"
31 : #include "Parallel/Info.hpp"
32 : #include "Parallel/Local.hpp"
33 : #include "Parallel/NodeLock.hpp"
34 : #include "Parallel/Reduction.hpp"
35 : #include "Utilities/ErrorHandling/Assert.hpp"
36 : #include "Utilities/ErrorHandling/Error.hpp"
37 : #include "Utilities/Functional.hpp"
38 : #include "Utilities/Gsl.hpp"
39 : #include "Utilities/TaggedTuple.hpp"
40 :
41 0 : namespace Parallel::Actions {
42 : /*!
43 : * \brief Creates the `DgElementArrayMember`s on the (node)group component.
44 : *
45 : * First the distribution of elements is computed using
46 : * `Parallel::create_elements_using_distribution()`, they are inserted on
47 : * each (node)group element. A reduction is done over the (node)group before
48 : * initializing the `DgElementArrayMember`s themselves, since they are allowed
49 : * to communicate with each other during their initialization. The reduction
50 : * target is `Parallel::Actions::SpawnInitializeElementsInCollection`.
51 : *
52 : * Uses:
53 : * - DataBox:
54 : * - `domain::Tags::Domain<Dim>`
55 : * - `domain::Tags::InitialRefinementLevels<Dim>`
56 : * - `domain::Tags::InitialExtents<Dim>`
57 : * - `evolution::dg::Tags::Quadrature`
58 : * - `domain::Tags::ElementDistribution`
59 : *
60 : * DataBox changes:
61 : * - Adds:
62 : * - `Parallel::Tags::ElementCollection`
63 : * - `Parallel::Tags::ElementLocations<Dim>`
64 : * - `Parallel::Tags::NumberOfElementsTerminated`
65 : * - Removes: nothing
66 : * - Modifies:
67 : * - `Parallel::Tags::ElementCollection`
68 : * - `Parallel::Tags::ElementLocations<Dim>`
69 : * - `Parallel::Tags::NumberOfElementsTerminated`
70 : */
71 : template <size_t Dim, class Metavariables, class PhaseDepActionList,
72 : typename SimpleTagsFromOptions>
73 1 : struct CreateElementCollection {
74 0 : using simple_tags = tmpl::list<
75 : Parallel::Tags::ElementCollection<Dim, Metavariables, PhaseDepActionList,
76 : SimpleTagsFromOptions>,
77 : Parallel::Tags::ElementLocations<Dim>, Tags::NumberOfElementsTerminated>;
78 0 : using compute_tags = tmpl::list<>;
79 0 : using const_global_cache_tags =
80 : tmpl::list<::domain::Tags::Domain<Dim>,
81 : ::domain::Tags::ElementDistribution>;
82 :
83 0 : using return_tag_list = tmpl::append<simple_tags, compute_tags>;
84 :
85 : template <typename DbTagsList, typename... InboxTags, typename ArrayIndex,
86 : typename ActionList, typename ParallelComponent>
87 0 : static Parallel::iterable_action_return_t apply(
88 : db::DataBox<DbTagsList>& box,
89 : const tuples::TaggedTuple<InboxTags...>& /*inboxes*/,
90 : Parallel::GlobalCache<Metavariables>& local_cache,
91 : const ArrayIndex& /*array_index*/, const ActionList /*meta*/,
92 : const ParallelComponent* const /*meta*/) {
93 : const std::unordered_set<size_t> procs_to_ignore{};
94 :
95 : const auto& domain = Parallel::get<domain::Tags::Domain<Dim>>(local_cache);
96 : const auto& initial_refinement_levels =
97 : get<domain::Tags::InitialRefinementLevels<Dim>>(box);
98 : const auto& initial_extents = get<domain::Tags::InitialExtents<Dim>>(box);
99 : const auto i1_basis = Spectral::Basis::Legendre;
100 : const auto& i1_quadrature = get<evolution::dg::Tags::Quadrature>(box);
101 : const std::optional<domain::ElementWeight>& element_weight =
102 : Parallel::get<domain::Tags::ElementDistribution>(local_cache);
103 :
104 : const size_t number_of_procs =
105 : Parallel::number_of_procs<size_t>(local_cache);
106 : const size_t number_of_nodes =
107 : Parallel::number_of_nodes<size_t>(local_cache);
108 : const size_t num_of_procs_to_use = number_of_procs - procs_to_ignore.size();
109 :
110 : const auto& blocks = domain.blocks();
111 :
112 : const size_t total_num_elements = [&blocks, &initial_refinement_levels]() {
113 : size_t result = 0;
114 : for (const auto& block : blocks) {
115 : const auto& initial_ref_levs = initial_refinement_levels[block.id()];
116 : for (const size_t ref_lev : initial_ref_levs) {
117 : result += two_to_the(ref_lev);
118 : }
119 : }
120 : return result;
121 : }();
122 : std::vector<std::pair<ElementId<Dim>, size_t>> my_elements_and_cores{};
123 : my_elements_and_cores.reserve(total_num_elements / number_of_nodes + 1);
124 : std::unordered_map<ElementId<Dim>, size_t> node_of_elements{};
125 : const size_t my_node = Parallel::my_node<size_t>(local_cache);
126 :
127 : Parallel::create_elements_using_distribution(
128 : [&my_elements_and_cores, my_node, &node_of_elements](
129 : const ElementId<Dim>& element_id, const size_t target_proc,
130 : const size_t target_node) {
131 : node_of_elements.insert(std::pair{element_id, target_node});
132 : if (target_node == my_node) {
133 : my_elements_and_cores.push_back(std::pair{element_id, target_proc});
134 : }
135 : },
136 : element_weight, blocks, initial_extents, initial_refinement_levels,
137 : i1_basis, i1_quadrature,
138 : // The below arguments control how the elements are mapped to the
139 : // hardware.
140 : procs_to_ignore, number_of_procs, number_of_nodes, num_of_procs_to_use,
141 : local_cache, my_node == 0);
142 :
143 : tuples::tagged_tuple_from_typelist<SimpleTagsFromOptions>
144 : initialization_items = db::copy_items<SimpleTagsFromOptions>(box);
145 :
146 : const gsl::not_null<Parallel::NodeLock*> node_lock = make_not_null(
147 : &Parallel::local_branch(
148 : Parallel::get_parallel_component<ParallelComponent>(local_cache))
149 : ->get_node_lock());
150 : db::mutate<Tags::ElementLocations<Dim>,
151 : Tags::ElementCollection<Dim, Metavariables, PhaseDepActionList,
152 : SimpleTagsFromOptions>,
153 : Tags::NumberOfElementsTerminated>(
154 : [&local_cache, &initialization_items, &my_elements_and_cores,
155 : &node_of_elements](
156 : const auto element_locations_ptr, const auto collection_ptr,
157 : const gsl::not_null<size_t*> number_of_elements_terminated) {
158 : *number_of_elements_terminated = 0;
159 : const auto serialized_initialization_items =
160 : serialize(initialization_items);
161 : *element_locations_ptr = std::move(node_of_elements);
162 : for (const auto& element_id_and_core : my_elements_and_cores) {
163 : const auto& element_id = element_id_and_core.first;
164 : const auto core = element_id_and_core.second;
165 : if (not collection_ptr
166 : ->emplace(
167 : std::piecewise_construct,
168 : std::forward_as_tuple(element_id),
169 : std::forward_as_tuple(
170 : local_cache.get_this_proxy(),
171 : deserialize<tuples::tagged_tuple_from_typelist<
172 : SimpleTagsFromOptions>>(
173 : serialized_initialization_items.data()),
174 : element_id))
175 : .second) {
176 : ERROR("Failed to insert element with ID: " << element_id);
177 : }
178 : if (collection_ptr->at(element_id).get_terminate() == true) {
179 : ++(*number_of_elements_terminated);
180 : } else {
181 : ERROR("Inserted element with ID "
182 : << element_id
183 : << " was not initialized in a terminated state. This is a "
184 : "bug.");
185 : }
186 : collection_ptr->at(element_id).set_core(core);
187 : }
188 : if (*number_of_elements_terminated != collection_ptr->size()) {
189 : ERROR(
190 : "The number of elements inserted must match the number of "
191 : "elements set to terminate since the default state of an "
192 : "inserted element is terminated. This is a bug.");
193 : }
194 : },
195 : make_not_null(&box));
196 :
197 : Parallel::contribute_to_reduction<
198 : Parallel::Actions::SpawnInitializeElementsInCollection>(
199 : Parallel::ReductionData<
200 : Parallel::ReductionDatum<int, funcl::AssertEqual<>>>{0},
201 : Parallel::get_parallel_component<ParallelComponent>(
202 : local_cache)[my_node],
203 : Parallel::get_parallel_component<ParallelComponent>(local_cache));
204 :
205 : return {Parallel::AlgorithmExecution::Continue, std::nullopt};
206 : }
207 : };
208 : } // namespace Parallel::Actions
|
