Line data Source code
1 0 : // Distributed under the MIT License.
2 : // See LICENSE.txt for details.
3 :
4 : #pragma once
5 :
6 : #include <cstddef>
7 : #include <optional>
8 : #include <tuple>
9 : #include <unordered_set>
10 : #include <utility>
11 : #include <vector>
12 :
13 : #include "DataStructures/DataBox/DataBox.hpp"
14 : #include "Domain/Block.hpp"
15 : #include "Domain/Creators/Tags/Domain.hpp"
16 : #include "Domain/Creators/Tags/InitialExtents.hpp"
17 : #include "Domain/Creators/Tags/InitialRefinementLevels.hpp"
18 : #include "Domain/Domain.hpp"
19 : #include "Domain/Structure/ElementId.hpp"
20 : #include "Domain/Tags/ElementDistribution.hpp"
21 : #include "Evolution/DiscontinuousGalerkin/Initialization/QuadratureTag.hpp"
22 : #include "Parallel/AlgorithmExecution.hpp"
23 : #include "Parallel/ArrayCollection/SpawnInitializeElementsInCollection.hpp"
24 : #include "Parallel/ArrayCollection/Tags/ElementCollection.hpp"
25 : #include "Parallel/ArrayCollection/Tags/ElementLocations.hpp"
26 : #include "Parallel/ArrayCollection/Tags/NumberOfElementsTerminated.hpp"
27 : #include "Parallel/CreateElementsUsingDistribution.hpp"
28 : #include "Parallel/GlobalCache.hpp"
29 : #include "Parallel/Info.hpp"
30 : #include "Parallel/Local.hpp"
31 : #include "Parallel/NodeLock.hpp"
32 : #include "Parallel/Reduction.hpp"
33 : #include "Utilities/ErrorHandling/Assert.hpp"
34 : #include "Utilities/ErrorHandling/Error.hpp"
35 : #include "Utilities/Functional.hpp"
36 : #include "Utilities/Gsl.hpp"
37 : #include "Utilities/TaggedTuple.hpp"
38 :
39 0 : namespace Parallel::Actions {
40 : /*!
41 : * \brief Creates the `DgElementArrayMember`s on the (node)group component.
42 : *
43 : * First the distribution of elements is computed using
44 : * `Parallel::create_elements_using_distribution()`, they are inserted on
45 : * each (node)group element. A reduction is done over the (node)group before
46 : * initializing the `DgElementArrayMember`s themselves, since they are allowed
47 : * to communicate with each other during their initialization. The reduction
48 : * target is `Parallel::Actions::SpawnInitializeElementsInCollection`.
49 : *
50 : * Uses:
51 : * - DataBox:
52 : * - `domain::Tags::Domain<Dim>`
53 : * - `domain::Tags::InitialRefinementLevels<Dim>`
54 : * - `domain::Tags::InitialExtents<Dim>`
55 : * - `evolution::dg::Tags::Quadrature`
56 : * - `domain::Tags::ElementDistribution`
57 : *
58 : * DataBox changes:
59 : * - Adds:
60 : * - `Parallel::Tags::ElementCollection`
61 : * - `Parallel::Tags::ElementLocations<Dim>`
62 : * - `Parallel::Tags::NumberOfElementsTerminated`
63 : * - Removes: nothing
64 : * - Modifies:
65 : * - `Parallel::Tags::ElementCollection`
66 : * - `Parallel::Tags::ElementLocations<Dim>`
67 : * - `Parallel::Tags::NumberOfElementsTerminated`
68 : */
69 : template <size_t Dim, class Metavariables, class PhaseDepActionList,
70 : typename SimpleTagsFromOptions>
71 1 : struct CreateElementCollection {
72 0 : using simple_tags = tmpl::list<
73 : Parallel::Tags::ElementCollection<Dim, Metavariables, PhaseDepActionList,
74 : SimpleTagsFromOptions>,
75 : Parallel::Tags::ElementLocations<Dim>, Tags::NumberOfElementsTerminated>;
76 0 : using compute_tags = tmpl::list<>;
77 0 : using const_global_cache_tags =
78 : tmpl::list<::domain::Tags::Domain<Dim>,
79 : ::domain::Tags::ElementDistribution>;
80 :
81 0 : using return_tag_list = tmpl::append<simple_tags, compute_tags>;
82 :
83 : template <typename DbTagsList, typename... InboxTags, typename ArrayIndex,
84 : typename ActionList, typename ParallelComponent>
85 0 : static Parallel::iterable_action_return_t apply(
86 : db::DataBox<DbTagsList>& box,
87 : const tuples::TaggedTuple<InboxTags...>& /*inboxes*/,
88 : Parallel::GlobalCache<Metavariables>& local_cache,
89 : const ArrayIndex& /*array_index*/, const ActionList /*meta*/,
90 : const ParallelComponent* const /*meta*/) {
91 : const std::unordered_set<size_t> procs_to_ignore{};
92 :
93 : const auto& domain = Parallel::get<domain::Tags::Domain<Dim>>(local_cache);
94 : const auto& initial_refinement_levels =
95 : get<domain::Tags::InitialRefinementLevels<Dim>>(box);
96 : const auto& initial_extents = get<domain::Tags::InitialExtents<Dim>>(box);
97 : const auto& quadrature = get<evolution::dg::Tags::Quadrature>(box);
98 : const std::optional<domain::ElementWeight>& element_weight =
99 : Parallel::get<domain::Tags::ElementDistribution>(local_cache);
100 :
101 : const size_t number_of_procs =
102 : Parallel::number_of_procs<size_t>(local_cache);
103 : const size_t number_of_nodes =
104 : Parallel::number_of_nodes<size_t>(local_cache);
105 : const size_t num_of_procs_to_use = number_of_procs - procs_to_ignore.size();
106 :
107 : const auto& blocks = domain.blocks();
108 :
109 : const size_t total_num_elements = [&blocks, &initial_refinement_levels]() {
110 : size_t result = 0;
111 : for (const auto& block : blocks) {
112 : const auto& initial_ref_levs = initial_refinement_levels[block.id()];
113 : for (const size_t ref_lev : initial_ref_levs) {
114 : result += two_to_the(ref_lev);
115 : }
116 : }
117 : return result;
118 : }();
119 : std::vector<std::pair<ElementId<Dim>, size_t>> my_elements_and_cores{};
120 : my_elements_and_cores.reserve(total_num_elements / number_of_nodes + 1);
121 : std::unordered_map<ElementId<Dim>, size_t> node_of_elements{};
122 : const size_t my_node = Parallel::my_node<size_t>(local_cache);
123 :
124 : Parallel::create_elements_using_distribution(
125 : [&my_elements_and_cores, my_node, &node_of_elements](
126 : const ElementId<Dim>& element_id, const size_t target_proc,
127 : const size_t target_node) {
128 : node_of_elements.insert(std::pair{element_id, target_node});
129 : if (target_node == my_node) {
130 : my_elements_and_cores.push_back(std::pair{element_id, target_proc});
131 : }
132 : },
133 : element_weight, blocks, initial_extents, initial_refinement_levels,
134 : quadrature,
135 : // The below arguments control how the elements are mapped to the
136 : // hardware.
137 : procs_to_ignore, number_of_procs, number_of_nodes, num_of_procs_to_use,
138 : local_cache, my_node == 0);
139 :
140 : tuples::tagged_tuple_from_typelist<SimpleTagsFromOptions>
141 : initialization_items = db::copy_items<SimpleTagsFromOptions>(box);
142 :
143 : const gsl::not_null<Parallel::NodeLock*> node_lock = make_not_null(
144 : &Parallel::local_branch(
145 : Parallel::get_parallel_component<ParallelComponent>(local_cache))
146 : ->get_node_lock());
147 : db::mutate<Tags::ElementLocations<Dim>,
148 : Tags::ElementCollection<Dim, Metavariables, PhaseDepActionList,
149 : SimpleTagsFromOptions>,
150 : Tags::NumberOfElementsTerminated>(
151 : [&local_cache, &initialization_items, &my_elements_and_cores,
152 : &node_of_elements](
153 : const auto element_locations_ptr, const auto collection_ptr,
154 : const gsl::not_null<size_t*> number_of_elements_terminated) {
155 : *number_of_elements_terminated = 0;
156 : const auto serialized_initialization_items =
157 : serialize(initialization_items);
158 : *element_locations_ptr = std::move(node_of_elements);
159 : for (const auto& element_id_and_core : my_elements_and_cores) {
160 : const auto& element_id = element_id_and_core.first;
161 : const auto core = element_id_and_core.second;
162 : if (not collection_ptr
163 : ->emplace(
164 : std::piecewise_construct,
165 : std::forward_as_tuple(element_id),
166 : std::forward_as_tuple(
167 : local_cache.get_this_proxy(),
168 : deserialize<tuples::tagged_tuple_from_typelist<
169 : SimpleTagsFromOptions>>(
170 : serialized_initialization_items.data()),
171 : element_id))
172 : .second) {
173 : ERROR("Failed to insert element with ID: " << element_id);
174 : }
175 : if (collection_ptr->at(element_id).get_terminate() == true) {
176 : ++(*number_of_elements_terminated);
177 : } else {
178 : ERROR("Inserted element with ID "
179 : << element_id
180 : << " was not initialized in a terminated state. This is a "
181 : "bug.");
182 : }
183 : collection_ptr->at(element_id).set_core(core);
184 : }
185 : ASSERT(*number_of_elements_terminated == collection_ptr->size(),
186 : "The number of elements inserted must match the number of "
187 : "elements set to terminate since the default state of an "
188 : "inserted element is terminated. This is a bug.");
189 : },
190 : make_not_null(&box));
191 :
192 : Parallel::contribute_to_reduction<
193 : Parallel::Actions::SpawnInitializeElementsInCollection>(
194 : Parallel::ReductionData<
195 : Parallel::ReductionDatum<int, funcl::AssertEqual<>>>{0},
196 : Parallel::get_parallel_component<ParallelComponent>(
197 : local_cache)[my_node],
198 : Parallel::get_parallel_component<ParallelComponent>(local_cache));
199 :
200 : return {Parallel::AlgorithmExecution::Continue, std::nullopt};
201 : }
202 : };
203 : } // namespace Parallel::Actions
|
