Line data Source code
1 0 : // Distributed under the MIT License.
2 : // See LICENSE.txt for details.
3 :
4 : #pragma once
5 :
6 : #include <array>
7 : #include <cstddef>
8 : #include <functional>
9 : #include <optional>
10 : #include <unordered_map>
11 : #include <unordered_set>
12 : #include <vector>
13 :
14 : #include "Domain/Block.hpp"
15 : #include "Domain/ElementDistribution.hpp"
16 : #include "Domain/Structure/ElementId.hpp"
17 : #include "Domain/Structure/InitialElementIds.hpp"
18 : #include "Parallel/DomainDiagnosticInfo.hpp"
19 : #include "Parallel/GlobalCache.hpp"
20 : #include "Utilities/Numeric.hpp"
21 :
22 : /// \cond
23 : namespace Spectral {
24 : enum class Basis : uint8_t;
25 : enum class Quadrature : uint8_t;
26 : } // namespace Spectral
27 : /// \endcond
28 :
29 : namespace Parallel {
30 : /*!
31 : * \brief Creates elements using a chosen distribution.
32 : *
33 : * The `func` is called with `(element_id, target_proc, target_node)` allowing
34 : * the `func` to insert the element with `element_id` on the target processor
35 : * and node.
36 : *
37 : * The `weighting_extents_override`, if present, gives per-element extents to
38 : * use instead of `initial_extents` only when computing the cost of elements
39 : * for the distribution (e.g. to weight elements that run the finite-difference
40 : * subcell scheme by their finite-difference grid rather than their DG grid).
41 : * Only elements whose weighting extents differ from `initial_extents` need an
42 : * entry; elements not found in the map use their block's `initial_extents`.
43 : * The actual `initial_extents` are always used for everything else, e.g. the
44 : * diagnostics below.
45 : */
46 : template <typename F, size_t Dim, typename Metavariables>
47 1 : void create_elements_using_distribution(
48 : const F& func, const std::optional<domain::ElementWeight>& element_weight,
49 : const std::vector<Block<Dim>>& blocks,
50 : const std::vector<std::array<size_t, Dim>>& initial_extents,
51 : const std::vector<std::array<size_t, Dim>>& initial_refinement_levels,
52 : const Spectral::Basis i1_basis, const Spectral::Quadrature i1_quadrature,
53 :
54 : const std::unordered_set<size_t>& procs_to_ignore,
55 : const size_t number_of_procs, const size_t number_of_nodes,
56 : const size_t num_of_procs_to_use,
57 : const Parallel::GlobalCache<Metavariables>& local_cache,
58 : const bool print_diagnostics,
59 : const std::optional<
60 : std::unordered_map<ElementId<Dim>, std::array<size_t, Dim>>>&
61 : weighting_extents_override = std::nullopt) {
62 : // Only need the element distribution if the element weight has a value
63 : // because then we have to use the space filling curve and not just use round
64 : // robin.
65 : domain::BlockZCurveProcDistribution<Dim> element_distribution{};
66 : if (element_weight.has_value()) {
67 : const std::unordered_map<ElementId<Dim>, std::array<size_t, Dim>>
68 : no_override{};
69 : const std::unordered_map<ElementId<Dim>, double> element_costs =
70 : domain::get_element_costs(blocks, initial_refinement_levels,
71 : initial_extents, element_weight.value(),
72 : i1_basis, i1_quadrature,
73 : weighting_extents_override.has_value()
74 : ? weighting_extents_override.value()
75 : : no_override);
76 : element_distribution = domain::BlockZCurveProcDistribution<Dim>{
77 : element_costs, num_of_procs_to_use, blocks, initial_refinement_levels,
78 : procs_to_ignore};
79 : }
80 :
81 : // Will be used to print domain diagnostic info
82 : std::vector<size_t> elements_per_core(number_of_procs, 0_st);
83 : std::vector<size_t> elements_per_node(number_of_nodes, 0_st);
84 : std::vector<size_t> grid_points_per_core(number_of_procs, 0_st);
85 : std::vector<size_t> grid_points_per_node(number_of_nodes, 0_st);
86 :
87 : size_t which_proc = 0;
88 : for (const auto& block : blocks) {
89 : const auto& initial_ref_levs = initial_refinement_levels[block.id()];
90 : const size_t grid_points_per_element =
91 : alg::accumulate(initial_extents[block.id()], 1_st, std::multiplies<>());
92 :
93 : const std::vector<ElementId<Dim>> element_ids =
94 : initial_element_ids(block.id(), initial_ref_levs);
95 :
96 : // Value means ZCurve. nullopt means round robin
97 : if (element_weight.has_value()) {
98 : for (const auto& element_id : element_ids) {
99 : const size_t target_proc =
100 : element_distribution.get_proc_for_element(element_id);
101 : const size_t target_node =
102 : Parallel::node_of<size_t>(target_proc, local_cache);
103 : func(element_id, target_proc, target_node);
104 :
105 : ++elements_per_core[target_proc];
106 : ++elements_per_node[target_node];
107 : grid_points_per_core[target_proc] += grid_points_per_element;
108 : grid_points_per_node[target_node] += grid_points_per_element;
109 : }
110 : } else {
111 : for (size_t i = 0; i < element_ids.size(); ++i) {
112 : while (procs_to_ignore.find(which_proc) != procs_to_ignore.end()) {
113 : which_proc = which_proc + 1 == number_of_procs ? 0 : which_proc + 1;
114 : }
115 : const size_t target_proc = which_proc;
116 : const size_t target_node =
117 : Parallel::node_of<size_t>(which_proc, local_cache);
118 : const ElementId<Dim> element_id(element_ids[i]);
119 : func(element_id, target_proc, target_node);
120 :
121 : ++elements_per_core[which_proc];
122 : ++elements_per_node[target_node];
123 : grid_points_per_core[which_proc] += grid_points_per_element;
124 : grid_points_per_node[target_node] += grid_points_per_element;
125 :
126 : which_proc = which_proc + 1 == number_of_procs ? 0 : which_proc + 1;
127 : }
128 : }
129 : }
130 :
131 : if (print_diagnostics) {
132 : Parallel::printf("\n%s\n", domain::diagnostic_info(
133 : blocks.size(), local_cache,
134 : elements_per_core, elements_per_node,
135 : grid_points_per_core, grid_points_per_node));
136 : }
137 : }
138 : } // namespace Parallel
|
