Line data Source code
1 0 : // Distributed under the MIT License.
2 : // See LICENSE.txt for details.
3 :
4 : #pragma once
5 :
6 : #include <cstddef>
7 : #include <optional>
8 : #include <pup.h>
9 : #include <string>
10 : #include <type_traits>
11 : #include <utility>
12 : #include <vector>
13 :
14 : #include "Domain/Creators/Tags/Domain.hpp"
15 : #include "Domain/Domain.hpp"
16 : #include "Domain/Structure/Element.hpp"
17 : #include "Domain/Structure/ElementId.hpp"
18 : #include "Domain/Tags.hpp"
19 : #include "IO/Observer/Helpers.hpp"
20 : #include "IO/Observer/ObservationId.hpp"
21 : #include "IO/Observer/ObserverComponent.hpp"
22 : #include "IO/Observer/Protocols/ReductionDataFormatter.hpp"
23 : #include "IO/Observer/ReductionActions.hpp"
24 : #include "IO/Observer/TypeOfObservation.hpp"
25 : #include "NumericalAlgorithms/Spectral/Mesh.hpp"
26 : #include "Options/Auto.hpp"
27 : #include "Options/String.hpp"
28 : #include "Parallel/ArrayComponentId.hpp"
29 : #include "Parallel/ArrayIndex.hpp"
30 : #include "Parallel/GlobalCache.hpp"
31 : #include "Parallel/Info.hpp"
32 : #include "Parallel/Invoke.hpp"
33 : #include "Parallel/Local.hpp"
34 : #include "Parallel/Reduction.hpp"
35 : #include "Parallel/TypeTraits.hpp"
36 : #include "ParallelAlgorithms/Events/ReductionScope.hpp"
37 : #include "ParallelAlgorithms/Events/Tags.hpp"
38 : #include "ParallelAlgorithms/EventsAndTriggers/Event.hpp"
39 : #include "Utilities/Functional.hpp"
40 : #include "Utilities/ProtocolHelpers.hpp"
41 : #include "Utilities/Serialization/CharmPupable.hpp"
42 : #include "Utilities/Serialization/PupStlCpp17.hpp"
43 : #include "Utilities/TMPL.hpp"
44 :
45 : namespace Events {
46 : /// \cond
47 : template <size_t Dim, typename NonTensorComputeTags>
48 : class ObserveGridStats;
49 : /// \endcond
50 :
51 : namespace ObserveGridStats_detail {
52 : using ReductionData = Parallel::ReductionData<
53 : // Observation value
54 : Parallel::ReductionDatum<double, funcl::AssertEqual<>>,
55 : // Number of elements
56 : Parallel::ReductionDatum<size_t, funcl::Plus<>>,
57 : // Number of grid points
58 : Parallel::ReductionDatum<size_t, funcl::Plus<>>,
59 : // Fraction of elements doing FD
60 : Parallel::ReductionDatum<double, funcl::Plus<>, funcl::Divides<>,
61 : std::index_sequence<1>>,
62 : // Fraction of grid points doing FD
63 : Parallel::ReductionDatum<double, funcl::Plus<>, funcl::Divides<>,
64 : std::index_sequence<2>>,
65 : // Grid points per dimension: average, min, max
66 : Parallel::ReductionDatum<
67 : std::vector<double>, funcl::ElementWise<funcl::Plus<>>,
68 : funcl::ElementWise<funcl::Divides<>>, std::index_sequence<1>>,
69 : Parallel::ReductionDatum<std::vector<size_t>,
70 : funcl::ElementWise<funcl::Min<>>>,
71 : Parallel::ReductionDatum<std::vector<size_t>,
72 : funcl::ElementWise<funcl::Max<>>>,
73 : // Refinement levels per dimension: average, min, max
74 : Parallel::ReductionDatum<
75 : std::vector<double>, funcl::ElementWise<funcl::Plus<>>,
76 : funcl::ElementWise<funcl::Divides<>>, std::index_sequence<1>>,
77 : Parallel::ReductionDatum<std::vector<size_t>,
78 : funcl::ElementWise<funcl::Min<>>>,
79 : Parallel::ReductionDatum<std::vector<size_t>,
80 : funcl::ElementWise<funcl::Max<>>>>;
81 :
82 : // The column names of the reduction data
83 : std::vector<std::string> legend(const std::string& observation_value_name,
84 : size_t dim);
85 :
86 : // The contribution of a single element to the reduction
87 : template <size_t Dim>
88 : ReductionData reduction_data(double observation_value,
89 : const Mesh<Dim>& observer_mesh,
90 : const ElementId<Dim>& element_id);
91 :
92 : // The path of the subfile (without extension) that the element contributes
93 : // to, or `std::nullopt` if the element is not observed.
94 : template <size_t Dim>
95 : std::optional<std::string> subfile_path(
96 : ReductionScope reduction_scope,
97 : const std::optional<std::vector<std::string>>& blocks_to_observe,
98 : const Domain<Dim>& domain, const ElementId<Dim>& element_id);
99 :
100 : // Formats the reduced statistics of one subfile for printing to the terminal
101 : struct FormatGridStatsOutput
102 : : tt::ConformsTo<observers::protocols::ReductionDataFormatter> {
103 : using reduction_data = ReductionData;
104 :
105 : FormatGridStatsOutput() = default;
106 : FormatGridStatsOutput(std::string in_observation_value_name,
107 : std::string in_subfile_path);
108 :
109 : std::string operator()(double observation_value, size_t num_elements,
110 : size_t num_points, double fraction_of_elements_fd,
111 : double fraction_of_points_fd,
112 : const std::vector<double>& avg_points_per_dim,
113 : const std::vector<size_t>& min_points_per_dim,
114 : const std::vector<size_t>& max_points_per_dim,
115 : const std::vector<double>& avg_refinement_level,
116 : const std::vector<size_t>& min_refinement_level,
117 : const std::vector<size_t>& max_refinement_level) const;
118 :
119 : // NOLINTNEXTLINE(google-runtime-references)
120 : void pup(PUP::er& p);
121 :
122 : std::string observation_value_name{};
123 : std::string subfile_path{};
124 : };
125 : } // namespace ObserveGridStats_detail
126 :
127 : /*!
128 : * \brief Observe statistics of the computational grid.
129 : *
130 : * Each observation writes a row with the following columns:
131 : * - The observation value (e.g. `Time`)
132 : * - `NumElements`: The number of elements
133 : * - `NumPoints`: The number of grid points
134 : * - `FractionOfElementsFd`: The fraction of elements doing FD
135 : * - `FractionOfPointsFd`: The fraction of grid points in elements doing FD
136 : * - `{Avg,Min,Max}PointsPerDim_{d}`: The average, min, and max number of grid
137 : * points in each logical dimension `d` over the elements
138 : * - `{Avg,Min,Max}RefinementLevel_{d}`: The average, min, and max refinement
139 : * level in each logical dimension `d` over the elements
140 : *
141 : * All averages are over elements, i.e. each element is weighted equally.
142 : *
143 : * The grid points are counted on the `Events::Tags::ObserverMesh`, which must
144 : * be provided by the `NonTensorComputeTags`. For hybrid DG-FD methods this is
145 : * the subcell mesh in elements doing FD, so elements doing FD contribute their
146 : * subcell grid points. An element is considered to be doing FD if its
147 : * `ObserverMesh` has a `Spectral::Basis::FiniteDifference` basis in any
148 : * dimension. Without subcell the FD fractions are always zero.
149 : *
150 : * \par Observing a subset of blocks
151 : * The `BlocksToObserve` option restricts the observation to the listed blocks
152 : * and block groups. Set it to `All` to observe the whole domain.
153 : *
154 : * \par Printing to the terminal
155 : * Set `PrintToTerminal` to also print a summary of the statistics to the
156 : * terminal for each observation. With a `ReductionScope` other than
157 : * `WholeDomain` a summary is printed for each block or group.
158 : *
159 : * \par Reduction scope
160 : * The `ReductionScope` option (see `Events::ReductionScope`) controls whether
161 : * the observed elements are reduced together or split up by block:
162 : * - `WholeDomain`: a single reduction over all observed elements, written to
163 : * the subfile `GridStats.dat`. In addition, the same statistics restricted
164 : * to the elements on each core are written to the subfile
165 : * `Core{core_id}/GridStats.dat` in one H5 file per node (see
166 : * `observers::Actions::ContributeReductionData`). This is useful to assess
167 : * load-balancing. For nodegroup components the statistics are per node
168 : * instead of per core.
169 : * - `ByBlock`: a separate reduction for each observed block, written to
170 : * `GridStats/<BlockName>.dat`.
171 : * - `ByBlockOrGroup`: a separate reduction for each entry of `BlocksToObserve`,
172 : * written to `GridStats/<BlockOrGroupName>.dat`.
173 : *
174 : * To observe both the whole domain and individual blocks, add two
175 : * `ObserveGridStats` events, one with `ReductionScope: WholeDomain` and one
176 : * with `ReductionScope: ByBlock`.
177 : */
178 : template <size_t Dim, typename... NonTensorComputeTags>
179 1 : class ObserveGridStats<Dim, tmpl::list<NonTensorComputeTags...>>
180 : : public Event {
181 : static_assert(
182 : (std::is_base_of_v<::Events::Tags::ObserverMesh<Dim>,
183 : NonTensorComputeTags> or
184 : ...),
185 : "The NonTensorComputeTags of ObserveGridStats must contain a tag that "
186 : "derives from Events::Tags::ObserverMesh<Dim>, e.g. "
187 : "Events::Tags::ObserverMeshCompute<Dim>.");
188 :
189 : public:
190 : /// \brief Whether to print the statistics to the terminal.
191 1 : struct PrintToTerminal {
192 0 : using type = bool;
193 0 : static constexpr Options::String help = {
194 : "Whether to print the statistics to the terminal."};
195 : };
196 : /// \brief A list of block or group names on which to observe.
197 : ///
198 : /// Set to `All` to observe everywhere.
199 1 : struct BlocksToObserve {
200 0 : using type =
201 : Options::Auto<std::vector<std::string>, Options::AutoLabel::All>;
202 0 : static constexpr Options::String help = {
203 : "A list of block and group names on which to observe."};
204 : };
205 : /// \brief Whether to reduce over all observed elements or separately for
206 : /// each block or group.
207 1 : struct ReductionScope {
208 0 : using type = ::Events::ReductionScope;
209 0 : static constexpr Options::String help = {
210 : "How to group the observed elements into reductions: WholeDomain, "
211 : "ByBlock, or ByBlockOrGroup. WholeDomain also writes the statistics "
212 : "on each core."};
213 : };
214 :
215 : /// \cond
216 : explicit ObserveGridStats(CkMigrateMessage* msg) : Event(msg) {}
217 : using PUP::able::register_constructor;
218 : WRAPPED_PUPable_decl_template(ObserveGridStats); // NOLINT
219 : /// \endcond
220 :
221 0 : using options = tmpl::list<PrintToTerminal, BlocksToObserve, ReductionScope>;
222 0 : static constexpr Options::String help =
223 : "Observe statistics of the computational grid, such as the number of "
224 : "elements and grid points, their resolution and refinement levels, and "
225 : "the fraction of the grid that is doing finite-difference.";
226 :
227 0 : ObserveGridStats() = default;
228 0 : ObserveGridStats(bool print_to_terminal,
229 : std::optional<std::vector<std::string>> blocks_to_observe,
230 : ::Events::ReductionScope reduction_scope)
231 : : print_to_terminal_(print_to_terminal),
232 : blocks_to_observe_(std::move(blocks_to_observe)),
233 : reduction_scope_(reduction_scope) {}
234 :
235 0 : using observed_reduction_data_tags = observers::make_reduction_data_tags<
236 : tmpl::list<ObserveGridStats_detail::ReductionData>>;
237 :
238 0 : using compute_tags_for_observation_box = tmpl::list<NonTensorComputeTags...>;
239 0 : using argument_tags =
240 : tmpl::list<::Events::Tags::ObserverMesh<Dim>, domain::Tags::Domain<Dim>>;
241 0 : using return_tags = tmpl::list<>;
242 :
243 : template <typename Metavariables, typename ParallelComponent>
244 0 : void operator()(const Mesh<Dim>& observer_mesh, const Domain<Dim>& domain,
245 : Parallel::GlobalCache<Metavariables>& cache,
246 : const ElementId<Dim>& element_id,
247 : const ParallelComponent* const /*meta*/,
248 : const ObservationValue& observation_value) const {
249 : const std::optional<std::string> subfile_path =
250 : ObserveGridStats_detail::subfile_path(
251 : reduction_scope_, blocks_to_observe_, domain, element_id);
252 : if (not subfile_path.has_value()) {
253 : return;
254 : }
255 : auto& local_observer = *Parallel::local_branch(
256 : Parallel::get_parallel_component<
257 : tmpl::conditional_t<Parallel::is_nodegroup_v<ParallelComponent>,
258 : observers::ObserverWriter<Metavariables>,
259 : observers::Observer<Metavariables>>>(cache));
260 : observers::ObservationId observation_id{observation_value.value,
261 : subfile_path.value() + ".dat"};
262 : Parallel::ArrayComponentId array_component_id{
263 : std::add_pointer_t<ParallelComponent>{nullptr},
264 : Parallel::ArrayIndex<ElementId<Dim>>(element_id)};
265 : auto legend = ObserveGridStats_detail::legend(observation_value.name, Dim);
266 : auto reduction_data = ObserveGridStats_detail::reduction_data(
267 : observation_value.value, observer_mesh, element_id);
268 : // Choosing to only provide per-core granularity when reducing over entire
269 : // domain to avoid a subfile for every core x block combination
270 : const bool observe_per_core =
271 : reduction_scope_ == ::Events::ReductionScope::WholeDomain;
272 : auto formatter =
273 : print_to_terminal_
274 : ? std::make_optional(ObserveGridStats_detail::FormatGridStatsOutput{
275 : observation_value.name, subfile_path.value()})
276 : : std::nullopt;
277 : if constexpr (Parallel::is_nodegroup_v<ParallelComponent>) {
278 : const std::optional<int> observe_with_core_id =
279 : observe_per_core
280 : ? std::make_optional(Parallel::my_node<int>(local_observer))
281 : : std::nullopt;
282 : Parallel::threaded_action<
283 : observers::ThreadedActions::CollectReductionDataOnNode>(
284 : local_observer, std::move(observation_id),
285 : std::move(array_component_id), subfile_path.value(),
286 : std::move(legend), std::move(reduction_data), std::move(formatter),
287 : observe_with_core_id);
288 : } else {
289 : Parallel::simple_action<observers::Actions::ContributeReductionData>(
290 : local_observer, std::move(observation_id),
291 : std::move(array_component_id), subfile_path.value(),
292 : std::move(legend), std::move(reduction_data), std::move(formatter),
293 : observe_per_core);
294 : }
295 : }
296 :
297 0 : using observation_registration_tags =
298 : tmpl::list<domain::Tags::Domain<Dim>, domain::Tags::Element<Dim>>;
299 :
300 : std::optional<
301 : std::pair<observers::TypeOfObservation, observers::ObservationKey>>
302 0 : get_observation_type_and_key_for_registration(
303 : const Domain<Dim>& domain, const Element<Dim>& element) const {
304 : const std::optional<std::string> subfile_path =
305 : ObserveGridStats_detail::subfile_path(
306 : reduction_scope_, blocks_to_observe_, domain, element.id());
307 : if (not subfile_path.has_value()) {
308 : return std::nullopt;
309 : }
310 : return {{observers::TypeOfObservation::Reduction,
311 : observers::ObservationKey(subfile_path.value() + ".dat")}};
312 : }
313 :
314 0 : using is_ready_argument_tags = tmpl::list<>;
315 :
316 : template <typename Metavariables, typename ArrayIndex, typename Component>
317 0 : bool is_ready(Parallel::GlobalCache<Metavariables>& /*cache*/,
318 : const ArrayIndex& /*array_index*/,
319 : const Component* const /*meta*/) const {
320 : return true;
321 : }
322 :
323 1 : bool needs_evolved_variables() const override { return false; }
324 :
325 : // NOLINTNEXTLINE(google-runtime-references)
326 0 : void pup(PUP::er& p) override {
327 : Event::pup(p);
328 : p | print_to_terminal_;
329 : p | blocks_to_observe_;
330 : p | reduction_scope_;
331 : }
332 :
333 : private:
334 0 : bool print_to_terminal_{false};
335 0 : std::optional<std::vector<std::string>> blocks_to_observe_{};
336 0 : ::Events::ReductionScope reduction_scope_{
337 : ::Events::ReductionScope::WholeDomain};
338 : };
339 :
340 : /// \cond
341 : // NOLINTBEGIN(cppcoreguidelines-avoid-non-const-global-variables)
342 : template <size_t Dim, typename... NonTensorComputeTags>
343 : PUP::able::PUP_ID
344 : ObserveGridStats<Dim, tmpl::list<NonTensorComputeTags...>>::my_PUP_ID = 0;
345 : // NOLINTEND(cppcoreguidelines-avoid-non-const-global-variables)
346 : /// \endcond
347 : } // namespace Events
|
