Line data Source code
1 0 : // Distributed under the MIT License.
2 : // See LICENSE.txt for details.
3 :
4 : #pragma once
5 :
6 : #include <algorithm>
7 : #include <cstddef>
8 : #include <cstdint>
9 : #include <limits>
10 : #include <map>
11 : #include <memory>
12 : #include <pup.h>
13 : #include <pup_stl.h>
14 : #include <typeindex>
15 : #include <unordered_map>
16 : #include <utility>
17 : #include <vector>
18 :
19 : #include "DataStructures/DataBox/DataBox.hpp"
20 : #include "Options/Context.hpp"
21 : #include "Options/ParseError.hpp"
22 : #include "Options/String.hpp"
23 : #include "Parallel/ArrayCollection/IsDgElementCollection.hpp"
24 : #include "Parallel/GlobalCache.hpp"
25 : #include "Parallel/Reduction.hpp"
26 : #include "ParallelAlgorithms/EventsAndTriggers/Event.hpp"
27 : #include "Time/ChangeSlabSize/Tags.hpp"
28 : #include "Time/LtsMode.hpp"
29 : #include "Time/RequestsStepperErrorTolerances.hpp"
30 : #include "Time/StepChoosers/StepChooser.hpp"
31 : #include "Time/StepperErrorTolerances.hpp"
32 : #include "Time/TimeStepId.hpp"
33 : #include "Time/TimeStepRequest.hpp"
34 : #include "Time/TimeStepRequestProcessor.hpp"
35 : #include "Utilities/ErrorHandling/Error.hpp"
36 : #include "Utilities/Functional.hpp"
37 : #include "Utilities/PrettyType.hpp"
38 : #include "Utilities/Serialization/CharmPupable.hpp"
39 : #include "Utilities/TMPL.hpp"
40 :
41 : /// \cond
42 : namespace Tags {
43 : struct DataBox;
44 : struct LtsMode;
45 : struct TimeStepId;
46 : } // namespace Tags
47 : /// \endcond
48 :
49 : namespace Events {
50 : namespace ChangeSlabSize_detail {
51 : struct StoreNewSlabSize {
52 : template <typename ParallelComponent, typename DbTags, typename Metavariables,
53 : typename ArrayIndex>
54 : static void apply(db::DataBox<DbTags>& box,
55 : Parallel::GlobalCache<Metavariables>& /*cache*/,
56 : const ArrayIndex& /*array_index*/,
57 : const int64_t slab_number,
58 : const TimeStepRequestProcessor& requests) {
59 : db::mutate<::Tags::ChangeSlabSize::NewSlabSize>(
60 : [&](const gsl::not_null<
61 : std::map<int64_t, std::vector<TimeStepRequestProcessor>>*>
62 : sizes) { (*sizes)[slab_number].emplace_back(requests); },
63 : make_not_null(&box));
64 : }
65 : };
66 : } // namespace ChangeSlabSize_detail
67 :
68 : /// \ingroup TimeGroup
69 : /// %Trigger a slab size change.
70 : ///
71 : /// The new size will be the minimum suggested by any of the provided
72 : /// step choosers on any element. This requires a global reduction,
73 : /// so it is possible to delay the change until a later slab to avoid
74 : /// a global synchronization. The actual change is carried out by
75 : /// Actions::ChangeSlabSize.
76 : ///
77 : /// When running with global time-stepping, the slab size and step
78 : /// size are the same, so this adjusts the step size used by the time
79 : /// integration. With local time-stepping this controls the interval
80 : /// between times when the sequences of steps on all elements are
81 : /// forced to align.
82 1 : class ChangeSlabSize : public Event, public RequestsStepperErrorTolerances {
83 0 : using ReductionData = Parallel::ReductionData<
84 : Parallel::ReductionDatum<int64_t, funcl::AssertEqual<>>,
85 : Parallel::ReductionDatum<TimeStepRequestProcessor, funcl::Plus<>>>;
86 :
87 : public:
88 : /// \cond
89 : explicit ChangeSlabSize(CkMigrateMessage* /*unused*/) {}
90 : using PUP::able::register_constructor;
91 : WRAPPED_PUPable_decl_template(ChangeSlabSize); // NOLINT
92 : /// \endcond
93 :
94 0 : struct StepChoosers {
95 0 : static constexpr Options::String help = "Limits on slab size";
96 0 : using type =
97 : std::vector<std::unique_ptr<StepChooser<StepChooserUse::Slab>>>;
98 0 : static size_t lower_bound_on_size() { return 1; }
99 : };
100 :
101 0 : struct DelayChange {
102 0 : static constexpr Options::String help = "Slabs to wait before changing";
103 0 : using type = uint64_t;
104 : };
105 :
106 0 : using options = tmpl::list<StepChoosers, DelayChange>;
107 0 : static constexpr Options::String help =
108 : "Trigger a slab size change chosen by the provided step choosers.\n"
109 : "The actual changing of the slab size can be delayed until a later\n"
110 : "slab to improve parallelization.";
111 :
112 0 : ChangeSlabSize() = default;
113 0 : ChangeSlabSize(std::vector<std::unique_ptr<StepChooser<StepChooserUse::Slab>>>
114 : step_choosers,
115 : const uint64_t delay_change, const Options::Context& context)
116 : : step_choosers_(std::move(step_choosers)), delay_change_(delay_change) {
117 : if (delay_change != 0) {
118 : for (const auto& chooser : step_choosers_) {
119 : if (not chooser->can_be_delayed()) {
120 : // The runtime name might not be exactly the same as the one
121 : // used by the factory, but hopefully it's close enough that
122 : // the user can figure it out.
123 : PARSE_ERROR(context,
124 : "The " << pretty_type::get_runtime_type_name(*chooser)
125 : << " StepChooser cannot be applied with a delay.");
126 : }
127 : }
128 : }
129 : }
130 :
131 0 : using compute_tags_for_observation_box = tmpl::list<>;
132 :
133 : // Need a const version of the full box for the step choosers, but
134 : // can't get a const version while mutating other tags, so request a
135 : // mutable version.
136 0 : using return_tags = tmpl::list<::Tags::DataBox>;
137 0 : using argument_tags = tmpl::list<::Tags::TimeStepId, ::Tags::LtsMode>;
138 :
139 : template <typename DbTags, typename Metavariables, typename ArrayIndex,
140 : typename ParallelComponent>
141 0 : void operator()(const gsl::not_null<db::DataBox<DbTags>*> box,
142 : const TimeStepId& time_step_id, const LtsMode lts_mode,
143 : Parallel::GlobalCache<Metavariables>& cache,
144 : const ArrayIndex& array_index,
145 : const ParallelComponent* const /*meta*/,
146 : const ObservationValue& /*observation_value*/) const {
147 : const auto next_changable_slab = time_step_id.is_at_slab_boundary()
148 : ? time_step_id.slab_number()
149 : : time_step_id.slab_number() + 1;
150 : const auto slab_to_change =
151 : next_changable_slab + static_cast<int64_t>(delay_change_);
152 : const auto slab_start = time_step_id.step_time();
153 : const auto current_slab_size = (time_step_id.time_runs_forward() ? 1 : -1) *
154 : slab_start.slab().duration();
155 :
156 : TimeStepRequestProcessor step_requests(time_step_id.time_runs_forward());
157 : bool synchronization_required = false;
158 : for (const auto& step_chooser : step_choosers_) {
159 : if (lts_mode != LtsMode::Off and step_chooser->must_set_step_size()) {
160 : // The runtime name might not be exactly the same as the one
161 : // used by the factory, but hopefully it's close enough that
162 : // the user can figure it out.
163 : ERROR_NO_TRACE("The "
164 : << pretty_type::get_runtime_type_name(*step_chooser)
165 : << " StepChooser cannot be used for the slab size in "
166 : "an LTS evolution.");
167 : }
168 : step_requests.process(
169 : step_chooser->desired_step(current_slab_size.value(), *box));
170 : // We must synchronize if any step chooser requires it, not just
171 : // the limiting one, because choosers requiring synchronization
172 : // can be limiting on some processors and not others.
173 : if (not synchronization_required) {
174 : synchronization_required = step_chooser->uses_local_data();
175 : }
176 : }
177 :
178 : db::mutate<::Tags::ChangeSlabSize::NumberOfExpectedMessages>(
179 : [&](const gsl::not_null<std::map<int64_t, size_t>*> expected) {
180 : ++(*expected)[slab_to_change];
181 : },
182 : box);
183 :
184 : if constexpr (Parallel::is_dg_element_collection_v<ParallelComponent>) {
185 : ERROR(
186 : "Slab changing is not yet implemented for the DgElementCollection "
187 : "parallel component. Specifically, the ability to do reductions "
188 : "using Charm++-style reductions has not been implemented.");
189 : } else {
190 : const auto& component_proxy =
191 : Parallel::get_parallel_component<ParallelComponent>(cache);
192 : const auto& self_proxy = component_proxy[array_index];
193 : if (synchronization_required) {
194 : Parallel::contribute_to_reduction<
195 : ChangeSlabSize_detail::StoreNewSlabSize>(
196 : ReductionData(slab_to_change, step_requests), self_proxy,
197 : component_proxy);
198 : } else {
199 : db::mutate<::Tags::ChangeSlabSize::NewSlabSize>(
200 : [&](const gsl::not_null<
201 : std::map<int64_t, std::vector<TimeStepRequestProcessor>>*>
202 : sizes) {
203 : (*sizes)[slab_to_change].push_back(step_requests);
204 : },
205 : box);
206 : }
207 : }
208 : (void)cache, (void)array_index;
209 : }
210 :
211 0 : using is_ready_argument_tags = tmpl::list<>;
212 :
213 : template <typename Metavariables, typename ArrayIndex, typename Component>
214 0 : bool is_ready(Parallel::GlobalCache<Metavariables>& /*cache*/,
215 : const ArrayIndex& /*array_index*/,
216 : const Component* const /*meta*/) const {
217 : return true;
218 : }
219 :
220 1 : bool needs_evolved_variables() const override {
221 : // This depends on the chosen StepChoosers, but they don't have a
222 : // way to report this information so we just return true to be
223 : // safe.
224 : return true;
225 : }
226 :
227 1 : std::unordered_map<std::type_index, StepperErrorTolerances> tolerances()
228 : const override;
229 :
230 : // NOLINTNEXTLINE(google-runtime-references)
231 0 : void pup(PUP::er& p) override {
232 : Event::pup(p);
233 : p | step_choosers_;
234 : p | delay_change_;
235 : }
236 :
237 : private:
238 : std::vector<std::unique_ptr<StepChooser<StepChooserUse::Slab>>>
239 0 : step_choosers_;
240 0 : uint64_t delay_change_ = std::numeric_limits<uint64_t>::max();
241 : };
242 : } // namespace Events
|
