ChangeSlabSize.hpp
1 // Distributed under the MIT License.
2 // See LICENSE.txt for details.
3 
4 #pragma once
5 
6 #include <boost/functional/hash.hpp> // IWYU pragma: keep
7 #include <cstddef>
8 #include <cstdint>
9 #include <limits>
10 #include <map>
11 #include <memory>
12 #include <ostream>
13 #include <pup.h>
14 #include <tuple>
15 #include <unordered_set>
16 #include <vector>
17 
19 #include "Options/Options.hpp"
20 #include "Parallel/AlgorithmMetafunctions.hpp"
22 #include "Parallel/GlobalCache.hpp"
23 #include "Parallel/InboxInserters.hpp"
24 #include "Parallel/Invoke.hpp"
25 #include "Parallel/Reduction.hpp"
26 #include "ParallelAlgorithms/EventsAndTriggers/Event.hpp"
27 #include "Time/Slab.hpp"
28 #include "Time/Tags.hpp"
29 #include "Time/Time.hpp"
30 #include "Time/TimeStepId.hpp"
31 #include "Time/TimeSteppers/TimeStepper.hpp"
32 #include "Utilities/Algorithm.hpp"
34 #include "Utilities/Functional.hpp"
35 #include "Utilities/Gsl.hpp"
36 #include "Utilities/Registration.hpp"
37 #include "Utilities/TMPL.hpp"
38 #include "Utilities/TaggedTuple.hpp"
39 
40 /// \cond
41 template <typename StepChooserRegistrars>
42 class StepChooser;
43 namespace Tags {
44 struct DataBox;
45 template <typename Tag>
46 struct Next;
47 } // namespace Tags
48 // IWYU pragma: no_forward_declare db::DataBox
49 /// \endcond
50 
51 namespace ChangeSlabSize_detail {
52 struct NewSlabSizeInbox
53  : public Parallel::InboxInserters::MemberInsert<NewSlabSizeInbox> {
54  using temporal_id = int64_t;
56 };
57 
58 // This inbox doesn't receive any data, it just counts messages (using
59 // the size of the multiset). Whenever a message is sent to the
60 // NewSlabSizeInbox, another message is sent here synchronously, so
61 // the count here is the number of messages we expect in the
62 // NewSlabSizeInbox.
63 struct NumberOfExpectedMessagesInbox
65  NumberOfExpectedMessagesInbox> {
66  using temporal_id = int64_t;
67  using NoData = std::tuple<>;
68  using type = std::map<temporal_id,
70 };
71 
72 struct StoreNewSlabSize {
73  template <typename ParallelComponent, typename DbTags, typename Metavariables,
74  typename ArrayIndex>
75  static void apply(const db::DataBox<DbTags>& /*box*/,
77  const ArrayIndex& array_index, const int64_t slab_number,
78  const double slab_size) noexcept {
79  Parallel::receive_data<ChangeSlabSize_detail::NewSlabSizeInbox>(
80  *Parallel::get_parallel_component<ParallelComponent>(cache)[array_index]
81  .ckLocal(),
82  slab_number, slab_size);
83  }
84 };
85 } // namespace ChangeSlabSize_detail
86 
87 namespace Actions {
88 /// \ingroup ActionsGroup
89 /// \ingroup TimeGroup
90 /// Adjust the slab size based on previous executions of
91 /// Events::ChangeSlabSize
92 ///
93 /// Uses:
94 /// - GlobalCache:
95 /// - Tags::TimeStepperBase
96 /// - DataBox:
97 /// - Tags::HistoryEvolvedVariables
98 /// - Tags::TimeStep
99 /// - Tags::TimeStepId
100 ///
101 /// DataBox changes:
102 /// - Adds: nothing
103 /// - Removes: nothing
104 /// - Modifies:
105 /// - Tags::Next<Tags::TimeStepId>
106 /// - Tags::TimeStep
107 /// - Tags::TimeStepId
109  using inbox_tags =
110  tmpl::list<ChangeSlabSize_detail::NumberOfExpectedMessagesInbox,
111  ChangeSlabSize_detail::NewSlabSizeInbox>;
112 
113  template <typename DbTags, typename... InboxTags, typename Metavariables,
114  typename ArrayIndex, typename ActionList,
115  typename ParallelComponent>
117  db::DataBox<DbTags>& box, tuples::TaggedTuple<InboxTags...>& inboxes,
118  const Parallel::GlobalCache<Metavariables>& /*cache*/,
119  const ArrayIndex& /*array_index*/, const ActionList /*meta*/,
120  const ParallelComponent* const /*meta*/) noexcept {
121  const auto& time_step_id = db::get<Tags::TimeStepId>(box);
122  if (not time_step_id.is_at_slab_boundary()) {
123  return {std::move(box), Parallel::AlgorithmExecution::Continue};
124  }
125 
126  auto& message_count_inbox =
127  tuples::get<ChangeSlabSize_detail::NumberOfExpectedMessagesInbox>(
128  inboxes);
129  if (message_count_inbox.empty() or
130  message_count_inbox.begin()->first != time_step_id.slab_number()) {
131  return {std::move(box), Parallel::AlgorithmExecution::Continue};
132  }
133 
134  auto& new_slab_size_inbox =
135  tuples::get<ChangeSlabSize_detail::NewSlabSizeInbox>(inboxes);
136 
137  const auto slab_number = time_step_id.slab_number();
138  const auto number_of_changes = [&slab_number](
139  const auto& inbox) noexcept->size_t {
140  if (inbox.empty()) {
141  return 0;
142  }
143  if (inbox.begin()->first == slab_number) {
144  return inbox.begin()->second.size();
145  }
146  ASSERT(inbox.begin()->first >= slab_number,
147  "Received data for a change at slab " << inbox.begin()->first
148  << " but it is already slab " << slab_number);
149  return 0;
150  };
151 
152  const size_t expected_messages = number_of_changes(message_count_inbox);
153  const size_t received_messages = number_of_changes(new_slab_size_inbox);
154  ASSERT(expected_messages >= received_messages,
155  "Received " << received_messages << " size change messages at slab "
156  << slab_number << ", but only expected "
157  << expected_messages);
158  if (expected_messages != received_messages) {
159  return {std::move(box), Parallel::AlgorithmExecution::Retry};
160  }
161 
162  message_count_inbox.erase(message_count_inbox.begin());
163 
164  const double new_slab_size =
165  *alg::min_element(new_slab_size_inbox.begin()->second);
166  new_slab_size_inbox.erase(new_slab_size_inbox.begin());
167 
168  const TimeStepper& time_stepper = db::get<Tags::TimeStepper<>>(box);
169 
170  // Sometimes time steppers need to run with a fixed step size.
171  // This is generally at the start of an evolution when the history
172  // is in an unusual state.
173  if (not time_stepper.can_change_step_size(
174  time_step_id, db::get<Tags::HistoryEvolvedVariables<>>(box))) {
175  return {std::move(box), Parallel::AlgorithmExecution::Continue};
176  }
177 
178  const auto& current_step = db::get<Tags::TimeStep>(box);
179  const auto& current_slab = current_step.slab();
180 
181  const auto new_slab =
182  current_step.is_positive()
183  ? current_slab.with_duration_from_start(new_slab_size)
184  : current_slab.with_duration_to_end(new_slab_size);
185 
186  const auto new_step = current_step.with_slab(new_slab);
187 
188  // We are at a slab boundary, so the substep is 0.
189  const auto new_time_step_id =
190  TimeStepId(time_step_id.time_runs_forward(), time_step_id.slab_number(),
191  time_step_id.step_time().with_slab(new_slab));
192 
193  const auto new_next_time_step_id =
194  time_stepper.next_time_id(new_time_step_id, new_step);
195 
196  db::mutate<Tags::Next<Tags::TimeStepId>, Tags::TimeStep,
198  make_not_null(&box),
199  [&new_next_time_step_id, &new_step, &new_time_step_id](
200  const gsl::not_null<TimeStepId*> next_time_step_id,
201  const gsl::not_null<TimeDelta*> time_step,
202  const gsl::not_null<TimeDelta*> next_time_step,
203  const gsl::not_null<TimeStepId*> local_time_step_id) noexcept {
204  *next_time_step_id = new_next_time_step_id;
205  *time_step = new_step;
206  *next_time_step = new_step;
207  *local_time_step_id = new_time_step_id;
208  });
209 
210  return {std::move(box), Parallel::AlgorithmExecution::Continue};
211  }
212 };
213 } // namespace Actions
214 
215 namespace Events {
216 template <typename StepChooserRegistrars, typename EventRegistrars>
218 
219 namespace Registrars {
220 template <typename StepChooserRegistrars>
221 using ChangeSlabSize =
223 } // namespace Registrars
224 
225 /// \ingroup TimeGroup
226 /// %Trigger a slab size change.
227 ///
228 /// The new size will be the minimum suggested by any of the provided
229 /// step choosers on any element. This requires a global reduction,
230 /// so it is possible to delay the change until a later slab to avoid
231 /// a global synchronization. The actual change is carried out by
232 /// Actions::ChangeSlabSize.
233 ///
234 /// When running with global time-stepping, the slab size and step
235 /// size are the same, so this adjusts the step size used by the time
236 /// integration. With local time-stepping this controls the interval
237 /// between times when the sequences of steps on all elements are
238 /// forced to align.
239 template <typename StepChooserRegistrars,
240  typename EventRegistrars =
241  tmpl::list<Registrars::ChangeSlabSize<StepChooserRegistrars>>>
242 class ChangeSlabSize : public Event<EventRegistrars> {
243  using ReductionData = Parallel::ReductionData<
246 
247  public:
248  /// \cond
249  explicit ChangeSlabSize(CkMigrateMessage* /*unused*/) noexcept {}
250  using PUP::able::register_constructor;
251  WRAPPED_PUPable_decl_template(ChangeSlabSize); // NOLINT
252  /// \endcond
253 
254  struct StepChoosers {
255  static constexpr Options::String help = "Limits on slab size";
256  using type =
258  static size_t lower_bound_on_size() noexcept { return 1; }
259  };
260 
261  struct DelayChange {
262  static constexpr Options::String help = "Slabs to wait before changing";
263  using type = uint64_t;
264  };
265 
266  using options = tmpl::list<StepChoosers, DelayChange>;
267  static constexpr Options::String help =
268  "Trigger a slab size change chosen by the provided step choosers.\n"
269  "The actual changing of the slab size can be delayed until a later\n"
270  "slab to improve parallelization.";
271 
272  ChangeSlabSize() = default;
275  step_choosers,
276  const uint64_t delay_change) noexcept
277  : step_choosers_(std::move(step_choosers)), delay_change_(delay_change) {}
278 
279  using argument_tags = tmpl::list<Tags::TimeStepId, Tags::DataBox>;
280 
281  template <typename DbTags, typename Metavariables, typename ArrayIndex,
282  typename ParallelComponent>
283  void operator()(const TimeStepId& time_step_id,
284  const db::DataBox<DbTags>& box_for_step_choosers,
286  const ArrayIndex& array_index,
287  const ParallelComponent* const /*meta*/) const noexcept {
288  const auto next_changable_slab = time_step_id.is_at_slab_boundary()
289  ? time_step_id.slab_number()
290  : time_step_id.slab_number() + 1;
291  const auto slab_to_change =
292  next_changable_slab + static_cast<int64_t>(delay_change_);
293 
294  double desired_slab_size = std::numeric_limits<double>::infinity();
295  for (const auto& step_chooser : step_choosers_) {
296  desired_slab_size =
297  std::min(desired_slab_size,
298  step_chooser->desired_slab(
299  time_step_id.step_time().slab().duration().value(),
300  box_for_step_choosers, cache));
301  }
302 
303  const auto& component_proxy =
304  Parallel::get_parallel_component<ParallelComponent>(cache);
305  const auto& self_proxy = component_proxy[array_index];
306  // This message is sent synchronously, so it is guaranteed to
307  // arrive before the ChangeSlabSize action is called.
309  ChangeSlabSize_detail::NumberOfExpectedMessagesInbox>(
310  *self_proxy.ckLocal(), slab_to_change,
312  Parallel::contribute_to_reduction<ChangeSlabSize_detail::StoreNewSlabSize>(
313  ReductionData(slab_to_change, desired_slab_size), self_proxy,
314  component_proxy);
315  }
316 
317  bool needs_evolved_variables() const noexcept override {
318  // This depends on the chosen StepChoosers, but they don't have a
319  // way to report this information so we just return true to be
320  // safe.
321  return true;
322  }
323 
324  // NOLINTNEXTLINE(google-runtime-references)
325  void pup(PUP::er& p) noexcept override {
327  p | step_choosers_;
328  p | delay_change_;
329  }
330 
331  private:
333  step_choosers_;
334  uint64_t delay_change_ = std::numeric_limits<uint64_t>::max();
335 };
336 
337 /// \cond
338 template <typename StepChooserRegistrars, typename EventRegistrars>
339 PUP::able::PUP_ID
340  ChangeSlabSize<StepChooserRegistrars, EventRegistrars>::my_PUP_ID =
341  0; // NOLINT
342 /// \endcond
343 } // namespace Events
std::apply
T apply(T... args)
CharmPupable.hpp
StepChoosers
Definition: ByBlock.hpp:33
Parallel::AlgorithmExecution::Retry
@ Retry
Temporarily stop executing iterable actions, but try the same action again after receiving data from ...
Tags::TimeStepId
Tag for TimeStepId for the algorithm state.
Definition: Tags.hpp:33
Parallel::GlobalCache
Definition: ElementReceiveInterpPoints.hpp:15
Parallel::ReductionDatum
The data to be reduced, and invokables to be called whenever two reduction messages are combined and ...
Definition: Reduction.hpp:63
unordered_set
GlobalCache.hpp
Options.hpp
vector
Tags::TimeStep
Tag for step size.
Definition: Tags.hpp:42
tuple
db::get
const auto & get(const DataBox< TagList > &box) noexcept
Retrieve the item with tag Tag from the DataBox.
Definition: DataBox.hpp:786
Tags::Next
Prefix indicating the value a quantity will take on the next iteration of the algorithm.
Definition: Prefixes.hpp:118
TimeStepper::next_time_id
virtual TimeStepId next_time_id(const TimeStepId &current_id, const TimeDelta &time_step) const noexcept=0
The TimeStepId after the current substep.
Events::ChangeSlabSize::DelayChange
Definition: ChangeSlabSize.hpp:261
Registration::Registrar
A template for defining a registrar.
Definition: Registration.hpp:42
Actions::ChangeSlabSize
Definition: ChangeSlabSize.hpp:108
WRAPPED_PUPable_decl_template
#define WRAPPED_PUPable_decl_template(className)
Mark derived classes as serializable.
Definition: CharmPupable.hpp:22
Event
Definition: Event.hpp:30
Parallel::AlgorithmExecution::Continue
@ Continue
Leave the algorithm termination flag in its current state.
DataBox.hpp
std::numeric_limits::infinity
T infinity(T... args)
cstddef
Assert.hpp
Parallel::InboxInserters::MemberInsert
Inserter for inserting data that is received as the value_type of a data structure that has an insert...
Definition: InboxInserters.hpp:67
TimeStepper
Definition: TimeStepper.hpp:47
cstdint
alg::min_element
decltype(auto) min_element(const Container &c)
Convenience wrapper around std::min_element.
Definition: Algorithm.hpp:343
tuples::TaggedTuple
An associative container that is indexed by structs.
Definition: TaggedTuple.hpp:271
map
Time.hpp
Parallel::AlgorithmExecution
AlgorithmExecution
The possible options for altering the current execution of the algorithm, used in the return type of ...
Definition: AlgorithmMetafunctions.hpp:31
memory
TimeStepId
Definition: TimeStepId.hpp:25
std::unordered_multiset
ASSERT
#define ASSERT(a, m)
Assert that an expression should be true.
Definition: Assert.hpp:49
Parallel::receive_data
void receive_data(Proxy &&proxy, typename ReceiveTag::temporal_id temporal_id, ReceiveDataType &&receive_data, const bool enable_if_disabled=false) noexcept
Send the data args... to the algorithm running on proxy, and tag the message with the identifier temp...
Definition: Invoke.hpp:32
TimeStepper::can_change_step_size
bool can_change_step_size(const TimeStepId &time_id, const TimeSteppers::History< Vars, DerivVars > &history) const noexcept
Whether a change in the step size is allowed before taking a step.
Definition: TimeStepper.hpp:138
TimeStepId.hpp
ActionTesting::cache
Parallel::GlobalCache< Metavariables > & cache(MockRuntimeSystem< Metavariables > &runner, const ArrayIndex &array_index) noexcept
Returns the GlobalCache of Component with index array_index.
Definition: MockRuntimeSystemFreeFunctions.hpp:382
limits
Gsl.hpp
Options::String
const char *const String
The string used in option structs.
Definition: Options.hpp:32
Slab.hpp
make_not_null
gsl::not_null< T * > make_not_null(T *ptr) noexcept
Construct a not_null from a pointer. Often this will be done as an implicit conversion,...
Definition: Gsl.hpp:880
std::numeric_limits::max
T max(T... args)
Tags::HistoryEvolvedVariables
Definition: Tags.hpp:85
ostream
std::unique_ptr
StepChooser
Definition: StepChooser.hpp:98
Events::ChangeSlabSize
Definition: ChangeSlabSize.hpp:217
TMPL.hpp
gsl::not_null
Require a pointer to not be a nullptr
Definition: ReadSpecThirdOrderPiecewisePolynomial.hpp:13