ChangeSlabSize.hpp
1 // Distributed under the MIT License.
2 // See LICENSE.txt for details.
3 
4 #pragma once
5 
6 #include <boost/functional/hash.hpp> // IWYU pragma: keep
7 #include <cstddef>
8 #include <cstdint>
9 #include <limits>
10 #include <map>
11 #include <memory>
12 #include <ostream>
13 #include <pup.h>
14 #include <tuple>
15 #include <unordered_set>
16 #include <vector>
17 
19 #include "ErrorHandling/Assert.hpp"
20 #include "Options/Options.hpp"
23 #include "Parallel/Invoke.hpp"
24 #include "Parallel/Reduction.hpp"
25 #include "ParallelAlgorithms/EventsAndTriggers/Event.hpp"
26 #include "Time/Slab.hpp"
27 #include "Time/Tags.hpp"
28 #include "Time/Time.hpp"
29 #include "Time/TimeStepId.hpp"
30 #include "Time/TimeSteppers/TimeStepper.hpp"
31 #include "Utilities/Algorithm.hpp"
32 #include "Utilities/Functional.hpp"
33 #include "Utilities/Gsl.hpp"
34 #include "Utilities/Registration.hpp"
35 #include "Utilities/TMPL.hpp"
37 
38 /// \cond
39 template <typename StepChooserRegistrars>
40 class StepChooser;
41 namespace Tags {
42 struct DataBox;
43 template <typename Tag>
44 struct Next;
45 } // namespace Tags
46 // IWYU pragma: no_forward_declare db::DataBox
47 /// \endcond
48 
49 namespace ChangeSlabSize_detail {
50 struct NewSlabSizeInbox {
51  using temporal_id = int64_t;
53 };
54 
55 // This inbox doesn't receive any data, it just counts messages (using
56 // the size of the multiset). Whenever a message is sent to the
57 // NewSlabSizeInbox, another message is sent here synchronously, so
58 // the count here is the number of messages we expect in the
59 // NewSlabSizeInbox.
60 struct NumberOfExpectedMessagesInbox {
61  using temporal_id = int64_t;
62  using NoData = std::tuple<>;
63  using type = std::map<temporal_id,
65 };
66 
67 struct StoreNewSlabSize {
68  template <typename ParallelComponent, typename DbTags, typename Metavariables,
69  typename ArrayIndex>
70  static void apply(const db::DataBox<DbTags>& /*box*/,
72  const ArrayIndex& array_index, const int64_t slab_number,
73  const double slab_size) noexcept {
74  Parallel::receive_data<ChangeSlabSize_detail::NewSlabSizeInbox>(
75  *Parallel::get_parallel_component<ParallelComponent>(cache)[array_index]
76  .ckLocal(),
77  slab_number, slab_size);
78  }
79 };
80 } // namespace ChangeSlabSize_detail
81 
82 namespace Actions {
83 /// \ingroup ActionsGroup
84 /// \ingroup TimeGroup
85 /// Adjust the slab size based on previous executions of
86 /// Events::ChangeSlabSize
87 ///
88 /// Uses:
89 /// - ConstGlobalCache:
90 /// - Tags::TimeStepperBase
91 /// - DataBox:
92 /// - Tags::HistoryEvolvedVariables
93 /// - Tags::TimeStep
94 /// - Tags::TimeStepId
95 ///
96 /// DataBox changes:
97 /// - Adds: nothing
98 /// - Removes: nothing
99 /// - Modifies:
100 /// - Tags::Next<Tags::TimeStepId>
101 /// - Tags::TimeStep
102 /// - Tags::TimeStepId
104  using inbox_tags =
105  tmpl::list<ChangeSlabSize_detail::NumberOfExpectedMessagesInbox,
106  ChangeSlabSize_detail::NewSlabSizeInbox>;
107 
108  template <typename DbTags, typename... InboxTags, typename Metavariables,
109  typename ArrayIndex, typename ActionList,
110  typename ParallelComponent>
114  const ArrayIndex& /*array_index*/, const ActionList /*meta*/,
115  const ParallelComponent* const /*meta*/) noexcept {
116  const auto& time_step_id = db::get<Tags::TimeStepId>(box);
117 
118  if (not time_step_id.is_at_slab_boundary()) {
119  return std::forward_as_tuple(std::move(box));
120  }
121 
122  auto& message_count_inbox =
123  tuples::get<ChangeSlabSize_detail::NumberOfExpectedMessagesInbox>(
124  inboxes);
125  if (message_count_inbox.empty() or
126  message_count_inbox.begin()->first != time_step_id.slab_number()) {
127  return std::forward_as_tuple(std::move(box));
128  }
129  message_count_inbox.erase(message_count_inbox.begin());
130 
131  auto& new_slab_size_inbox =
132  tuples::get<ChangeSlabSize_detail::NewSlabSizeInbox>(inboxes);
133  const double new_slab_size =
134  *alg::min_element(new_slab_size_inbox.begin()->second);
135  new_slab_size_inbox.erase(new_slab_size_inbox.begin());
136 
137  const TimeStepper& time_stepper =
138  Parallel::get<Tags::TimeStepperBase>(cache);
139 
140  // Sometimes time steppers need to run with a fixed step size.
141  // This is generally at the start of an evolution when the history
142  // is in an unusual state.
143  if (not time_stepper.can_change_step_size(
144  time_step_id, db::get<Tags::HistoryEvolvedVariables<>>(box))) {
145  return std::forward_as_tuple(std::move(box));
146  }
147 
148  const auto& current_step = db::get<Tags::TimeStep>(box);
149  const auto& current_slab = current_step.slab();
150 
151  const auto new_slab =
152  current_step.is_positive()
153  ? current_slab.with_duration_from_start(new_slab_size)
154  : current_slab.with_duration_to_end(new_slab_size);
155 
156  const auto new_step = current_step.with_slab(new_slab);
157 
158  // We are at a slab boundary, so the substep is 0.
159  const auto new_time_step_id =
160  TimeStepId(time_step_id.time_runs_forward(), time_step_id.slab_number(),
161  time_step_id.step_time().with_slab(new_slab));
162 
163  const auto new_next_time_step_id =
164  time_stepper.next_time_id(new_time_step_id, new_step);
165 
166  db::mutate<Tags::Next<Tags::TimeStepId>, Tags::TimeStep, Tags::TimeStepId>(
167  make_not_null(&box),
168  [&new_next_time_step_id, &new_step, &new_time_step_id](
169  const gsl::not_null<TimeStepId*> next_time_step_id,
170  const gsl::not_null<TimeDelta*> time_step,
171  const gsl::not_null<TimeStepId*> local_time_step_id) noexcept {
172  *next_time_step_id = new_next_time_step_id;
173  *time_step = new_step;
174  *local_time_step_id = new_time_step_id;
175  });
176 
177  return std::forward_as_tuple(std::move(box));
178  }
179 
180  template <typename DbTags, typename... InboxTags, typename Metavariables,
181  typename ArrayIndex>
182  static bool is_ready(
183  const db::DataBox<DbTags>& box,
184  const tuples::TaggedTuple<InboxTags...>& inboxes,
186  const ArrayIndex& /*array_index*/) noexcept {
187  const auto& time_step_id = db::get<Tags::TimeStepId>(box);
188  if (not time_step_id.is_at_slab_boundary()) {
189  return true;
190  }
191 
192  const auto& message_count_inbox =
193  tuples::get<ChangeSlabSize_detail::NumberOfExpectedMessagesInbox>(
194  inboxes);
195  const auto& new_slab_size_inbox =
196  tuples::get<ChangeSlabSize_detail::NewSlabSizeInbox>(inboxes);
197 
198  const auto slab_number = time_step_id.slab_number();
199  const auto number_of_changes = [&slab_number](
200  const auto& inbox) noexcept->size_t {
201  if (inbox.empty()) {
202  return 0;
203  }
204  if (inbox.begin()->first == slab_number) {
205  return inbox.begin()->second.size();
206  }
207  ASSERT(inbox.begin()->first >= slab_number,
208  "Received data for a change at slab " << inbox.begin()->first
209  << " but it is already slab " << slab_number);
210  return 0;
211  };
212 
213  const size_t expected_messages = number_of_changes(message_count_inbox);
214  const size_t received_messages = number_of_changes(new_slab_size_inbox);
215  ASSERT(expected_messages >= received_messages,
216  "Received " << received_messages << " size change messages at slab "
217  << slab_number << ", but only expected "
218  << expected_messages);
219  return expected_messages == received_messages;
220  }
221 };
222 } // namespace Actions
223 
224 namespace Events {
225 template <typename StepChooserRegistrars, typename EventRegistrars>
227 
228 namespace Registrars {
229 template <typename StepChooserRegistrars>
230 using ChangeSlabSize =
232 } // namespace Registrars
233 
234 /// \ingroup TimeGroup
235 /// %Trigger a slab size change.
236 ///
237 /// The new size will be the minimum suggested by any of the provided
238 /// step choosers on any element. This requires a global reduction,
239 /// so it is possible to delay the change until a later slab to avoid
240 /// a global synchronization. The actual change is carried out by
241 /// Actions::ChangeSlabSize.
242 ///
243 /// When running with global time-stepping, the slab size and step
244 /// size are the same, so this adjusts the step size used by the time
245 /// integration. With local time-stepping this controls the interval
246 /// between times when the sequences of steps on all elements are
247 /// forced to align.
248 template <typename StepChooserRegistrars,
249  typename EventRegistrars =
250  tmpl::list<Registrars::ChangeSlabSize<StepChooserRegistrars>>>
251 class ChangeSlabSize : public Event<EventRegistrars> {
252  using ReductionData = Parallel::ReductionData<
255 
256  public:
257  /// \cond
258  explicit ChangeSlabSize(CkMigrateMessage* /*unused*/) noexcept {}
259  using PUP::able::register_constructor;
261  /// \endcond
262 
263  struct StepChoosers {
264  static constexpr OptionString help = "Limits on slab size";
265  using type =
267  static size_t lower_bound_on_size() noexcept { return 1; }
268  };
269 
270  struct DelayChange {
271  static constexpr OptionString help = "Slabs to wait before changing";
272  using type = uint64_t;
273  };
274 
275  using options = tmpl::list<StepChoosers, DelayChange>;
276  static constexpr OptionString help =
277  "Trigger a slab size change chosen by the provided step choosers.\n"
278  "The actual changing of the slab size can be delayed until a later\n"
279  "slab to improve parallelization.";
280 
281  ChangeSlabSize() = default;
284  step_choosers,
285  const uint64_t delay_change) noexcept
286  : step_choosers_(std::move(step_choosers)), delay_change_(delay_change) {}
287 
288  using argument_tags = tmpl::list<Tags::TimeStepId, Tags::DataBox>;
289 
290  template <typename DbTags, typename Metavariables, typename ArrayIndex,
291  typename ParallelComponent>
292  void operator()(const TimeStepId& time_step_id,
293  const db::DataBox<DbTags>& box_for_step_choosers,
295  const ArrayIndex& array_index,
296  const ParallelComponent* const /*meta*/) const noexcept {
297  const auto next_changable_slab = time_step_id.is_at_slab_boundary()
298  ? time_step_id.slab_number()
299  : time_step_id.slab_number() + 1;
300  const auto slab_to_change =
301  next_changable_slab + static_cast<int64_t>(delay_change_);
302 
303  double desired_slab_size = std::numeric_limits<double>::infinity();
304  for (const auto& step_chooser : step_choosers_) {
305  desired_slab_size =
306  std::min(desired_slab_size,
307  step_chooser->desired_step(
308  time_step_id.step_time().slab().duration().value(),
309  box_for_step_choosers, cache));
310  }
311 
312  const auto& component_proxy =
313  Parallel::get_parallel_component<ParallelComponent>(cache);
314  const auto& self_proxy = component_proxy[array_index];
315  // This message is sent synchronously, so it is guaranteed to
316  // arrive before the ChangeSlabSize action is called.
318  ChangeSlabSize_detail::NumberOfExpectedMessagesInbox>(
319  *self_proxy.ckLocal(), slab_to_change,
321  Parallel::contribute_to_reduction<ChangeSlabSize_detail::StoreNewSlabSize>(
322  ReductionData(slab_to_change, desired_slab_size), self_proxy,
323  component_proxy);
324  }
325 
326  // NOLINTNEXTLINE(google-runtime-references)
327  void pup(PUP::er& p) noexcept override {
329  p | step_choosers_;
330  p | delay_change_;
331  }
332 
333  private:
335  step_choosers_;
336  uint64_t delay_change_ = std::numeric_limits<uint64_t>::max();
337 };
338 
339 /// \cond
340 template <typename StepChooserRegistrars, typename EventRegistrars>
341 PUP::able::PUP_ID
343  0; // NOLINT
344 /// \endcond
345 } // namespace Events
decltype(auto) min_element(const Container &c)
Convenience wrapper around std::min_element.
Definition: Algorithm.hpp:293
Tag for step size.
Definition: Tags.hpp:42
Defines class tuples::TaggedTuple.
Definition: ChangeSlabSize.hpp:49
void receive_data(Proxy &&proxy, typename ReceiveTag::temporal_id temporal_id, ReceiveDataType &&receive_data, const bool enable_if_disabled) noexcept
Send the data args... to the algorithm running on proxy, and tag the message with the identifier temp...
Definition: Invoke.hpp:51
Defines class TimeStepId.
Abstract base class for TimeSteppers.
Definition: TimeStepper.hpp:47
Definition: ChangeSlabSize.hpp:270
Defines classes and functions for making classes creatable from input files.
Defines macros to allow serialization of abstract template base classes.
#define ASSERT(a, m)
Assert that an expression should be true.
Definition: Assert.hpp:51
Definition: Completion.hpp:13
virtual TimeStepId next_time_id(const TimeStepId &current_id, const TimeDelta &time_step) const noexcept=0
The TimeStepId after the current substep.
#define WRAPPED_PUPable_decl_template(className)
Mark derived classes as serializable.
Definition: CharmPupable.hpp:22
Defines Time and TimeDelta.
StepChoosers suggest upper bounds on step sizes. Concrete StepChoosers should define operator() retur...
Definition: StepChooser.hpp:43
Tag for TimeStepId for the algorithm state.
Definition: Tags.hpp:32
A unique identifier for the temporal state of an integrated system.
Definition: TimeStepId.hpp:25
const char *const OptionString
The string used in option structs.
Definition: Options.hpp:29
Tag for the TimeStepper history.
Definition: Tags.hpp:86
An associative container that is indexed by structs.
Definition: TaggedTuple.hpp:273
Defines classes and functions used for manipulating DataBox&#39;s.
Trigger a slab size change.
Definition: ChangeSlabSize.hpp:226
T infinity(T... args)
T max(T... args)
A template for defining a registrar.
Definition: Registration.hpp:42
Defines class Slab.
Definition: InterpolationTargetWedgeSectionTorus.hpp:24
bool can_change_step_size(const TimeStepId &time_id, const TimeSteppers::History< Vars, DerivVars > &history) const noexcept
Whether a change in the step size is allowed before taking a step.
Definition: TimeStepper.hpp:99
Definition: DataBoxTag.hpp:29
A Charm++ chare that caches constant data once per Charm++ node.
Definition: ConstGlobalCache.hpp:135
constexpr auto apply(F &&f, const DataBox< BoxTags > &box, Args &&... args) noexcept
Apply the invokable f with argument Tags TagsList from DataBox box
Definition: DataBox.hpp:1628
const Time & step_time() const noexcept
Time at the start of the current step.
Definition: TimeStepId.hpp:58
Defines macro ASSERT.
const auto & get(const DataBox< TagList > &box) noexcept
Retrieve the item with tag Tag from the DataBox.
Definition: DataBox.hpp:1211
Wraps the template metaprogramming library used (brigand)
Base class for something that can happen during a simulation (such as an observation).
Definition: Event.hpp:30
Defines functions and classes from the GSL.
gsl::not_null< T * > make_not_null(T *ptr) noexcept
Construct a not_null from a pointer. Often this will be done as an implicit conversion, but it may be necessary to perform the conversion explicitly when type deduction is desired.
Definition: Gsl.hpp:879
Defines class template ConstGlobalCache.
double value() const noexcept
Approximate numerical length of the interval.
Definition: Time.cpp:111
The data to be reduced, and invokables to be called whenever two reduction messages are combined and ...
Definition: Reduction.hpp:64
Defines tags related to Time quantities.
Definition: ComputeTimeDerivative.hpp:28
Require a pointer to not be a nullptr
Definition: Gsl.hpp:182
Holds all the StepChoosers.
Definition: ByBlock.hpp:31
Adjust the slab size based on previous executions of Events::ChangeSlabSize.
Definition: ChangeSlabSize.hpp:103