ChangeSlabSize.hpp
1 // Distributed under the MIT License.
2 // See LICENSE.txt for details.
3 
4 #pragma once
5 
6 #include <boost/functional/hash.hpp> // IWYU pragma: keep
7 #include <cstddef>
8 #include <cstdint>
9 #include <limits>
10 #include <map>
11 #include <memory>
12 #include <ostream>
13 #include <pup.h>
14 #include <tuple>
15 #include <unordered_set>
16 #include <vector>
17 
19 #include "ErrorHandling/Assert.hpp"
20 #include "Options/Options.hpp"
22 #include "Parallel/GlobalCache.hpp"
23 #include "Parallel/InboxInserters.hpp"
24 #include "Parallel/Invoke.hpp"
25 #include "Parallel/Reduction.hpp"
26 #include "ParallelAlgorithms/EventsAndTriggers/Event.hpp"
27 #include "Time/Slab.hpp"
28 #include "Time/Tags.hpp"
29 #include "Time/Time.hpp"
30 #include "Time/TimeStepId.hpp"
31 #include "Time/TimeSteppers/TimeStepper.hpp"
32 #include "Utilities/Algorithm.hpp"
33 #include "Utilities/Functional.hpp"
34 #include "Utilities/Gsl.hpp"
35 #include "Utilities/Registration.hpp"
36 #include "Utilities/TMPL.hpp"
37 #include "Utilities/TaggedTuple.hpp"
38 
39 /// \cond
40 template <typename StepChooserRegistrars>
41 class StepChooser;
42 namespace Tags {
43 struct DataBox;
44 template <typename Tag>
45 struct Next;
46 } // namespace Tags
47 // IWYU pragma: no_forward_declare db::DataBox
48 /// \endcond
49 
50 namespace ChangeSlabSize_detail {
51 struct NewSlabSizeInbox
52  : public Parallel::InboxInserters::MemberInsert<NewSlabSizeInbox> {
53  using temporal_id = int64_t;
55 };
56 
57 // This inbox doesn't receive any data, it just counts messages (using
58 // the size of the multiset). Whenever a message is sent to the
59 // NewSlabSizeInbox, another message is sent here synchronously, so
60 // the count here is the number of messages we expect in the
61 // NewSlabSizeInbox.
62 struct NumberOfExpectedMessagesInbox
64  NumberOfExpectedMessagesInbox> {
65  using temporal_id = int64_t;
66  using NoData = std::tuple<>;
67  using type = std::map<temporal_id,
69 };
70 
71 struct StoreNewSlabSize {
72  template <typename ParallelComponent, typename DbTags, typename Metavariables,
73  typename ArrayIndex>
74  static void apply(const db::DataBox<DbTags>& /*box*/,
76  const ArrayIndex& array_index, const int64_t slab_number,
77  const double slab_size) noexcept {
78  Parallel::receive_data<ChangeSlabSize_detail::NewSlabSizeInbox>(
79  *Parallel::get_parallel_component<ParallelComponent>(cache)[array_index]
80  .ckLocal(),
81  slab_number, slab_size);
82  }
83 };
84 } // namespace ChangeSlabSize_detail
85 
86 namespace Actions {
87 /// \ingroup ActionsGroup
88 /// \ingroup TimeGroup
89 /// Adjust the slab size based on previous executions of
90 /// Events::ChangeSlabSize
91 ///
92 /// Uses:
93 /// - GlobalCache:
94 /// - Tags::TimeStepperBase
95 /// - DataBox:
96 /// - Tags::HistoryEvolvedVariables
97 /// - Tags::TimeStep
98 /// - Tags::TimeStepId
99 ///
100 /// DataBox changes:
101 /// - Adds: nothing
102 /// - Removes: nothing
103 /// - Modifies:
104 /// - Tags::Next<Tags::TimeStepId>
105 /// - Tags::TimeStep
106 /// - Tags::TimeStepId
108  using inbox_tags =
109  tmpl::list<ChangeSlabSize_detail::NumberOfExpectedMessagesInbox,
110  ChangeSlabSize_detail::NewSlabSizeInbox>;
111 
112  template <typename DbTags, typename... InboxTags, typename Metavariables,
113  typename ArrayIndex, typename ActionList,
114  typename ParallelComponent>
115  static std::tuple<db::DataBox<DbTags>&&> apply(
117  const Parallel::GlobalCache<Metavariables>& /*cache*/,
118  const ArrayIndex& /*array_index*/, const ActionList /*meta*/,
119  const ParallelComponent* const /*meta*/) noexcept {
120  const auto& time_step_id = db::get<Tags::TimeStepId>(box);
121 
122  if (not time_step_id.is_at_slab_boundary()) {
123  return std::forward_as_tuple(std::move(box));
124  }
125 
126  auto& message_count_inbox =
127  tuples::get<ChangeSlabSize_detail::NumberOfExpectedMessagesInbox>(
128  inboxes);
129  if (message_count_inbox.empty() or
130  message_count_inbox.begin()->first != time_step_id.slab_number()) {
131  return std::forward_as_tuple(std::move(box));
132  }
133  message_count_inbox.erase(message_count_inbox.begin());
134 
135  auto& new_slab_size_inbox =
136  tuples::get<ChangeSlabSize_detail::NewSlabSizeInbox>(inboxes);
137  const double new_slab_size =
138  *alg::min_element(new_slab_size_inbox.begin()->second);
139  new_slab_size_inbox.erase(new_slab_size_inbox.begin());
140 
141  const TimeStepper& time_stepper = db::get<Tags::TimeStepper<>>(box);
142 
143  // Sometimes time steppers need to run with a fixed step size.
144  // This is generally at the start of an evolution when the history
145  // is in an unusual state.
146  if (not time_stepper.can_change_step_size(
147  time_step_id, db::get<Tags::HistoryEvolvedVariables<>>(box))) {
148  return std::forward_as_tuple(std::move(box));
149  }
150 
151  const auto& current_step = db::get<Tags::TimeStep>(box);
152  const auto& current_slab = current_step.slab();
153 
154  const auto new_slab =
155  current_step.is_positive()
156  ? current_slab.with_duration_from_start(new_slab_size)
157  : current_slab.with_duration_to_end(new_slab_size);
158 
159  const auto new_step = current_step.with_slab(new_slab);
160 
161  // We are at a slab boundary, so the substep is 0.
162  const auto new_time_step_id =
163  TimeStepId(time_step_id.time_runs_forward(), time_step_id.slab_number(),
164  time_step_id.step_time().with_slab(new_slab));
165 
166  const auto new_next_time_step_id =
167  time_stepper.next_time_id(new_time_step_id, new_step);
168 
169  db::mutate<Tags::Next<Tags::TimeStepId>, Tags::TimeStep, Tags::TimeStepId>(
170  make_not_null(&box),
171  [&new_next_time_step_id, &new_step, &new_time_step_id](
172  const gsl::not_null<TimeStepId*> next_time_step_id,
173  const gsl::not_null<TimeDelta*> time_step,
174  const gsl::not_null<TimeStepId*> local_time_step_id) noexcept {
175  *next_time_step_id = new_next_time_step_id;
176  *time_step = new_step;
177  *local_time_step_id = new_time_step_id;
178  });
179 
180  return std::forward_as_tuple(std::move(box));
181  }
182 
183  template <typename DbTags, typename... InboxTags, typename Metavariables,
184  typename ArrayIndex>
185  static bool is_ready(
186  const db::DataBox<DbTags>& box,
187  const tuples::TaggedTuple<InboxTags...>& inboxes,
188  const Parallel::GlobalCache<Metavariables>& /*cache*/,
189  const ArrayIndex& /*array_index*/) noexcept {
190  const auto& time_step_id = db::get<Tags::TimeStepId>(box);
191  if (not time_step_id.is_at_slab_boundary()) {
192  return true;
193  }
194 
195  const auto& message_count_inbox =
196  tuples::get<ChangeSlabSize_detail::NumberOfExpectedMessagesInbox>(
197  inboxes);
198  const auto& new_slab_size_inbox =
199  tuples::get<ChangeSlabSize_detail::NewSlabSizeInbox>(inboxes);
200 
201  const auto slab_number = time_step_id.slab_number();
202  const auto number_of_changes = [&slab_number](
203  const auto& inbox) noexcept->size_t {
204  if (inbox.empty()) {
205  return 0;
206  }
207  if (inbox.begin()->first == slab_number) {
208  return inbox.begin()->second.size();
209  }
210  ASSERT(inbox.begin()->first >= slab_number,
211  "Received data for a change at slab " << inbox.begin()->first
212  << " but it is already slab " << slab_number);
213  return 0;
214  };
215 
216  const size_t expected_messages = number_of_changes(message_count_inbox);
217  const size_t received_messages = number_of_changes(new_slab_size_inbox);
218  ASSERT(expected_messages >= received_messages,
219  "Received " << received_messages << " size change messages at slab "
220  << slab_number << ", but only expected "
221  << expected_messages);
222  return expected_messages == received_messages;
223  }
224 };
225 } // namespace Actions
226 
227 namespace Events {
228 template <typename StepChooserRegistrars, typename EventRegistrars>
230 
231 namespace Registrars {
232 template <typename StepChooserRegistrars>
233 using ChangeSlabSize =
235 } // namespace Registrars
236 
237 /// \ingroup TimeGroup
238 /// %Trigger a slab size change.
239 ///
240 /// The new size will be the minimum suggested by any of the provided
241 /// step choosers on any element. This requires a global reduction,
242 /// so it is possible to delay the change until a later slab to avoid
243 /// a global synchronization. The actual change is carried out by
244 /// Actions::ChangeSlabSize.
245 ///
246 /// When running with global time-stepping, the slab size and step
247 /// size are the same, so this adjusts the step size used by the time
248 /// integration. With local time-stepping this controls the interval
249 /// between times when the sequences of steps on all elements are
250 /// forced to align.
251 template <typename StepChooserRegistrars,
252  typename EventRegistrars =
253  tmpl::list<Registrars::ChangeSlabSize<StepChooserRegistrars>>>
254 class ChangeSlabSize : public Event<EventRegistrars> {
255  using ReductionData = Parallel::ReductionData<
258 
259  public:
260  /// \cond
261  explicit ChangeSlabSize(CkMigrateMessage* /*unused*/) noexcept {}
262  using PUP::able::register_constructor;
263  WRAPPED_PUPable_decl_template(ChangeSlabSize); // NOLINT
264  /// \endcond
265 
266  struct StepChoosers {
267  static constexpr Options::String help = "Limits on slab size";
268  using type =
270  static size_t lower_bound_on_size() noexcept { return 1; }
271  };
272 
273  struct DelayChange {
274  static constexpr Options::String help = "Slabs to wait before changing";
275  using type = uint64_t;
276  };
277 
278  using options = tmpl::list<StepChoosers, DelayChange>;
279  static constexpr Options::String help =
280  "Trigger a slab size change chosen by the provided step choosers.\n"
281  "The actual changing of the slab size can be delayed until a later\n"
282  "slab to improve parallelization.";
283 
284  ChangeSlabSize() = default;
287  step_choosers,
288  const uint64_t delay_change) noexcept
289  : step_choosers_(std::move(step_choosers)), delay_change_(delay_change) {}
290 
291  using argument_tags = tmpl::list<Tags::TimeStepId, Tags::DataBox>;
292 
293  template <typename DbTags, typename Metavariables, typename ArrayIndex,
294  typename ParallelComponent>
295  void operator()(const TimeStepId& time_step_id,
296  const db::DataBox<DbTags>& box_for_step_choosers,
298  const ArrayIndex& array_index,
299  const ParallelComponent* const /*meta*/) const noexcept {
300  const auto next_changable_slab = time_step_id.is_at_slab_boundary()
301  ? time_step_id.slab_number()
302  : time_step_id.slab_number() + 1;
303  const auto slab_to_change =
304  next_changable_slab + static_cast<int64_t>(delay_change_);
305 
306  double desired_slab_size = std::numeric_limits<double>::infinity();
307  for (const auto& step_chooser : step_choosers_) {
308  desired_slab_size =
309  std::min(desired_slab_size,
310  step_chooser->desired_step(
311  time_step_id.step_time().slab().duration().value(),
312  box_for_step_choosers, cache));
313  }
314 
315  const auto& component_proxy =
316  Parallel::get_parallel_component<ParallelComponent>(cache);
317  const auto& self_proxy = component_proxy[array_index];
318  // This message is sent synchronously, so it is guaranteed to
319  // arrive before the ChangeSlabSize action is called.
321  ChangeSlabSize_detail::NumberOfExpectedMessagesInbox>(
322  *self_proxy.ckLocal(), slab_to_change,
324  Parallel::contribute_to_reduction<ChangeSlabSize_detail::StoreNewSlabSize>(
325  ReductionData(slab_to_change, desired_slab_size), self_proxy,
326  component_proxy);
327  }
328 
329  // NOLINTNEXTLINE(google-runtime-references)
330  void pup(PUP::er& p) noexcept override {
332  p | step_choosers_;
333  p | delay_change_;
334  }
335 
336  private:
338  step_choosers_;
339  uint64_t delay_change_ = std::numeric_limits<uint64_t>::max();
340 };
341 
342 /// \cond
343 template <typename StepChooserRegistrars, typename EventRegistrars>
344 PUP::able::PUP_ID
345  ChangeSlabSize<StepChooserRegistrars, EventRegistrars>::my_PUP_ID =
346  0; // NOLINT
347 /// \endcond
348 } // namespace Events
CharmPupable.hpp
StepChoosers
Definition: ByBlock.hpp:33
Tags::TimeStepId
Tag for TimeStepId for the algorithm state.
Definition: Tags.hpp:34
Parallel::GlobalCache
Definition: ElementReceiveInterpPoints.hpp:16
Parallel::ReductionDatum
The data to be reduced, and invokables to be called whenever two reduction messages are combined and ...
Definition: Reduction.hpp:63
unordered_set
GlobalCache.hpp
Options.hpp
vector
Tags::TimeStep
Tag for step size.
Definition: Tags.hpp:43
tuple
db::get
const auto & get(const DataBox< TagList > &box) noexcept
Retrieve the item with tag Tag from the DataBox.
Definition: DataBox.hpp:1116
TimeStepper::next_time_id
virtual TimeStepId next_time_id(const TimeStepId &current_id, const TimeDelta &time_step) const noexcept=0
The TimeStepId after the current substep.
Events::ChangeSlabSize::DelayChange
Definition: ChangeSlabSize.hpp:273
Registration::Registrar
A template for defining a registrar.
Definition: Registration.hpp:42
Actions::ChangeSlabSize
Definition: ChangeSlabSize.hpp:107
db::apply
constexpr auto apply(F &&f, const DataBox< BoxTags > &box, Args &&... args) noexcept
Apply the invokable f with argument Tags TagsList from DataBox box
Definition: DataBox.hpp:1424
WRAPPED_PUPable_decl_template
#define WRAPPED_PUPable_decl_template(className)
Mark derived classes as serializable.
Definition: CharmPupable.hpp:22
Event
Definition: Event.hpp:30
DataBox.hpp
std::numeric_limits::infinity
T infinity(T... args)
cstddef
Assert.hpp
Parallel::InboxInserters::MemberInsert
Inserter for inserting data that is received as the value_type of a data structure that has an insert...
Definition: InboxInserters.hpp:67
TimeStepper
Definition: TimeStepper.hpp:47
cstdint
alg::min_element
decltype(auto) min_element(const Container &c)
Convenience wrapper around std::min_element.
Definition: Algorithm.hpp:343
tuples::TaggedTuple
An associative container that is indexed by structs.
Definition: TaggedTuple.hpp:271
map
Time.hpp
memory
TimeStepId
Definition: TimeStepId.hpp:25
std::unordered_multiset
ASSERT
#define ASSERT(a, m)
Assert that an expression should be true.
Definition: Assert.hpp:51
TimeStepper::can_change_step_size
bool can_change_step_size(const TimeStepId &time_id, const TimeSteppers::History< Vars, DerivVars > &history) const noexcept
Whether a change in the step size is allowed before taking a step.
Definition: TimeStepper.hpp:99
TimeStepId.hpp
limits
Parallel::receive_data
void receive_data(Proxy &&proxy, typename ReceiveTag::temporal_id temporal_id, ReceiveDataType &&receive_data, const bool enable_if_disabled=false) noexcept
Send the data args... to the algorithm running on proxy, and tag the message with the identifier temp...
Definition: Invoke.hpp:47
Gsl.hpp
Options::String
const char *const String
The string used in option structs.
Definition: Options.hpp:32
Slab.hpp
make_not_null
gsl::not_null< T * > make_not_null(T *ptr) noexcept
Construct a not_null from a pointer. Often this will be done as an implicit conversion,...
Definition: Gsl.hpp:880
std::numeric_limits::max
T max(T... args)
Tags::HistoryEvolvedVariables
Definition: Tags.hpp:86
ostream
std::unique_ptr
StepChooser
Definition: StepChooser.hpp:43
db::DataBox
Definition: InterpolationTargetWedgeSectionTorus.hpp:24
Events::ChangeSlabSize
Definition: ChangeSlabSize.hpp:229
TMPL.hpp
gsl::not_null
Require a pointer to not be a nullptr
Definition: Gsl.hpp:183