ChangeSlabSize.hpp
1 // Distributed under the MIT License.
2 // See LICENSE.txt for details.
3 
4 #pragma once
5 
6 #include <boost/functional/hash.hpp> // IWYU pragma: keep
7 #include <cstddef>
8 #include <cstdint>
9 #include <limits>
10 #include <map>
11 #include <memory>
12 #include <ostream>
13 #include <pup.h>
14 #include <tuple>
15 #include <unordered_set>
16 #include <vector>
17 
19 #include "Options/Options.hpp"
20 #include "Parallel/AlgorithmMetafunctions.hpp"
22 #include "Parallel/GlobalCache.hpp"
23 #include "Parallel/InboxInserters.hpp"
24 #include "Parallel/Invoke.hpp"
25 #include "Parallel/Reduction.hpp"
26 #include "ParallelAlgorithms/EventsAndTriggers/Event.hpp"
27 #include "Time/Slab.hpp"
28 #include "Time/StepChoosers/StepChooser.hpp"
29 #include "Time/Tags.hpp"
30 #include "Time/Time.hpp"
31 #include "Time/TimeStepId.hpp"
32 #include "Time/TimeSteppers/TimeStepper.hpp"
33 #include "Utilities/Algorithm.hpp"
35 #include "Utilities/Functional.hpp"
36 #include "Utilities/Gsl.hpp"
37 #include "Utilities/TMPL.hpp"
38 #include "Utilities/TaggedTuple.hpp"
39 
40 /// \cond
41 namespace Tags {
42 struct DataBox;
43 template <typename Tag>
44 struct Next;
45 } // namespace Tags
46 // IWYU pragma: no_forward_declare db::DataBox
47 /// \endcond
48 
49 namespace ChangeSlabSize_detail {
50 struct NewSlabSizeInbox
51  : public Parallel::InboxInserters::MemberInsert<NewSlabSizeInbox> {
52  using temporal_id = int64_t;
54 };
55 
56 // This inbox doesn't receive any data, it just counts messages (using
57 // the size of the multiset). Whenever a message is sent to the
58 // NewSlabSizeInbox, another message is sent here synchronously, so
59 // the count here is the number of messages we expect in the
60 // NewSlabSizeInbox.
61 struct NumberOfExpectedMessagesInbox
63  NumberOfExpectedMessagesInbox> {
64  using temporal_id = int64_t;
65  using NoData = std::tuple<>;
66  using type = std::map<temporal_id,
68 };
69 
70 struct StoreNewSlabSize {
71  template <typename ParallelComponent, typename DbTags, typename Metavariables,
72  typename ArrayIndex>
73  static void apply(const db::DataBox<DbTags>& /*box*/,
75  const ArrayIndex& array_index, const int64_t slab_number,
76  const double slab_size) noexcept {
77  Parallel::receive_data<ChangeSlabSize_detail::NewSlabSizeInbox>(
78  *Parallel::get_parallel_component<ParallelComponent>(cache)[array_index]
79  .ckLocal(),
80  slab_number, slab_size);
81  }
82 };
83 } // namespace ChangeSlabSize_detail
84 
85 namespace Actions {
86 /// \ingroup ActionsGroup
87 /// \ingroup TimeGroup
88 /// Adjust the slab size based on previous executions of
89 /// Events::ChangeSlabSize
90 ///
91 /// Uses:
92 /// - GlobalCache:
93 /// - Tags::TimeStepperBase
94 /// - DataBox:
95 /// - Tags::HistoryEvolvedVariables
96 /// - Tags::TimeStep
97 /// - Tags::TimeStepId
98 ///
99 /// DataBox changes:
100 /// - Adds: nothing
101 /// - Removes: nothing
102 /// - Modifies:
103 /// - Tags::Next<Tags::TimeStepId>
104 /// - Tags::TimeStep
105 /// - Tags::TimeStepId
107  using inbox_tags =
108  tmpl::list<ChangeSlabSize_detail::NumberOfExpectedMessagesInbox,
109  ChangeSlabSize_detail::NewSlabSizeInbox>;
110 
111  template <typename DbTags, typename... InboxTags, typename Metavariables,
112  typename ArrayIndex, typename ActionList,
113  typename ParallelComponent>
115  db::DataBox<DbTags>& box, tuples::TaggedTuple<InboxTags...>& inboxes,
116  const Parallel::GlobalCache<Metavariables>& /*cache*/,
117  const ArrayIndex& /*array_index*/, const ActionList /*meta*/,
118  const ParallelComponent* const /*meta*/) noexcept {
119  const auto& time_step_id = db::get<Tags::TimeStepId>(box);
120  if (not time_step_id.is_at_slab_boundary()) {
121  return {std::move(box), Parallel::AlgorithmExecution::Continue};
122  }
123 
124  auto& message_count_inbox =
125  tuples::get<ChangeSlabSize_detail::NumberOfExpectedMessagesInbox>(
126  inboxes);
127  if (message_count_inbox.empty() or
128  message_count_inbox.begin()->first != time_step_id.slab_number()) {
129  return {std::move(box), Parallel::AlgorithmExecution::Continue};
130  }
131 
132  auto& new_slab_size_inbox =
133  tuples::get<ChangeSlabSize_detail::NewSlabSizeInbox>(inboxes);
134 
135  const auto slab_number = time_step_id.slab_number();
136  const auto number_of_changes = [&slab_number](
137  const auto& inbox) noexcept->size_t {
138  if (inbox.empty()) {
139  return 0;
140  }
141  if (inbox.begin()->first == slab_number) {
142  return inbox.begin()->second.size();
143  }
144  ASSERT(inbox.begin()->first >= slab_number,
145  "Received data for a change at slab " << inbox.begin()->first
146  << " but it is already slab " << slab_number);
147  return 0;
148  };
149 
150  const size_t expected_messages = number_of_changes(message_count_inbox);
151  const size_t received_messages = number_of_changes(new_slab_size_inbox);
152  ASSERT(expected_messages >= received_messages,
153  "Received " << received_messages << " size change messages at slab "
154  << slab_number << ", but only expected "
155  << expected_messages);
156  if (expected_messages != received_messages) {
157  return {std::move(box), Parallel::AlgorithmExecution::Retry};
158  }
159 
160  message_count_inbox.erase(message_count_inbox.begin());
161 
162  const double new_slab_size =
163  *alg::min_element(new_slab_size_inbox.begin()->second);
164  new_slab_size_inbox.erase(new_slab_size_inbox.begin());
165 
166  const TimeStepper& time_stepper = db::get<Tags::TimeStepper<>>(box);
167 
168  // Sometimes time steppers need to run with a fixed step size.
169  // This is generally at the start of an evolution when the history
170  // is in an unusual state.
171  if (not time_stepper.can_change_step_size(
172  time_step_id, db::get<Tags::HistoryEvolvedVariables<>>(box))) {
173  return {std::move(box), Parallel::AlgorithmExecution::Continue};
174  }
175 
176  const auto& current_step = db::get<Tags::TimeStep>(box);
177  const auto& current_slab = current_step.slab();
178 
179  const auto new_slab =
180  current_step.is_positive()
181  ? current_slab.with_duration_from_start(new_slab_size)
182  : current_slab.with_duration_to_end(new_slab_size);
183 
184  const auto new_step = current_step.with_slab(new_slab);
185 
186  // We are at a slab boundary, so the substep is 0.
187  const auto new_time_step_id =
188  TimeStepId(time_step_id.time_runs_forward(), time_step_id.slab_number(),
189  time_step_id.step_time().with_slab(new_slab));
190 
191  const auto new_next_time_step_id =
192  time_stepper.next_time_id(new_time_step_id, new_step);
193 
194  db::mutate<Tags::Next<Tags::TimeStepId>, Tags::TimeStep,
196  make_not_null(&box),
197  [&new_next_time_step_id, &new_step, &new_time_step_id](
198  const gsl::not_null<TimeStepId*> next_time_step_id,
199  const gsl::not_null<TimeDelta*> time_step,
200  const gsl::not_null<TimeDelta*> next_time_step,
201  const gsl::not_null<TimeStepId*> local_time_step_id) noexcept {
202  *next_time_step_id = new_next_time_step_id;
203  *time_step = new_step;
204  *next_time_step = new_step;
205  *local_time_step_id = new_time_step_id;
206  });
207 
208  return {std::move(box), Parallel::AlgorithmExecution::Continue};
209  }
210 };
211 } // namespace Actions
212 
213 namespace Events {
214 /// \ingroup TimeGroup
215 /// %Trigger a slab size change.
216 ///
217 /// The new size will be the minimum suggested by any of the provided
218 /// step choosers on any element. This requires a global reduction,
219 /// so it is possible to delay the change until a later slab to avoid
220 /// a global synchronization. The actual change is carried out by
221 /// Actions::ChangeSlabSize.
222 ///
223 /// When running with global time-stepping, the slab size and step
224 /// size are the same, so this adjusts the step size used by the time
225 /// integration. With local time-stepping this controls the interval
226 /// between times when the sequences of steps on all elements are
227 /// forced to align.
228 class ChangeSlabSize : public Event {
229  using ReductionData = Parallel::ReductionData<
232 
233  public:
234  /// \cond
235  explicit ChangeSlabSize(CkMigrateMessage* /*unused*/) noexcept {}
236  using PUP::able::register_constructor;
238  /// \endcond
239 
240  struct StepChoosers {
241  static constexpr Options::String help = "Limits on slab size";
242  using type =
244  static size_t lower_bound_on_size() noexcept { return 1; }
245  };
246 
247  struct DelayChange {
248  static constexpr Options::String help = "Slabs to wait before changing";
249  using type = uint64_t;
250  };
251 
252  using options = tmpl::list<StepChoosers, DelayChange>;
253  static constexpr Options::String help =
254  "Trigger a slab size change chosen by the provided step choosers.\n"
255  "The actual changing of the slab size can be delayed until a later\n"
256  "slab to improve parallelization.";
257 
258  ChangeSlabSize() = default;
261  step_choosers,
262  const uint64_t delay_change) noexcept
263  : step_choosers_(std::move(step_choosers)), delay_change_(delay_change) {}
264 
265  using argument_tags = tmpl::list<Tags::TimeStepId, Tags::DataBox>;
266 
267  template <typename DbTags, typename Metavariables, typename ArrayIndex,
268  typename ParallelComponent>
269  void operator()(const TimeStepId& time_step_id,
270  const db::DataBox<DbTags>& box_for_step_choosers,
272  const ArrayIndex& array_index,
273  const ParallelComponent* const /*meta*/) const noexcept {
274  const auto next_changable_slab = time_step_id.is_at_slab_boundary()
275  ? time_step_id.slab_number()
276  : time_step_id.slab_number() + 1;
277  const auto slab_to_change =
278  next_changable_slab + static_cast<int64_t>(delay_change_);
279 
280  double desired_slab_size = std::numeric_limits<double>::infinity();
281  for (const auto& step_chooser : step_choosers_) {
282  desired_slab_size =
283  std::min(desired_slab_size,
284  step_chooser->desired_slab(
285  time_step_id.step_time().slab().duration().value(),
286  box_for_step_choosers, cache));
287  }
288 
289  const auto& component_proxy =
290  Parallel::get_parallel_component<ParallelComponent>(cache);
291  const auto& self_proxy = component_proxy[array_index];
292  // This message is sent synchronously, so it is guaranteed to
293  // arrive before the ChangeSlabSize action is called.
295  ChangeSlabSize_detail::NumberOfExpectedMessagesInbox>(
296  *self_proxy.ckLocal(), slab_to_change,
298  Parallel::contribute_to_reduction<ChangeSlabSize_detail::StoreNewSlabSize>(
299  ReductionData(slab_to_change, desired_slab_size), self_proxy,
300  component_proxy);
301  }
302 
303  using is_ready_argument_tags = tmpl::list<>;
304 
305  template <typename Metavariables, typename ArrayIndex, typename Component>
306  bool is_ready(Parallel::GlobalCache<Metavariables>& /*cache*/,
307  const ArrayIndex& /*array_index*/,
308  const Component* const /*meta*/) const noexcept {
309  return true;
310  }
311 
312  bool needs_evolved_variables() const noexcept override {
313  // This depends on the chosen StepChoosers, but they don't have a
314  // way to report this information so we just return true to be
315  // safe.
316  return true;
317  }
318 
319  // NOLINTNEXTLINE(google-runtime-references)
320  void pup(PUP::er& p) noexcept override {
321  Event::pup(p);
322  p | step_choosers_;
323  p | delay_change_;
324  }
325 
326  private:
328  step_choosers_;
329  uint64_t delay_change_ = std::numeric_limits<uint64_t>::max();
330 };
331 } // namespace Events
std::apply
T apply(T... args)
CharmPupable.hpp
StepChoosers
Definition: ByBlock.hpp:33
Parallel::AlgorithmExecution::Retry
@ Retry
Temporarily stop executing iterable actions, but try the same action again after receiving data from ...
Tags::TimeStepId
Tag for TimeStepId for the algorithm state.
Definition: Tags.hpp:33
Parallel::GlobalCache
Definition: ElementReceiveInterpPoints.hpp:15
Parallel::ReductionDatum
The data to be reduced, and invokables to be called whenever two reduction messages are combined and ...
Definition: Reduction.hpp:65
unordered_set
GlobalCache.hpp
Options.hpp
vector
Tags::TimeStep
Tag for step size.
Definition: Tags.hpp:42
tuple
db::get
const auto & get(const DataBox< TagList > &box) noexcept
Retrieve the item with tag Tag from the DataBox.
Definition: DataBox.hpp:791
Tags::Next
Prefix indicating the value a quantity will take on the next iteration of the algorithm.
Definition: Prefixes.hpp:118
TimeStepper::next_time_id
virtual TimeStepId next_time_id(const TimeStepId &current_id, const TimeDelta &time_step) const noexcept=0
The TimeStepId after the current substep.
Events::ChangeSlabSize::DelayChange
Definition: ChangeSlabSize.hpp:247
Actions::ChangeSlabSize
Definition: ChangeSlabSize.hpp:106
WRAPPED_PUPable_decl_template
#define WRAPPED_PUPable_decl_template(className)
Mark derived classes as serializable.
Definition: CharmPupable.hpp:22
StepChooser< StepChooserUse::Slab >
Definition: StepChooser.hpp:94
Event
Definition: Event.hpp:19
Parallel::AlgorithmExecution::Continue
@ Continue
Leave the algorithm termination flag in its current state.
DataBox.hpp
alg::min_element
constexpr decltype(auto) min_element(const Container &c)
Convenience wrapper around std::min_element.
Definition: Algorithm.hpp:343
std::numeric_limits::infinity
T infinity(T... args)
cstddef
Assert.hpp
Parallel::InboxInserters::MemberInsert
Inserter for inserting data that is received as the value_type of a data structure that has an insert...
Definition: InboxInserters.hpp:67
TimeStepper
Definition: TimeStepper.hpp:47
cstdint
tuples::TaggedTuple
An associative container that is indexed by structs.
Definition: TaggedTuple.hpp:271
map
Time.hpp
Parallel::AlgorithmExecution
AlgorithmExecution
The possible options for altering the current execution of the algorithm, used in the return type of ...
Definition: AlgorithmMetafunctions.hpp:31
memory
TimeStepId
Definition: TimeStepId.hpp:25
std::unordered_multiset
ASSERT
#define ASSERT(a, m)
Assert that an expression should be true.
Definition: Assert.hpp:49
Parallel::receive_data
void receive_data(Proxy &&proxy, typename ReceiveTag::temporal_id temporal_id, ReceiveDataType &&receive_data, const bool enable_if_disabled=false) noexcept
Send the data args... to the algorithm running on proxy, and tag the message with the identifier temp...
Definition: Invoke.hpp:32
TimeStepper::can_change_step_size
bool can_change_step_size(const TimeStepId &time_id, const TimeSteppers::History< Vars, DerivVars > &history) const noexcept
Whether a change in the step size is allowed before taking a step.
Definition: TimeStepper.hpp:138
TimeStepId.hpp
ActionTesting::cache
Parallel::GlobalCache< Metavariables > & cache(MockRuntimeSystem< Metavariables > &runner, const ArrayIndex &array_index) noexcept
Returns the GlobalCache of Component with index array_index.
Definition: MockRuntimeSystemFreeFunctions.hpp:382
limits
Gsl.hpp
Options::String
const char *const String
The string used in option structs.
Definition: Options.hpp:32
Slab.hpp
make_not_null
gsl::not_null< T * > make_not_null(T *ptr) noexcept
Construct a not_null from a pointer. Often this will be done as an implicit conversion,...
Definition: Gsl.hpp:880
std::numeric_limits::max
T max(T... args)
Tags::HistoryEvolvedVariables
Definition: Tags.hpp:85
Events::ChangeSlabSize::needs_evolved_variables
bool needs_evolved_variables() const noexcept override
Whether the event uses anything depending on the evolved_variables. If this returns false,...
Definition: ChangeSlabSize.hpp:312
ostream
std::unique_ptr
Events::ChangeSlabSize
Definition: ChangeSlabSize.hpp:228
TMPL.hpp
gsl::not_null
Require a pointer to not be a nullptr
Definition: ReadSpecPiecewisePolynomial.hpp:13