Line data    Source code

       1           0 : // Distributed under the MIT License.
       2             : // See LICENSE.txt for details.
       3             : 
       4             : #pragma once
       5             : 
       6             : #include <algorithm>
       7             : #include <cstddef>
       8             : #include <cstdint>
       9             : #include <limits>
      10             : #include <memory>
      11             : #include <pup.h>
      12             : #include <pup_stl.h>
      13             : #include <utility>
      14             : #include <vector>
      15             : 
      16             : #include "DataStructures/DataBox/DataBox.hpp"
      17             : #include "Options/String.hpp"
      18             : #include "Parallel/GlobalCache.hpp"
      19             : #include "Parallel/Reduction.hpp"
      20             : #include "ParallelAlgorithms/EventsAndTriggers/Event.hpp"
      21             : #include "Time/ChangeSlabSize/Tags.hpp"
      22             : #include "Time/StepChoosers/StepChooser.hpp"
      23             : #include "Time/TimeStepId.hpp"
      24             : #include "Utilities/Functional.hpp"
      25             : #include "Utilities/Serialization/CharmPupable.hpp"
      26             : #include "Utilities/TMPL.hpp"
      27             : 
      28             : /// \cond
      29             : namespace Tags {
      30             : struct DataBox;
      31             : struct TimeStepId;
      32             : }  // namespace Tags
      33             : /// \endcond
      34             : 
      35             : namespace Events {
      36             : namespace ChangeSlabSize_detail {
      37             : struct StoreNewSlabSize {
      38             :   template <typename ParallelComponent, typename DbTags, typename Metavariables,
      39             :             typename ArrayIndex>
      40             :   static void apply(db::DataBox<DbTags>& box,
      41             :                     Parallel::GlobalCache<Metavariables>& /*cache*/,
      42             :                     const ArrayIndex& /*array_index*/,
      43             :                     const int64_t slab_number, const double slab_size) {
      44             :     db::mutate<::Tags::ChangeSlabSize::NewSlabSize>(
      45             :         [&](const gsl::not_null<
      46             :             std::map<int64_t, std::unordered_multiset<double>>*>
      47             :                 sizes) { (*sizes)[slab_number].insert(slab_size); },
      48             :         make_not_null(&box));
      49             :   }
      50             : };
      51             : }  // namespace ChangeSlabSize_detail
      52             : 
      53             : /// \ingroup TimeGroup
      54             : /// %Trigger a slab size change.
      55             : ///
      56             : /// The new size will be the minimum suggested by any of the provided
      57             : /// step choosers on any element.  This requires a global reduction,
      58             : /// so it is possible to delay the change until a later slab to avoid
      59             : /// a global synchronization.  The actual change is carried out by
      60             : /// Actions::ChangeSlabSize.
      61             : ///
      62             : /// When running with global time-stepping, the slab size and step
      63             : /// size are the same, so this adjusts the step size used by the time
      64             : /// integration.  With local time-stepping this controls the interval
      65             : /// between times when the sequences of steps on all elements are
      66             : /// forced to align.
      67           1 : class ChangeSlabSize : public Event {
      68           0 :   using ReductionData = Parallel::ReductionData<
      69             :       Parallel::ReductionDatum<int64_t, funcl::AssertEqual<>>,
      70             :       Parallel::ReductionDatum<double, funcl::Min<>>>;
      71             : 
      72             :  public:
      73             :   /// \cond
      74             :   explicit ChangeSlabSize(CkMigrateMessage* /*unused*/) {}
      75             :   using PUP::able::register_constructor;
      76             :   WRAPPED_PUPable_decl_template(ChangeSlabSize);  // NOLINT
      77             :   /// \endcond
      78             : 
      79           0 :   struct StepChoosers {
      80           0 :     static constexpr Options::String help = "Limits on slab size";
      81           0 :     using type =
      82             :         std::vector<std::unique_ptr<StepChooser<StepChooserUse::Slab>>>;
      83           0 :     static size_t lower_bound_on_size() { return 1; }
      84             :   };
      85             : 
      86           0 :   struct DelayChange {
      87           0 :     static constexpr Options::String help = "Slabs to wait before changing";
      88           0 :     using type = uint64_t;
      89             :   };
      90             : 
      91           0 :   using options = tmpl::list<StepChoosers, DelayChange>;
      92           0 :   static constexpr Options::String help =
      93             :       "Trigger a slab size change chosen by the provided step choosers.\n"
      94             :       "The actual changing of the slab size can be delayed until a later\n"
      95             :       "slab to improve parallelization.";
      96             : 
      97           0 :   ChangeSlabSize() = default;
      98           0 :   ChangeSlabSize(std::vector<std::unique_ptr<StepChooser<StepChooserUse::Slab>>>
      99             :                      step_choosers,
     100             :                  const uint64_t delay_change)
     101             :       : step_choosers_(std::move(step_choosers)), delay_change_(delay_change) {}
     102             : 
     103           0 :   using compute_tags_for_observation_box = tmpl::list<>;
     104             : 
     105             :   // Need a const version of the full box for the step choosers, but
     106             :   // can't get a const version while mutating other tags, so request a
     107             :   // mutable version.
     108           0 :   using return_tags = tmpl::list<::Tags::DataBox>;
     109           0 :   using argument_tags = tmpl::list<::Tags::TimeStepId>;
     110             : 
     111             :   template <typename DbTags, typename Metavariables, typename ArrayIndex,
     112             :             typename ParallelComponent>
     113           0 :   void operator()(const gsl::not_null<db::DataBox<DbTags>*> box,
     114             :                   const TimeStepId& time_step_id,
     115             :                   Parallel::GlobalCache<Metavariables>& cache,
     116             :                   const ArrayIndex& array_index,
     117             :                   const ParallelComponent* const /*meta*/,
     118             :                   const ObservationValue& /*observation_value*/) const {
     119             :     const auto next_changable_slab = time_step_id.is_at_slab_boundary()
     120             :                                          ? time_step_id.slab_number()
     121             :                                          : time_step_id.slab_number() + 1;
     122             :     const auto slab_to_change =
     123             :         next_changable_slab + static_cast<int64_t>(delay_change_);
     124             : 
     125             :     double desired_slab_size = std::numeric_limits<double>::infinity();
     126             :     bool synchronization_required = false;
     127             :     for (const auto& step_chooser : step_choosers_) {
     128             :       desired_slab_size = std::min(
     129             :           desired_slab_size,
     130             :           step_chooser
     131             :               ->desired_step(time_step_id.step_time().slab().duration().value(),
     132             :                              *box)
     133             :               .first);
     134             :       // We must synchronize if any step chooser requires it, not just
     135             :       // the limiting one, because choosers requiring synchronization
     136             :       // can be limiting on some processors and not others.
     137             :       if (not synchronization_required) {
     138             :         synchronization_required = step_chooser->uses_local_data();
     139             :       }
     140             :     }
     141             : 
     142             :     db::mutate<::Tags::ChangeSlabSize::NumberOfExpectedMessages>(
     143             :         [&](const gsl::not_null<std::map<int64_t, size_t>*> expected) {
     144             :           ++(*expected)[slab_to_change];
     145             :         },
     146             :         box);
     147             : 
     148             :     const auto& component_proxy =
     149             :         Parallel::get_parallel_component<ParallelComponent>(cache);
     150             :     const auto& self_proxy = component_proxy[array_index];
     151             :     if (synchronization_required) {
     152             :       Parallel::contribute_to_reduction<
     153             :           ChangeSlabSize_detail::StoreNewSlabSize>(
     154             :           ReductionData(slab_to_change, desired_slab_size), self_proxy,
     155             :           component_proxy);
     156             :     } else {
     157             :       db::mutate<::Tags::ChangeSlabSize::NewSlabSize>(
     158             :           [&](const gsl::not_null<
     159             :               std::map<int64_t, std::unordered_multiset<double>>*>
     160             :                   sizes) {
     161             :             (*sizes)[slab_to_change].insert(desired_slab_size);
     162             :           },
     163             :           box);
     164             :     }
     165             :   }
     166             : 
     167           0 :   using is_ready_argument_tags = tmpl::list<>;
     168             : 
     169             :   template <typename Metavariables, typename ArrayIndex, typename Component>
     170           0 :   bool is_ready(Parallel::GlobalCache<Metavariables>& /*cache*/,
     171             :                 const ArrayIndex& /*array_index*/,
     172             :                 const Component* const /*meta*/) const {
     173             :     return true;
     174             :   }
     175             : 
     176           1 :   bool needs_evolved_variables() const override {
     177             :     // This depends on the chosen StepChoosers, but they don't have a
     178             :     // way to report this information so we just return true to be
     179             :     // safe.
     180             :     return true;
     181             :   }
     182             : 
     183             :   template <typename F>
     184           0 :   void for_each_step_chooser(F&& f) const {
     185             :     for (const auto& step_chooser : step_choosers_) {
     186             :       f(*step_chooser);
     187             :     }
     188             :   }
     189             : 
     190             :   // NOLINTNEXTLINE(google-runtime-references)
     191           0 :   void pup(PUP::er& p) override {
     192             :     Event::pup(p);
     193             :     p | step_choosers_;
     194             :     p | delay_change_;
     195             :   }
     196             : 
     197             :  private:
     198             :   std::vector<std::unique_ptr<StepChooser<StepChooserUse::Slab>>>
     199           0 :       step_choosers_;
     200           0 :   uint64_t delay_change_ = std::numeric_limits<uint64_t>::max();
     201             : };
     202             : }  // namespace Events