Line data Source code
1 0 : // Distributed under the MIT License. 2 : // See LICENSE.txt for details. 3 : 4 : #pragma once 5 : 6 : #include <algorithm> 7 : #include <cstddef> 8 : #include <cstdint> 9 : #include <limits> 10 : #include <map> 11 : #include <memory> 12 : #include <pup.h> 13 : #include <pup_stl.h> 14 : #include <utility> 15 : #include <vector> 16 : 17 : #include "DataStructures/DataBox/DataBox.hpp" 18 : #include "Options/Context.hpp" 19 : #include "Options/ParseError.hpp" 20 : #include "Options/String.hpp" 21 : #include "Parallel/ArrayCollection/IsDgElementCollection.hpp" 22 : #include "Parallel/GlobalCache.hpp" 23 : #include "Parallel/Reduction.hpp" 24 : #include "ParallelAlgorithms/EventsAndTriggers/Event.hpp" 25 : #include "Time/ChangeSlabSize/Tags.hpp" 26 : #include "Time/StepChoosers/StepChooser.hpp" 27 : #include "Time/TimeStepId.hpp" 28 : #include "Time/TimeStepRequest.hpp" 29 : #include "Time/TimeStepRequestProcessor.hpp" 30 : #include "Utilities/Functional.hpp" 31 : #include "Utilities/PrettyType.hpp" 32 : #include "Utilities/Serialization/CharmPupable.hpp" 33 : #include "Utilities/TMPL.hpp" 34 : 35 : /// \cond 36 : namespace Tags { 37 : struct DataBox; 38 : struct TimeStepId; 39 : } // namespace Tags 40 : /// \endcond 41 : 42 : namespace Events { 43 : namespace ChangeSlabSize_detail { 44 : struct StoreNewSlabSize { 45 : template <typename ParallelComponent, typename DbTags, typename Metavariables, 46 : typename ArrayIndex> 47 : static void apply(db::DataBox<DbTags>& box, 48 : Parallel::GlobalCache<Metavariables>& /*cache*/, 49 : const ArrayIndex& /*array_index*/, 50 : const int64_t slab_number, 51 : const TimeStepRequestProcessor& requests) { 52 : db::mutate<::Tags::ChangeSlabSize::NewSlabSize>( 53 : [&](const gsl::not_null< 54 : std::map<int64_t, std::vector<TimeStepRequestProcessor>>*> 55 : sizes) { (*sizes)[slab_number].emplace_back(requests); }, 56 : make_not_null(&box)); 57 : } 58 : }; 59 : } // namespace ChangeSlabSize_detail 60 : 61 : /// \ingroup TimeGroup 62 : /// %Trigger a slab size change. 63 : /// 64 : /// The new size will be the minimum suggested by any of the provided 65 : /// step choosers on any element. This requires a global reduction, 66 : /// so it is possible to delay the change until a later slab to avoid 67 : /// a global synchronization. The actual change is carried out by 68 : /// Actions::ChangeSlabSize. 69 : /// 70 : /// When running with global time-stepping, the slab size and step 71 : /// size are the same, so this adjusts the step size used by the time 72 : /// integration. With local time-stepping this controls the interval 73 : /// between times when the sequences of steps on all elements are 74 : /// forced to align. 75 1 : class ChangeSlabSize : public Event { 76 0 : using ReductionData = Parallel::ReductionData< 77 : Parallel::ReductionDatum<int64_t, funcl::AssertEqual<>>, 78 : Parallel::ReductionDatum<TimeStepRequestProcessor, funcl::Plus<>>>; 79 : 80 : public: 81 : /// \cond 82 : explicit ChangeSlabSize(CkMigrateMessage* /*unused*/) {} 83 : using PUP::able::register_constructor; 84 : WRAPPED_PUPable_decl_template(ChangeSlabSize); // NOLINT 85 : /// \endcond 86 : 87 0 : struct StepChoosers { 88 0 : static constexpr Options::String help = "Limits on slab size"; 89 0 : using type = 90 : std::vector<std::unique_ptr<StepChooser<StepChooserUse::Slab>>>; 91 0 : static size_t lower_bound_on_size() { return 1; } 92 : }; 93 : 94 0 : struct DelayChange { 95 0 : static constexpr Options::String help = "Slabs to wait before changing"; 96 0 : using type = uint64_t; 97 : }; 98 : 99 0 : using options = tmpl::list<StepChoosers, DelayChange>; 100 0 : static constexpr Options::String help = 101 : "Trigger a slab size change chosen by the provided step choosers.\n" 102 : "The actual changing of the slab size can be delayed until a later\n" 103 : "slab to improve parallelization."; 104 : 105 0 : ChangeSlabSize() = default; 106 0 : ChangeSlabSize(std::vector<std::unique_ptr<StepChooser<StepChooserUse::Slab>>> 107 : step_choosers, 108 : const uint64_t delay_change, const Options::Context& context) 109 : : step_choosers_(std::move(step_choosers)), delay_change_(delay_change) { 110 : if (delay_change != 0) { 111 : for (const auto& chooser : step_choosers_) { 112 : if (not chooser->can_be_delayed()) { 113 : // The runtime name might not be exactly the same as the one 114 : // used by the factory, but hopefully it's close enough that 115 : // the user can figure it out. 116 : PARSE_ERROR(context, 117 : "The " << pretty_type::get_runtime_type_name(*chooser) 118 : << " StepChooser cannot be applied with a delay."); 119 : } 120 : } 121 : } 122 : } 123 : 124 0 : using compute_tags_for_observation_box = tmpl::list<>; 125 : 126 : // Need a const version of the full box for the step choosers, but 127 : // can't get a const version while mutating other tags, so request a 128 : // mutable version. 129 0 : using return_tags = tmpl::list<::Tags::DataBox>; 130 0 : using argument_tags = tmpl::list<::Tags::TimeStepId>; 131 : 132 : template <typename DbTags, typename Metavariables, typename ArrayIndex, 133 : typename ParallelComponent> 134 0 : void operator()(const gsl::not_null<db::DataBox<DbTags>*> box, 135 : const TimeStepId& time_step_id, 136 : Parallel::GlobalCache<Metavariables>& cache, 137 : const ArrayIndex& array_index, 138 : const ParallelComponent* const /*meta*/, 139 : const ObservationValue& /*observation_value*/) const { 140 : const auto next_changable_slab = time_step_id.is_at_slab_boundary() 141 : ? time_step_id.slab_number() 142 : : time_step_id.slab_number() + 1; 143 : const auto slab_to_change = 144 : next_changable_slab + static_cast<int64_t>(delay_change_); 145 : const auto slab_start = time_step_id.step_time(); 146 : const auto current_slab_size = (time_step_id.time_runs_forward() ? 1 : -1) * 147 : slab_start.slab().duration(); 148 : 149 : TimeStepRequestProcessor step_requests(time_step_id.time_runs_forward()); 150 : bool synchronization_required = false; 151 : for (const auto& step_chooser : step_choosers_) { 152 : step_requests.process( 153 : step_chooser->desired_step(current_slab_size.value(), *box).first); 154 : // We must synchronize if any step chooser requires it, not just 155 : // the limiting one, because choosers requiring synchronization 156 : // can be limiting on some processors and not others. 157 : if (not synchronization_required) { 158 : synchronization_required = step_chooser->uses_local_data(); 159 : } 160 : } 161 : 162 : db::mutate<::Tags::ChangeSlabSize::NumberOfExpectedMessages>( 163 : [&](const gsl::not_null<std::map<int64_t, size_t>*> expected) { 164 : ++(*expected)[slab_to_change]; 165 : }, 166 : box); 167 : 168 : if constexpr (Parallel::is_dg_element_collection_v<ParallelComponent>) { 169 : ERROR( 170 : "Slab changing is not yet implemented for the DgElementCollection " 171 : "parallel component. Specifically, the ability to do reductions " 172 : "using Charm++-style reductions has not been implemented."); 173 : } else { 174 : const auto& component_proxy = 175 : Parallel::get_parallel_component<ParallelComponent>(cache); 176 : const auto& self_proxy = component_proxy[array_index]; 177 : if (synchronization_required) { 178 : Parallel::contribute_to_reduction< 179 : ChangeSlabSize_detail::StoreNewSlabSize>( 180 : ReductionData(slab_to_change, step_requests), self_proxy, 181 : component_proxy); 182 : } else { 183 : db::mutate<::Tags::ChangeSlabSize::NewSlabSize>( 184 : [&](const gsl::not_null< 185 : std::map<int64_t, std::vector<TimeStepRequestProcessor>>*> 186 : sizes) { 187 : (*sizes)[slab_to_change].push_back(step_requests); 188 : }, 189 : box); 190 : } 191 : } 192 : (void)cache, (void)array_index; 193 : } 194 : 195 0 : using is_ready_argument_tags = tmpl::list<>; 196 : 197 : template <typename Metavariables, typename ArrayIndex, typename Component> 198 0 : bool is_ready(Parallel::GlobalCache<Metavariables>& /*cache*/, 199 : const ArrayIndex& /*array_index*/, 200 : const Component* const /*meta*/) const { 201 : return true; 202 : } 203 : 204 1 : bool needs_evolved_variables() const override { 205 : // This depends on the chosen StepChoosers, but they don't have a 206 : // way to report this information so we just return true to be 207 : // safe. 208 : return true; 209 : } 210 : 211 : template <typename F> 212 0 : void for_each_step_chooser(F&& f) const { 213 : for (const auto& step_chooser : step_choosers_) { 214 : f(*step_chooser); 215 : } 216 : } 217 : 218 : // NOLINTNEXTLINE(google-runtime-references) 219 0 : void pup(PUP::er& p) override { 220 : Event::pup(p); 221 : p | step_choosers_; 222 : p | delay_change_; 223 : } 224 : 225 : private: 226 : std::vector<std::unique_ptr<StepChooser<StepChooserUse::Slab>>> 227 0 : step_choosers_; 228 0 : uint64_t delay_change_ = std::numeric_limits<uint64_t>::max(); 229 : }; 230 : } // namespace Events