Line data    Source code

       1           0 : // Distributed under the MIT License.
       2             : // See LICENSE.txt for details.
       3             : 
       4             : #pragma once
       5             : 
       6             : #include <memory>
       7             : #include <optional>
       8             : #include <type_traits>
       9             : #include <utility>
      10             : #include <vector>
      11             : 
      12             : #include "DataStructures/DataBox/DataBox.hpp"
      13             : #include "DataStructures/TaggedTuple.hpp"
      14             : #include "Parallel/GlobalCache.hpp"
      15             : #include "Parallel/Phase.hpp"
      16             : #include "ParallelAlgorithms/EventsAndTriggers/Trigger.hpp"
      17             : #include "Utilities/CallWithDynamicType.hpp"
      18             : #include "Utilities/Gsl.hpp"
      19             : #include "Utilities/TMPL.hpp"
      20             : 
      21             : /// Contains utilities for determining control-flow among phases
      22             : namespace PhaseControl {
      23             : /// The possible options for instructing the Main chare in deciding the next
      24             : /// phase to jump to.
      25             : ///
      26             : /// An object of this enum type is packaged with a requested phase in the
      27             : /// `PhaseChange::arbitrate_phase_change` function.
      28           1 : enum ArbitrationStrategy {
      29             :   /// Jump to the requested phase immediately, before considering other
      30             :   /// requested phases.
      31             :   ///
      32             :   /// This will ensure that the requested phase is always run, where
      33             :   /// alternative methods could have 'double-jumps' where the Main chare
      34             :   /// replaces a requested phase immediately without actually entering the
      35             :   /// phase.
      36             :   RunPhaseImmediately,
      37             :   /// After the requested phase is considered, continue considering other
      38             :   /// requests, potentially replacing this request.
      39             :   ///
      40             :   /// This will permit reprocessing the phase-jumps to help with cases where
      41             :   /// multiple phases are simultaneously requested.
      42             :   /// The `PermitAdditionalJumps` permits 'double-jumps' where a requested phase
      43             :   /// is immediately replaced by another phase to jump to.
      44             :   PermitAdditionalJumps
      45             : };
      46             : }  // namespace PhaseControl
      47             : 
      48             : /*!
      49             :  * \brief `PhaseChange` objects determine the storage types and logic for
      50             :  * moving between phases based on runtime data.
      51             :  *
      52             :  * The phase control flow must have the cooperation of each parallel component,
      53             :  * but make phase decisions centrally so that at any point, all components are
      54             :  * in the same phase. The operations needed by the parallel components and by
      55             :  * the Main chare, are:
      56             :  *
      57             :  * 1. Parallel components must select and/or compute the runtime data necessary
      58             :  *    for choosing the next phase, then contribute it to a global reduction to
      59             :  *    the Main component.
      60             :  *    The components must then halt at a globally-valid state for the phase
      61             :  *    change.
      62             :  *    The requirements for the state will vary depending on the phase choices,
      63             :  *    so triggers must be selected appropriately for the `PhaseChange` object.
      64             :  *    For instance, selecting a common slab will usually represent a globally
      65             :  *    well-behaved state for a `DgElementArray`.
      66             :  * 2. On the Main chare, the `PhaseChange` objects must use the collected
      67             :  *    reduction data, or other persistent data stored in
      68             :  *    `phase_change_decision_data` to decide on a phase to request and an
      69             :  *    `PhaseControl::ArbitrationStrategy` to determine how to resolve multiple
      70             :  *    simultaneous requests.
      71             :  *    Additionally, the `PhaseChange` objects must specify initialization
      72             :  *    functions to set the starting state of the tags in
      73             :  *    `phase_change_decision_data` for which they are responsible.
      74             :  *
      75             :  * In addition to the `options` type alias and `static constexpr Options::String
      76             :  * help` variable needed to be option-creatable, a derived class of
      77             :  * `PhaseChange` must specify the type aliases:
      78             :  * - `argument_tags`: A `tmpl::list` of tags from the
      79             :  *   \ref DataBoxGroup "DataBox" to be passed to `contribute_phase_data_impl` as
      80             :  *   const references.
      81             :  * - `return_tags`: A `tmpl::list` of mutable tags from the
      82             :  *   \ref DataBoxGroup "DataBox" to be passed to `contribute_phase_data_impl` as
      83             :  *   `gsl::not_null` pointers. This should be used only for tags that may be
      84             :  *   altered during the `contribute_phase_data_impl` function.
      85             :  * - `phase_change_tags_and_combines`: A `tmpl::list` of tags for
      86             :  *   populating the `phase_change_decision_data` in the Main chare. Each tag in
      87             :  *   this list must also define a `combine_method` and a `main_combine_method`
      88             :  *   for performing the aggregation during reduction.
      89             :  * - `participating_components` (templated on `Metavariables`): A `tmpl::list`
      90             :  *   of components that contribute data during this reduction. This can be used
      91             :  *   to screen out components that will not have the necessary information to
      92             :  *   contribute to the reduction. If all components should participate, this
      93             :  *   type alias can be set to simply `typename Metavariables::component_list`.
      94             :  *
      95             :  * And member functions with signatures:
      96             :  *
      97             :  * ```
      98             :  * template <typename... DecisionTags>
      99             :  * void initialize_phase_data_impl(
     100             :  *     const gsl::not_null<tuples::TaggedTuple<DecisionTags...>*>
     101             :  *         phase_change_decision_data) const;
     102             :  * ```
     103             :  * - Must set all tags in `phase_change_tags_and_combines` to useful
     104             :  *   initial states in the `phase_change_decision_data`.
     105             :  *
     106             :  * ```
     107             :  * template <typename ParallelComponent, typename ArrayIndex>
     108             :  * void contribute_phase_data_impl(
     109             :  *     [DataBox return tags...], [DataBox argument tags...],
     110             :  *     Parallel::GlobalCache<Metavariables>& cache,
     111             :  *     const ArrayIndex& array_index) const;
     112             :  * ```
     113             :  * - Should send any data relevant for the associated phase change decision made
     114             :  *   in `arbitrate_phase_change_impl` to the Main chare via function
     115             :  *   `Parallel::contribute_to_phase_change_reduction`.
     116             :  *
     117             :  * ```
     118             :  * template <typename... DecisionTags, typename Metavariables>
     119             :  * typename std::optional<
     120             :  *     std::pair<Parallel::Phase, ArbitrationStrategy>>
     121             :  * arbitrate_phase_change_impl(
     122             :  *     const gsl::not_null<tuples::TaggedTuple<DecisionTags...>*>
     123             :  *         phase_change_decision_data,
     124             :  *     const Parallel::Phase current_phase,
     125             :  *     const Parallel::GlobalCache<Metavariables>& cache) const;
     126             :  * ```
     127             :  * - Should examine the collected data in `phase_change_decision_data` and
     128             :  *   optionally return a `std::pair` with the desired `Parallel::Phase` and
     129             :  *   an `PhaseControl::ArbitrationStrategy` indicating a method for arbitrating
     130             :  *   multiple simultaneous requests. Alternatively, it may return `std::nullopt`
     131             :  *   to abstain from the phase decision.
     132             :  *   The `arbitrate_phase_change_impl` may (and often will) mutate the
     133             :  *   `phase_change_decision_data`. For instance, it may be desirable to 'reset'
     134             :  *   the data to allow for future jumps associated with the same `PhaseChange`,
     135             :  *   or the `PhaseChange` will describe multiple changes in sequence, and the
     136             :  *   state of that sequential process can be recorded in
     137             :  *   `phase_change_decision_data`.
     138             :  */
     139           1 : struct PhaseChange : public PUP::able {
     140             :  protected:
     141             :   /// \cond
     142             :   PhaseChange() = default;
     143             :   PhaseChange(const PhaseChange&) = default;
     144             :   PhaseChange(PhaseChange&&) = default;
     145             :   PhaseChange& operator=(const PhaseChange&) = default;
     146             :   PhaseChange& operator=(PhaseChange&&) = default;
     147             :   /// \endcond
     148             : 
     149             :  public:
     150           0 :   PhaseChange(CkMigrateMessage* msg) : PUP::able(msg){};
     151             : 
     152           0 :   ~PhaseChange() override = default;
     153             : 
     154           0 :   WRAPPED_PUPable_abstract(PhaseChange);  // NOLINT
     155             : 
     156             :   /// Send data from all `participating_components` to the Main chare for
     157             :   /// determining the next phase.
     158             :   template <typename ParallelComponent, typename DbTags, typename Metavariables,
     159             :             typename ArrayIndex>
     160           1 :   void contribute_phase_data(const gsl::not_null<db::DataBox<DbTags>*> box,
     161             :                              Parallel::GlobalCache<Metavariables>& cache,
     162             :                              const ArrayIndex& array_index) const {
     163             :     using factory_classes =
     164             :         typename Metavariables::factory_creation::factory_classes;
     165             :     call_with_dynamic_type<void, tmpl::at<factory_classes, PhaseChange>>(
     166             :         this, [&box, &cache, &array_index](const auto* const phase_change) {
     167             :           using phase_change_t = typename std::decay_t<decltype(*phase_change)>;
     168             :           if constexpr (tmpl::list_contains_v<
     169             :                             typename phase_change_t::
     170             :                                 template participating_components<
     171             :                                     Metavariables>,
     172             :                             ParallelComponent>) {
     173             :             db::mutate_apply<typename phase_change_t::return_tags,
     174             :                              typename phase_change_t::argument_tags>(
     175             :                 [&phase_change, &cache, &array_index](auto&&... args) {
     176             :                   phase_change
     177             :                       ->template contribute_phase_data_impl<ParallelComponent>(
     178             :                           args..., cache, array_index);
     179             :                 },
     180             :                 box);
     181             :           }
     182             :         });
     183             :   }
     184             : 
     185             :   /// Determine a phase request and `PhaseControl::ArbitrationStrategy` based on
     186             :   /// aggregated `phase_change_decision_data` on the Main Chare.
     187             :   template <typename... DecisionTags, typename Metavariables>
     188             :   std::optional<std::pair<Parallel::Phase, PhaseControl::ArbitrationStrategy>>
     189           1 :   arbitrate_phase_change(
     190             :       const gsl::not_null<tuples::TaggedTuple<DecisionTags...>*>
     191             :           phase_change_decision_data,
     192             :       const Parallel::Phase current_phase,
     193             :       const Parallel::GlobalCache<Metavariables>& cache) const {
     194             :     using factory_classes =
     195             :         typename Metavariables::factory_creation::factory_classes;
     196             :     return call_with_dynamic_type<
     197             :         std::optional<
     198             :             std::pair<Parallel::Phase, PhaseControl::ArbitrationStrategy>>,
     199             :         tmpl::at<factory_classes, PhaseChange>>(
     200             :         this, [&current_phase, &phase_change_decision_data,
     201             :                &cache](const auto* const phase_change) {
     202             :           return phase_change->arbitrate_phase_change_impl(
     203             :               phase_change_decision_data, current_phase, cache);
     204             :         });
     205             :   }
     206             : 
     207             :   /// Initialize the `phase_change_decision_data` on the main chare to starting
     208             :   /// values.
     209             :   template <typename Metavariables, typename... Tags>
     210           1 :   void initialize_phase_data(const gsl::not_null<tuples::TaggedTuple<Tags...>*>
     211             :                                  phase_change_decision_data) const {
     212             :     using factory_classes =
     213             :         typename Metavariables::factory_creation::factory_classes;
     214             :     return call_with_dynamic_type<void, tmpl::at<factory_classes, PhaseChange>>(
     215             :         this, [&phase_change_decision_data](const auto* const phase_change) {
     216             :           return phase_change->initialize_phase_data_impl(
     217             :               phase_change_decision_data);
     218             :         });
     219             :   }
     220             : };