Line data    Source code

       1           0 : // Distributed under the MIT License.
       2             : // See LICENSE.txt for details.
       3             : 
       4             : #pragma once
       5             : 
       6             : #include <memory>
       7             : 
       8             : #include "DataStructures/DataBox/DataBox.hpp"
       9             : #include "Parallel/Algorithms/AlgorithmSingleton.hpp"
      10             : #include "Parallel/GlobalCache.hpp"
      11             : #include "Parallel/Local.hpp"
      12             : #include "Parallel/ParallelComponentHelpers.hpp"
      13             : #include "Parallel/Phase.hpp"
      14             : #include "Parallel/PhaseDependentActionList.hpp"
      15             : #include "ParallelAlgorithms/Actions/TerminatePhase.hpp"
      16             : #include "ParallelAlgorithms/Interpolation/Actions/InterpolationTargetSendPoints.hpp"
      17             : #include "ParallelAlgorithms/Interpolation/Protocols/InterpolationTargetTag.hpp"
      18             : #include "Utilities/PrettyType.hpp"
      19             : #include "Utilities/ProtocolHelpers.hpp"
      20             : #include "Utilities/TMPL.hpp"
      21             : #include "Utilities/TypeTraits.hpp"
      22             : 
      23             : /// \cond
      24             : namespace intrp {
      25             : namespace Actions {
      26             : template <typename Metavariables, typename InterpolationTargetTag>
      27             : struct InitializeInterpolationTarget;
      28             : }  // namespace Actions
      29             : }  // namespace intrp
      30             : /// \endcond
      31             : 
      32             : namespace intrp {
      33             : 
      34             : /// \brief ParallelComponent representing a set of points to be interpolated
      35             : /// to and a function to call upon interpolation to those points.
      36             : ///
      37             : /// Each InterpolationTarget will communicate with the `Interpolator`.
      38             : ///
      39             : /// `InterpolationTargetTag` must conform to the
      40             : /// intrp::protocols::InterpolationTargetTag protocol.
      41             : ///
      42             : /// The metavariables must contain the following type aliases:
      43             : /// - interpolator_source_vars:
      44             : ///      A `tmpl::list` of tags that define a `Variables` sent from all
      45             : ///      `Element`s to the local `Interpolator`.
      46             : /// - interpolation_target_tags:
      47             : ///      A `tmpl::list` of all `InterpolationTargetTag`s.
      48             : /// - temporal_id:
      49             : ///      The type held by ::intrp::Tags::TemporalIds.
      50             : ///
      51             : /// `Metavariables` must contain the following static constexpr members:
      52             : /// - size_t volume_dim:
      53             : ///      The dimension of the Domain.
      54             : ///
      55             : /// ### Interpolation with time-dependent CoordinateMaps
      56             : ///
      57             : /// Each set of points to be interpolated onto is labeled by a
      58             : /// `temporal_id`.  If any step of the interpolation procedure ever
      59             : /// uses a time-dependent `CoordinateMap`, then it needs to grab
      60             : /// `FunctionOfTime`s from the `GlobalCache`.  Before doing so, it
      61             : /// must verify that those `FunctionOfTime`s are up-to-date for the
      62             : /// given `temporal_id`.
      63             : ///
      64             : /// Note that once the `FunctionOfTime` has been verified to be
      65             : /// up-to-date for a particular `temporal_id` at one step in the
      66             : /// interpolation procedure, all subsequent steps of the interpolation
      67             : /// procedure for that same `temporal_id` need not worry about
      68             : /// re-verifying the `FunctionOfTime`.  Therefore, we need only focus
      69             : /// on the first step in the interpolation procedure that needs
      70             : /// `FunctionOfTime`s: computing the points on which to interpolate.
      71             : ///
      72             : /// Each `InterpolationTarget` has a function
      73             : /// `InterpolationTargetTag::compute_target_points` that returns the
      74             : /// points to be interpolated onto, expressed in the frame
      75             : /// `InterpolationTargetTag::compute_target_points::frame`.  Then the
      76             : /// function `block_logical_coordinates` (and eventually
      77             : /// `element_logical_coordinates`) is called to convert those points
      78             : /// to the element logical frame to do the interpolation.  If
      79             : /// `InterpolationTargetTag::compute_target_points::frame` is
      80             : /// different from the grid frame, and if the `CoordinateMap` is
      81             : /// time-dependent, then `block_logical_coordinates` grabs
      82             : /// `FunctionOfTime`s from the `GlobalCache`.  So therefore any Action
      83             : /// calling `block_logical_coordinates` must wait until the
      84             : /// `FunctionOfTime`s in the `GlobalCache` are up-to-date for the
      85             : /// `temporal_id` being passed into `block_logical_coordinates`.
      86             : ///
      87             : /// Here we describe the logic used in all the Actions that call
      88             : /// `block_logical_coordinates`.
      89             : ///
      90             : /// #### Interpolation using the Interpolator ParallelComponent
      91             : ///
      92             : /// Recall that `InterpolationTarget` can be used with the
      93             : /// `Interpolator` ParallelComponent (as for the horizon finder), or
      94             : /// by having the `Element`s interpolate directly (as for most
      95             : /// Observers).  Here we discuss the case when the `Interpolator`
      96             : /// is used; the other case is discussed below.
      97             : ///
      98             : /// Ensuring the `FunctionOfTime`s are up-to-date is done via two Tags
      99             : /// in the DataBox and a helper Action.  When interpolation is
     100             : /// requested for a new `temporal_id` (e.g. by
     101             : /// `intrp::Events::Interpolate`), the `temporal_id` is added to
     102             : /// `Tags::PendingTemporalIds`, which holds a
     103             : /// `std::deque<temporal_id>`, and represents `temporal_ids` that we
     104             : /// want to interpolate onto, but for which `FunctionOfTime`s are not
     105             : /// necessarily up-to-date.  We also keep another list of
     106             : /// `temporal_ids`: `Tags::TemporalIds`, for which `FunctionOfTime`s
     107             : /// are guaranteed to be up-to-date.
     108             : ///
     109             : /// The action `Actions::VerifyTemporalIdsAndSendPoints` moves
     110             : /// `temporal_id`s from `PendingTemporalIds` to `TemporalIds` as
     111             : /// appropriate, and if any `temporal_id`s have been so moved, it
     112             : /// generates the `block_logical_coordinates` and sends them to the
     113             : /// `Interpolator` `ParallelComponent`.  The logic is illustrated in
     114             : /// pseudocode below.  Recall that some InterpolationTargets are
     115             : /// sequential, (i.e. you cannot interpolate onto one temporal_id until
     116             : /// interpolation on previous ones are done, like the apparent horizon
     117             : /// finder), and some are non-sequential (i.e. you can interpolate in
     118             : /// any order).
     119             : ///
     120             : /// ```
     121             : /// if (map is time-independent or frame is grid frame) {
     122             : ///   move all PendingTemporalIds to TemporalIds
     123             : ///   if (sequential) {
     124             : ///     call block_logical_coords and interpolate for first temporal_id.
     125             : ///     // when interpolation is complete,
     126             : ///     // Actions::InterpolationTargetReceiveVars will begin interpolation
     127             : ///     // on the next temporal_id.
     128             : ///   } else {
     129             : ///     call block_logical_coords and interpolate for all temporal_ids.
     130             : ///   }
     131             : ///   return;
     132             : /// }
     133             : /// if (FunctionOfTimes are up-to-date for any PendingTemporalIds) {
     134             : ///   move up-to-date PendingTemporalIds to TemporalIds
     135             : ///   if (sequential) {
     136             : ///     call block_logical_coords and interpolate for first temporal_id.
     137             : ///     // when interpolation is complete,
     138             : ///     // Actions::InterpolationTargetReceiveVars will begin interpolation
     139             : ///     // on the next temporal_id.
     140             : ///   } else {
     141             : ///     call block_logical_coords and interpolate for all temporal_ids.
     142             : ///     if (PendingTemporalIds is non-empty) {
     143             : ///       call myself (i.e. execute VerifyTemporalIdsAndSendPoints)
     144             : ///     }
     145             : ///   }
     146             : /// } else {
     147             : ///   set VerifyTemporalIdsAndSendPoints as a callback for the
     148             : ///   `domain::Tags::FunctionsOfTime` in the mutable part of the GlobalCache, so
     149             : ///   that once `FunctionsOfTime` are updated, VerifyTemporalIdsAndSendPoints is
     150             : ///   called.
     151             : /// }
     152             : /// ```
     153             : ///
     154             : /// Note that VerifyTemporalIdsAndSendPoints always exits in one of three
     155             : /// ways:
     156             : /// - It has set itself up as a callback, so it will be called again.
     157             : /// - It started a sequential interpolation that will automatically
     158             : ///   start another sequential interpolation when finished.
     159             : /// - It started non-sequential interpolations on all
     160             : ///   `TemporalIds`, and there are no `PendingTemporalIds` left.
     161             : ///
     162             : /// We now describe the logic of the Actions that use
     163             : /// VerifyTemporalIdsAndSendPoints.
     164             : ///
     165             : /// ##### Actions::AddTemporalIdsToInterpolationTarget
     166             : ///
     167             : /// `Actions::AddTemporalIdsToInterpolationTarget` is called by
     168             : /// `intrp::Events::Interpolate` to trigger interpolation for new
     169             : /// `temporal_id`s. Its logic is as follows, in pseudocode:
     170             : ///
     171             : /// ```
     172             : /// Add passed-in temporal_ids to PendingTemporalIds
     173             : /// if (sequential) {
     174             : ///   if (TemporalIds is empty and
     175             : ///       PendingTemporalIds was empty before it was appended above) {
     176             : ///     call VerifyTemporalIdsAndSendPoints
     177             : ///   } else {
     178             : ///     Do nothing, because there is already a sequential interpolation in
     179             : ///     progress, or there is a callback already waiting.
     180             : ///   }
     181             : /// } else { // not sequential
     182             : ///   if (TemporalIds is non-empty) {
     183             : ///     call block_logical_coordinates and begin interpolating for
     184             : ///     all TemporalIds.
     185             : ///   }
     186             : ///   if (PendingTemporalIds is non-empty) {
     187             : ///     call VerifyTemporalIdsAndSendPoints
     188             : ///   }
     189             : ///}
     190             : ///```
     191             : ///
     192             : /// ##### Actions::InterpolationTargetReceiveVars
     193             : ///
     194             : /// `Actions::InterpolationTargetReceiveVars` is called by the
     195             : /// `Interpolator` when it is finished interpolating the current
     196             : /// `temporal_id`.  For the sequential case, it needs to start
     197             : /// interpolating for the next `temporal_id`.  The logic is, in pseudocode:
     198             : ///
     199             : ///```
     200             : /// if (sequential) {
     201             : ///   if (TemporalIds is not empty) {
     202             : ///     call block_logical_coordinates and interpolate for next temporal_id
     203             : ///   } else if (PendingTemporalIds is not empty) {
     204             : ///     call VerifyTemporalIdsAndSendPoints
     205             : ///   }
     206             : /// }
     207             : ///```
     208             : ///
     209             : /// ##### intrp::callbacks::FindApparentHorizon
     210             : ///
     211             : /// `intrp::callbacks::FindApparentHorizon calls`
     212             : /// `block_logical_coordinates` when it needs to start a new iteration
     213             : /// of the horizon finder at the same `temporal_id`, so one might
     214             : /// think you need to worry about up-to-date `FunctionOfTime`s. But
     215             : /// since `intrp::callbacks::FindApparentHorizon` always works on the same
     216             : /// `temporal_id` for which the `FunctionOfTime`s have already been
     217             : /// verified as up-to-date from the last iteration, no special consideration
     218             : /// of `FunctionOfTime`s need be done here.
     219             : ///
     220             : /// #### Interpolation without using the Interpolator ParallelComponent
     221             : ///
     222             : /// This case is easier than the case with the `Interpolator`, because
     223             : /// the target points are always time-independent in the frame
     224             : /// `compute_target_points::frame`.
     225             : ///
     226             : /// ##### Actions::EnsureFunctionOfTimeUpToDate
     227             : ///
     228             : /// `Actions::EnsureFunctionOfTimeUpToDate` verifies that the
     229             : ///  `FunctionOfTime`s are up-to-date at the `DgElementArray`s current
     230             : ///  time.
     231             : ///
     232             : /// ###### Current logic:
     233             : ///
     234             : /// > `Actions::EnsureFunctionOfTimeUpToDate` is placed in `DgElementArray`s
     235             : /// >  PDAL before any use of interpolation.
     236             : ///
     237             : /// ##### Actions::InterpolationTargetSendTimeIndepPointsToElements
     238             : ///
     239             : /// `Actions::InterpolationTargetSendTimeIndepPointsToElements`
     240             : /// is invoked on `InterpolationTarget` during the Registration PDAL,
     241             : /// to send time-independent point information to `Element`s.
     242             : ///
     243             : /// ###### Current logic:
     244             : ///
     245             : /// > Send the result of `compute_target_points` to all `Element`s.
     246             : ///
     247             : /// Note that this may need to be revisited because every `Element` has
     248             : /// a copy of every target point, which may use a lot of memory.  An
     249             : /// alternative is for each Element to invoke an Action on each
     250             : /// `InterpolationTarget` (presumably from an `Event`) at each time,
     251             : /// and then the InterpolationTarget invokes another Action to send points
     252             : /// to only those `Elements` that contain the points; this alternative
     253             : /// uses less memory but much more communication. Another alternative would
     254             : /// be to place the points in the GlobalCache (one copy per node) since the
     255             : /// points need be computed only once.
     256             : ///
     257             : template <class Metavariables, typename InterpolationTargetTag>
     258           1 : struct InterpolationTarget {
     259           0 :   using interpolation_target_tag = InterpolationTargetTag;
     260             :   static_assert(
     261             :       tt::assert_conforms_to_v<interpolation_target_tag,
     262             :                                intrp::protocols::InterpolationTargetTag>);
     263           0 :   static std::string name() {
     264             :     return pretty_type::name<InterpolationTargetTag>();
     265             :   }
     266           0 :   using chare_type = ::Parallel::Algorithms::Singleton;
     267           0 :   using const_global_cache_tags =
     268             :       Parallel::get_const_global_cache_tags_from_actions<
     269             :           tmpl::flatten<tmpl::list<
     270             :               typename InterpolationTargetTag::compute_target_points,
     271             :               typename InterpolationTargetTag::post_interpolation_callbacks>>>;
     272           0 :   using metavariables = Metavariables;
     273           0 :   using phase_dependent_action_list = tmpl::list<
     274             :       Parallel::PhaseActions<
     275             :           Parallel::Phase::Initialization,
     276             :           tmpl::list<intrp::Actions::InitializeInterpolationTarget<
     277             :                          Metavariables, InterpolationTargetTag>,
     278             :                      Parallel::Actions::TerminatePhase>>,
     279             :       Parallel::PhaseActions<
     280             :           Parallel::Phase::Register,
     281             :           tmpl::list<
     282             :               tmpl::conditional_t<
     283             :                   InterpolationTargetTag::compute_target_points::is_sequential::
     284             :                       value,
     285             :                   tmpl::list<>,
     286             :                   tmpl::list<
     287             :                       Actions::InterpolationTargetSendTimeIndepPointsToElements<
     288             :                           InterpolationTargetTag>>>,
     289             :               Parallel::Actions::TerminatePhase>>>;
     290             : 
     291           0 :   using simple_tags_from_options = Parallel::get_simple_tags_from_options<
     292             :       Parallel::get_initialization_actions_list<phase_dependent_action_list>>;
     293             : 
     294           0 :   static void execute_next_phase(
     295             :       Parallel::Phase next_phase,
     296             :       Parallel::CProxy_GlobalCache<metavariables>& global_cache) {
     297             :     auto& local_cache = *Parallel::local_branch(global_cache);
     298             :     Parallel::get_parallel_component<
     299             :         InterpolationTarget<metavariables, InterpolationTargetTag>>(local_cache)
     300             :         .start_phase(next_phase);
     301             :   };
     302             : };
     303             : }  // namespace intrp