Line data    Source code

       1           0 : // Distributed under the MIT License.
       2             : // See LICENSE.txt for details.
       3             : 
       4             : #pragma once
       5             : 
       6             : #include <cstddef>
       7             : #include <mutex>
       8             : 
       9             : #include "DataStructures/DataBox/DataBox.hpp"
      10             : #include "Domain/Structure/ElementId.hpp"
      11             : #include "Parallel/GlobalCache.hpp"
      12             : #include "Parallel/Invoke.hpp"
      13             : #include "Parallel/Local.hpp"
      14             : #include "Parallel/NodeLock.hpp"
      15             : #include "Parallel/Phase.hpp"
      16             : #include "Utilities/ErrorHandling/Error.hpp"
      17             : #include "Utilities/Gsl.hpp"
      18             : #include "Utilities/TaggedTuple.hpp"
      19             : 
      20           0 : namespace Parallel::Actions {
      21             : /// \brief Receive data for a specific element on the nodegroup.
      22             : ///
      23             : /// If `StartPhase` is `true` then `start_phase(phase)` is called on the
      24             : /// `element_to_execute_on`, otherwise `perform_algorithm()` is called on the
      25             : /// `element_to_execute_on`.
      26             : template <bool StartPhase = false>
      27           1 : struct ReceiveDataForElement {
      28             :   /// \brief Entry method called when receiving data from another node.
      29             :   template <typename ParallelComponent, typename DbTagsList,
      30             :             typename Metavariables, typename ArrayIndex, typename ReceiveData,
      31             :             typename ReceiveTag, size_t Dim, typename DistributedObject>
      32           1 :   static void apply(db::DataBox<DbTagsList>& box,
      33             :                     Parallel::GlobalCache<Metavariables>& cache,
      34             :                     const ArrayIndex& /*array_index*/,
      35             :                     const gsl::not_null<Parallel::NodeLock*> /*node_lock*/,
      36             :                     const DistributedObject* /*distributed_object*/,
      37             :                     const ReceiveTag& /*meta*/,
      38             :                     const ElementId<Dim>& element_to_execute_on,
      39             :                     typename ReceiveTag::temporal_id instance,
      40             :                     ReceiveData receive_data) {
      41             :     ERROR(
      42             :         "The multi-node code hasn't been tested. It should work, but be aware "
      43             :         "that I haven't tried yet.");
      44             :     auto& element_collection = db::get_mutable_reference<
      45             :         typename ParallelComponent::element_collection_tag>(
      46             :         make_not_null(&box));
      47             :     // Note: We'll be able to do a counter-based check here too once that
      48             :     // works for LTS in `SendDataToElement`
      49             :     ReceiveTag::insert_into_inbox(
      50             :         make_not_null(&tuples::get<ReceiveTag>(
      51             :             element_collection.at(element_to_execute_on).inboxes())),
      52             :         instance, std::move(receive_data));
      53             : 
      54             :     apply_impl<ParallelComponent>(cache, element_to_execute_on,
      55             :                                   make_not_null(&element_collection));
      56             :   }
      57             : 
      58             :   /// \brief Entry method call when receiving from same node.
      59             :   template <typename ParallelComponent, typename DbTagsList,
      60             :             typename Metavariables, typename ArrayIndex, size_t Dim,
      61             :             typename DistributedObject>
      62           1 :   static void apply(db::DataBox<DbTagsList>& box,
      63             :                     Parallel::GlobalCache<Metavariables>& cache,
      64             :                     const ArrayIndex& /*array_index*/,
      65             :                     const gsl::not_null<Parallel::NodeLock*> /*node_lock*/,
      66             :                     const DistributedObject* /*distributed_object*/,
      67             :                     const ElementId<Dim>& element_to_execute_on) {
      68             :     auto& element_collection = db::get_mutable_reference<
      69             :         typename ParallelComponent::element_collection_tag>(
      70             :         make_not_null(&box));
      71             :     apply_impl<ParallelComponent>(cache, element_to_execute_on,
      72             :                                   make_not_null(&element_collection));
      73             :   }
      74             : 
      75             :  private:
      76             :   template <typename ParallelComponent, typename Metavariables,
      77             :             typename ElementCollection, size_t Dim>
      78           0 :   static void apply_impl(
      79             :       Parallel::GlobalCache<Metavariables>& cache,
      80             :       const ElementId<Dim>& element_to_execute_on,
      81             :       const gsl::not_null<ElementCollection*> element_collection) {
      82             :     const size_t my_node = Parallel::my_node<size_t>(cache);
      83             :     auto& my_proxy = Parallel::get_parallel_component<ParallelComponent>(cache);
      84             : 
      85             :     if constexpr (StartPhase) {
      86             :       const Phase current_phase =
      87             :           Parallel::local_branch(
      88             :               Parallel::get_parallel_component<ParallelComponent>(cache))
      89             :               ->phase();
      90             :       auto& element = element_collection->at(element_to_execute_on);
      91             :       const std::lock_guard element_lock(element.element_lock());
      92             :       element.start_phase(current_phase);
      93             :     } else {
      94             :       auto& element = element_collection->at(element_to_execute_on);
      95             :       std::unique_lock element_lock(element.element_lock(), std::defer_lock);
      96             :       if (element_lock.try_lock()) {
      97             :         element.perform_algorithm();
      98             :       } else {
      99             :         Parallel::threaded_action<Parallel::Actions::ReceiveDataForElement<>>(
     100             :             my_proxy[my_node], element_to_execute_on);
     101             :       }
     102             :     }
     103             :   }
     104             : };
     105             : }  // namespace Parallel::Actions