Line data    Source code

       1            0 : // Distributed under the MIT License.
       2              : // See LICENSE.txt for details.
       3              : 
       4              : #pragma once
       5              : 
       6              : #include <cstddef>
       7              : #include <mutex>
       8              : #include <type_traits>
       9              : #include <utility>
      10              : 
      11              : #include "DataStructures/DataBox/DataBox.hpp"
      12              : #include "DataStructures/TaggedTuple.hpp"
      13              : #include "Domain/Structure/ElementId.hpp"
      14              : #include "Evolution/DiscontinuousGalerkin/AtomicInboxBoundaryData.hpp"
      15              : #include "Parallel/ArrayCollection/ReceiveDataForElement.hpp"
      16              : #include "Parallel/ArrayCollection/Tags/ElementLocations.hpp"
      17              : #include "Parallel/GlobalCache.hpp"
      18              : #include "Parallel/Info.hpp"
      19              : #include "Parallel/NodeLock.hpp"
      20              : #include "Utilities/ErrorHandling/Assert.hpp"
      21              : #include "Utilities/Gsl.hpp"
      22              : 
      23              : namespace Parallel::Actions {
      24              : /*!
      25              :  * \brief A local synchronous action where data is communicated to neighbor
      26              :  * elements.
      27              :  *
      28              :  * If the inbox tag type is an `evolution::dg::AtomicInboxBoundaryData` then
      29              :  * remote insert for elements on the same node is done in a lock-free manner
      30              :  * between the sender and receiver elements, and in a wait-free manner between
      31              :  * different sender elements to the same receiver element.
      32              :  *
      33              :  * The number of messages needed to take the next time step on the receiver
      34              :  * element is kept track of and a message is sent to the parallel runtime
      35              :  * system (e.g. Charm++) only when the receiver/neighbor element has all the
      36              :  * data it needs to take the next time step. This is done so as to reduce
      37              :  * pressure on the runtime system by sending fewer messages.
      38              :  */
      39            1 : struct SendDataToElement {
      40            0 :   using return_type = void;
      41              : 
      42              :   template <typename ParallelComponent, typename DbTagList, size_t Dim,
      43              :             typename ReceiveTag, typename ReceiveData, typename Metavariables>
      44            0 :   static return_type apply(
      45              :       db::DataBox<DbTagList>& box,
      46              :       const gsl::not_null<Parallel::NodeLock*> /*node_lock*/,
      47              :       const gsl::not_null<Parallel::GlobalCache<Metavariables>*> cache,
      48              :       const ReceiveTag& /*meta*/, const ElementId<Dim>& element_to_execute_on,
      49              :       typename ReceiveTag::temporal_id instance, ReceiveData&& receive_data) {
      50              :     const size_t my_node = Parallel::my_node<size_t>(*cache);
      51              :     // While we don't mutate the value, we want to avoid locking the DataBox
      52              :     // and the nodegroup by using `db::get_mutable_reference`. If/when we
      53              :     // start dynamically inserting and removing elements, we'll need to update
      54              :     // how we handle this. For example, we might need the containers to have
      55              :     // strong stability guarantees.
      56              :     ASSERT(db::get_mutable_reference<Parallel::Tags::ElementLocations<Dim>>(
      57              :                make_not_null(&box))
      58              :                    .count(element_to_execute_on) == 1,
      59              :            "Could not find ElementId " << element_to_execute_on
      60              :                                        << " in the list of element locations");
      61              :     const size_t node_of_element =
      62              :         db::get_mutable_reference<Parallel::Tags::ElementLocations<Dim>>(
      63              :             make_not_null(&box))
      64              :             .at(element_to_execute_on);
      65              :     auto& my_proxy =
      66              :         Parallel::get_parallel_component<ParallelComponent>(*cache);
      67              :     if (node_of_element == my_node) {
      68              :       bool run_algorithm = false;
      69              :       ASSERT(db::get_mutable_reference<
      70              :                  typename ParallelComponent::element_collection_tag>(
      71              :                  make_not_null(&box))
      72              :                      .count(element_to_execute_on) == 1,
      73              :              "The element with ID "
      74              :                  << element_to_execute_on << " is not on node " << my_node
      75              :                  << ". We should be sending data to node " << node_of_element);
      76              :       auto& element = db::get_mutable_reference<
      77              :                           typename ParallelComponent::element_collection_tag>(
      78              :                           make_not_null(&box))
      79              :                           .at(element_to_execute_on);
      80              :       if constexpr (std::is_same_v<evolution::dg::AtomicInboxBoundaryData<Dim>,
      81              :                                    typename ReceiveTag::type>) {
      82              :         run_algorithm = ReceiveTag::insert_into_inbox(
      83              :             make_not_null(&tuples::get<ReceiveTag>(element.inboxes())),
      84              :             instance, std::forward<ReceiveData>(receive_data));
      85              :       } else {
      86              :         // Scope so that we minimize how long we lock the inbox.
      87              :         std::lock_guard inbox_lock(element.inbox_lock());
      88              :         run_algorithm = ReceiveTag::insert_into_inbox(
      89              :             make_not_null(&tuples::get<ReceiveTag>(element.inboxes())),
      90              :             instance, std::forward<ReceiveData>(receive_data));
      91              :       }
      92              :       if (run_algorithm) {
      93              :         Parallel::threaded_action<Parallel::Actions::ReceiveDataForElement<>>(
      94              :             my_proxy[node_of_element], element_to_execute_on);
      95              :       }
      96              :     } else {
      97              :       Parallel::threaded_action<Parallel::Actions::ReceiveDataForElement<>>(
      98              :           my_proxy[node_of_element], ReceiveTag{}, element_to_execute_on,
      99              :           instance, std::forward<ReceiveData>(receive_data));
     100              :     }
     101              :   }
     102              : };
     103              : }  // namespace Parallel::Actions