Line data    Source code

       1           1 : // Distributed under the MIT License.
       2             : // See LICENSE.txt for details.
       3             : 
       4             : /// \file
       5             : /// Actions for communicating data on regions that overlap with the subdomains
       6             : /// of other elements
       7             : 
       8             : #pragma once
       9             : 
      10             : #include <cstddef>
      11             : #include <map>
      12             : #include <optional>
      13             : #include <tuple>
      14             : #include <utility>
      15             : 
      16             : #include "DataStructures/DataBox/DataBox.hpp"
      17             : #include "Domain/Structure/Element.hpp"
      18             : #include "Domain/Tags.hpp"
      19             : #include "IO/Logging/Tags.hpp"
      20             : #include "IO/Logging/Verbosity.hpp"
      21             : #include "NumericalAlgorithms/Convergence/Tags.hpp"
      22             : #include "NumericalAlgorithms/DiscontinuousGalerkin/HasReceivedFromAllMortars.hpp"
      23             : #include "Parallel/AlgorithmExecution.hpp"
      24             : #include "Parallel/GlobalCache.hpp"
      25             : #include "Parallel/InboxInserters.hpp"
      26             : #include "Parallel/Invoke.hpp"
      27             : #include "Parallel/Printf/Printf.hpp"
      28             : #include "ParallelAlgorithms/LinearSolver/Schwarz/OverlapHelpers.hpp"
      29             : #include "ParallelAlgorithms/LinearSolver/Schwarz/Tags.hpp"
      30             : #include "Utilities/PrettyType.hpp"
      31             : #include "Utilities/TMPL.hpp"
      32             : #include "Utilities/TaggedTuple.hpp"
      33             : 
      34             : /// \cond
      35             : template <size_t Dim>
      36             : struct ElementId;
      37             : /// \endcond
      38             : 
      39           1 : namespace LinearSolver::Schwarz {
      40             : /// Actions related to the Schwarz solver
      41           1 : namespace Actions {
      42             : 
      43             : namespace detail {
      44             : template <size_t Dim, typename OverlapFields, typename OptionsGroup>
      45             : struct OverlapFieldsTag
      46             :     : public Parallel::InboxInserters::Map<
      47             :           OverlapFieldsTag<Dim, OverlapFields, OptionsGroup>> {
      48             :   using temporal_id = size_t;
      49             :   using type = std::map<
      50             :       temporal_id,
      51             :       OverlapMap<Dim, tmpl::conditional_t<
      52             :                           (tmpl::size<OverlapFields>::value > 1),
      53             :                           tuples::tagged_tuple_from_typelist<OverlapFields>,
      54             :                           typename tmpl::front<OverlapFields>::type>>>;
      55             : };
      56             : }  // namespace detail
      57             : 
      58             : /*!
      59             :  * \brief Send data on regions that overlap with other subdomains to their
      60             :  * corresponding elements
      61             :  *
      62             :  * Collect the `OverlapFields` on "intruding overlaps", i.e. regions that
      63             :  * overlap with the subdomains of other elements, and send the data to those
      64             :  * elements. The `OverlapFields` can be tags holding either `Variables` or
      65             :  * `Tensor`s. The `RestrictToOverlap` flag controls whether the tags are simply
      66             :  * retrieved from the element and sent as-is (`false`) or only the data that
      67             :  * intersect the overlap region are sent (`true`). If `RestrictToOverlap` is
      68             :  * `false` this action can also be used to communicate non-tensor data.
      69             :  *
      70             :  * This actions should be followed by
      71             :  * `LinearSolver::Schwarz::Actions::ReceiveOverlapFields` in the action list.
      72             :  */
      73             : template <typename OverlapFields, typename OptionsGroup, bool RestrictToOverlap,
      74             :           typename TemporalIdTag = Convergence::Tags::IterationId<OptionsGroup>>
      75           1 : struct SendOverlapFields;
      76             : 
      77             : /// \cond
      78             : template <typename... OverlapFields, typename OptionsGroup,
      79             :           bool RestrictToOverlap, typename TemporalIdTag>
      80             : struct SendOverlapFields<tmpl::list<OverlapFields...>, OptionsGroup,
      81             :                          RestrictToOverlap, TemporalIdTag> {
      82             :   using const_global_cache_tags =
      83             :       tmpl::list<Tags::MaxOverlap<OptionsGroup>,
      84             :                  logging::Tags::Verbosity<OptionsGroup>>;
      85             : 
      86             :   template <typename DbTagsList, typename... InboxTags, typename Metavariables,
      87             :             size_t Dim, typename ActionList, typename ParallelComponent>
      88             :   static Parallel::iterable_action_return_t apply(
      89             :       db::DataBox<DbTagsList>& box,
      90             :       const tuples::TaggedTuple<InboxTags...>& /*inboxes*/,
      91             :       Parallel::GlobalCache<Metavariables>& cache,
      92             :       const ElementId<Dim>& element_id, const ActionList /*meta*/,
      93             :       const ParallelComponent* const /*meta*/) {
      94             :     const auto& element = get<domain::Tags::Element<Dim>>(box);
      95             : 
      96             :     // Skip communicating if the overlap is empty and we RestrictToOverlap,
      97             :     // because we would end up with empty tensor data.
      98             :     if (UNLIKELY((RestrictToOverlap and
      99             :                   db::get<Tags::MaxOverlap<OptionsGroup>>(box) == 0) or
     100             :                  element.number_of_neighbors() == 0)) {
     101             :       return {Parallel::AlgorithmExecution::Continue, std::nullopt};
     102             :     }
     103             : 
     104             :     // Do some logging
     105             :     const auto& iteration_id = get<TemporalIdTag>(box);
     106             :     if (UNLIKELY(get<logging::Tags::Verbosity<OptionsGroup>>(box) >=
     107             :                  ::Verbosity::Debug)) {
     108             :       Parallel::printf("%s %s(%zu): Send overlap fields\n", element_id,
     109             :                        pretty_type::name<OptionsGroup>(), iteration_id);
     110             :     }
     111             : 
     112             :     // Send data on intruding overlaps to the corresponding neighbors
     113             :     auto& receiver_proxy =
     114             :         Parallel::get_parallel_component<ParallelComponent>(cache);
     115             :     for (const auto& direction_and_neighbors : element.neighbors()) {
     116             :       const auto& direction = direction_and_neighbors.first;
     117             :       const auto& neighbors = direction_and_neighbors.second;
     118             :       // Collect the data on intruding overlaps
     119             :       tuples::TaggedTuple<OverlapFields...> overlap_fields{};
     120             :       if constexpr (RestrictToOverlap) {
     121             :         const auto& element_extents =
     122             :             get<domain::Tags::Mesh<Dim>>(box).extents();
     123             :         const size_t intruding_extent =
     124             :             gsl::at(get<Tags::IntrudingExtents<Dim, OptionsGroup>>(box),
     125             :                     direction.dimension());
     126             :         expand_pack((get<OverlapFields>(overlap_fields) =
     127             :                          LinearSolver::Schwarz::data_on_overlap(
     128             :                              db::get<OverlapFields>(box), element_extents,
     129             :                              intruding_extent, direction))...);
     130             :       } else {
     131             :         expand_pack((get<OverlapFields>(overlap_fields) =
     132             :                          db::get<OverlapFields>(box))...);
     133             :       }
     134             :       // We elide the tagged tuple in the inbox tag if only a single tag is
     135             :       // communicated. This optimization allows moving the overlap-map into the
     136             :       // DataBox in one piece.
     137             :       auto& collapsed_overlap_fields = [&overlap_fields]() -> auto& {
     138             :         if constexpr (sizeof...(OverlapFields) > 1) {
     139             :           return overlap_fields;
     140             :         } else {
     141             :           return get<OverlapFields...>(overlap_fields);
     142             :         }
     143             :       }
     144             :       ();
     145             :       // Copy data to send to neighbors, but move it for the last one
     146             :       const auto direction_from_neighbor =
     147             :           neighbors.orientation()(direction.opposite());
     148             :       for (auto neighbor = neighbors.begin(); neighbor != neighbors.end();
     149             :            ++neighbor) {
     150             :         Parallel::receive_data<detail::OverlapFieldsTag<
     151             :             Dim, tmpl::list<OverlapFields...>, OptionsGroup>>(
     152             :             receiver_proxy[*neighbor], iteration_id,
     153             :             std::make_pair(
     154             :                 OverlapId<Dim>{direction_from_neighbor, element.id()},
     155             :                 (std::next(neighbor) == neighbors.end())
     156             :                     // NOLINTNEXTLINE(bugprone-use-after-move)
     157             :                     ? std::move(collapsed_overlap_fields)
     158             :                     : collapsed_overlap_fields));
     159             :       }
     160             :     }
     161             :     return {Parallel::AlgorithmExecution::Continue, std::nullopt};
     162             :   }
     163             : };
     164             : /// \endcond
     165             : 
     166             : /*!
     167             :  * \brief Receive data from regions of this element's subdomain that overlap
     168             :  * with other elements
     169             :  *
     170             :  * This action waits until overlap data from all neighboring elements has been
     171             :  * received and then moves the data into the DataBox as
     172             :  * `LinearSolver::Schwarz::Tags::Overlaps<OverlapFields...>`.
     173             :  *
     174             :  * This actions should be preceded by
     175             :  * `LinearSolver::Schwarz::Actions::SendOverlapFields` in the action list.
     176             :  */
     177             : template <size_t Dim, typename OverlapFields, typename OptionsGroup,
     178             :           bool RestrictToOverlap,
     179             :           typename TemporalIdTag = Convergence::Tags::IterationId<OptionsGroup>>
     180           1 : struct ReceiveOverlapFields;
     181             : 
     182             : /// \cond
     183             : template <size_t Dim, typename... OverlapFields, typename OptionsGroup,
     184             :           bool RestrictToOverlap, typename TemporalIdTag>
     185             : struct ReceiveOverlapFields<Dim, tmpl::list<OverlapFields...>, OptionsGroup,
     186             :                             RestrictToOverlap, TemporalIdTag> {
     187             :  private:
     188             :   using overlap_fields_tag =
     189             :       detail::OverlapFieldsTag<Dim, tmpl::list<OverlapFields...>, OptionsGroup>;
     190             : 
     191             :  public:
     192             :   using simple_tags =
     193             :       tmpl::list<Tags::Overlaps<OverlapFields, Dim, OptionsGroup>...>;
     194             :   using const_global_cache_tags =
     195             :       tmpl::list<Tags::MaxOverlap<OptionsGroup>,
     196             :                  logging::Tags::Verbosity<OptionsGroup>>;
     197             :   using inbox_tags = tmpl::list<overlap_fields_tag>;
     198             : 
     199             :   template <typename DbTagsList, typename... InboxTags, typename Metavariables,
     200             :             typename ActionList, typename ParallelComponent>
     201             :   static Parallel::iterable_action_return_t apply(
     202             :       db::DataBox<DbTagsList>& box, tuples::TaggedTuple<InboxTags...>& inboxes,
     203             :       const Parallel::GlobalCache<Metavariables>& /*cache*/,
     204             :       const ElementId<Dim>& element_id, const ActionList /*meta*/,
     205             :       const ParallelComponent* const /*meta*/) {
     206             :     const auto& iteration_id = get<TemporalIdTag>(box);
     207             :     const auto& element = get<domain::Tags::Element<Dim>>(box);
     208             : 
     209             :     // Nothing to receive if nothing was sent in the action above
     210             :     if (UNLIKELY((RestrictToOverlap and
     211             :                   db::get<Tags::MaxOverlap<OptionsGroup>>(box) == 0) or
     212             :                  element.number_of_neighbors() == 0)) {
     213             :       return {Parallel::AlgorithmExecution::Continue, std::nullopt};
     214             :     }
     215             : 
     216             :     if (not dg::has_received_from_all_mortars<overlap_fields_tag>(
     217             :             iteration_id, element, inboxes)) {
     218             :       return {Parallel::AlgorithmExecution::Retry, std::nullopt};
     219             :     }
     220             : 
     221             :     // Do some logging
     222             :     if (UNLIKELY(get<logging::Tags::Verbosity<OptionsGroup>>(box) >=
     223             :                  ::Verbosity::Debug)) {
     224             :       Parallel::printf("%s %s(%zu): Receive overlap fields\n", element_id,
     225             :                        pretty_type::name<OptionsGroup>(), iteration_id);
     226             :     }
     227             : 
     228             :     // Move received overlap data into DataBox
     229             :     auto received_overlap_fields =
     230             :         std::move(tuples::get<overlap_fields_tag>(inboxes)
     231             :                       .extract(iteration_id)
     232             :                       .mapped());
     233             :     db::mutate<Tags::Overlaps<OverlapFields, Dim, OptionsGroup>...>(
     234             :         [&received_overlap_fields](const auto... local_overlap_fields) {
     235             :           if constexpr (sizeof...(OverlapFields) > 1) {
     236             :             (local_overlap_fields->clear(), ...);
     237             :             for (auto& [overlap_id, overlap_fields] : received_overlap_fields) {
     238             :               expand_pack((*local_overlap_fields)[overlap_id] =
     239             :                               std::move(get<OverlapFields>(overlap_fields))...);
     240             :             }
     241             :           } else {
     242             :             expand_pack((*local_overlap_fields =
     243             :                              std::move(received_overlap_fields))...);
     244             :           }
     245             :         },
     246             :         make_not_null(&box));
     247             : 
     248             :     return {Parallel::AlgorithmExecution::Continue, std::nullopt};
     249             :   }
     250             : };
     251             : /// \endcond
     252             : 
     253             : }  // namespace Actions
     254             : }  // namespace LinearSolver::Schwarz