SpECTRE Documentation Coverage Report
Current view: top level - ParallelAlgorithms/DiscontinuousGalerkin - FluxCommunication.hpp Hit Total Coverage
Commit: ebec864322c50bab8dca0a90baf8d01875114261 Lines: 3 20 15.0 %
Date: 2020-11-25 20:28:50
Legend: Lines: hit not hit

          Line data    Source code
       1           0 : // Distributed under the MIT License.
       2             : // See LICENSE.txt for details.
       3             : 
       4             : #pragma once
       5             : 
       6             : #include <algorithm>
       7             : #include <cstddef>
       8             : #include <tuple>
       9             : #include <utility>
      10             : 
      11             : #include "DataStructures/DataBox/DataBox.hpp"
      12             : #include "DataStructures/FixedHashMap.hpp"
      13             : #include "Domain/Structure/Direction.hpp"
      14             : #include "Domain/Structure/Element.hpp"
      15             : #include "Domain/Structure/ElementId.hpp"
      16             : #include "Domain/Structure/MaxNumberOfNeighbors.hpp"
      17             : #include "Domain/Structure/OrientationMap.hpp"
      18             : #include "Domain/Tags.hpp"
      19             : #include "ErrorHandling/Assert.hpp"
      20             : #include "NumericalAlgorithms/DiscontinuousGalerkin/MortarHelpers.hpp"
      21             : #include "NumericalAlgorithms/DiscontinuousGalerkin/Tags.hpp"
      22             : #include "Parallel/GlobalCache.hpp"
      23             : #include "Parallel/InboxInserters.hpp"
      24             : #include "Parallel/Invoke.hpp"
      25             : #include "ParallelAlgorithms/DiscontinuousGalerkin/HasReceivedFromAllMortars.hpp"
      26             : #include "Utilities/Gsl.hpp"
      27             : #include "Utilities/Requires.hpp"
      28             : #include "Utilities/TMPL.hpp"
      29             : #include "Utilities/TaggedTuple.hpp"
      30             : 
      31             : namespace dg {
      32             : 
      33             : /// The inbox tag for flux communication
      34             : template <typename BoundaryScheme>
      35             : struct FluxesInboxTag
      36           1 :     : public Parallel::InboxInserters::Map<FluxesInboxTag<BoundaryScheme>> {
      37             :  private:
      38           0 :   static constexpr size_t volume_dim = BoundaryScheme::volume_dim;
      39             : 
      40             :  public:
      41           0 :   using temporal_id = typename BoundaryScheme::temporal_id_tag::type;
      42           0 :   using type = std::map<
      43             :       temporal_id,
      44             :       FixedHashMap<
      45             :           maximum_number_of_neighbors(volume_dim), MortarId<volume_dim>,
      46             :           std::pair<temporal_id, typename BoundaryScheme::BoundaryData>,
      47             :           boost::hash<MortarId<volume_dim>>>>;
      48             : };
      49             : 
      50             : namespace Actions {
      51             : 
      52             : /*!
      53             :  * \ingroup ActionsGroup
      54             :  * \ingroup DiscontinuousGalerkinGroup
      55             :  * \brief Send local boundary data needed for fluxes to neighbors.
      56             :  *
      57             :  * Sends the data in `Tags::Mortars<BoundaryScheme::mortar_data_tag,
      58             :  * volume_dim>` to neighbors. Note that this action does not collect the data
      59             :  * from element interfaces or compute anything, it only sends the available data
      60             :  * to neighbors. Make sure to collect the data that should be sent before
      61             :  * invoking this action and store them in
      62             :  * `Tags::Mortars<BoundaryScheme::mortar_data_tag, volume_dim>`, including
      63             :  * projecting them to the mortar if necessary. See
      64             :  * `dg::Actions::CollectDataForFluxes` for an action that uses the interface
      65             :  * tag mechanism to do so.
      66             :  *
      67             :  * Uses:
      68             :  * - DataBox:
      69             :  *   - `Tags::Mortars<typename BoundaryScheme::mortar_data_tag, volume_dim>`
      70             :  *   - `Tags::Element<volume_dim>`
      71             :  *   - `BoundaryScheme::temporal_id_tag`
      72             :  *   - `BoundaryScheme::receive_temporal_id_tag`
      73             :  *   - `Tags::Mortars<Tags::Mesh<volume_dim - 1>, volume_dim>`
      74             :  *
      75             :  * \see `dg::Actions::ReceiveDataForFluxes`
      76             :  */
      77             : template <typename BoundaryScheme>
      78           1 : struct SendDataForFluxes {
      79             :  private:
      80           0 :   static constexpr size_t volume_dim = BoundaryScheme::volume_dim;
      81           0 :   using temporal_id_tag = typename BoundaryScheme::temporal_id_tag;
      82           0 :   using receive_temporal_id_tag =
      83             :       typename BoundaryScheme::receive_temporal_id_tag;
      84           0 :   using fluxes_inbox_tag = dg::FluxesInboxTag<BoundaryScheme>;
      85           0 :   using all_mortar_data_tag =
      86             :       ::Tags::Mortars<typename BoundaryScheme::mortar_data_tag, volume_dim>;
      87             : 
      88             :  public:
      89             :   template <typename DbTags, typename... InboxTags, typename Metavariables,
      90             :             typename ArrayIndex, typename ActionList,
      91             :             typename ParallelComponent>
      92           0 :   static std::tuple<db::DataBox<DbTags>&&> apply(
      93             :       db::DataBox<DbTags>& box, tuples::TaggedTuple<InboxTags...>& /*inboxes*/,
      94             :       Parallel::GlobalCache<Metavariables>& cache,
      95             :       const ArrayIndex& /*array_index*/, const ActionList /*meta*/,
      96             :       const ParallelComponent* const /*meta*/) noexcept {
      97             :     auto& receiver_proxy =
      98             :         Parallel::get_parallel_component<ParallelComponent>(cache);
      99             : 
     100             :     const auto& all_mortar_data = get<all_mortar_data_tag>(box);
     101             :     const auto& element = db::get<domain::Tags::Element<volume_dim>>(box);
     102             :     const auto& temporal_id = db::get<temporal_id_tag>(box);
     103             :     const auto& receive_temporal_id = db::get<receive_temporal_id_tag>(box);
     104             :     const auto& mortar_meshes = db::get<
     105             :         ::Tags::Mortars<domain::Tags::Mesh<volume_dim - 1>, volume_dim>>(box);
     106             : 
     107             :     // Iterate over neighbors
     108             :     for (const auto& direction_and_neighbors : element.neighbors()) {
     109             :       const auto& direction = direction_and_neighbors.first;
     110             :       const size_t dimension = direction.dimension();
     111             :       const auto& neighbors_in_direction = direction_and_neighbors.second;
     112             :       const auto& orientation = neighbors_in_direction.orientation();
     113             :       const auto direction_from_neighbor = orientation(direction.opposite());
     114             : 
     115             :       for (const auto& neighbor : neighbors_in_direction) {
     116             :         const MortarId<volume_dim> mortar_id{direction, neighbor};
     117             : 
     118             :         // Make a copy of the local boundary data on the mortar to send to the
     119             :         // neighbor
     120             :         ASSERT(all_mortar_data.find(mortar_id) != all_mortar_data.end(),
     121             :                "Mortar data on mortar "
     122             :                    << mortar_id
     123             :                    << " not available for sending. Did you forget to collect "
     124             :                       "the data on mortars?");
     125             :         auto remote_boundary_data_on_mortar =
     126             :             all_mortar_data.at(mortar_id).local_data(temporal_id);
     127             : 
     128             :         // Reorient the data to the neighbor orientation if necessary
     129             :         if (not orientation.is_aligned()) {
     130             :           remote_boundary_data_on_mortar.orient_on_slice(
     131             :               mortar_meshes.at(mortar_id).extents(), dimension, orientation);
     132             :         }
     133             : 
     134             :         // Send remote data to neighbor
     135             :         Parallel::receive_data<fluxes_inbox_tag>(
     136             :             receiver_proxy[neighbor], temporal_id,
     137             :             std::make_pair(
     138             :                 MortarId<volume_dim>{direction_from_neighbor, element.id()},
     139             :                 std::make_pair(receive_temporal_id,
     140             :                                std::move(remote_boundary_data_on_mortar))));
     141             :       }
     142             :     }
     143             :     return {std::move(box)};
     144             :   }
     145             : };
     146             : 
     147             : /*!
     148             :  * \ingroup ActionsGroup
     149             :  * \ingroup DiscontinuousGalerkinGroup
     150             :  * \brief Receive boundary data needed for fluxes from neighbors.
     151             :  *
     152             :  * Waits until remote boundary data from all internal mortars has arrived, then
     153             :  * contributes it to the data in `Tags::Mortars<BoundaryScheme::mortar_data_tag,
     154             :  * volume_dim>` that already holds the local boundary data.
     155             :  *
     156             :  * For boundary schemes that communicate asynchronously, in the sense that
     157             :  * neighboring elements may not send and receive data at the same temporal IDs
     158             :  * (e.g. for local time-stepping), see the documentation of this action's
     159             :  * template specialization.
     160             :  *
     161             :  * Uses:
     162             :  * - DataBox:
     163             :  *   - `Tags::Element<volume_dim>`
     164             :  *   - `BoundaryScheme::temporal_id`
     165             :  *
     166             :  * DataBox changes:
     167             :  * - Adds: nothing
     168             :  * - Removes: nothing
     169             :  * - Modifies:
     170             :  *   - `Tags::Mortars<BoundaryScheme::mortar_data_tag, volume_dim>`
     171             :  *
     172             :  * \see `dg::Actions::SendDataForFluxes`
     173             :  */
     174             : template <typename BoundaryScheme, typename = std::nullptr_t>
     175           1 : struct ReceiveDataForFluxes {
     176             :  private:
     177           0 :   static constexpr size_t volume_dim = BoundaryScheme::volume_dim;
     178           0 :   using temporal_id_tag = typename BoundaryScheme::temporal_id_tag;
     179           0 :   using fluxes_inbox_tag = dg::FluxesInboxTag<BoundaryScheme>;
     180           0 :   using all_mortar_data_tag =
     181             :       ::Tags::Mortars<typename BoundaryScheme::mortar_data_tag, volume_dim>;
     182             : 
     183             :  public:
     184           0 :   using inbox_tags = tmpl::list<fluxes_inbox_tag>;
     185             : 
     186             :   template <typename DbTags, typename... InboxTags, typename Metavariables,
     187             :             typename ArrayIndex, typename ActionList,
     188             :             typename ParallelComponent>
     189           0 :   static std::tuple<db::DataBox<DbTags>&&> apply(
     190             :       db::DataBox<DbTags>& box, tuples::TaggedTuple<InboxTags...>& inboxes,
     191             :       const Parallel::GlobalCache<Metavariables>& /*cache*/,
     192             :       const ArrayIndex& /*array_index*/, const ActionList /*meta*/,
     193             :       const ParallelComponent* const /*meta*/) noexcept {
     194             :     if (UNLIKELY(
     195             :             get<domain::Tags::Element<volume_dim>>(box).number_of_neighbors() ==
     196             :             0)) {
     197             :       return {std::move(box)};
     198             :     }
     199             :     auto& inbox = tuples::get<fluxes_inbox_tag>(inboxes);
     200             :     const auto& temporal_id = get<temporal_id_tag>(box);
     201             :     const auto temporal_received = inbox.find(temporal_id);
     202             :     db::mutate<all_mortar_data_tag>(
     203             :         make_not_null(&box), [&temporal_received](
     204             :                                  const gsl::not_null<
     205             :                                      typename all_mortar_data_tag::type*>
     206             :                                      mortar_data) noexcept {
     207             :           for (auto& received_mortar_data : temporal_received->second) {
     208             :             const auto& mortar_id = received_mortar_data.first;
     209             :             mortar_data->at(mortar_id).remote_insert(
     210             :                 temporal_received->first,
     211             :                 std::move(received_mortar_data.second.second));
     212             :           }
     213             :         });
     214             :     inbox.erase(temporal_received);
     215             :     return {std::move(box)};
     216             :   }
     217             : 
     218             :   template <typename DbTags, typename... InboxTags, typename Metavariables,
     219             :             typename ArrayIndex>
     220           0 :   static bool is_ready(
     221             :       const db::DataBox<DbTags>& box,
     222             :       const tuples::TaggedTuple<InboxTags...>& inboxes,
     223             :       const Parallel::GlobalCache<Metavariables>& /*cache*/,
     224             :       const ArrayIndex& /*array_index*/) noexcept {
     225             :     return has_received_from_all_mortars<fluxes_inbox_tag>(
     226             :         db::get<temporal_id_tag>(box),
     227             :         get<domain::Tags::Element<volume_dim>>(box), inboxes);
     228             :   }
     229             : };
     230             : 
     231             : /*!
     232             :  * \ingroup ActionsGroup
     233             :  * \ingroup DiscontinuousGalerkinGroup
     234             :  * \brief Receive asynchronous boundary data needed for fluxes from neighbors.
     235             :  *
     236             :  * Waits until remote boundary data from all internal mortars has arrived, then
     237             :  * contributes it to the data in `Tags::Mortars<BoundaryScheme::mortar_data_tag,
     238             :  * volume_dim>` that already holds the local boundary data.
     239             :  *
     240             :  * This template specialization handles asynchronous boundary schemes, e.g.
     241             :  * for local time-stepping. It proceeds only once the received boundary data
     242             :  * on every mortar has caught up with the `BoundaryScheme::receive_temporal_id`
     243             :  * on the element.
     244             :  *
     245             :  * Uses:
     246             :  * - DataBox:
     247             :  *   - `Tags::Element<volume_dim>`
     248             :  *   - `BoundaryScheme::receive_temporal_id`
     249             :  *
     250             :  * DataBox changes:
     251             :  * - Adds: nothing
     252             :  * - Removes: nothing
     253             :  * - Modifies:
     254             :  *   - `Tags::Mortars<BoundaryScheme::mortar_data_tag, volume_dim>`
     255             :  *   - `Tags::Mortars<BoundaryScheme::receive_temporal_id_tag, volume_dim>`
     256             :  *
     257             :  * \see `dg::Actions::SendDataForFluxes`
     258             :  * \see `dg::Actions::ReceiveDataForFluxes`
     259             :  */
     260             : template <typename BoundaryScheme>
     261             : struct ReceiveDataForFluxes<
     262             :     BoundaryScheme, Requires<not std::is_same_v<
     263             :                         typename BoundaryScheme::receive_temporal_id_tag,
     264             :                         typename BoundaryScheme::temporal_id_tag>>> {
     265             :  private:
     266             :   static constexpr size_t volume_dim = BoundaryScheme::volume_dim;
     267             :   using temporal_id_tag = typename BoundaryScheme::temporal_id_tag;
     268             :   using receive_temporal_id_tag =
     269             :       typename BoundaryScheme::receive_temporal_id_tag;
     270             :   using fluxes_inbox_tag = dg::FluxesInboxTag<BoundaryScheme>;
     271             :   using all_mortar_data_tag =
     272             :       ::Tags::Mortars<typename BoundaryScheme::mortar_data_tag, volume_dim>;
     273             : 
     274             :  public:
     275             :   using inbox_tags = tmpl::list<fluxes_inbox_tag>;
     276             : 
     277             :   template <typename DbTags, typename... InboxTags, typename Metavariables,
     278             :             typename ArrayIndex, typename ActionList,
     279             :             typename ParallelComponent>
     280             :   static std::tuple<db::DataBox<DbTags>&&> apply(
     281             :       db::DataBox<DbTags>& box, tuples::TaggedTuple<InboxTags...>& inboxes,
     282             :       const Parallel::GlobalCache<Metavariables>& /*cache*/,
     283             :       const ArrayIndex& /*array_index*/, const ActionList /*meta*/,
     284             :       const ParallelComponent* const /*meta*/) noexcept {
     285             :     db::mutate<all_mortar_data_tag,
     286             :                ::Tags::Mortars<receive_temporal_id_tag, volume_dim>>(
     287             :         make_not_null(&box),
     288             :         [&inboxes](const gsl::not_null<typename all_mortar_data_tag::type*>
     289             :                        mortar_data,
     290             :                    const gsl::not_null<typename ::Tags::Mortars<
     291             :                        receive_temporal_id_tag, volume_dim>::type*>
     292             :                        neighbor_next_temporal_ids,
     293             :                    const typename receive_temporal_id_tag::type&
     294             :                        local_next_temporal_id) noexcept {
     295             :           auto& inbox = tuples::get<fluxes_inbox_tag>(inboxes);
     296             :           for (auto received_data = inbox.begin();
     297             :                received_data != inbox.end() and
     298             :                    received_data->first < local_next_temporal_id;
     299             :                received_data = inbox.erase(received_data)) {
     300             :             const auto& receive_temporal_id = received_data->first;
     301             :             for (auto& received_mortar_data : received_data->second) {
     302             :               const auto mortar_id = received_mortar_data.first;
     303             :               ASSERT(neighbor_next_temporal_ids->at(mortar_id) ==
     304             :                          receive_temporal_id,
     305             :                      "Expected data at "
     306             :                      << neighbor_next_temporal_ids->at(mortar_id)
     307             :                      << " but received at " << receive_temporal_id);
     308             :               neighbor_next_temporal_ids->at(mortar_id) =
     309             :                   received_mortar_data.second.first;
     310             :               mortar_data->at(mortar_id).remote_insert(
     311             :                   receive_temporal_id,
     312             :                   std::move(received_mortar_data.second.second));
     313             :             }
     314             :           }
     315             : 
     316             :           // The apparently pointless lambda wrapping this check
     317             :           // prevents gcc-7.3.0 from segfaulting.
     318             :           ASSERT(([
     319             :                    &neighbor_next_temporal_ids, &local_next_temporal_id
     320             :                  ]() noexcept {
     321             :                    return std::all_of(
     322             :                        neighbor_next_temporal_ids->begin(),
     323             :                        neighbor_next_temporal_ids->end(),
     324             :                        [&local_next_temporal_id](const auto& next) noexcept {
     325             :                          return next.first.second ==
     326             :                                     ElementId<
     327             :                                         volume_dim>::external_boundary_id() or
     328             :                                 next.second >= local_next_temporal_id;
     329             :                        });
     330             :                  }()),
     331             :                  "apply called before all data received");
     332             :           ASSERT(
     333             :               inbox.empty() or (inbox.size() == 1 and
     334             :                                 inbox.begin()->first == local_next_temporal_id),
     335             :               "Shouldn't have received data that depended upon the step being "
     336             :               "taken: Received data at " << inbox.begin()->first
     337             :               << " while stepping to " << local_next_temporal_id);
     338             :         },
     339             :         db::get<receive_temporal_id_tag>(box));
     340             : 
     341             :     return std::forward_as_tuple(std::move(box));
     342             :   }
     343             : 
     344             :   template <typename DbTags, typename... InboxTags, typename Metavariables,
     345             :             typename ArrayIndex>
     346             :   static bool is_ready(
     347             :       const db::DataBox<DbTags>& box,
     348             :       const tuples::TaggedTuple<InboxTags...>& inboxes,
     349             :       const Parallel::GlobalCache<Metavariables>& /*cache*/,
     350             :       const ArrayIndex& /*array_index*/) noexcept {
     351             :     const auto& inbox = tuples::get<fluxes_inbox_tag>(inboxes);
     352             :     const auto& local_next_temporal_id = db::get<receive_temporal_id_tag>(box);
     353             :     const auto& mortars_next_temporal_id =
     354             :         db::get<::Tags::Mortars<receive_temporal_id_tag, volume_dim>>(box);
     355             :     for (const auto& mortar_id_next_temporal_id : mortars_next_temporal_id) {
     356             :       const auto& mortar_id = mortar_id_next_temporal_id.first;
     357             :       // If on an external boundary
     358             :       if (mortar_id.second == ElementId<volume_dim>::external_boundary_id()) {
     359             :         continue;
     360             :       }
     361             :       auto next_temporal_id = mortar_id_next_temporal_id.second;
     362             :       while (next_temporal_id < local_next_temporal_id) {
     363             :         const auto temporal_received = inbox.find(next_temporal_id);
     364             :         if (temporal_received == inbox.end()) {
     365             :           return false;
     366             :         }
     367             :         const auto mortar_received = temporal_received->second.find(mortar_id);
     368             :         if (mortar_received == temporal_received->second.end()) {
     369             :           return false;
     370             :         }
     371             :         next_temporal_id = mortar_received->second.first;
     372             :       }
     373             :     }
     374             :     return true;
     375             :   }
     376             : };
     377             : 
     378             : }  // namespace Actions
     379             : }  // namespace dg

Generated by: LCOV version 1.14