Line data    Source code

       1           1 : // Distributed under the MIT License.
       2             : // See LICENSE.txt for details.
       3             : 
       4             : /// \file
       5             : /// The parallel component and actions in this file keep track of all element
       6             : /// IDs during AMR and update the array sections for the multigrid hierarchy.
       7             : 
       8             : #pragma once
       9             : 
      10             : #include <algorithm>
      11             : #include <charm++.h>
      12             : #include <cstddef>
      13             : #include <optional>
      14             : #include <unordered_map>
      15             : #include <unordered_set>
      16             : #include <vector>
      17             : 
      18             : #include "Domain/Structure/ElementId.hpp"
      19             : #include "Parallel/AlgorithmExecution.hpp"
      20             : #include "Parallel/GlobalCache.hpp"
      21             : #include "Parallel/Local.hpp"
      22             : #include "Parallel/Printf/Printf.hpp"
      23             : #include "Parallel/Protocols/ElementRegistrar.hpp"
      24             : #include "Parallel/Section.hpp"
      25             : #include "Parallel/Tags/Section.hpp"
      26             : #include "ParallelAlgorithms/Amr/Protocols/Projector.hpp"
      27             : #include "ParallelAlgorithms/LinearSolver/Multigrid/Tags.hpp"
      28             : #include "Utilities/ProtocolHelpers.hpp"
      29             : #include "Utilities/TMPL.hpp"
      30             : 
      31             : namespace LinearSolver::multigrid {
      32             : 
      33           1 : namespace Tags {
      34             : // All element IDs grouped by multigrid level are stored in this tag on the
      35             : // singleton component below. The element IDs are registered and deregistered
      36             : // during AMR.
      37             : template <size_t Dim>
      38           0 : struct AllElementIds : db::SimpleTag {
      39           0 :   using type = std::unordered_map<size_t, std::unordered_set<ElementId<Dim>>>;
      40             : };
      41             : }  // namespace Tags
      42             : 
      43             : namespace detail {
      44             : 
      45             : // The projector for the sections does nothing, as the sections are updated by a
      46             : // simple action broadcast below.
      47             : template <typename Metavariables>
      48             : struct ProjectMultigridSections : tt::ConformsTo<amr::protocols::Projector> {
      49             :  private:
      50             :   using element_array = typename Metavariables::amr::element_array;
      51             : 
      52             :  public:
      53             :   using argument_tags = tmpl::list<>;
      54             :   using return_tags =
      55             :       tmpl::list<Parallel::Tags::Section<element_array, Tags::MultigridLevel>,
      56             :                  Parallel::Tags::Section<element_array, Tags::IsFinestGrid>>;
      57             : 
      58             :   template <typename AmrData>
      59             :   static void apply(
      60             :       const gsl::not_null<std::optional<
      61             :           Parallel::Section<element_array, Tags::MultigridLevel>>*>
      62             :       /*multigrid_level_section*/,
      63             :       const gsl::not_null<
      64             :           std::optional<Parallel::Section<element_array, Tags::IsFinestGrid>>*>
      65             :       /*finest_grid_section*/,
      66             :       const AmrData& /*amr_data*/) {}
      67             : };
      68             : 
      69             : template <size_t Dim>
      70             : struct InitializeElementsRegistration {
      71             :   using simple_tags = tmpl::list<Tags::AllElementIds<Dim>>;
      72             :   using compute_tags = tmpl::list<>;
      73             : 
      74             :   template <typename DbTagsList, typename... InboxTags, typename ArrayIndex,
      75             :             typename Metavariables, typename ActionList,
      76             :             typename ParallelComponent>
      77             :   static Parallel::iterable_action_return_t apply(
      78             :       db::DataBox<DbTagsList>& /*box*/,
      79             :       const tuples::TaggedTuple<InboxTags...>& /*inboxes*/,
      80             :       const Parallel::GlobalCache<Metavariables>& /*cache*/,
      81             :       const ArrayIndex& /*array_index*/, const ActionList /*meta*/,
      82             :       const ParallelComponent* const /*meta*/) {
      83             :     return {Parallel::AlgorithmExecution::Pause, std::nullopt};
      84             :   }
      85             : };
      86             : 
      87             : struct RegisterOrDeregisterElement {
      88             :   template <typename ParallelComponent, typename DbTagsList,
      89             :             typename Metavariables, typename ArrayIndex, size_t Dim>
      90             :   static void apply(db::DataBox<DbTagsList>& box,
      91             :                     const Parallel::GlobalCache<Metavariables>& /*cache*/,
      92             :                     const ArrayIndex& /*array_index*/,
      93             :                     const ElementId<Dim>& element_id,
      94             :                     const bool register_or_deregister) {
      95             :     db::mutate<Tags::AllElementIds<Dim>>(
      96             :         [&element_id, register_or_deregister](const auto all_element_ids) {
      97             :           auto& element_ids = (*all_element_ids)[element_id.grid_index()];
      98             :           if (register_or_deregister) {
      99             :             element_ids.insert(element_id);
     100             :           } else {
     101             :             element_ids.erase(element_id);
     102             :           }
     103             :         },
     104             :         make_not_null(&box));
     105             :   }
     106             : };
     107             : 
     108             : struct UpdateSectionsOnElement {
     109             :   template <typename ParallelComponent, typename DbTagsList,
     110             :             typename Metavariables, size_t Dim>
     111             :   static void apply(
     112             :       db::DataBox<DbTagsList>& box,
     113             :       const Parallel::GlobalCache<Metavariables>& /*cache*/,
     114             :       const ElementId<Dim>& element_id,
     115             :       Parallel::Section<ParallelComponent, Tags::MultigridLevel>
     116             :           multigrid_level_section,
     117             :       std::optional<Parallel::Section<ParallelComponent, Tags::IsFinestGrid>>
     118             :           finest_grid_section) {
     119             :     if (multigrid_level_section.id() != element_id.grid_index()) {
     120             :       // Discard broadcast to elements that are not part of the section. This
     121             :       // happens because we broadcast to all elements, not just to the section.
     122             :       // Broadcasting to the section fails with a segfault for some reason.
     123             :       return;
     124             :     }
     125             :     db::mutate<Parallel::Tags::Section<ParallelComponent, Tags::MultigridLevel>,
     126             :                Parallel::Tags::Section<ParallelComponent, Tags::IsFinestGrid>>(
     127             :         [&multigrid_level_section, &finest_grid_section](
     128             :             const auto stored_multigrid_level_section,
     129             :             const auto stored_finest_grid_section) {
     130             :           *stored_multigrid_level_section = std::move(multigrid_level_section);
     131             :           *stored_finest_grid_section = std::move(finest_grid_section);
     132             :         },
     133             :         make_not_null(&box));
     134             :   }
     135             : };
     136             : 
     137             : template <size_t Dim, typename ElementArray, typename OptionsGroup>
     138             : struct UpdateSections {
     139             :   template <typename DbTagList, typename... InboxTags, typename Metavariables,
     140             :             typename ArrayIndex, typename ActionList,
     141             :             typename ParallelComponent>
     142             :   static Parallel::iterable_action_return_t apply(
     143             :       db::DataBox<DbTagList>& box,
     144             :       const tuples::TaggedTuple<InboxTags...>& /*inboxes*/,
     145             :       Parallel::GlobalCache<Metavariables>& cache,
     146             :       const ArrayIndex& /*array_index*/, const ActionList /*meta*/,
     147             :       const ParallelComponent* const /*meta*/) {
     148             :     const auto& all_element_ids = db::get<Tags::AllElementIds<Dim>>(box);
     149             :     auto& element_array = Parallel::get_parallel_component<ElementArray>(cache);
     150             :     for (const auto& [multigrid_level, element_ids] : all_element_ids) {
     151             :       Parallel::printf("%s level %zu has %zu elements.\n",
     152             :                        pretty_type::name<OptionsGroup>(), multigrid_level,
     153             :                        element_ids.size());
     154             :       std::vector<CkArrayIndex> array_indices(element_ids.size());
     155             :       std::transform(
     156             :           element_ids.begin(), element_ids.end(), array_indices.begin(),
     157             :           [](const ElementId<Dim>& local_element_id) {
     158             :             return Parallel::ArrayIndex<ElementId<Dim>>(local_element_id);
     159             :           });
     160             :       using MultigridLevelSection =
     161             :           Parallel::Section<ElementArray, Tags::MultigridLevel>;
     162             :       const MultigridLevelSection multigrid_level_section{
     163             :           multigrid_level, MultigridLevelSection::cproxy_section::ckNew(
     164             :                                element_array.ckGetArrayID(),
     165             :                                array_indices.data(), array_indices.size())};
     166             :       using FinestGridSection =
     167             :           Parallel::Section<ElementArray, Tags::IsFinestGrid>;
     168             :       const std::optional<FinestGridSection> finest_grid_section =
     169             :           multigrid_level == 0
     170             :               ? std::make_optional(FinestGridSection{
     171             :                     true, FinestGridSection::cproxy_section::ckNew(
     172             :                               element_array.ckGetArrayID(),
     173             :                               array_indices.data(), array_indices.size())})
     174             :               : std::nullopt;
     175             :       // Send new sections to all elements. Broadcasting to the section fails
     176             :       // with a segfault for some reason.
     177             :       Parallel::simple_action<UpdateSectionsOnElement>(
     178             :           element_array, multigrid_level_section, finest_grid_section);
     179             :     }
     180             :     return {Parallel::AlgorithmExecution::Pause, std::nullopt};
     181             :   }
     182             : };
     183             : 
     184             : template <typename Metavariables, typename OptionsGroup>
     185             : struct ElementsRegistrationComponent {
     186             :   using chare_type = Parallel::Algorithms::Singleton;
     187             :   using const_global_cache_tags = tmpl::list<>;
     188             :   using metavariables = Metavariables;
     189             :   static constexpr size_t Dim = metavariables::volume_dim;
     190             :   using phase_dependent_action_list = tmpl::list<
     191             :       Parallel::PhaseActions<Parallel::Phase::Initialization,
     192             :                              tmpl::list<InitializeElementsRegistration<Dim>>>,
     193             :       Parallel::PhaseActions<
     194             :           // Update sections in the `CheckDomain` phase, which runs after the
     195             :           // AMR phase.
     196             :           Parallel::Phase::CheckDomain,
     197             :           tmpl::list<UpdateSections<
     198             :               Dim, typename metavariables::amr::element_array, OptionsGroup>>>>;
     199             :   using simple_tags_from_options = Parallel::get_simple_tags_from_options<
     200             :       Parallel::get_initialization_actions_list<phase_dependent_action_list>>;
     201             : 
     202             :   static void execute_next_phase(
     203             :       const Parallel::Phase next_phase,
     204             :       Parallel::CProxy_GlobalCache<Metavariables>& global_cache) {
     205             :     auto& local_cache = *Parallel::local_branch(global_cache);
     206             :     Parallel::get_parallel_component<ElementsRegistrationComponent>(local_cache)
     207             :         .start_phase(next_phase);
     208             :   }
     209             : };
     210             : 
     211             : template <size_t Dim, typename OptionsGroup>
     212             : struct RegisterElement : tt::ConformsTo<Parallel::protocols::ElementRegistrar> {
     213             :  public:  // ElementRegistrar protocol
     214             :   template <typename ParallelComponent, typename DbTagList,
     215             :             typename Metavariables>
     216             :   static void perform_registration(const db::DataBox<DbTagList>& /*box*/,
     217             :                                    Parallel::GlobalCache<Metavariables>& cache,
     218             :                                    const ElementId<Dim>& element_id) {
     219             :     Parallel::simple_action<RegisterOrDeregisterElement>(
     220             :         Parallel::get_parallel_component<
     221             :             ElementsRegistrationComponent<Metavariables, OptionsGroup>>(cache),
     222             :         element_id, true);
     223             :   }
     224             : 
     225             :   template <typename ParallelComponent, typename DbTagList,
     226             :             typename Metavariables>
     227             :   static void perform_deregistration(
     228             :       const db::DataBox<DbTagList>& /*box*/,
     229             :       Parallel::GlobalCache<Metavariables>& cache,
     230             :       const ElementId<Dim>& element_id) {
     231             :     Parallel::simple_action<RegisterOrDeregisterElement>(
     232             :         Parallel::get_parallel_component<
     233             :             ElementsRegistrationComponent<Metavariables, OptionsGroup>>(cache),
     234             :         element_id, false);
     235             :   }
     236             : 
     237             :  public:  // Iterable action
     238             :   template <typename DbTagList, typename... InboxTags, typename Metavariables,
     239             :             typename ActionList, typename ParallelComponent>
     240             :   static Parallel::iterable_action_return_t apply(
     241             :       db::DataBox<DbTagList>& box,
     242             :       const tuples::TaggedTuple<InboxTags...>& /*inboxes*/,
     243             :       Parallel::GlobalCache<Metavariables>& cache,
     244             :       const ElementId<Dim>& element_id, const ActionList /*meta*/,
     245             :       const ParallelComponent* const /*meta*/) {
     246             :     perform_registration<ParallelComponent>(box, cache, element_id);
     247             :     return {Parallel::AlgorithmExecution::Continue, std::nullopt};
     248             :   }
     249             : };
     250             : 
     251             : }  // namespace detail
     252             : }  // namespace LinearSolver::multigrid