Line data    Source code

       1           0 : // Distributed under the MIT License.
       2             : // See LICENSE.txt for details.
       3             : 
       4             : #pragma once
       5             : 
       6             : #include <array>
       7             : #include <charm++.h>
       8             : #include <cstddef>
       9             : #include <optional>
      10             : #include <utility>
      11             : #include <vector>
      12             : 
      13             : #include "DataStructures/DataBox/DataBox.hpp"
      14             : #include "Evolution/DiscontinuousGalerkin/EqualRateLts/Tags/EqualRateRegionId.hpp"
      15             : #include "Evolution/DiscontinuousGalerkin/EqualRateLts/Tags/EqualRateRegions.hpp"
      16             : #include "Parallel/AlgorithmExecution.hpp"
      17             : #include "Parallel/GlobalCache.hpp"
      18             : #include "Parallel/Invoke.hpp"
      19             : #include "Parallel/Section.hpp"
      20             : #include "Parallel/Tags/Section.hpp"
      21             : #include "Parallel/TypeTraits.hpp"
      22             : #include "Time/LtsMode.hpp"
      23             : #include "Time/Tags/FixedLtsRatio.hpp"
      24             : #include "Utilities/ErrorHandling/Assert.hpp"
      25             : #include "Utilities/Gsl.hpp"
      26             : #include "Utilities/TMPL.hpp"
      27             : 
      28             : /// \cond
      29             : template <size_t Dim, typename T>
      30             : class DirectionalIdMap;
      31             : template <size_t VolumeDim>
      32             : class Element;
      33             : class TimeDelta;
      34             : namespace Tags {
      35             : struct LtsMode;
      36             : struct TimeStep;
      37             : }  // namespace Tags
      38             : namespace domain::Tags {
      39             : template <size_t VolumeDim>
      40             : struct Element;
      41             : template <size_t Dim>
      42             : struct InitialRefinementLevels;
      43             : }  // namespace domain::Tags
      44             : namespace evolution::dg {
      45             : template <size_t Dim>
      46             : class EqualRateRegionsBase;
      47             : template <size_t VolumeDim>
      48             : class MortarInfo;
      49             : }  // namespace evolution::dg
      50             : namespace evolution::dg::Tags {
      51             : template <size_t Dim>
      52             : struct MortarInfo;
      53             : }  // namespace evolution::dg::Tags
      54             : namespace tuples {
      55             : template <class... Tags>
      56             : class TaggedTuple;
      57             : }  // namespace tuples
      58             : /// \endcond
      59             : 
      60             : namespace evolution::dg::Initialization {
      61             : /// Set up `FixedLtsRatio` and mortar time-stepping policies to
      62             : /// disable local time-stepping in equal rate regions.
      63             : template <size_t Dim>
      64           1 : struct SetupLocalEqualRateRegion {
      65           0 :   using return_tags =
      66             :       tmpl::list<::Tags::FixedLtsRatio, evolution::dg::Tags::MortarInfo<Dim>>;
      67           0 :   using argument_tags =
      68             :       tmpl::list<domain::Tags::Element<Dim>,
      69             :                  evolution::dg::Tags::EqualRateRegions<Dim>, ::Tags::TimeStep>;
      70             : 
      71           0 :   static void apply(
      72             :       gsl::not_null<std::optional<size_t>*> fixed_lts_ratio,
      73             :       gsl::not_null<DirectionalIdMap<Dim, ::evolution::dg::MortarInfo<Dim>>*>
      74             :           mortar_infos,
      75             :       const Element<Dim>& element,
      76             :       const EqualRateRegionsBase<Dim>& equal_rate_regions,
      77             :       const TimeDelta& time_step);
      78             : };
      79             : 
      80           0 : namespace Actions {
      81             : template <size_t Dim>
      82           0 : struct SetEqualRateSection {
      83             :   template <typename ParallelComponent, typename DbTags, typename Metavariables,
      84             :             typename ArrayIndex>
      85           0 :   static void apply(
      86             :       db::DataBox<DbTags>& box, Parallel::GlobalCache<Metavariables>& /*cache*/,
      87             :       const ArrayIndex& /*array_index*/,
      88             :       Parallel::Section<ParallelComponent, Tags::EqualRateRegionId> section) {
      89             :     const auto& equal_rate_regions =
      90             :         db::get<evolution::dg::Tags::EqualRateRegions<Dim>>(box);
      91             :     const auto& element = db::get<domain::Tags::Element<Dim>>(box);
      92             :     if (equal_rate_regions.is_in_region(section.id(), element.id())) {
      93             :       db::mutate<
      94             :           Parallel::Tags::Section<ParallelComponent, Tags::EqualRateRegionId>>(
      95             :           [&](const gsl::not_null<std::optional<Parallel::Section<
      96             :                   ParallelComponent, Tags::EqualRateRegionId>>*>
      97             :                   box_section) {
      98             :             ASSERT(not box_section->has_value(),
      99             :                    "Already have an equal-rate section");
     100             :             box_section->emplace(std::move(section));
     101             :           },
     102             :           make_not_null(&box));
     103             :     }
     104             :   }
     105             : };
     106             : 
     107             : namespace SetupEqualRateRegions_detail {
     108             : template <size_t Dim>
     109             : struct SectionsToCreate {
     110             :   using argument_tags = tmpl::list<domain::Tags::Element<Dim>,
     111             :                                    evolution::dg::Tags::EqualRateRegions<Dim>,
     112             :                                    domain::Tags::InitialRefinementLevels<Dim>>;
     113             : 
     114             :   static std::vector<std::pair<EqualRateRegionId, std::vector<CkArrayIndex>>>
     115             :   apply(const Element<Dim>& element,
     116             :         const EqualRateRegionsBase<Dim>& equal_rate_regions,
     117             :         const std::vector<std::array<size_t, Dim>>& initial_refinement_levels);
     118             : };
     119             : }  // namespace SetupEqualRateRegions_detail
     120             : 
     121             : /// Set up `FixedLtsRatio` and mortar time-stepping policies to
     122             : /// disable local time-stepping in equal rate regions.
     123             : template <typename Metavariables, size_t Dim, typename RegionGenerators>
     124           1 : struct SetupEqualRateRegions {
     125             :  private:
     126           0 :   using array_components = tmpl::filter<typename Metavariables::component_list,
     127             :                                         Parallel::is_array<tmpl::_1>>;
     128             :   static_assert(tmpl::size<array_components>::value == 1,
     129             :                 "Cannot identify element array.");
     130             : 
     131             :  public:
     132           0 :   using ParallelComponent = tmpl::front<array_components>;
     133             : 
     134           0 :   using const_global_cache_tags = tmpl::list<
     135             :       evolution::dg::Tags::ConcreteEqualRateRegions<Dim, RegionGenerators>>;
     136           0 :   using simple_tags = tmpl::list<
     137             :       ::Tags::FixedLtsRatio,
     138             :       Parallel::Tags::Section<ParallelComponent, Tags::EqualRateRegionId>>;
     139           0 :   using compute_tags = tmpl::list<
     140             :       evolution::dg::Tags::EqualRateRegionsRef<Dim, RegionGenerators>>;
     141             : 
     142             :   template <typename DbTags, typename... InboxTags, typename ArrayIndex,
     143             :             typename ActionList>
     144           0 :   static Parallel::iterable_action_return_t apply(
     145             :       db::DataBox<DbTags>& box,
     146             :       const tuples::TaggedTuple<InboxTags...>& /*inboxes*/,
     147             :       Parallel::GlobalCache<Metavariables>& cache,
     148             :       const ArrayIndex& /*array_index*/, const ActionList /*meta*/,
     149             :       const ParallelComponent* const /*meta*/) {
     150             :     if (db::get<::Tags::LtsMode>(box) == LtsMode::Off) {
     151             :       return {Parallel::AlgorithmExecution::Continue, std::nullopt};
     152             :     }
     153             : 
     154             :     db::mutate_apply<SetupLocalEqualRateRegion<Dim>>(make_not_null(&box));
     155             : 
     156             :     // Set up sections.  This will be empty everywhere except element 0.
     157             :     const auto sections_to_create =
     158             :         db::apply<SetupEqualRateRegions_detail::SectionsToCreate<Dim>>(box);
     159             :     for (const auto& [region_id, section_elements] : sections_to_create) {
     160             :       auto& component =
     161             :           Parallel::get_parallel_component<ParallelComponent>(cache);
     162             :       using Section =
     163             :           Parallel::Section<ParallelComponent, Tags::EqualRateRegionId>;
     164             :       const Section section{
     165             :           region_id, Section::cproxy_section::ckNew(component.ckGetArrayID(),
     166             :                                                     section_elements)};
     167             : 
     168             :       // Charm sections are buggy.  Most of their features only work
     169             :       // when manually sending charm messages.  The higher-level
     170             :       // interfaces exist, but fail to actually transmit data to the
     171             :       // receiving end and pass in garbage instead.  This is mentioned
     172             :       // in an offhand remark in a section of the charm documentation
     173             :       // on "optimized multicast".  As a result we avoid doing
     174             :       // anything other than reductions (which seem to work) on
     175             :       // sections.  Here we use an array broadcast and ignore the
     176             :       // message on most of the elements because section broadcasts
     177             :       // don't work.
     178             :       Parallel::simple_action<SetEqualRateSection<Dim>>(component, section);
     179             :     }
     180             : 
     181             :     return {Parallel::AlgorithmExecution::Continue, std::nullopt};
     182             :   }
     183             : };
     184             : }  // namespace Actions
     185             : }  // namespace evolution::dg::Initialization