Line data Source code
1 0 : // Distributed under the MIT License.
2 : // See LICENSE.txt for details.
3 :
4 : #pragma once
5 :
6 : #include <array>
7 : #include <charm++.h>
8 : #include <cstddef>
9 : #include <optional>
10 : #include <utility>
11 : #include <vector>
12 :
13 : #include "DataStructures/DataBox/DataBox.hpp"
14 : #include "Evolution/DiscontinuousGalerkin/EqualRateLts/Tags/EqualRateRegionId.hpp"
15 : #include "Evolution/DiscontinuousGalerkin/EqualRateLts/Tags/EqualRateRegions.hpp"
16 : #include "Parallel/AlgorithmExecution.hpp"
17 : #include "Parallel/GlobalCache.hpp"
18 : #include "Parallel/Invoke.hpp"
19 : #include "Parallel/Section.hpp"
20 : #include "Parallel/Tags/Section.hpp"
21 : #include "Parallel/TypeTraits.hpp"
22 : #include "Time/LtsMode.hpp"
23 : #include "Time/Tags/FixedLtsRatio.hpp"
24 : #include "Utilities/ErrorHandling/Assert.hpp"
25 : #include "Utilities/Gsl.hpp"
26 : #include "Utilities/TMPL.hpp"
27 :
28 : /// \cond
29 : template <size_t Dim, typename T>
30 : class DirectionalIdMap;
31 : template <size_t VolumeDim>
32 : class Element;
33 : class TimeDelta;
34 : namespace Tags {
35 : struct LtsMode;
36 : struct TimeStep;
37 : } // namespace Tags
38 : namespace domain::Tags {
39 : template <size_t VolumeDim>
40 : struct Element;
41 : template <size_t Dim>
42 : struct InitialRefinementLevels;
43 : } // namespace domain::Tags
44 : namespace evolution::dg {
45 : template <size_t Dim>
46 : class EqualRateRegionsBase;
47 : template <size_t VolumeDim>
48 : class MortarInfo;
49 : } // namespace evolution::dg
50 : namespace evolution::dg::Tags {
51 : template <size_t Dim>
52 : struct MortarInfo;
53 : } // namespace evolution::dg::Tags
54 : namespace tuples {
55 : template <class... Tags>
56 : class TaggedTuple;
57 : } // namespace tuples
58 : /// \endcond
59 :
60 : namespace evolution::dg::Initialization {
61 : /// Set up `FixedLtsRatio` and mortar time-stepping policies to
62 : /// disable local time-stepping in equal rate regions.
63 : template <size_t Dim>
64 1 : struct SetupLocalEqualRateRegion {
65 0 : using return_tags =
66 : tmpl::list<::Tags::FixedLtsRatio, evolution::dg::Tags::MortarInfo<Dim>>;
67 0 : using argument_tags =
68 : tmpl::list<domain::Tags::Element<Dim>,
69 : evolution::dg::Tags::EqualRateRegions<Dim>, ::Tags::TimeStep>;
70 :
71 0 : static void apply(
72 : gsl::not_null<std::optional<size_t>*> fixed_lts_ratio,
73 : gsl::not_null<DirectionalIdMap<Dim, ::evolution::dg::MortarInfo<Dim>>*>
74 : mortar_infos,
75 : const Element<Dim>& element,
76 : const EqualRateRegionsBase<Dim>& equal_rate_regions,
77 : const TimeDelta& time_step);
78 : };
79 :
80 0 : namespace Actions {
81 : template <size_t Dim>
82 0 : struct SetEqualRateSection {
83 : template <typename ParallelComponent, typename DbTags, typename Metavariables,
84 : typename ArrayIndex>
85 0 : static void apply(
86 : db::DataBox<DbTags>& box, Parallel::GlobalCache<Metavariables>& /*cache*/,
87 : const ArrayIndex& /*array_index*/,
88 : Parallel::Section<ParallelComponent, Tags::EqualRateRegionId> section) {
89 : const auto& equal_rate_regions =
90 : db::get<evolution::dg::Tags::EqualRateRegions<Dim>>(box);
91 : const auto& element = db::get<domain::Tags::Element<Dim>>(box);
92 : if (equal_rate_regions.is_in_region(section.id(), element.id())) {
93 : db::mutate<
94 : Parallel::Tags::Section<ParallelComponent, Tags::EqualRateRegionId>>(
95 : [&](const gsl::not_null<std::optional<Parallel::Section<
96 : ParallelComponent, Tags::EqualRateRegionId>>*>
97 : box_section) {
98 : ASSERT(not box_section->has_value(),
99 : "Already have an equal-rate section");
100 : box_section->emplace(std::move(section));
101 : },
102 : make_not_null(&box));
103 : }
104 : }
105 : };
106 :
107 : namespace SetupEqualRateRegions_detail {
108 : template <size_t Dim>
109 : struct SectionsToCreate {
110 : using argument_tags = tmpl::list<domain::Tags::Element<Dim>,
111 : evolution::dg::Tags::EqualRateRegions<Dim>,
112 : domain::Tags::InitialRefinementLevels<Dim>>;
113 :
114 : static std::vector<std::pair<EqualRateRegionId, std::vector<CkArrayIndex>>>
115 : apply(const Element<Dim>& element,
116 : const EqualRateRegionsBase<Dim>& equal_rate_regions,
117 : const std::vector<std::array<size_t, Dim>>& initial_refinement_levels);
118 : };
119 : } // namespace SetupEqualRateRegions_detail
120 :
121 : /// Set up `FixedLtsRatio` and mortar time-stepping policies to
122 : /// disable local time-stepping in equal rate regions.
123 : template <typename Metavariables, size_t Dim, typename RegionGenerators>
124 1 : struct SetupEqualRateRegions {
125 : private:
126 0 : using array_components = tmpl::filter<typename Metavariables::component_list,
127 : Parallel::is_array<tmpl::_1>>;
128 : static_assert(tmpl::size<array_components>::value == 1,
129 : "Cannot identify element array.");
130 :
131 : public:
132 0 : using ParallelComponent = tmpl::front<array_components>;
133 :
134 0 : using const_global_cache_tags = tmpl::list<
135 : evolution::dg::Tags::ConcreteEqualRateRegions<Dim, RegionGenerators>>;
136 0 : using simple_tags = tmpl::list<
137 : ::Tags::FixedLtsRatio,
138 : Parallel::Tags::Section<ParallelComponent, Tags::EqualRateRegionId>>;
139 0 : using compute_tags = tmpl::list<
140 : evolution::dg::Tags::EqualRateRegionsRef<Dim, RegionGenerators>>;
141 :
142 : template <typename DbTags, typename... InboxTags, typename ArrayIndex,
143 : typename ActionList>
144 0 : static Parallel::iterable_action_return_t apply(
145 : db::DataBox<DbTags>& box,
146 : const tuples::TaggedTuple<InboxTags...>& /*inboxes*/,
147 : Parallel::GlobalCache<Metavariables>& cache,
148 : const ArrayIndex& /*array_index*/, const ActionList /*meta*/,
149 : const ParallelComponent* const /*meta*/) {
150 : if (db::get<::Tags::LtsMode>(box) == LtsMode::Off) {
151 : return {Parallel::AlgorithmExecution::Continue, std::nullopt};
152 : }
153 :
154 : db::mutate_apply<SetupLocalEqualRateRegion<Dim>>(make_not_null(&box));
155 :
156 : // Set up sections. This will be empty everywhere except element 0.
157 : const auto sections_to_create =
158 : db::apply<SetupEqualRateRegions_detail::SectionsToCreate<Dim>>(box);
159 : for (const auto& [region_id, section_elements] : sections_to_create) {
160 : auto& component =
161 : Parallel::get_parallel_component<ParallelComponent>(cache);
162 : using Section =
163 : Parallel::Section<ParallelComponent, Tags::EqualRateRegionId>;
164 : const Section section{
165 : region_id, Section::cproxy_section::ckNew(component.ckGetArrayID(),
166 : section_elements)};
167 :
168 : // Charm sections are buggy. Most of their features only work
169 : // when manually sending charm messages. The higher-level
170 : // interfaces exist, but fail to actually transmit data to the
171 : // receiving end and pass in garbage instead. This is mentioned
172 : // in an offhand remark in a section of the charm documentation
173 : // on "optimized multicast". As a result we avoid doing
174 : // anything other than reductions (which seem to work) on
175 : // sections. Here we use an array broadcast and ignore the
176 : // message on most of the elements because section broadcasts
177 : // don't work.
178 : Parallel::simple_action<SetEqualRateSection<Dim>>(component, section);
179 : }
180 :
181 : return {Parallel::AlgorithmExecution::Continue, std::nullopt};
182 : }
183 : };
184 : } // namespace Actions
185 : } // namespace evolution::dg::Initialization
|
