Line data Source code
1 0 : // Distributed under the MIT License.
2 : // See LICENSE.txt for details.
3 :
4 : #pragma once
5 :
6 : #include "DataStructures/DataBox/DataBox.hpp"
7 : #include "Domain/Creators/Tags/Domain.hpp"
8 : #include "Domain/Domain.hpp"
9 : #include "Parallel/ArrayComponentId.hpp"
10 : #include "Parallel/GlobalCache.hpp"
11 : #include "Parallel/Invoke.hpp"
12 : #include "Parallel/ParallelComponentHelpers.hpp"
13 : #include "ParallelAlgorithms/Interpolation/Actions/SendPointsToInterpolator.hpp"
14 : #include "ParallelAlgorithms/Interpolation/InterpolationTargetDetail.hpp"
15 : #include "ParallelAlgorithms/Interpolation/Tags.hpp"
16 : #include "Utilities/Algorithm.hpp"
17 : #include "Utilities/Gsl.hpp"
18 :
19 : namespace intrp::Actions {
20 :
21 : template <typename InterpolationTargetTag>
22 : struct VerifyTemporalIdsAndSendPoints;
23 :
24 : namespace detail {
25 : template <typename InterpolationTargetTag, typename ParallelComponent,
26 : typename DbTags, typename Metavariables>
27 : void verify_temporal_ids_and_send_points_time_independent(
28 : const gsl::not_null<db::DataBox<DbTags>*> box,
29 : Parallel::GlobalCache<Metavariables>& cache) {
30 : using TemporalId = typename InterpolationTargetTag::temporal_id::type;
31 :
32 : // Move all PendingTemporalIds to TemporalIds, provided
33 : // that they are not already there, and fill new_temporal_ids
34 : // with the temporal_ids that were so moved.
35 : std::vector<TemporalId> new_temporal_ids{};
36 : db::mutate_apply<tmpl::list<Tags::TemporalIds<TemporalId>,
37 : Tags::PendingTemporalIds<TemporalId>>,
38 : tmpl::list<Tags::CompletedTemporalIds<TemporalId>>>(
39 : [&new_temporal_ids](
40 : const gsl::not_null<std::deque<TemporalId>*> ids,
41 : const gsl::not_null<std::deque<TemporalId>*> pending_ids,
42 : const std::deque<TemporalId>& completed_ids) {
43 : // This sort is needed because the ordering of these ids and pending ids
44 : // are not guaranteed. They aren't guaranteed because the elements must
45 : // send a communication to the this target with the the temporal id. So
46 : // it is possible that there are two interpolations that need to happen,
47 : // the earlier temporal id is sent first and the later temporal id is
48 : // sent second. But the later temporal id could arrive to this target
49 : // first because of charm communication latency. If this was it, then
50 : // there's nothing we could do and the later interpolation would happen
51 : // before the earlier one (even for sequential targets). However, there
52 : // is a scenario where this happens, except there is an interpolation in
53 : // progress so it doesn't send points to the interpolator when it
54 : // receives the later time first. In that case, we then receive the
55 : // earlier time second, and sort them here so they are in temporal
56 : // order. Note that this isn't a permanent actual fix, but so far works
57 : // in practice.
58 : alg::sort(*ids);
59 : alg::sort(*pending_ids);
60 : for (auto& id : *pending_ids) {
61 : if (std::find(completed_ids.begin(), completed_ids.end(), id) ==
62 : completed_ids.end() and
63 : std::find(ids->begin(), ids->end(), id) == ids->end()) {
64 : ids->push_back(id);
65 : new_temporal_ids.push_back(id);
66 : }
67 : }
68 : pending_ids->clear();
69 : },
70 : box);
71 : if (InterpolationTargetTag::compute_target_points::is_sequential::value) {
72 : // Sequential: start interpolation only for the first new_temporal_id.
73 : if (not new_temporal_ids.empty()) {
74 : auto& my_proxy =
75 : Parallel::get_parallel_component<ParallelComponent>(cache);
76 : Parallel::simple_action<
77 : Actions::SendPointsToInterpolator<InterpolationTargetTag>>(
78 : my_proxy, new_temporal_ids.front());
79 : }
80 : } else {
81 : // Non-sequential: start interpolation for all new_temporal_ids.
82 : auto& my_proxy = Parallel::get_parallel_component<ParallelComponent>(cache);
83 : for (const auto& id : new_temporal_ids) {
84 : Parallel::simple_action<
85 : Actions::SendPointsToInterpolator<InterpolationTargetTag>>(my_proxy,
86 : id);
87 : }
88 : }
89 : }
90 :
91 : template <typename InterpolationTargetTag, typename ParallelComponent,
92 : typename DbTags, typename Metavariables, typename ArrayIndex>
93 : void verify_temporal_ids_and_send_points_time_dependent(
94 : const gsl::not_null<db::DataBox<DbTags>*> box,
95 : Parallel::GlobalCache<Metavariables>& cache,
96 : const ArrayIndex& array_index) {
97 : using TemporalId = typename InterpolationTargetTag::temporal_id::type;
98 :
99 : const auto& pending_temporal_ids =
100 : db::get<Tags::PendingTemporalIds<TemporalId>>(*box);
101 : if (pending_temporal_ids.empty()) {
102 : return; // Nothing to do if there are no pending temporal_ids.
103 : }
104 :
105 : auto& this_proxy = Parallel::get_parallel_component<ParallelComponent>(cache);
106 : double min_expiration_time = std::numeric_limits<double>::max();
107 : const Parallel::ArrayComponentId array_component_id =
108 : Parallel::make_array_component_id<ParallelComponent>(array_index);
109 : const bool at_least_one_pending_temporal_id_is_ready =
110 : [&cache, &array_component_id, &this_proxy, &pending_temporal_ids,
111 : &min_expiration_time]() {
112 : if constexpr (Parallel::is_in_mutable_global_cache<
113 : Metavariables, domain::Tags::FunctionsOfTime>) {
114 : return ::Parallel::mutable_cache_item_is_ready<
115 : domain::Tags::FunctionsOfTime>(
116 : cache, array_component_id,
117 : [&this_proxy, &pending_temporal_ids, &min_expiration_time](
118 : const std::unordered_map<
119 : std::string,
120 : std::unique_ptr<domain::FunctionsOfTime::FunctionOfTime>>&
121 : functions_of_time)
122 : -> std::unique_ptr<Parallel::Callback> {
123 : min_expiration_time =
124 : std::min_element(functions_of_time.begin(),
125 : functions_of_time.end(),
126 : [](const auto& a, const auto& b) {
127 : return a.second->time_bounds()[1] <
128 : b.second->time_bounds()[1];
129 : })
130 : ->second->time_bounds()[1];
131 : for (const auto& pending_id : pending_temporal_ids) {
132 : if (InterpolationTarget_detail::get_temporal_id_value(
133 : pending_id) <= min_expiration_time) {
134 : // Success: at least one pending_temporal_id is ok.
135 : return std::unique_ptr<Parallel::Callback>{};
136 : }
137 : }
138 : // Failure: none of the pending_temporal_ids are ok.
139 : // Even though the GlobalCache docs say to only return a
140 : // PerformAlgorithmCallback, we return a SimpleActionCallback
141 : // here because it was already like this, and the effort to
142 : // change it is too great right now. This is alright because
143 : // this code should never be executed because the functions of
144 : // time should always be valid for times sent to the
145 : // interpolation target
146 : return std::unique_ptr<Parallel::Callback>(
147 : new Parallel::SimpleActionCallback<
148 : VerifyTemporalIdsAndSendPoints<InterpolationTargetTag>,
149 : decltype(this_proxy)>(this_proxy));
150 : });
151 : } else {
152 : (void)cache;
153 : (void)array_component_id;
154 : (void)this_proxy;
155 : (void)pending_temporal_ids;
156 : (void)min_expiration_time;
157 : return true;
158 : }
159 : }();
160 :
161 : if (not at_least_one_pending_temporal_id_is_ready) {
162 : // A callback has been set so that VerifyTemporalIdsAndSendPoints will
163 : // be called by the GlobalCache when domain::Tags::FunctionsOfTime is
164 : // updated. So we can exit now.
165 : return;
166 : }
167 :
168 : // Move up-to-date PendingTemporalIds to TemporalIds, provided
169 : // that they are not already there, and fill new_temporal_ids
170 : // with the temporal_ids that were so moved.
171 : std::vector<TemporalId> new_temporal_ids{};
172 : db::mutate_apply<tmpl::list<Tags::TemporalIds<TemporalId>,
173 : Tags::PendingTemporalIds<TemporalId>>,
174 : tmpl::list<Tags::CompletedTemporalIds<TemporalId>>>(
175 : [&min_expiration_time, &new_temporal_ids](
176 : const gsl::not_null<std::deque<TemporalId>*> ids,
177 : const gsl::not_null<std::deque<TemporalId>*> pending_ids,
178 : const std::deque<TemporalId>& completed_ids) {
179 : // This sort is needed because the ordering of these ids and pending ids
180 : // are not guaranteed. They aren't guaranteed because the elements must
181 : // send a communication to the this target with the the temporal id. So
182 : // it is possible that there are two interpolations that need to happen,
183 : // the earlier temporal id is sent first and the later temporal id is
184 : // sent second. But the later temporal id could arrive to this target
185 : // first because of charm communication latency. If this was it, then
186 : // there's nothing we could do and the later interpolation would happen
187 : // before the earlier one (even for sequential targets). However, there
188 : // is a scenario where this happens, except there is an interpolation in
189 : // progress so it doesn't send points to the interpolator when it
190 : // receives the later time first. In that case, we then receive the
191 : // earlier time second, and sort them here so they are in temporal
192 : // order. Note that this isn't a permanent actual fix, but so far works
193 : // in practice.
194 : alg::sort(*ids);
195 : alg::sort(*pending_ids);
196 : for (auto it = pending_ids->begin(); it != pending_ids->end();) {
197 : if (InterpolationTarget_detail::get_temporal_id_value(*it) <=
198 : min_expiration_time and
199 : std::find(completed_ids.begin(), completed_ids.end(), *it) ==
200 : completed_ids.end() and
201 : std::find(ids->begin(), ids->end(), *it) == ids->end()) {
202 : ids->push_back(*it);
203 : new_temporal_ids.push_back(*it);
204 : it = pending_ids->erase(it);
205 : } else {
206 : ++it;
207 : }
208 : }
209 : },
210 : box);
211 :
212 : if (InterpolationTargetTag::compute_target_points::is_sequential::value) {
213 : // Sequential: start interpolation only for the first new_temporal_id.
214 : if (not new_temporal_ids.empty()) {
215 : auto& my_proxy =
216 : Parallel::get_parallel_component<ParallelComponent>(cache);
217 : Parallel::simple_action<
218 : Actions::SendPointsToInterpolator<InterpolationTargetTag>>(
219 : my_proxy, new_temporal_ids.front());
220 : }
221 : } else {
222 : // Non-sequential: start interpolation for all new_temporal_ids.
223 : auto& my_proxy = Parallel::get_parallel_component<ParallelComponent>(cache);
224 : for (const auto& id : new_temporal_ids) {
225 : Parallel::simple_action<
226 : Actions::SendPointsToInterpolator<InterpolationTargetTag>>(my_proxy,
227 : id);
228 : }
229 : // If there are still pending temporal_ids, call
230 : // VerifyTemporalIdsAndSendPoints again, so that those pending
231 : // temporal_ids can be waited for.
232 : if (not db::get<Tags::PendingTemporalIds<TemporalId>>(*box).empty()) {
233 : Parallel::simple_action<
234 : VerifyTemporalIdsAndSendPoints<InterpolationTargetTag>>(my_proxy);
235 : }
236 : }
237 : }
238 : } // namespace detail
239 :
240 : /// \ingroup ActionsGroup
241 : /// \brief Sends points to an Interpolator for verified temporal_ids.
242 : ///
243 : /// VerifyTemporalIdsAndSendPoints is invoked on an InterpolationTarget.
244 : ///
245 : /// In more detail, does the following:
246 : /// - If any map is time-dependent:
247 : /// - Moves verified PendingTemporalIds to TemporalIds, where
248 : /// verified means that the FunctionsOfTime in the GlobalCache
249 : /// are up-to-date for that TemporalId. If no PendingTemporalIds are
250 : /// moved, then VerifyTemporalIdsAndSendPoints sets itself as a
251 : /// callback in the GlobalCache so that it is called again when the
252 : /// FunctionsOfTime are mutated.
253 : /// - If the InterpolationTarget is sequential, invokes
254 : /// intrp::Actions::SendPointsToInterpolator for the first TemporalId.
255 : /// (when interpolation is complete,
256 : /// intrp::Actions::InterpolationTargetReceiveVars will begin interpolation
257 : /// on the next TemporalId)
258 : /// - If the InterpolationTarget is not sequential, invokes
259 : /// intrp::Actions::SendPointsToInterpolator for all valid TemporalIds,
260 : /// and then if PendingTemporalIds is non-empty it invokes itself.
261 : ///
262 : /// - If all maps are time-independent:
263 : /// - Moves all PendingTemporalIds to TemporalIds
264 : /// - If the InterpolationTarget is sequential, invokes
265 : /// intrp::Actions::SendPointsToInterpolator for the first TemporalId.
266 : /// (when interpolation is complete,
267 : /// intrp::Actions::InterpolationTargetReceiveVars will begin interpolation
268 : /// on the next TemporalId)
269 : /// - If the InterpolationTarget is not sequential, invokes
270 : /// intrp::Actions::SendPointsToInterpolator for all TemporalIds.
271 : ///
272 : /// Uses:
273 : /// - DataBox:
274 : /// - `intrp::Tags::PendingTeporalIds`
275 : /// - `intrp::Tags::TeporalIds`
276 : ///
277 : /// DataBox changes:
278 : /// - Adds: nothing
279 : /// - Removes: nothing
280 : /// - Modifies:
281 : /// - `intrp::Tags::PendingTeporalIds`
282 : /// - `intrp::Tags::TeporalIds`
283 : ///
284 : /// For requirements on InterpolationTargetTag, see InterpolationTarget
285 : template <typename InterpolationTargetTag>
286 1 : struct VerifyTemporalIdsAndSendPoints {
287 : template <typename ParallelComponent, typename DbTags, typename Metavariables,
288 : typename ArrayIndex>
289 0 : static void apply(db::DataBox<DbTags>& box,
290 : Parallel::GlobalCache<Metavariables>& cache,
291 : const ArrayIndex& array_index) {
292 : if constexpr (std::is_same_v<typename InterpolationTargetTag::
293 : compute_target_points::frame,
294 : ::Frame::Grid>) {
295 : detail::verify_temporal_ids_and_send_points_time_independent<
296 : InterpolationTargetTag, ParallelComponent>(make_not_null(&box),
297 : cache);
298 : } else {
299 : const auto& domain =
300 : get<domain::Tags::Domain<Metavariables::volume_dim>>(cache);
301 : if (domain.is_time_dependent()) {
302 : if constexpr (Parallel::is_in_global_cache<
303 : Metavariables, domain::Tags::FunctionsOfTime>) {
304 : detail::verify_temporal_ids_and_send_points_time_dependent<
305 : InterpolationTargetTag, ParallelComponent>(make_not_null(&box),
306 : cache, array_index);
307 : } else {
308 : // We error here because the maps are time-dependent, yet
309 : // the cache does not contain FunctionsOfTime. It would be
310 : // nice to make this a compile-time error; however, we want
311 : // the code to compile for the completely time-independent
312 : // case where there are no FunctionsOfTime in the cache at
313 : // all. Unfortunately, checking whether the maps are
314 : // time-dependent is currently not constexpr.
315 : ERROR(
316 : "There is a time-dependent CoordinateMap in at least one "
317 : "of the Blocks, but FunctionsOfTime are not in the "
318 : "GlobalCache. If you intend to use a time-dependent "
319 : "CoordinateMap, please add FunctionsOfTime to the GlobalCache.");
320 : }
321 : } else {
322 : detail::verify_temporal_ids_and_send_points_time_independent<
323 : InterpolationTargetTag, ParallelComponent>(make_not_null(&box),
324 : cache);
325 : }
326 : }
327 : }
328 : };
329 : } // namespace intrp::Actions
|
