Line data Source code
1 0 : // Distributed under the MIT License.
2 : // See LICENSE.txt for details.
3 :
4 : #pragma once
5 :
6 : #include <charm++.h>
7 : #include <converse.h>
8 : #include <cstddef>
9 : #include <exception>
10 : #include <initializer_list>
11 : #include <limits>
12 : #include <mutex>
13 : #include <optional>
14 : #include <ostream>
15 : #include <pup.h>
16 : #include <sstream>
17 : #include <string>
18 : #include <tuple>
19 : #include <unordered_map>
20 : #include <unordered_set>
21 : #include <utility>
22 :
23 : #include "DataStructures/DataBox/DataBox.hpp"
24 : #include "DataStructures/DataBox/PrefixHelpers.hpp"
25 : #include "Parallel/AlgorithmExecution.hpp"
26 : #include "Parallel/AlgorithmMetafunctions.hpp"
27 : #include "Parallel/Algorithms/AlgorithmArrayDeclarations.hpp"
28 : #include "Parallel/Algorithms/AlgorithmGroupDeclarations.hpp"
29 : #include "Parallel/Algorithms/AlgorithmNodegroupDeclarations.hpp"
30 : #include "Parallel/Algorithms/AlgorithmSingletonDeclarations.hpp"
31 : #include "Parallel/Callback.hpp"
32 : #include "Parallel/CharmRegistration.hpp"
33 : #include "Parallel/ElementRegistration.hpp"
34 : #include "Parallel/GlobalCache.hpp"
35 : #include "Parallel/Info.hpp"
36 : #include "Parallel/Local.hpp"
37 : #include "Parallel/NodeLock.hpp"
38 : #include "Parallel/ParallelComponentHelpers.hpp"
39 : #include "Parallel/Phase.hpp"
40 : #include "Parallel/PhaseDependentActionList.hpp"
41 : #include "Parallel/Printf/Printf.hpp"
42 : #include "Parallel/Tags/ArrayIndex.hpp"
43 : #include "Parallel/Tags/DistributedObjectTags.hpp"
44 : #include "Parallel/Tags/Metavariables.hpp"
45 : #include "Parallel/TypeTraits.hpp"
46 : #include "ParallelAlgorithms/Initialization/MutateAssign.hpp"
47 : #include "Utilities/Algorithm.hpp"
48 : #include "Utilities/ErrorHandling/Assert.hpp"
49 : #include "Utilities/ErrorHandling/Error.hpp"
50 : #include "Utilities/ForceInline.hpp"
51 : #include "Utilities/Gsl.hpp"
52 : #include "Utilities/MakeString.hpp"
53 : #include "Utilities/NoSuchType.hpp"
54 : #include "Utilities/Overloader.hpp"
55 : #include "Utilities/PrettyType.hpp"
56 : #include "Utilities/Requires.hpp"
57 : #include "Utilities/Serialization/PupStlCpp11.hpp"
58 : #include "Utilities/Serialization/PupStlCpp17.hpp"
59 : #include "Utilities/System/ParallelInfo.hpp"
60 : #include "Utilities/TMPL.hpp"
61 : #include "Utilities/TaggedTuple.hpp"
62 : #include "Utilities/TypeTraits.hpp"
63 :
64 : /// \cond
65 : template <size_t Dim>
66 : class ElementId;
67 :
68 : template <size_t Dim>
69 : bool is_zeroth_element(const ElementId<Dim>&, const std::optional<size_t>&);
70 : /// \cond
71 :
72 : namespace Parallel {
73 : /// \cond
74 : template <typename ParallelComponent, typename PhaseDepActionList>
75 : class DistributedObject;
76 : /// \endcond
77 :
78 : /*!
79 : * \ingroup ParallelGroup
80 : * \brief A distributed object (Charm++ Chare) that executes a series of Actions
81 : * and is capable of sending and receiving data. Acts as an interface to
82 : * Charm++.
83 : *
84 : * ### Different Types of Algorithms
85 : * Charm++ chares can be one of four types, which is specified by the type alias
86 : * `chare_type` inside the `ParallelComponent`. The four available types of
87 : * Algorithms are:
88 : * 1. A Parallel::Algorithms::Singleton where there is only one
89 : * in the entire execution of the program.
90 : * 2. A Parallel::Algorithms::Array which holds zero or more
91 : * elements each of which is a distributed object on some core. An array can
92 : * grow and shrink in size dynamically if need be and can also be bound to
93 : * another array. That is, the bound array has the same number of elements as
94 : * the array it is bound to, and elements with the same ID are on the same core.
95 : * 3. A Parallel::Algorithms::Group, which is an array but there is
96 : * one element per core and they are not able to be moved around between cores.
97 : * These are typically useful for gathering data from array elements on their
98 : * core, and then processing or reducing it.
99 : * 4. A Parallel::Algorithms::Nodegroup, which is similar to a
100 : * group except that there is one element per node. For Charm++ SMP (shared
101 : * memory parallelism) builds a node corresponds to the usual definition of a
102 : * node on a supercomputer. However, for non-SMP builds nodes and cores are
103 : * equivalent. An important difference between groups and nodegroups is that
104 : * entry methods (remote calls to functions) are not threadsafe on nodegroups.
105 : * It is up to the person writing the Actions that will be executed on the
106 : * Nodegroup Algorithm to ensure they are threadsafe.
107 : *
108 : * ### What is an Algorithm?
109 : * An Algorithm is a distributed object, a Charm++ chare, that repeatedly
110 : * executes a series of Actions. An Action is a struct that has a `static` apply
111 : * function with signature:
112 : *
113 : * \code
114 : * template <typename... DbTags, typename... InboxTags, typename Metavariables,
115 : * typename ArrayIndex, typename ActionList>
116 : * static auto apply(db::DataBox<tmpl::list<DbTags...>>& box,
117 : * tuples::TaggedTuple<InboxTags...>& inboxes,
118 : * const GlobalCache<Metavariables>& cache,
119 : * const ArrayIndex& array_index,
120 : * const TemporalId& temporal_id, const ActionList meta);
121 : * \endcode
122 : *
123 : * Note that any of the arguments can be const or non-const references except
124 : * `array_index`, which must be a `const&`.
125 : *
126 : * ### Explicit instantiations of entry methods
127 : * The code in src/Parallel/CharmMain.tpp registers all entry methods, and if
128 : * one is not properly registered then a static_assert explains how to have it
129 : * be registered. If there is a bug in the implementation and an entry method
130 : * isn't being registered or hitting a static_assert then Charm++ will give an
131 : * error of the following form:
132 : *
133 : * \verbatim
134 : * registration happened after init Entry point: simple_action(), addr:
135 : * 0x555a3d0e2090
136 : * ------------- Processor 0 Exiting: Called CmiAbort ------------
137 : * Reason: Did you forget to instantiate a templated entry method in a .ci file?
138 : * \endverbatim
139 : *
140 : * If you encounter this issue please file a bug report supplying everything
141 : * necessary to reproduce the issue.
142 : */
143 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
144 1 : class DistributedObject<ParallelComponent,
145 : tmpl::list<PhaseDepActionListsPack...>>
146 : : public ParallelComponent::chare_type::template cbase<
147 : ParallelComponent,
148 : typename get_array_index<typename ParallelComponent::chare_type>::
149 : template f<ParallelComponent>> {
150 : static_assert(
151 : sizeof...(PhaseDepActionListsPack) > 0,
152 : "Must have at least one phase dependent action list "
153 : "(PhaseActions) in a parallel component. See the first template "
154 : "parameter of 'DistributedObject' in the error message to see which "
155 : "component doesn't have any phase dependent action lists.");
156 :
157 : public:
158 : /// List of Actions in the order that generates the DataBox types
159 1 : using all_actions_list = tmpl::flatten<
160 : tmpl::list<typename PhaseDepActionListsPack::action_list...>>;
161 : /// The metavariables class passed to the Algorithm
162 1 : using metavariables = typename ParallelComponent::metavariables;
163 : /// List off all the Tags that can be received into the Inbox
164 1 : using inbox_tags_list = Parallel::get_inbox_tags<all_actions_list>;
165 : /// The type of the object used to uniquely identify the element of the array,
166 : /// group, or nodegroup. The default depends on the component, see
167 : /// ParallelComponentHelpers.
168 1 : using array_index = typename get_array_index<
169 : typename ParallelComponent::chare_type>::template f<ParallelComponent>;
170 :
171 0 : using parallel_component = ParallelComponent;
172 : /// The type of the Chare
173 1 : using chare_type = typename parallel_component::chare_type;
174 : /// The Charm++ proxy object type
175 1 : using cproxy_type =
176 : typename chare_type::template cproxy<parallel_component, array_index>;
177 : /// The Charm++ base object type
178 1 : using cbase_type =
179 : typename chare_type::template cbase<parallel_component, array_index>;
180 :
181 0 : using phase_dependent_action_lists = tmpl::list<PhaseDepActionListsPack...>;
182 :
183 0 : using inbox_type = tuples::tagged_tuple_from_typelist<inbox_tags_list>;
184 0 : using all_cache_tags = get_const_global_cache_tags<metavariables>;
185 0 : using distributed_object_tags =
186 : typename Tags::distributed_object_tags<metavariables, array_index>;
187 0 : using databox_type = db::compute_databox_type<tmpl::flatten<tmpl::list<
188 : distributed_object_tags,
189 : typename parallel_component::simple_tags_from_options,
190 : Tags::GlobalCacheImplCompute<metavariables>,
191 : Tags::ResourceInfoReference<metavariables>,
192 : db::wrap_tags_in<Tags::FromGlobalCache, all_cache_tags>,
193 : Algorithm_detail::action_list_simple_tags<parallel_component>,
194 : Algorithm_detail::action_list_compute_tags<parallel_component>>>>;
195 :
196 : /// \cond
197 : // Needed for serialization
198 : DistributedObject();
199 : /// \endcond
200 :
201 : /// Constructor used by Main to initialize the algorithm
202 : template <class... InitializationTags>
203 1 : DistributedObject(
204 : const Parallel::CProxy_GlobalCache<metavariables>& global_cache_proxy,
205 : tuples::TaggedTuple<InitializationTags...> initialization_items);
206 :
207 : /// Constructor used to dynamically add a new element of an array
208 : /// The `callback` is executed after the element is created.
209 1 : DistributedObject(
210 : const Parallel::CProxy_GlobalCache<metavariables>& global_cache_proxy,
211 : Parallel::Phase current_phase,
212 : std::unordered_map<Parallel::Phase, size_t> phase_bookmarks,
213 : const std::unique_ptr<Parallel::Callback>& callback);
214 :
215 : /// Charm++ migration constructor, used after a chare is migrated
216 1 : explicit DistributedObject(CkMigrateMessage* /*msg*/);
217 :
218 : /// \cond
219 : ~DistributedObject() override;
220 :
221 : DistributedObject(const DistributedObject& /*unused*/) = delete;
222 : DistributedObject& operator=(const DistributedObject& /*unused*/) = delete;
223 : DistributedObject(DistributedObject&& /*unused*/) = delete;
224 : DistributedObject& operator=(DistributedObject&& /*unused*/) = delete;
225 : /// \endcond
226 :
227 : /// Print the expanded type aliases
228 1 : std::string print_types() const;
229 :
230 : /// Print the current state of the algorithm
231 1 : std::string print_state() const;
232 :
233 : /// Print the current contents of the inboxes
234 1 : std::string print_inbox() const;
235 :
236 : /// Print the current contents of the DataBox
237 1 : std::string print_databox() const;
238 :
239 : /// Get read access to all the inboxes
240 1 : const auto& get_inboxes() const { return inboxes_; }
241 :
242 0 : void pup(PUP::er& p) override; // NOLINT
243 :
244 : /*!
245 : * \brief Calls the `apply` function `Action` after a reduction has been
246 : * completed.
247 : *
248 : * The `apply` function must take `arg` as its last argument.
249 : */
250 : template <typename Action, typename Arg>
251 1 : void reduction_action(Arg arg);
252 :
253 : /// \brief Explicitly call the action `Action`.
254 : template <typename Action, typename... Args>
255 1 : void simple_action(std::tuple<Args...> args);
256 :
257 : template <typename Action>
258 0 : void simple_action();
259 :
260 : /// \brief Call the `Action` sychronously, returning a result without any
261 : /// parallelization. The action is called immediately and control flow returns
262 : /// to the caller immediately upon completion.
263 : ///
264 : /// \note `Action` must have a type alias `return_type` specifying its return
265 : /// type. This constraint is to simplify the variant visitation logic for the
266 : /// \ref DataBoxGroup "DataBox".
267 : template <typename Action, typename... Args>
268 1 : typename Action::return_type local_synchronous_action(Args&&... args);
269 :
270 : /// @{
271 : /// Call an Action on a local nodegroup requiring the Action to handle thread
272 : /// safety.
273 : ///
274 : /// The `Parallel::NodeLock` of the nodegroup is passed to the Action instead
275 : /// of the `action_list` as a `const gsl::not_null<Parallel::NodeLock*>&`. The
276 : /// node lock can be locked with the `Parallel::NodeLock::lock()` function,
277 : /// and unlocked with `Parallel::unlock()`. `Parallel::NodeLock::try_lock()`
278 : /// is also provided in case something useful can be done if the lock couldn't
279 : /// be acquired.
280 : template <
281 : typename Action, typename... Args,
282 : Requires<((void)sizeof...(Args),
283 : std::is_same_v<Parallel::Algorithms::Nodegroup, chare_type>)> =
284 : nullptr>
285 1 : void threaded_action(std::tuple<Args...> args) {
286 : // Note: this method is defined inline because GCC fails to compile when the
287 : // definition is out of line.
288 : (void)Parallel::charmxx::RegisterThreadedAction<ParallelComponent, Action,
289 : Args...>::registrar;
290 : forward_tuple_to_threaded_action<Action>(
291 : std::move(args), std::make_index_sequence<sizeof...(Args)>{});
292 : }
293 :
294 : template <typename Action>
295 1 : void threaded_action();
296 : /// @}
297 :
298 : /// \brief Receive data and store it in the Inbox, and try to continue
299 : /// executing the algorithm
300 : ///
301 : /// When an algorithm has terminated it can be restarted by passing
302 : /// `enable_if_disabled = true`. This allows long-term disabling and
303 : /// re-enabling of algorithms
304 : template <typename ReceiveTag, typename ReceiveDataType>
305 1 : void receive_data(typename ReceiveTag::temporal_id instance,
306 : ReceiveDataType&& t, bool enable_if_disabled = false);
307 :
308 : template <typename ReceiveTag, typename MessageType>
309 0 : void receive_data(MessageType* message);
310 :
311 : /// @{
312 : /// Start evaluating the algorithm until it is stopped by an action.
313 1 : void perform_algorithm();
314 :
315 1 : void perform_algorithm(const bool restart_if_terminated);
316 : /// @}
317 :
318 : /// Start execution of the phase-dependent action list in `next_phase`. If
319 : /// `next_phase` has already been visited, execution will resume at the point
320 : /// where the previous execution of the same phase left off.
321 1 : void start_phase(const Parallel::Phase next_phase);
322 :
323 : /// Get the current phase
324 1 : Phase phase() const { return phase_; }
325 :
326 : /// \brief Get the phase bookmarks
327 : ///
328 : /// \details These are used to allow a phase to be resumed at a specific
329 : /// step in its iterable action list after PhaseControl is used to temporarily
330 : /// switch to other phases.
331 1 : const std::unordered_map<Parallel::Phase, size_t>& phase_bookmarks() const {
332 : return phase_bookmarks_;
333 : }
334 :
335 : /// Tell the Algorithm it should no longer execute the algorithm. This does
336 : /// not mean that the execution of the program is terminated, but only that
337 : /// the algorithm has terminated. An algorithm can be restarted by passing
338 : /// `true` as the second argument to the `receive_data` method or by calling
339 : /// perform_algorithm(true).
340 1 : constexpr void set_terminate(const bool t) { terminate_ = t; }
341 :
342 : /// Check if an algorithm should continue being evaluated
343 1 : constexpr bool get_terminate() const { return terminate_; }
344 :
345 : /// @{
346 : /// Wrappers for charm++ informational functions.
347 :
348 : /// Number of processing elements
349 1 : inline int number_of_procs() const { return sys::number_of_procs(); }
350 :
351 : /// %Index of my processing element.
352 1 : inline int my_proc() const { return sys::my_proc(); }
353 :
354 : /// Number of nodes.
355 1 : inline int number_of_nodes() const { return sys::number_of_nodes(); }
356 :
357 : /// %Index of my node.
358 1 : inline int my_node() const { return sys::my_node(); }
359 :
360 : /// Number of processing elements on the given node.
361 1 : inline int procs_on_node(const int node_index) const {
362 : return sys::procs_on_node(node_index);
363 : }
364 :
365 : /// The local index of my processing element on my node.
366 : /// This is in the interval 0, ..., procs_on_node(my_node()) - 1.
367 1 : inline int my_local_rank() const { return sys::my_local_rank(); }
368 :
369 : /// %Index of first processing element on the given node.
370 1 : inline int first_proc_on_node(const int node_index) const {
371 : return sys::first_proc_on_node(node_index);
372 : }
373 :
374 : /// %Index of the node for the given processing element.
375 1 : inline int node_of(const int proc_index) const {
376 : return sys::node_of(proc_index);
377 : }
378 :
379 : /// The local index for the given processing element on its node.
380 1 : inline int local_rank_of(const int proc_index) const {
381 : return sys::local_rank_of(proc_index);
382 : }
383 : /// @}
384 :
385 : // Invoke the static `apply` method of `ThisAction`. The if constexprs are for
386 : // handling the cases where the `apply` method returns a tuple of one, two,
387 : // or three elements, in order:
388 : // 1. A DataBox
389 : // 2. Either:
390 : // 2a. A bool determining whether or not to terminate (and potentially move
391 : // to the next phase), or
392 : // 2b. An `AlgorithmExecution` object describing whether to continue,
393 : // pause, or halt.
394 : // 3. An unsigned integer corresponding to which action in the current phase's
395 : // algorithm to execute next.
396 : //
397 : // Returns whether the action ran successfully, i.e., did not return
398 : // AlgorithmExecution::Retry.
399 : //
400 : // Needs to be public for local entry methods to work.
401 : //
402 : // Note: The template parameters PhaseIndex and DataBoxIndex are used to
403 : // shorten the function name to make profiling easier.
404 : template <typename ThisAction, typename PhaseIndex, typename DataBoxIndex>
405 0 : bool invoke_iterable_action();
406 :
407 : /// Does a reduction over the component of the reduction status sending the
408 : /// result to Main's did_all_elements_terminate member function.
409 1 : void contribute_termination_status_to_main();
410 :
411 : /// Returns the name of the last "next iterable action" to be run before a
412 : /// deadlock occurred.
413 1 : const std::string& deadlock_analysis_next_iterable_action() const {
414 : return deadlock_analysis_next_iterable_action_;
415 : }
416 :
417 : private:
418 0 : void set_array_index();
419 :
420 : template <typename PhaseDepActions, size_t... Is>
421 0 : constexpr bool iterate_over_actions(std::index_sequence<Is...> /*meta*/);
422 :
423 : template <typename Action, typename... Args, size_t... Is>
424 0 : void forward_tuple_to_action(std::tuple<Args...>&& args,
425 : std::index_sequence<Is...> /*meta*/);
426 :
427 : template <typename Action, typename... Args, size_t... Is>
428 0 : void forward_tuple_to_threaded_action(
429 : std::tuple<Args...>&& args, std::index_sequence<Is...> /*meta*/);
430 :
431 0 : size_t number_of_actions_in_phase(const Parallel::Phase phase) const;
432 :
433 : // After catching an exception, shutdown the simulation
434 0 : void initiate_shutdown(const std::exception& exception);
435 :
436 : // Member variables
437 : #ifdef SPECTRE_CHARM_PROJECTIONS
438 : double non_action_time_start_;
439 : #endif
440 :
441 0 : Parallel::CProxy_GlobalCache<metavariables> global_cache_proxy_;
442 0 : bool performing_action_ = false;
443 0 : Parallel::Phase phase_{Parallel::Phase::Initialization};
444 0 : std::unordered_map<Parallel::Phase, size_t> phase_bookmarks_{};
445 0 : std::size_t algorithm_step_ = 0;
446 : tmpl::conditional_t<Parallel::is_node_group_proxy<cproxy_type>::value,
447 : Parallel::NodeLock, NoSuchType>
448 0 : node_lock_;
449 :
450 0 : bool terminate_{true};
451 0 : bool halt_algorithm_until_next_phase_{false};
452 :
453 : // Records the name of the next action to be called so that during deadlock
454 : // analysis we can print this out.
455 0 : std::string deadlock_analysis_next_iterable_action_{};
456 :
457 0 : databox_type box_;
458 0 : inbox_type inboxes_{};
459 0 : array_index array_index_;
460 : };
461 :
462 : ////////////////////////////////////////////////////////////////
463 : // Definitions
464 : ////////////////////////////////////////////////////////////////
465 :
466 : /// \cond
467 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
468 : DistributedObject<ParallelComponent,
469 : tmpl::list<PhaseDepActionListsPack...>>::DistributedObject() {
470 : set_array_index();
471 : }
472 :
473 : namespace detail {
474 : inline bool is_zeroth_element(const int array_index) {
475 : return 0 == array_index;
476 : }
477 :
478 : template <size_t Dim>
479 : bool is_zeroth_element(const ElementId<Dim>& array_index) {
480 : return ::is_zeroth_element(array_index, std::nullopt);
481 : }
482 : } // namespace detail
483 :
484 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
485 : template <class... InitializationTags>
486 : DistributedObject<ParallelComponent, tmpl::list<PhaseDepActionListsPack...>>::
487 : DistributedObject(
488 : const Parallel::CProxy_GlobalCache<metavariables>& global_cache_proxy,
489 : tuples::TaggedTuple<InitializationTags...> initialization_items)
490 : : DistributedObject() {
491 : try {
492 : if (detail::is_zeroth_element(array_index_)) {
493 : const auto check_for_phase = [](auto phase_dep_v) {
494 : using PhaseDep = decltype(phase_dep_v);
495 : constexpr Parallel::Phase phase = PhaseDep::phase;
496 : // PostFailureCleanup is never in the default phase order, but is
497 : // controlled by Main rather than PhaseControl
498 : if (alg::count(metavariables::default_phase_order, phase) == 0 and
499 : phase != Parallel::Phase::PostFailureCleanup) {
500 : Parallel::printf(
501 : "NOTE: Phase::%s is in the phase dependent action list of\n"
502 : "component %s,\nbut not in the default_phase_order specified by "
503 : "the metavariables.\nThis means that phase will not be executed "
504 : "unless chosen by PhaseControl.\n\n",
505 : phase, pretty_type::name<parallel_component>());
506 : }
507 : };
508 : EXPAND_PACK_LEFT_TO_RIGHT(check_for_phase(PhaseDepActionListsPack{}));
509 : }
510 : (void)initialization_items; // avoid potential compiler warnings if unused
511 : // When we are using the LoadBalancing phase, we want the Main component to
512 : // handle the synchronization, so the components do not participate in the
513 : // charm++ `AtSync` barrier.
514 : // The array parallel components are migratable so they get balanced
515 : // appropriately when load balancing is triggered by the LoadBalancing phase
516 : // in Main
517 : if constexpr (std::is_same_v<typename ParallelComponent::chare_type,
518 : Parallel::Algorithms::Array>) {
519 : this->usesAtSync = false;
520 : this->setMigratable(true);
521 : }
522 : global_cache_proxy_ = global_cache_proxy;
523 : ::Initialization::mutate_assign<
524 : tmpl::push_back<distributed_object_tags, InitializationTags...>>(
525 : make_not_null(&box_), metavariables{}, array_index_,
526 : global_cache_proxy_,
527 : std::move(get<InitializationTags>(initialization_items))...);
528 : } catch (const std::exception& exception) {
529 : initiate_shutdown(exception);
530 : }
531 : }
532 :
533 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
534 : DistributedObject<ParallelComponent, tmpl::list<PhaseDepActionListsPack...>>::
535 : DistributedObject(
536 : const Parallel::CProxy_GlobalCache<metavariables>& global_cache_proxy,
537 : Parallel::Phase current_phase,
538 : std::unordered_map<Parallel::Phase, size_t> phase_bookmarks,
539 : const std::unique_ptr<Parallel::Callback>& callback)
540 : : DistributedObject() {
541 : static_assert(Parallel::is_array_proxy<cproxy_type>::value,
542 : "Can only dynamically add elements to an array component");
543 : try {
544 : // When we are using the LoadBalancing phase, we want the Main component to
545 : // handle the synchronization, so the components do not participate in the
546 : // charm++ `AtSync` barrier.
547 : // The array parallel components are migratable so they get balanced
548 : // appropriately when load balancing is triggered by the LoadBalancing phase
549 : // in Main
550 : this->usesAtSync = false;
551 : this->setMigratable(true);
552 : global_cache_proxy_ = global_cache_proxy;
553 : phase_ = current_phase;
554 : phase_bookmarks_ = std::move(phase_bookmarks);
555 : ::Initialization::mutate_assign<distributed_object_tags>(
556 : make_not_null(&box_), metavariables{}, array_index_,
557 : global_cache_proxy_);
558 : callback->invoke();
559 : } catch (const std::exception& exception) {
560 : initiate_shutdown(exception);
561 : }
562 : }
563 :
564 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
565 : DistributedObject<ParallelComponent, tmpl::list<PhaseDepActionListsPack...>>::
566 : DistributedObject(CkMigrateMessage* msg)
567 : : cbase_type(msg) {}
568 :
569 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
570 : DistributedObject<ParallelComponent, tmpl::list<PhaseDepActionListsPack...>>::
571 : ~DistributedObject() {
572 : // We place the registrar in the destructor since every DistributedObject will
573 : // have a destructor, but we have different constructors so it's not clear
574 : // which will be instantiated.
575 : (void)Parallel::charmxx::RegisterParallelComponent<
576 : ParallelComponent>::registrar;
577 : }
578 :
579 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
580 : std::string
581 : DistributedObject<ParallelComponent,
582 : tmpl::list<PhaseDepActionListsPack...>>::print_types() const {
583 : std::ostringstream os;
584 : os << "Algorithm type aliases:\n";
585 : os << "using all_actions_list = " << pretty_type::get_name<all_actions_list>()
586 : << ";\n";
587 :
588 : os << "using metavariables = " << pretty_type::get_name<metavariables>()
589 : << ";\n";
590 : os << "using inbox_tags_list = " << pretty_type::get_name<inbox_tags_list>()
591 : << ";\n";
592 : os << "using array_index = " << pretty_type::get_name<array_index>() << ";\n";
593 : os << "using parallel_component = "
594 : << pretty_type::get_name<parallel_component>() << ";\n";
595 : os << "using chare_type = " << pretty_type::get_name<chare_type>() << ";\n";
596 : os << "using cproxy_type = " << pretty_type::get_name<cproxy_type>() << ";\n";
597 : os << "using cbase_type = " << pretty_type::get_name<cbase_type>() << ";\n";
598 : os << "using phase_dependent_action_lists = "
599 : << pretty_type::get_name<phase_dependent_action_lists>() << ";\n";
600 : os << "using all_cache_tags = " << pretty_type::get_name<all_cache_tags>()
601 : << ";\n";
602 : os << "using databox_type = " << pretty_type::get_name<databox_type>()
603 : << ";\n";
604 : return os.str();
605 : }
606 :
607 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
608 : std::string
609 : DistributedObject<ParallelComponent,
610 : tmpl::list<PhaseDepActionListsPack...>>::print_state() const {
611 : using ::operator<<;
612 : std::ostringstream os;
613 : os << "State:\n";
614 : os << "performing_action_ = " << std::boolalpha << performing_action_
615 : << ";\n";
616 : os << "phase_ = " << phase_ << ";\n";
617 : os << "phase_bookmarks_ = " << phase_bookmarks_ << ";\n";
618 : os << "algorithm_step_ = " << algorithm_step_ << ";\n";
619 : os << "terminate_ = " << terminate_ << ";\n";
620 : os << "halt_algorithm_until_next_phase_ = "
621 : << halt_algorithm_until_next_phase_ << ";\n";
622 : os << "array_index_ = " << array_index_ << ";\n";
623 : return os.str();
624 : }
625 :
626 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
627 : std::string
628 : DistributedObject<ParallelComponent,
629 : tmpl::list<PhaseDepActionListsPack...>>::print_inbox() const {
630 : std::ostringstream os;
631 : os << "inboxes_ = " << inboxes_ << ";\n";
632 : return os.str();
633 : }
634 :
635 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
636 : std::string DistributedObject<
637 : ParallelComponent, tmpl::list<PhaseDepActionListsPack...>>::print_databox()
638 : const {
639 : std::ostringstream os;
640 : os << "box_:\n" << box_;
641 : return os.str();
642 : }
643 :
644 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
645 : void DistributedObject<
646 : ParallelComponent,
647 : tmpl::list<PhaseDepActionListsPack...>>::pup(PUP::er& p) { // NOLINT
648 : #ifdef SPECTRE_CHARM_PROJECTIONS
649 : p | non_action_time_start_;
650 : #endif
651 : if (performing_action_ and not p.isSizing()) {
652 : ERROR("cannot serialize while performing action!");
653 : }
654 : p | performing_action_;
655 : p | phase_;
656 : p | phase_bookmarks_;
657 : p | algorithm_step_;
658 : if constexpr (Parallel::is_node_group_proxy<cproxy_type>::value) {
659 : p | node_lock_;
660 : }
661 : p | terminate_;
662 : p | halt_algorithm_until_next_phase_;
663 : p | box_;
664 : // After unpacking the DataBox, we "touch" the GlobalCache proxy inside.
665 : // This forces the DataBox to recompute the GlobalCache* the next time it
666 : // is needed, but delays this process until after the pupper is called.
667 : // (This delay is important: updating the pointer requires calling
668 : // ckLocalBranch() on the Charm++ proxy, and in a restart from checkpoint
669 : // this call may not be well-defined until after components are finished
670 : // unpacking.)
671 : if (p.isUnpacking()) {
672 : db::mutate<Tags::GlobalCacheProxy<metavariables>>(
673 : [](const gsl::not_null<CProxy_GlobalCache<metavariables>*> proxy) {
674 : (void)proxy;
675 : },
676 : make_not_null(&box_));
677 : }
678 : p | inboxes_;
679 : p | array_index_;
680 : p | global_cache_proxy_;
681 : // Note that `perform_registration_or_deregistration` passes the `box_` by
682 : // const reference. If mutable access is required to the box, this function
683 : // call needs to be carefully considered with respect to the `p | box_` call
684 : // in both packing and unpacking scenarios.
685 : //
686 : // Note also that we don't perform (de)registrations when pup'ing for a
687 : // checkpoint/restart. This enables a simpler first-pass implementation of
688 : // checkpointing, though it means the restart must occur on the same
689 : // hardware configuration (same number of nodes and same procs per node)
690 : // used when writing the checkpoint.
691 : if (phase_ == Parallel::Phase::LoadBalancing) {
692 : // The deregistration and registration below does not actually insert
693 : // anything into the PUP::er stream, so nothing is done on a sizing pup.
694 : if (p.isPacking()) {
695 : deregister_element<ParallelComponent>(
696 : box_, *Parallel::local_branch(global_cache_proxy_), array_index_);
697 : }
698 : if (p.isUnpacking()) {
699 : register_element<ParallelComponent>(
700 : box_, *Parallel::local_branch(global_cache_proxy_), array_index_);
701 : }
702 : }
703 : }
704 :
705 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
706 : template <typename Action, typename Arg>
707 : void DistributedObject<
708 : ParallelComponent,
709 : tmpl::list<PhaseDepActionListsPack...>>::reduction_action(Arg arg) {
710 : try {
711 : (void)Parallel::charmxx::RegisterReductionAction<
712 : ParallelComponent, Action, std::decay_t<Arg>>::registrar;
713 : {
714 : std::optional<std::lock_guard<Parallel::NodeLock>> hold_lock{};
715 : if constexpr (std::is_same_v<Parallel::NodeLock, decltype(node_lock_)>) {
716 : hold_lock.emplace(node_lock_);
717 : }
718 : if (performing_action_) {
719 : ERROR(
720 : "Already performing an Action and cannot execute additional "
721 : "Actions from inside of an Action. This is only possible if the "
722 : "reduction_action function is not invoked via a proxy, which makes "
723 : "no sense for a reduction.");
724 : }
725 : performing_action_ = true;
726 : arg.finalize();
727 : forward_tuple_to_action<Action>(
728 : std::move(arg.data()), std::make_index_sequence<Arg::pack_size()>{});
729 : performing_action_ = false;
730 : }
731 : perform_algorithm();
732 : } catch (const std::exception& exception) {
733 : initiate_shutdown(exception);
734 : }
735 : }
736 :
737 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
738 : template <typename Action, typename... Args>
739 : void DistributedObject<ParallelComponent,
740 : tmpl::list<PhaseDepActionListsPack...>>::
741 : simple_action(std::tuple<Args...> args) {
742 : try {
743 : (void)Parallel::charmxx::RegisterSimpleAction<ParallelComponent, Action,
744 : Args...>::registrar;
745 : {
746 : std::optional<std::lock_guard<Parallel::NodeLock>> hold_lock{};
747 : if constexpr (std::is_same_v<Parallel::NodeLock, decltype(node_lock_)>) {
748 : hold_lock.emplace(node_lock_);
749 : }
750 : if (performing_action_) {
751 : ERROR(
752 : "Already performing an Action and cannot execute additional "
753 : "Actions from inside of an Action. This is only possible if the "
754 : "simple_action function is not invoked via a proxy, which "
755 : "we do not allow.");
756 : }
757 : performing_action_ = true;
758 : forward_tuple_to_action<Action>(
759 : std::move(args), std::make_index_sequence<sizeof...(Args)>{});
760 : performing_action_ = false;
761 : }
762 : perform_algorithm();
763 : } catch (const std::exception& exception) {
764 : initiate_shutdown(exception);
765 : }
766 : }
767 :
768 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
769 : template <typename Action>
770 : void DistributedObject<
771 : ParallelComponent,
772 : tmpl::list<PhaseDepActionListsPack...>>::simple_action() {
773 : try {
774 : (void)Parallel::charmxx::RegisterSimpleAction<ParallelComponent,
775 : Action>::registrar;
776 : {
777 : std::optional<std::lock_guard<Parallel::NodeLock>> hold_lock{};
778 : if constexpr (std::is_same_v<Parallel::NodeLock, decltype(node_lock_)>) {
779 : hold_lock.emplace(node_lock_);
780 : }
781 : if (performing_action_) {
782 : ERROR(
783 : "Already performing an Action and cannot execute additional "
784 : "Actions from inside of an Action. This is only possible if the "
785 : "simple_action function is not invoked via a proxy, which "
786 : "we do not allow.");
787 : }
788 : performing_action_ = true;
789 : Action::template apply<ParallelComponent>(
790 : box_, *Parallel::local_branch(global_cache_proxy_),
791 : static_cast<const array_index&>(array_index_));
792 : performing_action_ = false;
793 : }
794 : perform_algorithm();
795 : } catch (const std::exception& exception) {
796 : initiate_shutdown(exception);
797 : }
798 : }
799 :
800 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
801 : template <typename Action, typename... Args>
802 : typename Action::return_type
803 : DistributedObject<ParallelComponent, tmpl::list<PhaseDepActionListsPack...>>::
804 : local_synchronous_action(Args&&... args) {
805 : static_assert(Parallel::is_node_group_proxy<cproxy_type>::value,
806 : "Cannot call a (blocking) local synchronous action on a "
807 : "chare that is not a NodeGroup");
808 : return Action::template apply<ParallelComponent>(
809 : box_, make_not_null(&node_lock_), std::forward<Args>(args)...);
810 : }
811 :
812 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
813 : template <typename Action>
814 : void DistributedObject<
815 : ParallelComponent,
816 : tmpl::list<PhaseDepActionListsPack...>>::threaded_action() {
817 : try {
818 : // NOLINTNEXTLINE(modernize-redundant-void-arg)
819 : (void)Parallel::charmxx::RegisterThreadedAction<ParallelComponent,
820 : Action>::registrar;
821 : Action::template apply<ParallelComponent>(
822 : box_, *Parallel::local_branch(global_cache_proxy_),
823 : static_cast<const array_index&>(array_index_),
824 : make_not_null(&node_lock_));
825 : } catch (const std::exception& exception) {
826 : initiate_shutdown(exception);
827 : }
828 : }
829 :
830 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
831 : template <typename ReceiveTag, typename ReceiveDataType>
832 : void DistributedObject<ParallelComponent,
833 : tmpl::list<PhaseDepActionListsPack...>>::
834 : receive_data(typename ReceiveTag::temporal_id instance, ReceiveDataType&& t,
835 : const bool enable_if_disabled) {
836 : try {
837 : (void)Parallel::charmxx::RegisterReceiveData<ParallelComponent, ReceiveTag,
838 : false>::registrar;
839 : {
840 : std::optional<std::lock_guard<Parallel::NodeLock>> hold_lock{};
841 : if constexpr (std::is_same_v<Parallel::NodeLock, decltype(node_lock_)>) {
842 : hold_lock.emplace(node_lock_);
843 : }
844 : if (enable_if_disabled) {
845 : set_terminate(false);
846 : }
847 : ReceiveTag::insert_into_inbox(
848 : make_not_null(&tuples::get<ReceiveTag>(inboxes_)), instance,
849 : std::forward<ReceiveDataType>(t));
850 : }
851 : perform_algorithm();
852 : } catch (const std::exception& exception) {
853 : initiate_shutdown(exception);
854 : }
855 : }
856 :
857 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
858 : template <typename ReceiveTag, typename MessageType>
859 : void DistributedObject<ParallelComponent,
860 : tmpl::list<PhaseDepActionListsPack...>>::
861 : receive_data(MessageType* message) {
862 : try {
863 : (void)Parallel::charmxx::RegisterReceiveData<ParallelComponent, ReceiveTag,
864 : true>::registrar;
865 : {
866 : std::optional<std::lock_guard<Parallel::NodeLock>> hold_lock{};
867 : if constexpr (std::is_same_v<Parallel::NodeLock, decltype(node_lock_)>) {
868 : hold_lock.emplace(node_lock_);
869 : }
870 : if (message->enable_if_disabled) {
871 : set_terminate(false);
872 : }
873 : ReceiveTag::insert_into_inbox(
874 : make_not_null(&tuples::get<ReceiveTag>(inboxes_)), message);
875 : // Cannot use message after this call because a std::unique_ptr now owns
876 : // it. Doing so would result in undefined behavior
877 : }
878 : perform_algorithm();
879 : } catch (const std::exception& exception) {
880 : initiate_shutdown(exception);
881 : }
882 : }
883 :
884 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
885 : void DistributedObject<
886 : ParallelComponent,
887 : tmpl::list<PhaseDepActionListsPack...>>::perform_algorithm() {
888 : try {
889 : if (performing_action_ or get_terminate() or
890 : halt_algorithm_until_next_phase_) {
891 : return;
892 : }
893 : #ifdef SPECTRE_CHARM_PROJECTIONS
894 : non_action_time_start_ = sys::wall_time();
895 : #endif
896 : {
897 : std::optional<std::lock_guard<Parallel::NodeLock>> hold_lock{};
898 : if constexpr (std::is_same_v<Parallel::NodeLock, decltype(node_lock_)>) {
899 : hold_lock.emplace(node_lock_);
900 : }
901 : const auto invoke_for_phase = [this](auto phase_dep_v) {
902 : using PhaseDep = decltype(phase_dep_v);
903 : constexpr Parallel::Phase phase = PhaseDep::phase;
904 : using actions_list = typename PhaseDep::action_list;
905 : if (phase_ == phase) {
906 : while (tmpl::size<actions_list>::value > 0 and not get_terminate() and
907 : not halt_algorithm_until_next_phase_ and
908 : iterate_over_actions<PhaseDep>(
909 : std::make_index_sequence<
910 : tmpl::size<actions_list>::value>{})) {
911 : }
912 : tmpl::for_each<actions_list>([this](auto action_v) {
913 : using action = tmpl::type_from<decltype(action_v)>;
914 : if (algorithm_step_ ==
915 : tmpl::index_of<actions_list, action>::value) {
916 : deadlock_analysis_next_iterable_action_ =
917 : pretty_type::name<action>();
918 : }
919 : });
920 : }
921 : };
922 : // Loop over all phases, once the current phase is found we perform the
923 : // algorithm in that phase until we are no longer able to because we are
924 : // waiting on data to be sent or because the algorithm has been marked as
925 : // terminated.
926 : EXPAND_PACK_LEFT_TO_RIGHT(invoke_for_phase(PhaseDepActionListsPack{}));
927 : }
928 : #ifdef SPECTRE_CHARM_PROJECTIONS
929 : traceUserBracketEvent(SPECTRE_CHARM_NON_ACTION_WALLTIME_EVENT_ID,
930 : non_action_time_start_, sys::wall_time());
931 : #endif
932 : } catch (const std::exception& exception) {
933 : initiate_shutdown(exception);
934 : }
935 : }
936 :
937 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
938 : void DistributedObject<ParallelComponent,
939 : tmpl::list<PhaseDepActionListsPack...>>::
940 : perform_algorithm(const bool restart_if_terminated) {
941 : try {
942 : if (restart_if_terminated) {
943 : set_terminate(false);
944 : }
945 : perform_algorithm();
946 : } catch (const std::exception& exception) {
947 : initiate_shutdown(exception);
948 : }
949 : }
950 :
951 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
952 : void DistributedObject<ParallelComponent,
953 : tmpl::list<PhaseDepActionListsPack...>>::
954 : start_phase(const Parallel::Phase next_phase) {
955 : try {
956 : // terminate should be true since we exited a phase previously.
957 : if (not get_terminate() and not halt_algorithm_until_next_phase_) {
958 : ERROR(
959 : "An algorithm must always be set to terminate at the beginning of a "
960 : "phase. Since this is not the case the previous phase did not end "
961 : "correctly. The previous phase is: "
962 : << phase_ << " and the next phase is: " << next_phase
963 : << ", The termination flag is: " << get_terminate()
964 : << ", and the halt flag is: " << halt_algorithm_until_next_phase_);
965 : }
966 : // set terminate to true if there are no actions in this PDAL
967 : set_terminate(number_of_actions_in_phase(next_phase) == 0);
968 :
969 : // Ideally, we'd set the bookmarks as we are leaving a phase, but there is
970 : // no 'clean-up' code that we run when departing a phase, so instead we set
971 : // the bookmark for the previous phase (still stored in `phase_` at this
972 : // point), before we update the member variable `phase_`.
973 : // Then, after updating `phase_`, we check if we've ever stored a bookmark
974 : // for the new phase previously. If so, we start from where we left off,
975 : // otherwise, start from the beginning of the action list.
976 : phase_bookmarks_[phase_] = algorithm_step_;
977 : phase_ = next_phase;
978 : if (phase_bookmarks_.count(phase_) != 0) {
979 : algorithm_step_ = phase_bookmarks_.at(phase_);
980 : } else {
981 : algorithm_step_ = 0;
982 : }
983 : halt_algorithm_until_next_phase_ = false;
984 : perform_algorithm();
985 : } catch (const std::exception& exception) {
986 : initiate_shutdown(exception);
987 : }
988 : }
989 :
990 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
991 : void DistributedObject<
992 : ParallelComponent,
993 : tmpl::list<PhaseDepActionListsPack...>>::set_array_index() {
994 : // down cast to the algorithm_type, so that the `thisIndex` method can be
995 : // called, which is defined in the CBase class
996 : array_index_ = static_cast<typename chare_type::template algorithm_type<
997 : ParallelComponent, array_index>&>(*this)
998 : .thisIndex;
999 : }
1000 :
1001 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
1002 : template <typename PhaseDepActions, size_t... Is>
1003 : constexpr bool
1004 : DistributedObject<ParallelComponent, tmpl::list<PhaseDepActionListsPack...>>::
1005 : iterate_over_actions(const std::index_sequence<Is...> /*meta*/) {
1006 : bool take_next_action = true;
1007 : const auto helper = [this, &take_next_action](auto iteration) {
1008 : constexpr size_t iter = decltype(iteration)::value;
1009 : if (not(take_next_action and not terminate_ and
1010 : not halt_algorithm_until_next_phase_ and algorithm_step_ == iter)) {
1011 : return;
1012 : }
1013 : using actions_list = typename PhaseDepActions::action_list;
1014 : using this_action = tmpl::at_c<actions_list, iter>;
1015 :
1016 : constexpr size_t phase_index =
1017 : tmpl::index_of<phase_dependent_action_lists, PhaseDepActions>::value;
1018 : performing_action_ = true;
1019 : ++algorithm_step_;
1020 : // While the overhead from using the local entry method to enable
1021 : // profiling is fairly small (<2%), we still avoid it when we aren't
1022 : // tracing.
1023 : #ifdef SPECTRE_CHARM_PROJECTIONS
1024 : if constexpr (Parallel::is_array<parallel_component>::value) {
1025 : if (not this->thisProxy[array_index_]
1026 : .template invoke_iterable_action<
1027 : this_action, std::integral_constant<size_t, phase_index>,
1028 : std::integral_constant<size_t, iter>>()) {
1029 : take_next_action = false;
1030 : --algorithm_step_;
1031 : }
1032 : } else {
1033 : #endif // SPECTRE_CHARM_PROJECTIONS
1034 : if (not invoke_iterable_action<
1035 : this_action, std::integral_constant<size_t, phase_index>,
1036 : std::integral_constant<size_t, iter>>()) {
1037 : take_next_action = false;
1038 : --algorithm_step_;
1039 : }
1040 : #ifdef SPECTRE_CHARM_PROJECTIONS
1041 : }
1042 : #endif // SPECTRE_CHARM_PROJECTIONS
1043 : performing_action_ = false;
1044 : // Wrap counter if necessary
1045 : if (algorithm_step_ >= tmpl::size<actions_list>::value) {
1046 : algorithm_step_ = 0;
1047 : }
1048 : };
1049 : // In case of no Actions avoid compiler warning.
1050 : (void)helper;
1051 : // This is a template for loop for Is
1052 : EXPAND_PACK_LEFT_TO_RIGHT(helper(std::integral_constant<size_t, Is>{}));
1053 : return take_next_action;
1054 : }
1055 :
1056 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
1057 : template <typename Action, typename... Args, size_t... Is>
1058 : void DistributedObject<ParallelComponent,
1059 : tmpl::list<PhaseDepActionListsPack...>>::
1060 : forward_tuple_to_action(std::tuple<Args...>&& args,
1061 : std::index_sequence<Is...> /*meta*/) {
1062 : Action::template apply<ParallelComponent>(
1063 : box_, *Parallel::local_branch(global_cache_proxy_),
1064 : static_cast<const array_index&>(array_index_),
1065 : std::forward<Args>(std::get<Is>(args))...);
1066 : }
1067 :
1068 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
1069 : template <typename Action, typename... Args, size_t... Is>
1070 : void DistributedObject<ParallelComponent,
1071 : tmpl::list<PhaseDepActionListsPack...>>::
1072 : forward_tuple_to_threaded_action(std::tuple<Args...>&& args,
1073 : std::index_sequence<Is...> /*meta*/) {
1074 : const gsl::not_null<Parallel::NodeLock*> node_lock{&node_lock_};
1075 : Action::template apply<ParallelComponent>(
1076 : box_, *Parallel::local_branch(global_cache_proxy_),
1077 : static_cast<const array_index&>(array_index_), node_lock,
1078 : std::forward<Args>(std::get<Is>(args))...);
1079 : }
1080 :
1081 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
1082 : size_t
1083 : DistributedObject<ParallelComponent, tmpl::list<PhaseDepActionListsPack...>>::
1084 : number_of_actions_in_phase(const Parallel::Phase phase) const {
1085 : size_t number_of_actions = 0;
1086 : const auto helper = [&number_of_actions, phase](auto pdal_v) {
1087 : if (pdal_v.phase == phase) {
1088 : number_of_actions = pdal_v.number_of_actions;
1089 : }
1090 : };
1091 : EXPAND_PACK_LEFT_TO_RIGHT(helper(PhaseDepActionListsPack{}));
1092 : return number_of_actions;
1093 : }
1094 :
1095 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
1096 : template <typename ThisAction, typename PhaseIndex, typename DataBoxIndex>
1097 : bool DistributedObject<
1098 : ParallelComponent,
1099 : tmpl::list<PhaseDepActionListsPack...>>::invoke_iterable_action() {
1100 : using phase_dep_action =
1101 : tmpl::at_c<phase_dependent_action_lists, PhaseIndex::value>;
1102 : using actions_list = typename phase_dep_action::action_list;
1103 :
1104 : #ifdef SPECTRE_CHARM_PROJECTIONS
1105 : if constexpr (Parallel::is_array<parallel_component>::value) {
1106 : (void)Parallel::charmxx::RegisterInvokeIterableAction<
1107 : ParallelComponent, ThisAction, PhaseIndex, DataBoxIndex>::registrar;
1108 : }
1109 : #endif // SPECTRE_CHARM_PROJECTIONS
1110 :
1111 : AlgorithmExecution requested_execution{};
1112 : std::optional<std::size_t> next_action_step{};
1113 : std::tie(requested_execution, next_action_step) = ThisAction::apply(
1114 : box_, inboxes_, *Parallel::local_branch(global_cache_proxy_),
1115 : std::as_const(array_index_), actions_list{},
1116 : std::add_pointer_t<ParallelComponent>{});
1117 :
1118 : if (next_action_step.has_value()) {
1119 : ASSERT(
1120 : AlgorithmExecution::Retry != requested_execution,
1121 : "Switching actions on Retry doesn't make sense. Specify std::nullopt "
1122 : "as the second argument of the iterable action return type");
1123 : algorithm_step_ = next_action_step.value();
1124 : }
1125 :
1126 : switch (requested_execution) {
1127 : case AlgorithmExecution::Continue:
1128 : return true;
1129 : case AlgorithmExecution::Retry:
1130 : return false;
1131 : case AlgorithmExecution::Pause:
1132 : terminate_ = true;
1133 : return true;
1134 : case AlgorithmExecution::Halt:
1135 : halt_algorithm_until_next_phase_ = true;
1136 : terminate_ = true;
1137 : return true;
1138 : default: // LCOV_EXCL_LINE
1139 : // LCOV_EXCL_START
1140 : ERROR("No case for a Parallel::AlgorithmExecution with integral value "
1141 : << static_cast<std::underlying_type_t<AlgorithmExecution>>(
1142 : requested_execution)
1143 : << "\n");
1144 : // LCOV_EXCL_STOP
1145 : }
1146 : }
1147 :
1148 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
1149 : void DistributedObject<ParallelComponent,
1150 : tmpl::list<PhaseDepActionListsPack...>>::
1151 : contribute_termination_status_to_main() {
1152 : auto* global_cache = Parallel::local_branch(global_cache_proxy_);
1153 : if (UNLIKELY(global_cache == nullptr)) {
1154 : // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
1155 : CkError(
1156 : "Global cache pointer is null. This is an internal inconsistency "
1157 : "error. Please file an issue.");
1158 : sys::abort("");
1159 : }
1160 : auto main_proxy = global_cache->get_main_proxy();
1161 : if (UNLIKELY(not main_proxy.has_value())) {
1162 : // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
1163 : CkError(
1164 : "The main proxy has not been set in the global cache when "
1165 : "checking that all components have terminated. This is an internal "
1166 : "inconsistency error. Please file an issue.");
1167 : sys::abort("");
1168 : }
1169 : CkCallback cb(
1170 : CkReductionTarget(Main<metavariables>, did_all_elements_terminate),
1171 : main_proxy.value());
1172 : this->contribute(sizeof(bool), &terminate_, CkReduction::logical_and_bool,
1173 : cb);
1174 : }
1175 :
1176 : template <typename ParallelComponent, typename... PhaseDepActionListsPack>
1177 : void DistributedObject<ParallelComponent,
1178 : tmpl::list<PhaseDepActionListsPack...>>::
1179 : initiate_shutdown(const std::exception& exception) {
1180 : // In order to make it so that we can later run other actions for cleanup
1181 : // (e.g. dumping data) we need to make sure that we enable running actions
1182 : // again
1183 : performing_action_ = false;
1184 : // Send message to `Main` that we received an exception and set termination.
1185 : auto* global_cache = Parallel::local_branch(global_cache_proxy_);
1186 : if (UNLIKELY(global_cache == nullptr)) {
1187 : // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
1188 : CkError(
1189 : "Global cache pointer is null. This is an internal inconsistency "
1190 : "error. Please file an issue.");
1191 : sys::abort("");
1192 : }
1193 : auto main_proxy = global_cache->get_main_proxy();
1194 : if (UNLIKELY(not main_proxy.has_value())) {
1195 : // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
1196 : CkError(
1197 : "The main proxy has not been set in the global cache when terminating "
1198 : "the component. This is an internal inconsistency error. Please file "
1199 : "an issue.");
1200 : sys::abort("");
1201 : }
1202 : const std::string message =
1203 : MakeString{} << "Component: " << pretty_type::name<parallel_component>()
1204 : << "\nArray Index: " << array_index_ << "\n"
1205 : << "Phase: " << phase_ << "\n"
1206 : << "Algorithm Step: " << algorithm_step_ << "\n"
1207 : << "Message: " << exception.what() << "\nType: "
1208 : << pretty_type::get_runtime_type_name(exception);
1209 : main_proxy.value().add_exception_message(message);
1210 : set_terminate(true);
1211 : }
1212 : /// \endcond
1213 :
1214 : template <typename ParallelComponent, typename PhaseDepActionLists>
1215 0 : std::ostream& operator<<(
1216 : std::ostream& os,
1217 : const DistributedObject<ParallelComponent, PhaseDepActionLists>&
1218 : algorithm_impl) {
1219 : os << algorithm_impl.print_types() << "\n";
1220 : os << algorithm_impl.print_state() << "\n";
1221 : os << algorithm_impl.print_inbox() << "\n";
1222 : os << algorithm_impl.print_databox() << "\n";
1223 : return os;
1224 : }
1225 : } // namespace Parallel
|
