Line data Source code
1 1 : // Distributed under the MIT License.
2 : // See LICENSE.txt for details.
3 :
4 : /// \file
5 : /// Defines class template GlobalCache.
6 :
7 : #pragma once
8 :
9 : #include <charm++.h>
10 : #include <memory>
11 : #include <mutex>
12 : #include <numeric>
13 : #include <optional>
14 : #include <pup.h>
15 : #include <string>
16 : #include <tuple>
17 : #include <unordered_map>
18 : #include <utility>
19 : #include <vector>
20 :
21 : #include "DataStructures/DataBox/Tag.hpp"
22 : #include "DataStructures/DataBox/TagTraits.hpp"
23 : #include "DataStructures/TaggedTuple.hpp"
24 : #include "Parallel/ArrayComponentId.hpp"
25 : #include "Parallel/Callback.hpp"
26 : #include "Parallel/CharmRegistration.hpp"
27 : #include "Parallel/Info.hpp"
28 : #include "Parallel/Local.hpp"
29 : #include "Parallel/ParallelComponentHelpers.hpp"
30 : #include "Parallel/ResourceInfo.hpp"
31 : #include "Parallel/Tags/ResourceInfo.hpp"
32 : #include "Utilities/Algorithm.hpp"
33 : #include "Utilities/ErrorHandling/Assert.hpp"
34 : #include "Utilities/ErrorHandling/Error.hpp"
35 : #include "Utilities/Gsl.hpp"
36 : #include "Utilities/Numeric.hpp"
37 : #include "Utilities/PrettyType.hpp"
38 : #include "Utilities/Requires.hpp"
39 : #include "Utilities/Serialization/PupStlCpp17.hpp"
40 : #include "Utilities/Serialization/Serialize.hpp"
41 : #include "Utilities/StdHelpers/RetrieveUniquePtr.hpp"
42 : #include "Utilities/System/ParallelInfo.hpp"
43 : #include "Utilities/TMPL.hpp"
44 : #include "Utilities/TypeTraits/CreateGetTypeAliasOrDefault.hpp"
45 : #include "Utilities/TypeTraits/IsA.hpp"
46 :
47 : #include "Parallel/GlobalCache.decl.h"
48 : #include "Parallel/Main.decl.h"
49 :
50 : /// \cond
51 : namespace Parallel {
52 : template <typename Metavariables>
53 : struct GlobalCache;
54 : } // namespace Parallel
55 : namespace mem_monitor {
56 : template <typename Metavariables>
57 : struct MemoryMonitor;
58 : template <typename ContributingComponent>
59 : struct ContributeMemoryData;
60 : } // namespace mem_monitor
61 : /// \endcond
62 :
63 : namespace Parallel {
64 :
65 : namespace GlobalCache_detail {
66 :
67 : template <class GlobalCacheTag, class Metavariables>
68 : using get_matching_tag = typename matching_tag_helper<
69 : GlobalCacheTag,
70 : tmpl::append<get_const_global_cache_tags<Metavariables>,
71 : get_mutable_global_cache_tags<Metavariables>>>::type;
72 :
73 : template <class GlobalCacheTag, class Metavariables>
74 : using type_for_get = typename type_for_get_helper<
75 : typename get_matching_tag<GlobalCacheTag, Metavariables>::type>::type;
76 :
77 : CREATE_GET_TYPE_ALIAS_OR_DEFAULT(component_being_mocked)
78 :
79 : template <typename... Tags>
80 : auto make_mutable_cache_tag_storage(tuples::TaggedTuple<Tags...>&& input) {
81 : return tuples::TaggedTuple<MutableCacheTag<Tags>...>(std::make_tuple(
82 : std::move(tuples::get<Tags>(input)),
83 : std::unordered_map<Parallel::ArrayComponentId,
84 : std::vector<std::unique_ptr<Callback>>>{})...);
85 : }
86 :
87 : template <typename ParallelComponent, typename ComponentList>
88 : auto get_component_if_mocked_impl() {
89 : if constexpr (tmpl::list_contains_v<ComponentList, ParallelComponent>) {
90 : return ParallelComponent{};
91 : } else {
92 : using mock_find = tmpl::find<
93 : ComponentList,
94 : std::is_same<get_component_being_mocked_or_default<tmpl::_1, void>,
95 : tmpl::pin<ParallelComponent>>>;
96 : static_assert(
97 : tmpl::size<mock_find>::value > 0,
98 : "The requested parallel component (first template argument) is not a "
99 : "known parallel component (second template argument) or a component "
100 : "being mocked by one of those components.");
101 : return tmpl::front<mock_find>{};
102 : }
103 : }
104 :
105 : /// In order to be able to use a mock action testing framework we need to be
106 : /// able to get the correct parallel component from the global cache even when
107 : /// the correct component is a mock. We do this by having the mocked
108 : /// components have a member type alias `component_being_mocked`, and having
109 : /// `Parallel::get_component` check if the component to be retrieved is in the
110 : /// `metavariables::component_list`. If it is not in the `component_list` then
111 : /// we search for a mock component that is mocking the component we are trying
112 : /// to retrieve.
113 : template <typename ComponentList, typename ParallelComponent>
114 : using get_component_if_mocked =
115 : decltype(get_component_if_mocked_impl<ParallelComponent, ComponentList>());
116 :
117 : /// This class replaces the Charm++ base class of the GlobalCache in unit tests
118 : /// that don't have a Charm++ main function.
119 : struct MockGlobalCache {
120 : MockGlobalCache() = default;
121 : explicit MockGlobalCache(CkMigrateMessage* /*msg*/) {}
122 : virtual ~MockGlobalCache() = default;
123 : virtual void pup(PUP::er& /*p*/) {}
124 :
125 : static bool isIrreducible() { return true; }
126 :
127 : #if defined(__GNUC__) && !defined(__clang__)
128 : #pragma GCC diagnostic push
129 : #pragma GCC diagnostic ignored "-Wsuggest-attribute=noreturn"
130 : #elif defined(__clang__)
131 : #pragma GCC diagnostic push
132 : #pragma GCC diagnostic ignored "-Wmissing-noreturn"
133 : #endif // defined(__GNUC__) && !defined(__clang__)
134 : template <typename T>
135 : static void contribute(T&& /*unused*/) {
136 : ERROR("Not implemented.");
137 : }
138 : #if defined(__GNUC__) || defined(__clang__)
139 : #pragma GCC diagnostic pop
140 : #endif
141 : };
142 :
143 : #ifdef SPECTRE_CHARM_HAS_MAIN
144 : static constexpr bool mock_global_cache = false;
145 : #else
146 : static constexpr bool mock_global_cache = true;
147 : #endif // SPECTRE_CHARM_HAS_MAIN
148 :
149 : } // namespace GlobalCache_detail
150 :
151 : /// \cond
152 : template <typename ParallelComponentTag, typename Metavariables>
153 : auto get_parallel_component(GlobalCache<Metavariables>& cache)
154 : -> Parallel::proxy_from_parallel_component<
155 : GlobalCache_detail::get_component_if_mocked<
156 : typename Metavariables::component_list, ParallelComponentTag>>&;
157 :
158 : template <typename ParallelComponentTag, typename Metavariables>
159 : auto get_parallel_component(const GlobalCache<Metavariables>& cache)
160 : -> const Parallel::proxy_from_parallel_component<
161 : GlobalCache_detail::get_component_if_mocked<
162 : typename Metavariables::component_list, ParallelComponentTag>>&;
163 : /// \endcond
164 :
165 : /*!
166 : * \ingroup ParallelGroup
167 : * \brief A Charm++ chare that caches global data once per Charm++ node.
168 : *
169 : * \details There are two types of global data that are stored; const data and
170 : * mutable data. Once the GlobalCache is created, const data cannot be edited
171 : * but mutable data can be edited using `Parallel::mutate`.
172 : *
173 : * The template parameter `Metavariables` must define the following type
174 : * aliases:
175 : * - `component_list` typelist of ParallelComponents
176 : * - `const_global_cache_tags` (possibly empty) typelist of tags of
177 : * constant data
178 : * - `mutable_global_cache_tags` (possibly empty) typelist of tags of
179 : * non-constant data
180 : *
181 : * The tag lists for the const items added to the GlobalCache is created by
182 : * combining the following tag lists:
183 : * - `Metavariables::const_global_cache_tags` which should contain only those
184 : * tags that cannot be added from the other tag lists below.
185 : * - `Component::const_global_cache_tags` for each `Component` in
186 : * `Metavariables::component_list` which should contain the tags needed by
187 : * any simple actions called on the Component, as well as tags need by the
188 : * `allocate_array` function of an array component. The type alias may be
189 : * omitted for an empty list.
190 : * - `Action::const_global_cache_tags` for each `Action` in the
191 : * `phase_dependent_action_list` of each `Component` of
192 : * `Metavariables::component_list` which should contain the tags needed by
193 : * that Action. The type alias may be omitted for an empty list.
194 : *
195 : * The tag lists for the mutable items added to the GlobalCache is created
196 : * by combining exactly the same tag lists as for the const items, except with
197 : * `const_global_cache_tags` replaced by `mutable_global_cache_tags`.
198 : *
199 : * The tags in the `const_global_cache_tags` and
200 : * `mutable_global_cache_tags` type lists are db::SimpleTag%s that
201 : * have a `using option_tags` type alias and a static function
202 : * `create_from_options` that are used to create the constant data (or initial
203 : * mutable data) from input file options.
204 : *
205 : * References to const items in the GlobalCache are also added to the
206 : * db::DataBox of each `Component` in the
207 : * `Metavariables::component_list` with the same tag with which they
208 : * were inserted into the GlobalCache. References to mutable items
209 : * in the GlobalCache are not added to the db::DataBox.
210 : *
211 : * Since mutable data is stored once per Charm++ node, we require that
212 : * data structures held by mutable tags have some sort of thread-safety.
213 : * Particularly, we require data structures in mutable tags be Single
214 : * Producer-Multiple Consumer. This means that the data structure should be
215 : * readable/accessible by multiple threads at once, even while being mutated
216 : * (multiple consumer), but will not be edited/mutated simultaneously on
217 : * multiple threads (single producer).
218 : */
219 : template <typename Metavariables>
220 1 : class GlobalCache
221 : : public std::conditional_t<GlobalCache_detail::mock_global_cache,
222 : GlobalCache_detail::MockGlobalCache,
223 : CBase_GlobalCache<Metavariables>> {
224 0 : using Base = std::conditional_t<GlobalCache_detail::mock_global_cache,
225 : GlobalCache_detail::MockGlobalCache,
226 : CBase_GlobalCache<Metavariables>>;
227 0 : using parallel_component_tag_list = tmpl::transform<
228 : typename Metavariables::component_list,
229 : tmpl::bind<
230 : tmpl::type_,
231 : tmpl::bind<Parallel::proxy_from_parallel_component, tmpl::_1>>>;
232 0 : using ParallelComponentTuple =
233 : tuples::tagged_tuple_from_typelist<parallel_component_tag_list>;
234 :
235 : public:
236 0 : static constexpr bool is_mocked = GlobalCache_detail::mock_global_cache;
237 0 : using proxy_type = CProxy_GlobalCache<Metavariables>;
238 0 : using main_proxy_type = CProxy_Main<Metavariables>;
239 : /// Access to the Metavariables template parameter
240 1 : using metavariables = Metavariables;
241 : /// Typelist of the ParallelComponents stored in the GlobalCache
242 1 : using component_list = typename Metavariables::component_list;
243 : // Even though the GlobalCache doesn't run the Algorithm, this type alias
244 : // helps in identifying that the GlobalCache is a Nodegroup using
245 : // Parallel::is_nodegroup_v
246 0 : using chare_type = Parallel::Algorithms::Nodegroup;
247 0 : using const_tags_list = get_const_global_cache_tags<Metavariables>;
248 0 : using ConstTagsTuple = tuples::tagged_tuple_from_typelist<const_tags_list>;
249 0 : using ConstTagsStorage = ConstTagsTuple;
250 0 : using mutable_tags_list = get_mutable_global_cache_tags<Metavariables>;
251 0 : using MutableTagsTuple =
252 : tuples::tagged_tuple_from_typelist<mutable_tags_list>;
253 0 : using MutableTagsStorage = tuples::tagged_tuple_from_typelist<
254 : get_mutable_global_cache_tag_storage<Metavariables>>;
255 :
256 : /// Constructor meant to be used in the ActionTesting framework.
257 1 : GlobalCache(ConstTagsTuple const_global_cache,
258 : MutableTagsTuple mutable_global_cache = {},
259 : std::vector<size_t> procs_per_node = {1}, const int my_proc = 0,
260 : const int my_node = 0, const int my_local_rank = 0);
261 :
262 : /// Constructor meant to be used in charm-aware settings (with a Main proxy).
263 1 : GlobalCache(ConstTagsTuple const_global_cache,
264 : MutableTagsTuple mutable_global_cache,
265 : std::optional<main_proxy_type> main_proxy);
266 :
267 0 : explicit GlobalCache(CkMigrateMessage* msg) : Base(msg) {}
268 :
269 0 : ~GlobalCache() override {
270 : (void)Parallel::charmxx::RegisterChare<
271 : GlobalCache<Metavariables>,
272 : CkIndex_GlobalCache<Metavariables>>::registrar;
273 : }
274 : /// \cond
275 : GlobalCache() = default;
276 : GlobalCache(const GlobalCache&) = default;
277 : GlobalCache& operator=(const GlobalCache&) = default;
278 : GlobalCache(GlobalCache&&) = default;
279 : GlobalCache& operator=(GlobalCache&&) = default;
280 : /// \endcond
281 :
282 : /// Entry method to set the ParallelComponents (should only be called once)
283 1 : void set_parallel_components(ParallelComponentTuple&& parallel_components,
284 : const CkCallback& callback);
285 :
286 : /*!
287 : * \brief Returns whether the object referred to by `GlobalCacheTag`
288 : * (which must be a mutable cache tag) is ready to be accessed by a
289 : * `get` call.
290 : *
291 : * \details `function` is a user-defined invokable that:
292 : * - takes one argument: a const reference to the object referred to by the
293 : * `GlobalCacheTag`.
294 : * - if the data is ready, returns a default constructed
295 : * `std::unique_ptr<CallBack>`
296 : * - if the data is not ready, returns a `std::unique_ptr<CallBack>`,
297 : * where the `Callback` will re-invoke the current action on the
298 : * current parallel component. This callback should be a
299 : * `Parallel::PerformAlgorithmCallback`. Other types of callbacks are not
300 : * supported at this time.
301 : *
302 : * \parblock
303 : * \warning The `function` may be called twice so it should not modify any
304 : * state in its scope.
305 : * \endparblock
306 : *
307 : * \parblock
308 : * \warning If there has already been a callback registered for the given
309 : * `array_component_id`, then the callback returned by `function` will **not**
310 : * be registered or called.
311 : * \endparblock
312 : */
313 : template <typename GlobalCacheTag, typename Function>
314 1 : bool mutable_cache_item_is_ready(
315 : const Parallel::ArrayComponentId& array_component_id,
316 : const Function& function);
317 :
318 : /// Mutates the non-const object identified by GlobalCacheTag.
319 : /// \requires `GlobalCacheTag` is a tag in `mutable_global_cache_tags`
320 : /// defined by the Metavariables and in Actions.
321 : ///
322 : /// Internally calls `Function::apply()`, where `Function` is a
323 : /// user-defined struct and `Function::apply()` is a user-defined
324 : /// static function that mutates the object. `Function::apply()`
325 : /// takes as its first argument a `gsl::not_null` pointer to the
326 : /// object named by the GlobalCacheTag (or if that object is a
327 : /// `std::unique_ptr<T>`, a `gsl::not_null<T*>`), and takes the contents of
328 : /// `args` as subsequent arguments.
329 : template <typename GlobalCacheTag, typename Function, typename... Args>
330 1 : void mutate(const std::tuple<Args...>& args);
331 :
332 : /// Entry method that computes the size of the local branch of the
333 : /// GlobalCache and sends it to the MemoryMonitor parallel component.
334 : ///
335 : /// \note This can only be called if the MemoryMonitor component is in the
336 : /// `component_list` of the metavariables. Also can't be called in the testing
337 : /// framework. Trying to do either of these will result in an ERROR.
338 1 : void compute_size_for_memory_monitor(const double time);
339 :
340 : /// Entry method that will set the value of the Parallel::Tags::ResourceInfo
341 : /// tag to the value passed in (if the tag exists in the GlobalCache)
342 : ///
343 : /// This is only meant to be called once.
344 1 : void set_resource_info(
345 : const Parallel::ResourceInfo<Metavariables>& resource_info);
346 :
347 : /// Entry method to print the mutable global cache callbacks upon a deadlock
348 : /// to a file.
349 : ///
350 : /// For each mutable cache tag, this prints
351 : ///
352 : /// - Tag name
353 : /// - Node number
354 : /// - Number of callbacks registered
355 : /// - For each callback, the ArrayComponentId and the callback name
356 1 : void print_mutable_cache_callbacks(const std::string& file_name);
357 :
358 : /// Retrieve the resource_info
359 1 : const Parallel::ResourceInfo<Metavariables>& get_resource_info() const {
360 : return resource_info_;
361 : }
362 :
363 : /// Overlay new data onto a subset of the GlobalCache's tags.
364 : ///
365 : /// This is used when reparsing an input file to update option values during
366 : /// a restart from a checkpoint.
367 1 : void overlay_cache_data(tuples::tagged_tuple_from_typelist<
368 : Parallel::get_overlayable_tag_list<Metavariables>>
369 : data_to_overlay);
370 :
371 : /// Retrieve the proxy to the global cache
372 1 : proxy_type get_this_proxy();
373 :
374 0 : void pup(PUP::er& p) override; // NOLINT
375 :
376 : /// Retrieve the proxy to the Main chare (or std::nullopt if the proxy has not
377 : /// been set, i.e. we are not charm-aware).
378 1 : std::optional<main_proxy_type> get_main_proxy();
379 :
380 : /// @{
381 : /// Wrappers for charm++ informational functions.
382 :
383 : /// Number of processing elements
384 1 : int number_of_procs() const;
385 : /// Number of nodes.
386 1 : int number_of_nodes() const;
387 : /// Number of processing elements on the given node.
388 1 : int procs_on_node(const int node_index) const;
389 : /// %Index of first processing element on the given node.
390 1 : int first_proc_on_node(const int node_index) const;
391 : /// %Index of the node for the given processing element.
392 1 : int node_of(const int proc_index) const;
393 : /// The local index for the given processing element on its node.
394 1 : int local_rank_of(const int proc_index) const;
395 : /// %Index of my processing element.
396 1 : int my_proc() const;
397 : /// %Index of my node.
398 1 : int my_node() const;
399 : /// The local index of my processing element on my node.
400 : /// This is in the interval 0, ..., procs_on_node(my_node()) - 1.
401 1 : int my_local_rank() const;
402 : /// @}
403 :
404 : private:
405 : // clang-tidy: false positive, redundant declaration
406 : template <typename GlobalCacheTag, typename MV>
407 0 : friend auto get(const GlobalCache<MV>& cache) // NOLINT
408 : -> const GlobalCache_detail::type_for_get<GlobalCacheTag, MV>&;
409 :
410 : // clang-tidy: false positive, redundant declaration
411 : template <typename ParallelComponentTag, typename MV>
412 0 : friend auto get_parallel_component( // NOLINT
413 : GlobalCache<MV>& cache)
414 : -> Parallel::proxy_from_parallel_component<
415 : GlobalCache_detail::get_component_if_mocked<
416 : typename MV::component_list, ParallelComponentTag>>&;
417 :
418 : // clang-tidy: false positive, redundant declaration
419 : template <typename ParallelComponentTag, typename MV>
420 0 : friend auto get_parallel_component( // NOLINT
421 : const GlobalCache<MV>& cache)
422 : -> const Parallel::proxy_from_parallel_component<
423 : GlobalCache_detail::get_component_if_mocked<
424 : typename MV::component_list,
425 : ParallelComponentTag>>&; // NOLINT
426 :
427 0 : ConstTagsStorage const_global_cache_{};
428 0 : MutableTagsStorage mutable_global_cache_{};
429 : // Wrap mutable tags in Parallel::MutexTag. The type of MutexTag is a
430 : // pair<mutex, mutex>. The first mutex is for editing the value of the mutable
431 : // tag. The second mutex is for editing the vector of callbacks associated
432 : // with the mutable tag.
433 : tuples::tagged_tuple_from_typelist<
434 : tmpl::transform<MutableTagsStorage, tmpl::bind<MutexTag, tmpl::_1>>>
435 0 : mutexes_{};
436 0 : ParallelComponentTuple parallel_components_{};
437 0 : Parallel::ResourceInfo<Metavariables> resource_info_{};
438 0 : bool parallel_components_have_been_set_{false};
439 0 : bool resource_info_has_been_set_{false};
440 0 : std::optional<main_proxy_type> main_proxy_;
441 : // Defaults for testing framework
442 0 : int my_proc_{0};
443 0 : int my_node_{0};
444 0 : int my_local_rank_{0};
445 0 : std::vector<size_t> procs_per_node_{1};
446 : };
447 :
448 : template <typename Metavariables>
449 : GlobalCache<Metavariables>::GlobalCache(ConstTagsTuple const_global_cache,
450 : MutableTagsTuple mutable_global_cache,
451 : std::vector<size_t> procs_per_node,
452 : const int my_proc, const int my_node,
453 : const int my_local_rank)
454 : : const_global_cache_(std::move(const_global_cache)),
455 : mutable_global_cache_(GlobalCache_detail::make_mutable_cache_tag_storage(
456 : std::move(mutable_global_cache))),
457 : main_proxy_(std::nullopt),
458 : my_proc_(my_proc),
459 : my_node_(my_node),
460 : my_local_rank_(my_local_rank),
461 : procs_per_node_(std::move(procs_per_node)) {}
462 :
463 : template <typename Metavariables>
464 : GlobalCache<Metavariables>::GlobalCache(
465 : ConstTagsTuple const_global_cache, MutableTagsTuple mutable_global_cache,
466 : std::optional<main_proxy_type> main_proxy)
467 : : const_global_cache_(std::move(const_global_cache)),
468 : mutable_global_cache_(GlobalCache_detail::make_mutable_cache_tag_storage(
469 : std::move(mutable_global_cache))),
470 : main_proxy_(std::move(main_proxy)) {}
471 :
472 : template <typename Metavariables>
473 : void GlobalCache<Metavariables>::set_parallel_components(
474 : ParallelComponentTuple&& parallel_components, const CkCallback& callback) {
475 : ASSERT(!parallel_components_have_been_set_,
476 : "Can only set the parallel_components once");
477 : parallel_components_ = std::move(parallel_components);
478 : parallel_components_have_been_set_ = true;
479 : this->contribute(callback);
480 : }
481 :
482 : template <typename Metavariables>
483 : template <typename GlobalCacheTag, typename Function>
484 : bool GlobalCache<Metavariables>::mutable_cache_item_is_ready(
485 : const Parallel::ArrayComponentId& array_component_id,
486 : const Function& function) {
487 : using tag = MutableCacheTag<GlobalCache_detail::get_matching_mutable_tag<
488 : GlobalCacheTag, Metavariables>>;
489 : std::unique_ptr<Callback> optional_callback{};
490 : // Returns true if a callback was returned from `function`. Returns false if
491 : // nullptr was returned
492 : const auto callback_was_registered = [this, &function,
493 : &optional_callback]() -> bool {
494 : // Reads don't need a lock.
495 : if constexpr (tt::is_a_v<std::unique_ptr, typename tag::tag::type>) {
496 : optional_callback =
497 : function(*(std::get<0>(tuples::get<tag>(mutable_global_cache_))));
498 : } else {
499 : optional_callback =
500 : function(std::get<0>(tuples::get<tag>(mutable_global_cache_)));
501 : }
502 :
503 : return optional_callback != nullptr;
504 : };
505 :
506 : if (callback_was_registered()) {
507 : optional_callback->register_with_charm();
508 : // Second mutex is for vector of callbacks
509 : std::mutex& mutex = tuples::get<MutexTag<tag>>(mutexes_).second;
510 : const std::unique_ptr<Callback> clone_of_optional_callback =
511 : optional_callback->get_clone();
512 : {
513 : // Scoped for lock guard
514 : const std::lock_guard<std::mutex> lock(mutex);
515 : std::unordered_map<Parallel::ArrayComponentId,
516 : std::vector<std::unique_ptr<Callback>>>& callbacks =
517 : std::get<1>(tuples::get<tag>(mutable_global_cache_));
518 :
519 : if (callbacks.contains(array_component_id)) {
520 : // If this array component id already exists, we don't want to add
521 : // multiple of the same callback, so we loop over the existing callbacks
522 : // and only if none of the existing callbacks are equal to the optional
523 : // callback do we move the optional callback into the vector
524 : auto& vec_callbacks = callbacks.at(array_component_id);
525 : if (alg::none_of(vec_callbacks,
526 : [&](const std::unique_ptr<Callback>& local_callback) {
527 : return local_callback->is_equal_to(
528 : *optional_callback);
529 : })) {
530 : vec_callbacks.emplace_back(std::move(optional_callback));
531 : }
532 : } else {
533 : // If we don't have this array component id, then we create the vector
534 : // and move the optional callback into the vector
535 : callbacks[array_component_id] =
536 : std::vector<std::unique_ptr<Callback>>(1);
537 : callbacks.at(array_component_id)[0] = std::move(optional_callback);
538 : }
539 : }
540 :
541 : // We must check if the tag is ready again. Consider the following example:
542 : //
543 : // We have two writers, A and B preparing to make independent changes to a
544 : // cache object. We have an element E with a callback waiting for the change
545 : // B is going to make. Suppose the following sequence of events:
546 : //
547 : // 1. A mutates the object, copies the callback list (below in `mutate`),
548 : // and starts the callback for element E.
549 : // 2. E checks the current value and determines it is not ready.
550 : // 3. B mutates the object, copies the empty callback list, and returns with
551 : // nothing to do.
552 : // 4. E returns a new callback, which is added to the callback list.
553 : // 5. A returns.
554 : //
555 : // We now have E waiting on a change that has already happened. This will
556 : // most certainly result in a deadlock if E is blocking the Algorithm. Thus
557 : // we must do another check for whether the cache object is ready or not. In
558 : // the order of events, this check happens sometime after 4. When this check
559 : // happens, E concludes that the cache object *is* ready (because 3 is when
560 : // the object was mutated) and E can continue on.
561 : //
562 : // If this second check reveals that the object *is* ready, then we have an
563 : // unecessary callback in our map, so we remove it.
564 : //
565 : // If this second check reveals that the object *isn't* ready, then we don't
566 : // bother adding another callback to the map because one already exists. No
567 : // need to call a callback twice.
568 : //
569 : // This function returns true if no callback was registered and false if one
570 : // was registered.
571 : const bool cache_item_is_ready = not callback_was_registered();
572 : if (cache_item_is_ready) {
573 : const std::lock_guard<std::mutex> lock(mutex);
574 : std::unordered_map<Parallel::ArrayComponentId,
575 : std::vector<std::unique_ptr<Callback>>>& callbacks =
576 : std::get<1>(tuples::get<tag>(mutable_global_cache_));
577 :
578 : // It's possible that no new callbacks were registered, so make sure this
579 : // array component id still has callbacks before trying to remove them.
580 : if (callbacks.contains(array_component_id)) {
581 : // If this callback was a duplicate, we'll have to search through all
582 : // callbacks to determine which to remove. If it wasn't a duplicate,
583 : // then it'll just be the last callback in the vector.
584 : auto& vec_callbacks = callbacks.at(array_component_id);
585 : std::erase_if(vec_callbacks,
586 : [&clone_of_optional_callback](const auto& t) {
587 : return t->is_equal_to(*clone_of_optional_callback);
588 : });
589 :
590 : if (callbacks.at(array_component_id).empty()) {
591 : callbacks.erase(array_component_id);
592 : }
593 : }
594 : }
595 :
596 : return cache_item_is_ready;
597 : } else {
598 : // The user-defined `function` didn't specify a callback, which
599 : // means that the item is ready.
600 : return true;
601 : }
602 : }
603 :
604 : template <typename Metavariables>
605 : template <typename GlobalCacheTag, typename Function, typename... Args>
606 : void GlobalCache<Metavariables>::mutate(const std::tuple<Args...>& args) {
607 : (void)Parallel::charmxx::RegisterGlobalCacheMutate<
608 : Metavariables, GlobalCacheTag, Function, Args...>::registrar;
609 : using tag = MutableCacheTag<GlobalCache_detail::get_matching_mutable_tag<
610 : GlobalCacheTag, Metavariables>>;
611 :
612 : // Do the mutate.
613 : std::apply(
614 : [this](const auto&... local_args) {
615 : // First mutex is for value of mutable tag
616 : std::mutex& mutex = tuples::get<MutexTag<tag>>(mutexes_).first;
617 : const std::lock_guard<std::mutex> lock(mutex);
618 : Function::apply(make_not_null(&StdHelpers::retrieve(std::get<0>(
619 : tuples::get<tag>(mutable_global_cache_)))),
620 : local_args...);
621 : },
622 : args);
623 :
624 : // A callback might call mutable_cache_item_is_ready, which might add yet
625 : // another callback to the vector of callbacks. We don't want to immediately
626 : // invoke this new callback as it might just add another callback again (and
627 : // again in an infinite loop). And we don't want to remove it from the map of
628 : // callbacks before it is invoked otherwise we could get a deadlock.
629 : // Therefore, after locking it, we std::move the map of callbacks into a
630 : // temporary map, clear the original map, and invoke the callbacks in the
631 : // temporary map.
632 : std::unordered_map<Parallel::ArrayComponentId,
633 : std::vector<std::unique_ptr<Callback>>>
634 : callbacks{};
635 : // Second mutex is for map of callbacks
636 : std::mutex& mutex = tuples::get<MutexTag<tag>>(mutexes_).second;
637 : {
638 : // Scoped for lock guard
639 : const std::lock_guard<std::mutex> lock(mutex);
640 : callbacks = std::move(std::get<1>(tuples::get<tag>(mutable_global_cache_)));
641 : std::get<1>(tuples::get<tag>(mutable_global_cache_)).clear();
642 : }
643 :
644 : // Invoke the callbacks. Any new callbacks that are added to the
645 : // list (if a callback calls mutable_cache_item_is_ready) will be
646 : // saved and will not be invoked here.
647 : for (auto& [array_component_id, vec_callbacks] : callbacks) {
648 : for (auto& callback : vec_callbacks) {
649 : callback->invoke();
650 : }
651 : }
652 : }
653 :
654 : template <typename Metavariables>
655 : void GlobalCache<Metavariables>::overlay_cache_data(
656 : tuples::tagged_tuple_from_typelist<
657 : Parallel::get_overlayable_tag_list<Metavariables>>
658 : data_to_overlay) {
659 : tmpl::for_each<Parallel::get_overlayable_tag_list<Metavariables>>(
660 : [this, &data_to_overlay]<typename Tag>(tmpl::type_<Tag> /*meta*/) {
661 : tuples::get<Tag>(const_global_cache_) =
662 : std::move(tuples::get<Tag>(data_to_overlay));
663 : });
664 : }
665 :
666 : #if defined(__GNUC__) && !defined(__clang__)
667 : #pragma GCC diagnostic push
668 : #pragma GCC diagnostic ignored "-Wsuggest-attribute=noreturn"
669 : #endif // defined(__GNUC__) && !defined(__clang__)
670 : template <typename Metavariables>
671 : void GlobalCache<Metavariables>::compute_size_for_memory_monitor(
672 : const double time) {
673 : if constexpr (tmpl::list_contains_v<
674 : typename Metavariables::component_list,
675 : mem_monitor::MemoryMonitor<Metavariables>>) {
676 : const double size_in_bytes =
677 : static_cast<double>(size_of_object_in_bytes(*this));
678 : const double size_in_MB = size_in_bytes / 1.0e6;
679 :
680 : auto& mem_monitor_proxy = Parallel::get_parallel_component<
681 : mem_monitor::MemoryMonitor<Metavariables>>(*this);
682 :
683 : const int my_node = Parallel::my_node<int>(*this);
684 :
685 : Parallel::simple_action<
686 : mem_monitor::ContributeMemoryData<GlobalCache<Metavariables>>>(
687 : mem_monitor_proxy, time, my_node, size_in_MB);
688 : } else {
689 : (void)time;
690 : ERROR(
691 : "GlobalCache::compute_size_for_memory_monitor can only be called if "
692 : "the MemoryMonitor is in the component list in the metavariables.\n");
693 : }
694 : }
695 :
696 : template <typename Metavariables>
697 : void GlobalCache<Metavariables>::set_resource_info(
698 : const Parallel::ResourceInfo<Metavariables>& resource_info) {
699 : ASSERT(not resource_info_has_been_set_,
700 : "Can only set the resource info once");
701 : resource_info_ = resource_info;
702 : resource_info_has_been_set_ = true;
703 : }
704 :
705 : template <typename Metavariables>
706 : void GlobalCache<Metavariables>::print_mutable_cache_callbacks(
707 : const std::string& file_name) {
708 : std::stringstream ss{};
709 : ss << std::setprecision(std::numeric_limits<double>::digits10 + 4)
710 : << std::scientific;
711 :
712 : ss << "========== BEGIN CALLBACKS ON NODE " << this->my_node()
713 : << " ==========\n";
714 :
715 : tmpl::for_each<typename MutableTagsStorage::tags_list>(
716 : [this, &ss](auto tag_v) {
717 : using Tag = tmpl::type_from<decltype(tag_v)>;
718 :
719 : const auto& callbacks =
720 : std::get<1>(tuples::get<Tag>(mutable_global_cache_));
721 :
722 : ss << "For tag " << pretty_type::name<typename Tag::tag>()
723 : << ", callbacks ("
724 : << alg::accumulate(callbacks, 0_st,
725 : [](const size_t cur_size, const auto& v) {
726 : return cur_size + v.second.size();
727 : })
728 : << "):\n";
729 :
730 : for (const auto& [array_component_id, vec_callbacks] : callbacks) {
731 : for (const auto& callback : vec_callbacks) {
732 : ss << " ArrayComponentId " << array_component_id << ": "
733 : << callback->name() << "\n";
734 : }
735 : }
736 : });
737 :
738 : ss << "========== END CALLBACKS ON NODE " << this->my_node()
739 : << " ============\n";
740 :
741 : Parallel::fprintf(file_name, "%s\n", ss.str());
742 : }
743 :
744 : #if defined(__GNUC__) && !defined(__clang__)
745 : #pragma GCC diagnostic pop
746 : #endif // defined(__GNUC__) && !defined(__clang__)
747 :
748 : template <typename Metavariables>
749 : typename Parallel::GlobalCache<Metavariables>::proxy_type
750 : GlobalCache<Metavariables>::get_this_proxy() {
751 : if constexpr (is_mocked) {
752 : // The proxy is not used in the testing framework
753 : return Parallel::GlobalCache<Metavariables>::proxy_type{};
754 : } else {
755 : return this->thisProxy;
756 : }
757 : }
758 :
759 : template <typename Metavariables>
760 : std::optional<typename Parallel::GlobalCache<Metavariables>::main_proxy_type>
761 : GlobalCache<Metavariables>::get_main_proxy() {
762 : return main_proxy_;
763 : }
764 :
765 : // For all these functions, if the main proxy is set (meaning we are
766 : // charm-aware) then just call the sys:: functions. Otherwise, use the values
767 : // set for the testing framework (or the defaults).
768 : template <typename Metavariables>
769 : int GlobalCache<Metavariables>::number_of_procs() const {
770 : return main_proxy_.has_value()
771 : ? sys::number_of_procs()
772 : : static_cast<int>(alg::accumulate(procs_per_node_, 0_st));
773 : }
774 :
775 : template <typename Metavariables>
776 : int GlobalCache<Metavariables>::number_of_nodes() const {
777 : return main_proxy_.has_value() ? sys::number_of_nodes()
778 : : static_cast<int>(procs_per_node_.size());
779 : }
780 :
781 : template <typename Metavariables>
782 : int GlobalCache<Metavariables>::procs_on_node(const int node_index) const {
783 : return main_proxy_.has_value()
784 : ? sys::procs_on_node(node_index)
785 : : static_cast<int>(
786 : procs_per_node_[static_cast<size_t>(node_index)]);
787 : }
788 :
789 : template <typename Metavariables>
790 : int GlobalCache<Metavariables>::first_proc_on_node(const int node_index) const {
791 : return main_proxy_.has_value()
792 : ? sys::first_proc_on_node(node_index)
793 : : static_cast<int>(
794 : std::accumulate(procs_per_node_.begin(),
795 : procs_per_node_.begin() + node_index, 0_st));
796 : }
797 :
798 : template <typename Metavariables>
799 : int GlobalCache<Metavariables>::node_of(const int proc_index) const {
800 : if (main_proxy_.has_value()) {
801 : // For some reason gcov doesn't think this line is tested even though it is
802 : // in Test_AlgorithmGlobalCache.cpp
803 : return sys::node_of(proc_index); // LCOV_EXCL_LINE
804 : } else {
805 : size_t procs_so_far = 0;
806 : size_t node = 0;
807 : while (procs_so_far <= static_cast<size_t>(proc_index)) {
808 : procs_so_far += procs_per_node_[node];
809 : ++node;
810 : }
811 : return static_cast<int>(--node);
812 : }
813 : }
814 :
815 : template <typename Metavariables>
816 : int GlobalCache<Metavariables>::local_rank_of(const int proc_index) const {
817 : return main_proxy_.has_value()
818 : ? sys::local_rank_of(proc_index)
819 : : proc_index - first_proc_on_node(node_of(proc_index));
820 : }
821 :
822 : template <typename Metavariables>
823 : int GlobalCache<Metavariables>::my_proc() const {
824 : return main_proxy_.has_value() ? sys::my_proc() : my_proc_;
825 : }
826 :
827 : template <typename Metavariables>
828 : int GlobalCache<Metavariables>::my_node() const {
829 : return main_proxy_.has_value() ? sys::my_node() : my_node_;
830 : }
831 :
832 : template <typename Metavariables>
833 : int GlobalCache<Metavariables>::my_local_rank() const {
834 : return main_proxy_.has_value() ? sys::my_local_rank() : my_local_rank_;
835 : }
836 :
837 : template <typename Metavariables>
838 : void GlobalCache<Metavariables>::pup(PUP::er& p) {
839 : p | const_global_cache_;
840 : p | parallel_components_;
841 : p | mutable_global_cache_;
842 : p | main_proxy_;
843 : p | parallel_components_have_been_set_;
844 : p | resource_info_has_been_set_;
845 : p | my_proc_;
846 : p | my_node_;
847 : p | my_local_rank_;
848 : p | procs_per_node_;
849 : }
850 :
851 : /// @{
852 : /// \ingroup ParallelGroup
853 : /// \brief Access the Charm++ proxy associated with a ParallelComponent
854 : ///
855 : /// \requires ParallelComponentTag is a tag in component_list
856 : ///
857 : /// \returns a Charm++ proxy that can be used to call an entry method on the
858 : /// chare(s)
859 : template <typename ParallelComponentTag, typename Metavariables>
860 1 : auto get_parallel_component(GlobalCache<Metavariables>& cache)
861 : -> Parallel::proxy_from_parallel_component<
862 : GlobalCache_detail::get_component_if_mocked<
863 : typename Metavariables::component_list, ParallelComponentTag>>& {
864 : return tuples::get<tmpl::type_<Parallel::proxy_from_parallel_component<
865 : GlobalCache_detail::get_component_if_mocked<
866 : typename Metavariables::component_list, ParallelComponentTag>>>>(
867 : cache.parallel_components_);
868 : }
869 :
870 : template <typename ParallelComponentTag, typename Metavariables>
871 1 : auto get_parallel_component(const GlobalCache<Metavariables>& cache)
872 : -> const Parallel::proxy_from_parallel_component<
873 : GlobalCache_detail::get_component_if_mocked<
874 : typename Metavariables::component_list, ParallelComponentTag>>& {
875 : return tuples::get<tmpl::type_<Parallel::proxy_from_parallel_component<
876 : GlobalCache_detail::get_component_if_mocked<
877 : typename Metavariables::component_list, ParallelComponentTag>>>>(
878 : cache.parallel_components_);
879 : }
880 : /// @}
881 :
882 : /// @{
883 : /// \ingroup ParallelGroup
884 : /// \brief Access data in the cache
885 : ///
886 : /// \requires GlobalCacheTag is a tag in the `mutable_global_cache_tags`
887 : /// or `const_global_cache_tags` defined by the Metavariables and in Actions.
888 : ///
889 : /// \returns a constant reference to an object in the cache
890 : template <typename GlobalCacheTag, typename Metavariables>
891 1 : auto get(const GlobalCache<Metavariables>& cache)
892 : -> const GlobalCache_detail::type_for_get<GlobalCacheTag, Metavariables>& {
893 : constexpr bool is_mutable =
894 : is_in_mutable_global_cache<Metavariables, GlobalCacheTag>;
895 : // We check if the tag is to be retrieved directly or via a base class
896 : using tmp_tag =
897 : GlobalCache_detail::get_matching_tag<GlobalCacheTag, Metavariables>;
898 : using tag =
899 : tmpl::conditional_t<is_mutable, MutableCacheTag<tmp_tag>, tmp_tag>;
900 : if constexpr (is_mutable) {
901 : // Tag is not in the const tags, so use mutable_global_cache_. No locks here
902 : // because we require all mutable tags to be able to be read at all times
903 : // (even when being written to)
904 : if constexpr (tt::is_a_v<std::unique_ptr, typename tag::tag::type>) {
905 : return *std::get<0>(tuples::get<tag>(cache.mutable_global_cache_));
906 : } else {
907 : return std::get<0>(tuples::get<tag>(cache.mutable_global_cache_));
908 : }
909 : } else {
910 : // Tag is in the const tags, so use const_global_cache_
911 : if constexpr (tt::is_a_v<std::unique_ptr, typename tag::type>) {
912 : return *(tuples::get<tag>(cache.const_global_cache_));
913 : } else {
914 : return tuples::get<tag>(cache.const_global_cache_);
915 : }
916 : }
917 : }
918 :
919 : /// \ingroup ParallelGroup
920 : /// \brief Returns whether the object identified by `GlobalCacheTag`
921 : /// is ready to be accessed by `get`.
922 : ///
923 : /// \requires `GlobalCacheTag` is a tag in `mutable_global_cache_tags`
924 : /// defined by the Metavariables and in Actions.
925 : ///
926 : /// \requires `function` is a user-defined invokable that takes one argument:
927 : /// a const reference to the object referred to by the
928 : /// `GlobalCacheTag`. `function` returns a
929 : /// `std::unique_ptr<CallBack>` that determines the readiness. To
930 : /// indicate that the item is ready, the `std::unique_ptr` returned
931 : /// by `function` must be nullptr; in this case
932 : /// `mutable_cache_item_is_ready` returns true. To indicate that the
933 : /// item is not ready, the `std::unique_ptr` returned by `function`
934 : /// must be valid; in this case, `mutable_cache_item_is_ready`
935 : /// appends the `std::unique_ptr<Callback>` to an
936 : /// internal list of callbacks to be called on `mutate`, and then
937 : /// returns false.
938 : ///
939 : /// \note If `function` is returning a valid callback, it should only return a
940 : /// `Parallel::PerformAlgorithmCallback`. Other types of callbacks are not
941 : /// supported at this time.
942 : template <typename GlobalCacheTag, typename Function, typename Metavariables>
943 1 : bool mutable_cache_item_is_ready(
944 : GlobalCache<Metavariables>& cache,
945 : const Parallel::ArrayComponentId& array_component_id,
946 : const Function& function) {
947 : return cache.template mutable_cache_item_is_ready<GlobalCacheTag>(
948 : array_component_id, function);
949 : }
950 :
951 : /// \ingroup ParallelGroup
952 : ///
953 : /// \brief Mutates non-const data in the cache, by calling `Function::apply()`
954 : ///
955 : /// \requires `GlobalCacheTag` is a tag in tag_list.
956 : /// \requires `Function` is a struct with a static void `apply()`
957 : /// function that mutates the object. `Function::apply()` takes as its
958 : /// first argument a `gsl::not_null` pointer to the object named by
959 : /// the `GlobalCacheTag`, and takes `args` as
960 : /// subsequent arguments.
961 : template <typename GlobalCacheTag, typename Function, typename Metavariables,
962 : typename... Args>
963 1 : void mutate(GlobalCache<Metavariables>& cache, Args&&... args) {
964 : if (cache.get_main_proxy().has_value()) {
965 : if constexpr (not GlobalCache<Metavariables>::is_mocked) {
966 : cache.thisProxy.template mutate<GlobalCacheTag, Function>(
967 : std::make_tuple<Args...>(std::forward<Args>(args)...));
968 : } else {
969 : ERROR(
970 : "Main proxy is set but global cache is being mocked. This is "
971 : "currently not implemented.");
972 : }
973 : } else {
974 : cache.template mutate<GlobalCacheTag, Function>(
975 : std::make_tuple<Args...>(std::forward<Args>(args)...));
976 : }
977 : }
978 :
979 : namespace Tags {
980 : template <class Metavariables>
981 0 : struct GlobalCacheProxy : db::SimpleTag {
982 0 : using type = CProxy_GlobalCache<Metavariables>;
983 : };
984 :
985 : /// \ingroup DataBoxTagsGroup
986 : /// \ingroup ParallelGroup
987 : /// Tag to retrieve the `Parallel::GlobalCache` from the DataBox.
988 : template <class Metavariables>
989 1 : struct GlobalCache : db::SimpleTag {
990 0 : using type = Parallel::GlobalCache<Metavariables>*;
991 : };
992 :
993 : template <class Metavariables>
994 0 : struct GlobalCacheCompute : GlobalCache<Metavariables>, db::ComputeTag {
995 0 : using base = GlobalCache<Metavariables>;
996 0 : using argument_tags = tmpl::list<GlobalCacheProxy<Metavariables>>;
997 0 : using return_type = Parallel::GlobalCache<Metavariables>*;
998 0 : static void function(
999 : const gsl::not_null<Parallel::GlobalCache<Metavariables>**>
1000 : local_branch_of_global_cache,
1001 : const CProxy_GlobalCache<Metavariables>& global_cache_proxy) {
1002 : *local_branch_of_global_cache = Parallel::local_branch(global_cache_proxy);
1003 : }
1004 : };
1005 :
1006 : /// \ingroup DataBoxTagsGroup
1007 : /// \ingroup ParallelGroup
1008 : /// Tag used to retrieve data from the `Parallel::GlobalCache`. This is the
1009 : /// recommended way for compute tags to retrieve data out of the global cache.
1010 : template <class CacheTag, class Metavariables>
1011 1 : struct FromGlobalCache : CacheTag, db::ReferenceTag {
1012 : static_assert(db::is_simple_tag_v<CacheTag>);
1013 0 : using base = CacheTag;
1014 0 : using argument_tags = tmpl::list<GlobalCache<Metavariables>>;
1015 :
1016 0 : static const auto& get(
1017 : const Parallel::GlobalCache<Metavariables>* const& cache) {
1018 : return Parallel::get<CacheTag>(*cache);
1019 : }
1020 : };
1021 :
1022 : template <typename Metavariables>
1023 0 : struct ResourceInfoReference : ResourceInfo<Metavariables>, db::ReferenceTag {
1024 0 : using base = ResourceInfo<Metavariables>;
1025 0 : using argument_tags = tmpl::list<GlobalCache<Metavariables>>;
1026 :
1027 0 : static const auto& get(
1028 : const Parallel::GlobalCache<Metavariables>* const& cache) {
1029 : return cache->get_resource_info();
1030 : }
1031 : };
1032 : } // namespace Tags
1033 : } // namespace Parallel
1034 :
1035 : namespace PUP {
1036 : /// \cond
1037 : // Warning! This is an invalid and kludgey pupper, because when unpacking it
1038 : // produces a `nullptr` rather than a valid `GlobalCache*`. This function is
1039 : // provided _only_ to enable putting a `GlobalCache*` in the DataBox.
1040 : //
1041 : // SpECTRE parallel components with a `GlobalCache*` in their DataBox should
1042 : // set this pointer using a compute item that calls `Parallel::local_branch` on
1043 : // a stored proxy. When deserializing the DataBox, these components should call
1044 : // `db:mutate<GlobalCacheProxy>(box)` to force the DataBox to update the
1045 : // pointer from the new Charm++ proxy.
1046 : //
1047 : // We do not currently anticipate needing to (de)serialize a `GlobalCache*`
1048 : // outside of a DataBox. But if this need arises, it will be necessary to
1049 : // provide a non-kludgey pupper here.
1050 : //
1051 : // Correctly (de)serializing the `GlobalCache*` would require obtaining a
1052 : // `CProxy_GlobalCache` item and calling `Parallel::local_branch` on it ---
1053 : // just as in the workflow described above, but within the pupper vs in the
1054 : // DataBox. But this strategy fails when restarting from a checkpoint file,
1055 : // because calling `Parallel::local_branch` may not be well-defined in the
1056 : // unpacking pup call when all Charm++ components may not yet be fully restored.
1057 : // This difficulty is why we instead write an invalid pupper here.
1058 : //
1059 : // In future versions of Charm++, the pup function may know whether it is
1060 : // called in the context of checkpointing, load balancing, etc. This knowledge
1061 : // would enable us to write a valid pupper for non-checkpoint contexts, and
1062 : // return a `nullptr` only when restoring from checkpoint.
1063 : template <typename Metavariables>
1064 : inline void operator|(PUP::er& p, // NOLINT
1065 : Parallel::GlobalCache<Metavariables>*& t) {
1066 : if (p.isUnpacking()) {
1067 : t = nullptr;
1068 : }
1069 : }
1070 : /// \endcond
1071 : } // namespace PUP
1072 :
1073 0 : #define CK_TEMPLATES_ONLY
1074 : #include "Parallel/GlobalCache.def.h"
1075 : #undef CK_TEMPLATES_ONLY
|
