Line data Source code
1 1 : // Distributed under the MIT License.
2 : // See LICENSE.txt for details.
3 :
4 : /// \file
5 : /// Defines the Charm++ mainchare.
6 :
7 : #pragma once
8 :
9 : #include <array>
10 : #include <boost/program_options.hpp>
11 : #include <charm++.h>
12 : #include <cstddef>
13 : #include <initializer_list>
14 : #include <pup.h>
15 : #include <string>
16 : #include <tuple>
17 : #include <type_traits>
18 :
19 : #include "Informer/InfoFromBuild.hpp"
20 : #include "Informer/Informer.hpp"
21 : #include "Options/ParseOptions.hpp"
22 : #include "Options/Tags.hpp"
23 : #include "Parallel/AlgorithmMetafunctions.hpp"
24 : #include "Parallel/CharmRegistration.hpp"
25 : #include "Parallel/CreateFromOptions.hpp"
26 : #include "Parallel/ExitCode.hpp"
27 : #include "Parallel/GlobalCache.hpp"
28 : #include "Parallel/Local.hpp"
29 : #include "Parallel/ParallelComponentHelpers.hpp"
30 : #include "Parallel/Phase.hpp"
31 : #include "Parallel/PhaseControl/ExecutePhaseChange.hpp"
32 : #include "Parallel/PhaseControl/InitializePhaseChangeDecisionData.hpp"
33 : #include "Parallel/PhaseControl/PhaseControlTags.hpp"
34 : #include "Parallel/PhaseControlReductionHelpers.hpp"
35 : #include "Parallel/Printf/Printf.hpp"
36 : #include "Parallel/Reduction.hpp"
37 : #include "Parallel/ResourceInfo.hpp"
38 : #include "Parallel/Tags/ResourceInfo.hpp"
39 : #include "Parallel/TypeTraits.hpp"
40 : #include "Utilities/Algorithm.hpp"
41 : #include "Utilities/ErrorHandling/Error.hpp"
42 : #include "Utilities/FileSystem.hpp"
43 : #include "Utilities/Formaline.hpp"
44 : #include "Utilities/Kokkos/KokkosCore.hpp"
45 : #include "Utilities/MakeString.hpp"
46 : #include "Utilities/Overloader.hpp"
47 : #include "Utilities/StdHelpers.hpp"
48 : #include "Utilities/System/Exit.hpp"
49 : #include "Utilities/System/ParallelInfo.hpp"
50 : #include "Utilities/TMPL.hpp"
51 : #include "Utilities/TaggedTuple.hpp"
52 : #include "Utilities/TypeTraits/CreateGetTypeAliasOrDefault.hpp"
53 : #include "Utilities/TypeTraits/CreateIsCallable.hpp"
54 :
55 : #include "Parallel/Main.decl.h"
56 :
57 : namespace Parallel {
58 : namespace detail {
59 : CREATE_IS_CALLABLE(run_deadlock_analysis_simple_actions)
60 : CREATE_IS_CALLABLE_V(run_deadlock_analysis_simple_actions)
61 :
62 : // Return the dir name for the Charm++ checkpoints as well as the prefix for
63 : // checkpoint names and their padding. This is a "detail" function so that
64 : // these pieces can be defined in one place only.
65 : std::tuple<std::string, std::string, size_t> checkpoints_dir_prefix_pad();
66 :
67 : // Return the dir name for the next Charm++ checkpoint; check and error if
68 : // this name already exists and writing the checkpoint would be unsafe.
69 : std::string next_checkpoint_dir(size_t checkpoint_dir_counter);
70 :
71 : // Check if future checkpoint dirs are available; error if any already exist.
72 : void check_future_checkpoint_dirs_available(size_t checkpoint_dir_counter);
73 : } // namespace detail
74 :
75 : /// \ingroup ParallelGroup
76 : /// The main function of a Charm++ executable.
77 : /// See [the Parallelization documentation](group__ParallelGroup.html#details)
78 : /// for an overview of Metavariables, Phases, and parallel components.
79 : template <typename Metavariables>
80 1 : class Main : public CBase_Main<Metavariables> {
81 : public:
82 0 : using component_list = typename Metavariables::component_list;
83 0 : using const_global_cache_tags = get_const_global_cache_tags<Metavariables>;
84 0 : using mutable_global_cache_tags =
85 : get_mutable_global_cache_tags<Metavariables>;
86 :
87 0 : using phase_change_tags_and_combines_list =
88 : PhaseControl::get_phase_change_tags<Metavariables>;
89 : /// \cond HIDDEN_SYMBOLS
90 : /// The constructor used to register the class
91 : explicit Main(const Parallel::charmxx::
92 : MainChareRegistrationConstructor& /*used_for_reg*/) {}
93 : ~Main() override {
94 : (void)Parallel::charmxx::RegisterChare<
95 : Main<Metavariables>, CkIndex_Main<Metavariables>>::registrar;
96 : }
97 : Main(const Main&) = default;
98 : Main& operator=(const Main&) = default;
99 : Main(Main&&) = default;
100 : Main& operator=(Main&&) = default;
101 : /// \endcond
102 :
103 0 : explicit Main(CkArgMsg* msg);
104 0 : explicit Main(CkMigrateMessage* msg);
105 :
106 : // NOLINTNEXTLINE(google-runtime-references)
107 0 : void pup(PUP::er& p) override;
108 :
109 : /// Allocate singleton components and the initial elements of array
110 : /// components, then execute the initialization phase on each component
111 1 : void allocate_remaining_components_and_execute_initialization_phase();
112 :
113 : /// Determine the next phase of the simulation and execute it.
114 1 : void execute_next_phase();
115 :
116 : /// Place the Charm++ call that starts load balancing
117 : ///
118 : /// \details This call is wrapped within an entry method so that it may be
119 : /// used as the callback after a quiescence detection.
120 1 : void start_load_balance();
121 :
122 : /// Place the Charm++ call that starts writing a checkpoint
123 : /// Reset the checkpoint counter to zero if the checkpoints directory does not
124 : /// exist (this happens when the simulation continues in a new segment).
125 : ///
126 : /// \details This call is wrapped within an entry method so that it may be
127 : /// used as the callback after a quiescence detection.
128 1 : void start_write_checkpoint();
129 :
130 : /// Reduction target for data used in phase change decisions.
131 : ///
132 : /// It is required that the `Parallel::ReductionData` holds a single
133 : /// `tuples::TaggedTuple`.
134 : template <typename InvokeCombine, typename... Tags>
135 1 : void phase_change_reduction(
136 : ReductionData<ReductionDatum<tuples::TaggedTuple<Tags...>, InvokeCombine,
137 : funcl::Identity, std::index_sequence<>>>
138 : reduction_data);
139 :
140 : /// Add an exception to the list of exceptions. Upon a phase change we print
141 : /// all the received exceptions and exit.
142 : ///
143 : /// Upon receiving an exception all algorithms are terminated to guarantee
144 : /// quiescence occurs soon after the exception is reported.
145 1 : void add_exception_message(std::string exception_message);
146 :
147 : /// A reduction target used to determine if all the elements of the array,
148 : /// group, nodegroup, or singleton parallel components terminated
149 : /// successfully.
150 : ///
151 : /// This allows detecting deadlocks in iterable actions, but not simple or
152 : /// reduction action.
153 1 : void did_all_elements_terminate(bool all_elements_terminated);
154 :
155 : /// Prints exit info and stops the executable with failure if a deadlock was
156 : /// detected.
157 1 : void post_deadlock_analysis_termination();
158 :
159 : private:
160 : // Starts a reduction on the component specified by
161 : // the current_termination_check_index_ member variable, then increment
162 : // current_termination_check_index_
163 0 : void check_if_component_terminated_correctly();
164 :
165 : // After a restart, update the const global cache with new values constructed
166 : // from parsing an overlay input file.
167 0 : void update_const_global_cache_from_input_file();
168 :
169 : // Lists of all parallel component types
170 0 : using group_component_list =
171 : tmpl::filter<component_list, tmpl::or_<Parallel::is_group<tmpl::_1>,
172 : Parallel::is_nodegroup<tmpl::_1>>>;
173 0 : using all_array_component_list =
174 : tmpl::filter<component_list, Parallel::is_array<tmpl::_1>>;
175 0 : using non_bound_array_component_list =
176 : tmpl::filter<component_list,
177 : tmpl::and_<Parallel::is_array<tmpl::_1>,
178 : tmpl::not_<Parallel::is_bound_array<tmpl::_1>>>>;
179 0 : using bound_array_component_list =
180 : tmpl::filter<component_list,
181 : tmpl::and_<Parallel::is_array<tmpl::_1>,
182 : Parallel::is_bound_array<tmpl::_1>>>;
183 0 : using singleton_component_list =
184 : tmpl::filter<component_list, Parallel::is_singleton<tmpl::_1>>;
185 :
186 : template <typename ParallelComponent>
187 0 : using parallel_component_options = Parallel::get_option_tags<
188 : typename ParallelComponent::simple_tags_from_options, Metavariables>;
189 : template <typename ArrayComponent>
190 0 : using array_component_allocation_options =
191 : Parallel::get_option_tags<typename ArrayComponent::array_allocation_tags,
192 : Metavariables>;
193 0 : using option_list = tmpl::remove_duplicates<tmpl::flatten<tmpl::list<
194 : Parallel::OptionTags::ResourceInfo<Metavariables>,
195 : Parallel::get_option_tags<const_global_cache_tags, Metavariables>,
196 : Parallel::get_option_tags<mutable_global_cache_tags, Metavariables>,
197 : tmpl::transform<component_list,
198 : tmpl::bind<parallel_component_options, tmpl::_1>>,
199 : tmpl::transform<
200 : all_array_component_list,
201 : tmpl::bind<array_component_allocation_options, tmpl::_1>>>>>;
202 0 : using overlayable_option_list =
203 : Parallel::get_overlayable_option_list<Metavariables>;
204 :
205 0 : Parallel::Phase current_phase_{Parallel::Phase::Initialization};
206 0 : CProxy_GlobalCache<Metavariables> global_cache_proxy_;
207 0 : detail::CProxy_AtSyncIndicator<Metavariables> at_sync_indicator_proxy_;
208 0 : std::string input_file_{};
209 : Options::Parser<tmpl::remove<option_list, Options::Tags::InputSource>>
210 0 : parser_{Metavariables::help};
211 : // This is only used during startup, and will be cleared after all
212 : // the chares are created. It is a member variable because passing
213 : // local state through charm callbacks is painful.
214 0 : tuples::tagged_tuple_from_typelist<option_list> options_{};
215 : // type to be determined by the collection of available phase changers in the
216 : // Metavariables
217 : tuples::tagged_tuple_from_typelist<phase_change_tags_and_combines_list>
218 0 : phase_change_decision_data_;
219 0 : size_t checkpoint_dir_counter_ = 0_st;
220 0 : Parallel::ResourceInfo<Metavariables> resource_info_{};
221 : // All exception errors we've received so far.
222 0 : std::vector<std::string> exception_messages_{};
223 : // Used to keep track of which parallel component we are checking has
224 : // successfully terminated.
225 0 : size_t current_termination_check_index_{0};
226 0 : std::vector<std::string> components_that_did_not_terminate_{};
227 0 : bool just_restored_from_checkpoint_ = false;
228 : };
229 :
230 : namespace detail {
231 :
232 : // Charm++ AtSync effectively requires an additional global sync to the
233 : // quiescence detection we do for switching phases. However, AtSync only needs
234 : // to be called for one array to trigger the sync-based load balancing, so the
235 : // AtSyncIndicator is a silly hack to have a centralized indication to start
236 : // load-balancing. It participates in the `AtSync` barrier, but is not
237 : // migratable, and should only be constructed by the `Main` chare on the same
238 : // processor as the `Main` chare. When load-balancing occurs, main invokes the
239 : // member function `IndicateAtSync()`, and when `ResumeFromSync()` is called
240 : // from charm, `AtSyncIndicator` simply passes control back to the Main chare
241 : // via `execute_next_phase()`.
242 : template <typename Metavariables>
243 : class AtSyncIndicator : public CBase_AtSyncIndicator<Metavariables> {
244 : public:
245 : AtSyncIndicator(CProxy_Main<Metavariables> main_proxy);
246 : AtSyncIndicator(const AtSyncIndicator&) = default;
247 : AtSyncIndicator& operator=(const AtSyncIndicator&) = default;
248 : AtSyncIndicator(AtSyncIndicator&&) = default;
249 : AtSyncIndicator& operator=(AtSyncIndicator&&) = default;
250 : ~AtSyncIndicator() override {
251 : (void)Parallel::charmxx::RegisterChare<
252 : AtSyncIndicator<Metavariables>,
253 : CkIndex_AtSyncIndicator<Metavariables>>::registrar;
254 : }
255 :
256 : void IndicateAtSync();
257 :
258 : void ResumeFromSync() override;
259 :
260 : explicit AtSyncIndicator(CkMigrateMessage* msg)
261 : : CBase_AtSyncIndicator<Metavariables>(msg) {}
262 :
263 : void pup(PUP::er& p) override { p | main_proxy_; }
264 :
265 : private:
266 : CProxy_Main<Metavariables> main_proxy_;
267 : };
268 :
269 : template <typename Metavariables>
270 : AtSyncIndicator<Metavariables>::AtSyncIndicator(
271 : CProxy_Main<Metavariables> main_proxy)
272 : : main_proxy_{main_proxy} {
273 : this->usesAtSync = true;
274 : this->setMigratable(false);
275 : }
276 :
277 : template <typename Metavariables>
278 : void AtSyncIndicator<Metavariables>::IndicateAtSync() {
279 : this->AtSync();
280 : }
281 :
282 : template <typename Metavariables>
283 : void AtSyncIndicator<Metavariables>::ResumeFromSync() {
284 : main_proxy_.execute_next_phase();
285 : }
286 : } // namespace detail
287 :
288 : // ================================================================
289 :
290 : template <typename Metavariables>
291 : Main<Metavariables>::Main(CkArgMsg* msg) {
292 : #ifdef SPECTRE_KOKKOS
293 : Kokkos::initialize(msg->argc, msg->argv);
294 : #endif // SPECTRE_KOKKOS
295 :
296 : Informer::print_startup_info(msg);
297 :
298 : /// \todo detail::register_events_to_trace();
299 :
300 : namespace bpo = boost::program_options;
301 : try {
302 : bpo::options_description command_line_options;
303 : // disable clang-format because it combines the repeated call operator
304 : // invocations making the code more difficult to parse.
305 : // clang-format off
306 : command_line_options.add_options()
307 : ("help,h", "Describe program options")
308 : ("check-options", "Check input file options")
309 : ("dump-source-tree-as", bpo::value<std::string>(),
310 : "If specified, then a gzip archive of the source tree is dumped "
311 : "with the specified name. The archive can be expanded using "
312 : "'tar -xzf ARCHIVE.tar.gz'")
313 : ("dump-paths",
314 : "Dump the PATH, CPATH, LD_LIBRARY_PATH, LIBRARY_PATH, and "
315 : "CMAKE_PREFIX_PATH at compile time.")
316 : ("dump-environment",
317 : "Dump the result of printenv at compile time.")
318 : ("dump-build-info",
319 : "Dump the contents of SpECTRE's BuildInfo.txt")
320 : ("dump-only",
321 : "Exit after dumping requested information.")
322 : ("copyright-and-licenses",
323 : "Returns all of the copyright and license info for SpECTRE and "
324 : "its dependencies.")
325 : ;
326 : // clang-format on
327 :
328 : // False if there are no other options besides the explicitly added
329 : // Parallel::OptionTags::ResourceInfo<Metavariables>,
330 : constexpr bool has_options = tmpl::size<option_list>::value > 1;
331 : // Add input-file option if it makes sense
332 : Overloader{
333 : [&command_line_options](std::true_type /*meta*/, auto mv,
334 : int /*gcc_bug*/)
335 : -> std::void_t<
336 : decltype(tmpl::type_from<decltype(mv)>::input_file)> {
337 : // Metavariables has options and default input file name
338 : command_line_options.add_options()(
339 : "input-file",
340 : bpo::value<std::string>()->default_value(
341 : tmpl::type_from<decltype(mv)>::input_file),
342 : "Input file name");
343 : },
344 : [&command_line_options](std::true_type /*meta*/, auto /*mv*/,
345 : auto... /*unused*/) {
346 : // Metavariables has options and no default input file name
347 : command_line_options.add_options()(
348 : "input-file", bpo::value<std::string>(), "Input file name");
349 : },
350 : [](std::false_type /*meta*/, auto mv, int /*gcc_bug*/)
351 : -> std::void_t<
352 : decltype(tmpl::type_from<decltype(mv)>::input_file)> {
353 : // Metavariables has no options and default input file name
354 :
355 : // always false, but must depend on mv
356 : static_assert(std::is_same_v<decltype(mv), void>,
357 : "Metavariables supplies input file name, "
358 : "but there are no options");
359 : ERROR("This should have failed at compile time");
360 : },
361 : [](std::false_type /*meta*/, auto... /*unused*/) {
362 : // Metavariables has no options and no default input file name
363 : }}(std::bool_constant<has_options>{}, tmpl::type_<Metavariables>{}, 0);
364 :
365 : bpo::command_line_parser command_line_parser(msg->argc, msg->argv);
366 : command_line_parser.options(command_line_options);
367 :
368 : const bool ignore_unrecognized_command_line_options = Overloader{
369 : [](auto mv, int /*gcc_bug*/)
370 : -> decltype(tmpl::type_from<decltype(mv)>::
371 : ignore_unrecognized_command_line_options) {
372 : return tmpl::type_from<
373 : decltype(mv)>::ignore_unrecognized_command_line_options;
374 : },
375 : [](auto /*mv*/, auto... /*meta*/) { return false; }}(
376 : tmpl::type_<Metavariables>{}, 0);
377 : if (ignore_unrecognized_command_line_options) {
378 : // Allow unknown --options
379 : command_line_parser.allow_unregistered();
380 : } else {
381 : // Forbid positional parameters
382 : command_line_parser.positional({});
383 : }
384 :
385 : bpo::variables_map parsed_command_line_options;
386 : bpo::store(command_line_parser.run(), parsed_command_line_options);
387 : bpo::notify(parsed_command_line_options);
388 :
389 : if (parsed_command_line_options.count("help") != 0) {
390 : Parallel::printf("%s\n%s", command_line_options, parser_.help());
391 : sys::exit();
392 : }
393 :
394 : if (parsed_command_line_options.count("dump-source-tree-as") != 0) {
395 : formaline::write_to_file(
396 : parsed_command_line_options["dump-source-tree-as"].as<std::string>());
397 : Parallel::printf("Dumping archive of source tree at link time.\n");
398 : }
399 : if (parsed_command_line_options.count("dump-paths") != 0) {
400 : Parallel::printf("Paths at link time were:\n%s\n",
401 : formaline::get_paths());
402 : }
403 : if (parsed_command_line_options.count("dump-environment") != 0) {
404 : Parallel::printf("Environment variables at link time were:\n%s\n",
405 : formaline::get_environment_variables());
406 : }
407 : if (parsed_command_line_options.count("dump-build-info") != 0) {
408 : Parallel::printf("BuildInfo.txt at link time was:\n%s\n",
409 : formaline::get_build_info());
410 : }
411 : if (parsed_command_line_options.count("copyright-and-licenses") != 0) {
412 : Parallel::printf("%s\n", copyright_and_license_info());
413 : }
414 : if (parsed_command_line_options.count("dump-only") != 0) {
415 : sys::exit();
416 : }
417 :
418 : if (has_options) {
419 : if (parsed_command_line_options.count("input-file") == 0) {
420 : ERROR_NO_TRACE("No default input file name. Pass --input-file.");
421 : }
422 : input_file_ = parsed_command_line_options["input-file"].as<std::string>();
423 : parser_.parse_file(input_file_);
424 : } else {
425 : if constexpr (tmpl::size<singleton_component_list>::value > 0) {
426 : parser_.parse(
427 : "ResourceInfo:\n"
428 : " AvoidGlobalProc0: false\n"
429 : " Singletons: Auto\n");
430 : } else {
431 : parser_.parse(
432 : "ResourceInfo:\n"
433 : " AvoidGlobalProc0: false\n");
434 : }
435 : }
436 :
437 : if (parsed_command_line_options.count("check-options") != 0) {
438 : // Force all the options to be created.
439 : parser_.template apply<option_list, Metavariables>([](auto... args) {
440 : (void)std::initializer_list<char>{((void)args, '0')...};
441 : });
442 : if (has_options) {
443 : Parallel::printf("\n%s parsed successfully!\n", input_file_);
444 : } else {
445 : // This is still considered successful, since it means the
446 : // program would have started.
447 : Parallel::printf("\nNo options to check!\n");
448 : }
449 :
450 : // Include a check that the checkpoint dirs are available for writing as
451 : // part of checking the option parsing. Doing these checks together helps
452 : // catch more user errors before running the executable 'for real'.
453 : //
454 : // Note we don't do this check at the beginning of the Main chare
455 : // constructor because we don't _always_ want to error if checkpoint dirs
456 : // already exist. For example, running the executable with flags like
457 : // `--help` or `--dump-source-tree-as` should succeed even if checkpoints
458 : // were previously written.
459 : detail::check_future_checkpoint_dirs_available(checkpoint_dir_counter_);
460 :
461 : sys::exit();
462 : }
463 :
464 : options_ =
465 : parser_.template apply<option_list, Metavariables>([](auto... args) {
466 : return tuples::tagged_tuple_from_typelist<option_list>(
467 : std::move(args)...);
468 : });
469 :
470 : resource_info_ =
471 : tuples::get<Parallel::OptionTags::ResourceInfo<Metavariables>>(
472 : options_);
473 :
474 : Parallel::printf("\nOption parsing completed.\n");
475 : } catch (const bpo::error& e) {
476 : ERROR(e.what());
477 : }
478 :
479 : detail::check_future_checkpoint_dirs_available(checkpoint_dir_counter_);
480 :
481 : global_cache_proxy_ = CProxy_GlobalCache<Metavariables>::ckNew(
482 : Parallel::create_from_options<Metavariables>(options_,
483 : const_global_cache_tags{}),
484 : Parallel::create_from_options<Metavariables>(options_,
485 : mutable_global_cache_tags{}),
486 : this->thisProxy);
487 :
488 : // Now that the GlobalCache has been built, create the singleton map which
489 : // will be used to allocate all the singletons. We need to be careful here
490 : // because the parallel components have not been set at this point, so if we
491 : // try to Parallel::get_parallel_component here, an error will occur. This
492 : // call is OK though because build_singleton_map() only uses the parallel info
493 : // functions from the GlobalCache (like cache.number_of_procs()).
494 : resource_info_.build_singleton_map(
495 : *Parallel::local_branch(global_cache_proxy_));
496 :
497 : // Now that the singleton map has been built, set the resource info in the
498 : // GlobalCache (if the tags exist). Since this info will be constant
499 : // throughout a simulation, we opt for directly editing a const tag in the
500 : // GlobalCache before we pass it to any other parallel component rather than
501 : // having a mutable tag and using a mutate call to set it.
502 : global_cache_proxy_.set_resource_info(resource_info_);
503 :
504 : // Now that the singleton map has been built, we have to replace the
505 : // ResourceInfo that was created from options with the one that has all the
506 : // correct singleton assignments so simple tags can be created from options
507 : // with a valid ResourceInfo.
508 : get<Parallel::OptionTags::ResourceInfo<Metavariables>>(options_) =
509 : resource_info_;
510 :
511 : at_sync_indicator_proxy_ =
512 : detail::CProxy_AtSyncIndicator<Metavariables>::ckNew();
513 : at_sync_indicator_proxy_[0].insert(this->thisProxy, sys::my_proc());
514 : at_sync_indicator_proxy_.doneInserting();
515 :
516 : using parallel_component_tag_list = tmpl::transform<
517 : component_list,
518 : tmpl::bind<
519 : tmpl::type_,
520 : tmpl::bind<Parallel::proxy_from_parallel_component, tmpl::_1>>>;
521 : tuples::tagged_tuple_from_typelist<parallel_component_tag_list>
522 : the_parallel_components;
523 :
524 : // Print info on DataBox variants
525 : #ifdef SPECTRE_DEBUG
526 : Parallel::printf("\nParallel components:\n");
527 : tmpl::for_each<component_list>([](auto parallel_component_v) {
528 : using parallel_component = tmpl::type_from<decltype(parallel_component_v)>;
529 : using chare_type = typename parallel_component::chare_type;
530 : using charm_type = Parallel::charm_types_with_parameters<
531 : parallel_component, typename Parallel::get_array_index<
532 : chare_type>::template f<parallel_component>>;
533 : Parallel::printf(
534 : " %s (%s) has a DataBox with %u items.\n",
535 : pretty_type::name<parallel_component>(),
536 : pretty_type::name<chare_type>(),
537 : tmpl::size<
538 : typename charm_type::algorithm::databox_type::tags_list>::value);
539 : });
540 : Parallel::printf("\n");
541 : #endif // SPECTRE_DEBUG
542 :
543 : // Construct the group proxies with a dependency on the GlobalCache proxy
544 : CkEntryOptions global_cache_dependency;
545 : global_cache_dependency.setGroupDepID(global_cache_proxy_.ckGetGroupID());
546 :
547 : tmpl::for_each<group_component_list>([this, &the_parallel_components,
548 : &global_cache_dependency](
549 : auto parallel_component_v) {
550 : using parallel_component = tmpl::type_from<decltype(parallel_component_v)>;
551 : using ParallelComponentProxy =
552 : Parallel::proxy_from_parallel_component<parallel_component>;
553 : tuples::get<tmpl::type_<ParallelComponentProxy>>(the_parallel_components) =
554 : ParallelComponentProxy::ckNew(
555 : global_cache_proxy_,
556 : Parallel::create_from_options<Metavariables>(
557 : options_,
558 : typename parallel_component::simple_tags_from_options{}),
559 : &global_cache_dependency);
560 : });
561 :
562 : // Create proxies for empty array chares (whose elements will be created by
563 : // the allocate functions of the array components during
564 : // execute_initialization_phase)
565 : tmpl::for_each<non_bound_array_component_list>([&the_parallel_components](
566 : auto parallel_component) {
567 : using ParallelComponentProxy = Parallel::proxy_from_parallel_component<
568 : tmpl::type_from<decltype(parallel_component)>>;
569 : tuples::get<tmpl::type_<ParallelComponentProxy>>(the_parallel_components) =
570 : ParallelComponentProxy::ckNew();
571 : });
572 :
573 : // Create proxies for empty bound array chares
574 : tmpl::for_each<bound_array_component_list>([&the_parallel_components](
575 : auto parallel_component) {
576 : using ParallelComponentProxy = Parallel::proxy_from_parallel_component<
577 : tmpl::type_from<decltype(parallel_component)>>;
578 : CkArrayOptions opts;
579 : opts.bindTo(
580 : tuples::get<tmpl::type_<Parallel::proxy_from_parallel_component<
581 : typename tmpl::type_from<decltype(parallel_component)>::bind_to>>>(
582 : the_parallel_components));
583 : tuples::get<tmpl::type_<ParallelComponentProxy>>(the_parallel_components) =
584 : ParallelComponentProxy::ckNew(opts);
585 : });
586 :
587 : // Create proxies for singletons (which are single-element charm++ arrays)
588 : tmpl::for_each<singleton_component_list>([&the_parallel_components](
589 : auto parallel_component) {
590 : using ParallelComponentProxy = Parallel::proxy_from_parallel_component<
591 : tmpl::type_from<decltype(parallel_component)>>;
592 : tuples::get<tmpl::type_<ParallelComponentProxy>>(the_parallel_components) =
593 : ParallelComponentProxy::ckNew();
594 : });
595 :
596 : // Send the complete list of parallel_components to the GlobalCache on
597 : // each Charm++ node. After all nodes have finished, the callback is
598 : // executed.
599 : CkCallback callback(
600 : CkIndex_Main<Metavariables>::
601 : allocate_remaining_components_and_execute_initialization_phase(),
602 : this->thisProxy);
603 : global_cache_proxy_.set_parallel_components(the_parallel_components,
604 : callback);
605 :
606 : get<Tags::ExitCode>(phase_change_decision_data_) =
607 : Parallel::ExitCode::Complete;
608 : PhaseControl::initialize_phase_change_decision_data(
609 : make_not_null(&phase_change_decision_data_),
610 : *Parallel::local_branch(global_cache_proxy_));
611 :
612 : printer_chare = CProxy_PrinterChare::ckNew(1);
613 : printer_chare_is_set = true;
614 : }
615 :
616 : template <typename Metavariables>
617 : Main<Metavariables>::Main(CkMigrateMessage* msg)
618 : : CBase_Main<Metavariables>(msg) {}
619 :
620 : template <typename Metavariables>
621 : void Main<Metavariables>::pup(PUP::er& p) { // NOLINT
622 : p | current_phase_;
623 : p | global_cache_proxy_;
624 : p | at_sync_indicator_proxy_;
625 : p | input_file_;
626 : p | parser_;
627 : // Note: we don't serialize options_, because it's used as a temporary in
628 : // startup only (see comment in class definition).
629 : p | phase_change_decision_data_;
630 :
631 : p | checkpoint_dir_counter_;
632 : p | resource_info_;
633 : p | exception_messages_;
634 : p | current_termination_check_index_;
635 : p | components_that_did_not_terminate_;
636 : if (p.isUnpacking()) {
637 : detail::check_future_checkpoint_dirs_available(checkpoint_dir_counter_);
638 : // Main doesn't migrate unless checkpointing or restarting, so we can
639 : // indicate here that we've just restored from checkpoint.
640 : just_restored_from_checkpoint_ = true;
641 : // Initialize Kokkos on restart
642 : #ifdef SPECTRE_KOKKOS
643 : Kokkos::initialize();
644 : #endif // SPECTRE_KOKKOS
645 : }
646 :
647 : // For now we only support restarts on the same hardware configuration (same
648 : // number of nodes and same procs per node) used when writing the checkpoint.
649 : // We check this by adding counters to the pup stream.
650 : if (p.isUnpacking()) {
651 : int previous_nodes = 0;
652 : int previous_procs = 0;
653 : p | previous_nodes;
654 : p | previous_procs;
655 : if (previous_nodes != sys::number_of_nodes() or
656 : previous_procs != sys::number_of_procs()) {
657 : ERROR(
658 : "Must restart on the same hardware configuration used when writing "
659 : "the checkpoint.\n"
660 : "Checkpoint written with "
661 : << previous_nodes << " nodes, " << previous_procs
662 : << " procs.\n"
663 : "Restarted with "
664 : << sys::number_of_nodes() << " nodes, " << sys::number_of_procs()
665 : << " procs.");
666 : }
667 : } else {
668 : int current_nodes = sys::number_of_nodes();
669 : int current_procs = sys::number_of_procs();
670 : p | current_nodes;
671 : p | current_procs;
672 : }
673 : }
674 :
675 : template <typename Metavariables>
676 : void Main<Metavariables>::
677 : allocate_remaining_components_and_execute_initialization_phase() {
678 : if (current_phase_ != Parallel::Phase::Initialization) {
679 : ERROR("Must be in the Initialization phase.");
680 : }
681 : // Since singletons are actually single-element Charm++ arrays, we have to
682 : // allocate them here along with the other Charm++ arrays.
683 : tmpl::for_each<singleton_component_list>([this](auto singleton_component_v) {
684 : using singleton_component =
685 : tmpl::type_from<decltype(singleton_component_v)>;
686 : auto& local_cache = *Parallel::local_branch(global_cache_proxy_);
687 : auto& singleton_proxy =
688 : Parallel::get_parallel_component<singleton_component>(local_cache);
689 : auto options = Parallel::create_from_options<Metavariables>(
690 : options_, typename singleton_component::simple_tags_from_options{});
691 :
692 : const size_t proc = resource_info_.template proc_for<singleton_component>();
693 : singleton_proxy[0].insert(global_cache_proxy_, std::move(options), proc);
694 : singleton_proxy.doneInserting();
695 : });
696 :
697 : // These are Spectre array components built on Charm++ array chares. Each
698 : // component is in charge of allocating and distributing its elements over the
699 : // computing system.
700 : tmpl::for_each<all_array_component_list>([this](auto parallel_component_v) {
701 : using parallel_component = tmpl::type_from<decltype(parallel_component_v)>;
702 : parallel_component::allocate_array(
703 : global_cache_proxy_,
704 : Parallel::create_from_options<Metavariables>(
705 : options_, typename parallel_component::simple_tags_from_options{}),
706 : Parallel::create_from_options<Metavariables>(
707 : options_, typename parallel_component::array_allocation_tags{}),
708 : resource_info_.procs_to_ignore());
709 : });
710 :
711 : // Free any resources from the initial option parsing.
712 : options_ = decltype(options_){};
713 :
714 : tmpl::for_each<component_list>([this](auto parallel_component_v) {
715 : using parallel_component = tmpl::type_from<decltype(parallel_component_v)>;
716 : Parallel::get_parallel_component<parallel_component>(
717 : *Parallel::local_branch(global_cache_proxy_))
718 : .start_phase(current_phase_);
719 : });
720 : CkStartQD(CkCallback(CkIndex_Main<Metavariables>::execute_next_phase(),
721 : this->thisProxy));
722 : }
723 :
724 : template <typename Metavariables>
725 : void Main<Metavariables>::execute_next_phase() {
726 : if (not exception_messages_.empty()) {
727 : // Print exceptions whether we errored during execution or cleanup
728 : Parallel::printf(
729 : "\n\n###############################\n"
730 : "The following exceptions were reported during the phase: %s\n",
731 : current_phase_);
732 : for (const std::string& exception_message : exception_messages_) {
733 : Parallel::printf("%s\n\n", exception_message);
734 : }
735 : exception_messages_.clear();
736 : Parallel::printf(
737 : "To determine where an exception is thrown, run gdb and do\n"
738 : "catch throw EXCEPTION_TYPE\n"
739 : "run\n"
740 : "where EXCEPTION_TYPE is the Type of the exception above.\n"
741 : "You may have to type `continue` to skip some option parser\n"
742 : "exceptions until you get to the one you care about\n"
743 : "You may also have to type `up` or `down` to go up and down\n"
744 : "the function calls in order to find a useful line number.\n\n");
745 :
746 : // Errored during cleanup. Can't have this, so just abort
747 : if (current_phase_ == Parallel::Phase::PostFailureCleanup) {
748 : Parallel::printf(
749 : "Received termination while cleaning up a previous termination. This "
750 : "is cyclic behavior and cannot be supported. Cleanup must exit "
751 : "cleanly without errors.");
752 : sys::abort("");
753 : }
754 :
755 : // Errored during execution. Go to cleanup
756 : current_phase_ = Parallel::Phase::PostFailureCleanup;
757 : Parallel::printf("Entering phase: %s at time %s\n", current_phase_,
758 : sys::pretty_wall_time());
759 : } else {
760 : if (Parallel::Phase::Exit == current_phase_) {
761 : ERROR("Current phase is Exit, but program did not exit!");
762 : }
763 :
764 : if (current_phase_ == Parallel::Phase::PostFailureCleanup) {
765 : Parallel::printf("PostFailureCleanup phase complete. Aborting.\n");
766 : Informer::print_exit_info();
767 : sys::abort("");
768 : }
769 :
770 : const auto next_phase = PhaseControl::arbitrate_phase_change(
771 : make_not_null(&phase_change_decision_data_), current_phase_,
772 : *Parallel::local_branch(global_cache_proxy_));
773 : if (next_phase.has_value()) {
774 : // Only print info if there was an actual phase change.
775 : if (current_phase_ != next_phase.value()) {
776 : Parallel::printf("Entering phase from phase control: %s at time %s\n",
777 : next_phase.value(), sys::pretty_wall_time());
778 : current_phase_ = next_phase.value();
779 : }
780 : } else {
781 : if (current_phase_ ==
782 : Parallel::Phase::UpdateOptionsAtRestartFromCheckpoint) {
783 : ERROR(
784 : "Failed to find the next phase after overlaying options during a "
785 : "restart. This is caused by removing the phase change that caused "
786 : "the restart.");
787 : }
788 : const auto& default_order = Metavariables::default_phase_order;
789 : auto it = alg::find(default_order, current_phase_);
790 : using ::operator<<;
791 : if (it == std::end(default_order)) {
792 : ERROR("Cannot determine next phase as '"
793 : << current_phase_
794 : << "' is not in Metavariables::default_phase_order "
795 : << default_order << "\n");
796 : }
797 : if (std::next(it) == std::end(default_order)) {
798 : ERROR("Cannot determine next phase as '"
799 : << current_phase_
800 : << "' is last in Metavariables::default_phase_order "
801 : << default_order << "\n");
802 : }
803 : current_phase_ = *std::next(it);
804 :
805 : Parallel::printf("Entering phase: %s at time %s\n", current_phase_,
806 : sys::pretty_wall_time());
807 : }
808 : }
809 :
810 : if (Parallel::Phase::Exit == current_phase_) {
811 : check_if_component_terminated_correctly();
812 : #ifdef SPECTRE_KOKKOS
813 : Kokkos::finalize();
814 : #endif // SPECTRE_KOKKOS
815 : return;
816 : }
817 : tmpl::for_each<component_list>([this](auto parallel_component) {
818 : tmpl::type_from<decltype(parallel_component)>::execute_next_phase(
819 : current_phase_, global_cache_proxy_);
820 : });
821 :
822 : // Here we handle phases with direct Charm++ calls. By handling these phases
823 : // after calling each component's execute_next_phase entry method, we ensure
824 : // that each component knows what phase it is in. This is useful for pup
825 : // functions that need special handling that depends on the phase.
826 : //
827 : // Note that in future versions of Charm++ it may become possible for pup
828 : // functions to have knowledge of the migration type. At that point, it
829 : // should no longer be necessary to wait until after
830 : // component::execute_next_phase to make the direct charm calls. Instead, the
831 : // load balance or checkpoint work could be initiated *before* the call to
832 : // component::execute_next_phase and *without* the need for a quiescence
833 : // detection. This may be a slight optimization.
834 : if (current_phase_ == Parallel::Phase::LoadBalancing) {
835 : CkStartQD(CkCallback(CkIndex_Main<Metavariables>::start_load_balance(),
836 : this->thisProxy));
837 : return;
838 : }
839 : if (current_phase_ == Parallel::Phase::WriteCheckpoint) {
840 : CkStartQD(CkCallback(CkIndex_Main<Metavariables>::start_write_checkpoint(),
841 : this->thisProxy));
842 : return;
843 : }
844 :
845 : // We skip the reparsing and overlaying if there are no eligible tags
846 : if constexpr (tmpl::size<overlayable_option_list>::value > 0) {
847 : // Make sure we only update the options the first time we run the phase
848 : // UpdateOptionsAtRestartFromCheckpoint after restoring from checkpoint.
849 : // Else, the code might try to update options each time the phase was
850 : // encountered, even if there was no checkpoint/restart...
851 : if (just_restored_from_checkpoint_ and
852 : current_phase_ ==
853 : Parallel::Phase::UpdateOptionsAtRestartFromCheckpoint) {
854 : update_const_global_cache_from_input_file();
855 : just_restored_from_checkpoint_ = false;
856 : }
857 : }
858 :
859 : // The general case simply returns to execute_next_phase
860 : CkStartQD(CkCallback(CkIndex_Main<Metavariables>::execute_next_phase(),
861 : this->thisProxy));
862 : }
863 :
864 : template <typename Metavariables>
865 : void Main<Metavariables>::start_load_balance() {
866 : at_sync_indicator_proxy_.IndicateAtSync();
867 : // No need for a callback to return to execute_next_phase: this is done by
868 : // ResumeFromSync instead.
869 : }
870 :
871 : template <typename Metavariables>
872 : void Main<Metavariables>::start_write_checkpoint() {
873 : // Reset the counter if the checkpoints directory does not exist.
874 : // This happens when the simulation continues in a new segment.
875 : const auto [checkpoints_dir, prefix, pad] =
876 : detail::checkpoints_dir_prefix_pad();
877 : if (not file_system::check_if_dir_exists(checkpoints_dir)) {
878 : checkpoint_dir_counter_ = 0;
879 : }
880 : const std::string dir = detail::next_checkpoint_dir(checkpoint_dir_counter_);
881 : checkpoint_dir_counter_++;
882 : file_system::create_directory(dir);
883 : CkStartCheckpoint(
884 : dir.c_str(), CkCallback(CkIndex_Main<Metavariables>::execute_next_phase(),
885 : this->thisProxy));
886 : }
887 :
888 : template <typename Metavariables>
889 : template <typename InvokeCombine, typename... Tags>
890 : void Main<Metavariables>::phase_change_reduction(
891 : ReductionData<ReductionDatum<tuples::TaggedTuple<Tags...>, InvokeCombine,
892 : funcl::Identity, std::index_sequence<>>>
893 : reduction_data) {
894 : using tagged_tuple_type = std::decay_t<
895 : std::tuple_element_t<0, std::decay_t<decltype(reduction_data.data())>>>;
896 : (void)Parallel::charmxx::RegisterPhaseChangeReduction<
897 : Metavariables, InvokeCombine, Tags...>::registrar;
898 : static_assert(tt::is_a_v<tuples::TaggedTuple, tagged_tuple_type>,
899 : "The main chare expects a tagged tuple in the phase change "
900 : "reduction target.");
901 : reduction_data.finalize();
902 : PhaseControl::TaggedTupleMainCombine::apply(
903 : make_not_null(&phase_change_decision_data_),
904 : get<0>(reduction_data.data()));
905 : }
906 :
907 : template <typename Metavariables>
908 : void Main<Metavariables>::add_exception_message(std::string exception_message) {
909 : exception_messages_.push_back(std::move(exception_message));
910 : auto* global_cache = Parallel::local_branch(global_cache_proxy_);
911 : ASSERT(global_cache != nullptr, "Could not retrieve the local global cache.");
912 : // Set terminate_=true on all components to cause them to stop the current
913 : // phase.
914 : tmpl::for_each<component_list>([global_cache](auto component_tag_v) {
915 : using component_tag = tmpl::type_from<decltype(component_tag_v)>;
916 : Parallel::get_parallel_component<component_tag>(*global_cache)
917 : .set_terminate(true);
918 : });
919 : }
920 :
921 : template <typename Metavariables>
922 : void Main<Metavariables>::did_all_elements_terminate(
923 : const bool all_elements_terminated) {
924 : if (not all_elements_terminated) {
925 : tmpl::for_each<component_list>([this](auto component_tag_v) {
926 : using component_tag = tmpl::type_from<decltype(component_tag_v)>;
927 : if (tmpl::index_of<component_list, component_tag>::value ==
928 : current_termination_check_index_ - 1) {
929 : components_that_did_not_terminate_.push_back(
930 : pretty_type::name<component_tag>());
931 : }
932 : });
933 : }
934 : if (current_termination_check_index_ == tmpl::size<component_list>::value) {
935 : if (not components_that_did_not_terminate_.empty()) {
936 : using ::operator<<;
937 : // Need the MakeString to avoid GCC compilation failure that it can't
938 : // print out the vector...
939 : Parallel::printf(
940 : "\n############ ERROR ############\n"
941 : "The following components did not terminate cleanly:\n"
942 : "%s\n\n"
943 : "This means the executable stopped because of a hang/deadlock.\n"
944 : "############ ERROR ############\n\n",
945 : std::string{MakeString{} << components_that_did_not_terminate_});
946 : if constexpr (detail::is_run_deadlock_analysis_simple_actions_callable_v<
947 : Metavariables, Parallel::GlobalCache<Metavariables>&,
948 : const std::vector<std::string>&>) {
949 : Parallel::printf("Starting deadlock analysis.\n");
950 : Metavariables::run_deadlock_analysis_simple_actions(
951 : *Parallel::local_branch(global_cache_proxy_),
952 : components_that_did_not_terminate_);
953 : CkStartQD(CkCallback(
954 : CkIndex_Main<Metavariables>::post_deadlock_analysis_termination(),
955 : this->thisProxy));
956 : return;
957 : } else {
958 : Parallel::printf(
959 : "No deadlock analysis function found in metavariables. To enable "
960 : "deadlock analysis via simple actions add a function:\n"
961 : " static void run_deadlock_analysis_simple_actions(\n"
962 : " Parallel::GlobalCache<metavariables>& cache,\n"
963 : " const std::vector<std::string>& deadlocked_components);\n"
964 : "to your metavariables.\n");
965 : }
966 : }
967 : post_deadlock_analysis_termination();
968 : }
969 :
970 : check_if_component_terminated_correctly();
971 : }
972 :
973 : template <typename Metavariables>
974 : void Main<Metavariables>::check_if_component_terminated_correctly() {
975 : if constexpr (tmpl::size<component_list>::value == 0) {
976 : post_deadlock_analysis_termination();
977 : }
978 : auto* global_cache = Parallel::local_branch(global_cache_proxy_);
979 : ASSERT(global_cache != nullptr, "Could not retrieve the local global cache.");
980 :
981 : tmpl::for_each<component_list>([global_cache, this](auto component_tag_v) {
982 : using component_tag = tmpl::type_from<decltype(component_tag_v)>;
983 : if (tmpl::index_of<component_list, component_tag>::value ==
984 : current_termination_check_index_) {
985 : Parallel::get_parallel_component<component_tag>(*global_cache)
986 : .contribute_termination_status_to_main();
987 : }
988 : });
989 : current_termination_check_index_++;
990 : }
991 :
992 : namespace detail {
993 : CREATE_IS_CALLABLE(execute_next_phase)
994 : CREATE_IS_CALLABLE_V(execute_next_phase)
995 : } // namespace detail
996 :
997 : template <typename Metavariables>
998 : void Main<Metavariables>::post_deadlock_analysis_termination() {
999 : // If no components deadlocked, then we just exit.
1000 : if (components_that_did_not_terminate_.empty()) {
1001 : Informer::print_exit_info();
1002 : const Parallel::ExitCode exit_code =
1003 : get<Tags::ExitCode>(phase_change_decision_data_);
1004 : sys::exit(static_cast<int>(exit_code));
1005 : } else {
1006 : // Even though we didn't raise an exception, a deadlock is still a
1007 : // failure and some components may want to run things during cleanup so
1008 : // we switch phases here and start the PostFailureCleanup phase on each
1009 : // component.
1010 : current_phase_ = Parallel::Phase::PostFailureCleanup;
1011 : Parallel::printf("Entering phase: %s at time %s\n", current_phase_,
1012 : sys::pretty_wall_time());
1013 : tmpl::for_each<component_list>([this](auto component_tag_v) {
1014 : using component_tag = tmpl::type_from<decltype(component_tag_v)>;
1015 : // If a component deadlocked, then the terminate_ flag in the
1016 : // DistributedObject isn't correct and we can't start a new phase.
1017 : // Therefore, we have to manually set it so that it thinks it terminated
1018 : // cleanly.
1019 : if (alg::found(components_that_did_not_terminate_,
1020 : pretty_type::name<component_tag>())) {
1021 : Parallel::printf("Setting terminate to true on %s\n",
1022 : pretty_type::name<component_tag>());
1023 : }
1024 : if constexpr (detail::is_execute_next_phase_callable_v<
1025 : component_tag, Parallel::Phase,
1026 : CProxy_GlobalCache<Metavariables>, bool>) {
1027 : component_tag::execute_next_phase(current_phase_, global_cache_proxy_,
1028 : true);
1029 : } else {
1030 : component_tag::execute_next_phase(current_phase_, global_cache_proxy_);
1031 : }
1032 : });
1033 : // As a callback, we set execute_next_phase() because it will just
1034 : // immediately exit after the PostFailureCleanup phase is complete which is
1035 : // what we want
1036 : CkStartQD(CkCallback(CkIndex_Main<Metavariables>::execute_next_phase(),
1037 : this->thisProxy));
1038 : }
1039 : }
1040 :
1041 : template <typename Metavariables>
1042 : void Main<Metavariables>::update_const_global_cache_from_input_file() {
1043 : if (checkpoint_dir_counter_ < 1) {
1044 : ERROR("Executable is unaware of previous checkpoints, so can't reparse.");
1045 : }
1046 :
1047 : // Get the padded counter (e.g., 000XYZ) of the checkpoint we restarted from
1048 : const size_t restart_checkpoint = checkpoint_dir_counter_ - 1;
1049 : const std::string counter = std::to_string(restart_checkpoint);
1050 : const auto [checkpoints_dir, prefix, pad] =
1051 : detail::checkpoints_dir_prefix_pad();
1052 : const std::string padded_counter =
1053 : std::string(pad - counter.size(), '0').append(counter);
1054 : (void)checkpoints_dir;
1055 : (void)prefix;
1056 :
1057 : // Given input file "Input.yaml", overlay file is "Input.overlay_WXYZ.yaml"
1058 : const std::string dot_yaml = ".yaml";
1059 : const std::string overlay_counter = ".overlay_" + padded_counter;
1060 : const auto found = input_file_.find(dot_yaml);
1061 : std::string input_file_for_reparse = input_file_;
1062 : if (found != std::string::npos) {
1063 : input_file_for_reparse.insert(found, overlay_counter);
1064 : } else {
1065 : ERROR("Overlaying assumes the input file has a .yaml extension");
1066 : }
1067 :
1068 : Parallel::printf("Attempting to overlay input file: %s\n",
1069 : input_file_for_reparse);
1070 : if (file_system::check_if_file_exists(input_file_for_reparse)) {
1071 : parser_.template overlay_file<overlayable_option_list>(
1072 : input_file_for_reparse);
1073 : const auto data_to_overlay =
1074 : parser_.template apply<overlayable_option_list,
1075 : Metavariables>([](auto... args) {
1076 : return tuples::tagged_tuple_from_typelist<overlayable_option_list>(
1077 : std::move(args)...);
1078 : });
1079 : global_cache_proxy_.overlay_cache_data(data_to_overlay);
1080 : Parallel::printf("... success!\n");
1081 : } else {
1082 : Parallel::printf(
1083 : "... file not found. Continuing with values from previous run.\n");
1084 : }
1085 : }
1086 :
1087 : } // namespace Parallel
1088 :
1089 0 : #define CK_TEMPLATES_ONLY
1090 : #include "Parallel/Main.def.h"
1091 : #undef CK_TEMPLATES_ONLY
|
