Line data Source code
1 1 : // Distributed under the MIT License.
2 : // See LICENSE.txt for details.
3 :
4 : /// \file
5 : /// Defines the Charm++ mainchare.
6 :
7 : #pragma once
8 :
9 : #include <array>
10 : #include <boost/program_options.hpp>
11 : #include <charm++.h>
12 : #include <cstddef>
13 : #include <initializer_list>
14 : #include <pup.h>
15 : #include <string>
16 : #include <tuple>
17 : #include <type_traits>
18 :
19 : #include "DataStructures/TaggedTuple.hpp"
20 : #include "Informer/InfoFromBuild.hpp"
21 : #include "Informer/Informer.hpp"
22 : #include "Options/ParseOptions.hpp"
23 : #include "Options/Tags.hpp"
24 : #include "Parallel/AlgorithmMetafunctions.hpp"
25 : #include "Parallel/CharmRegistration.hpp"
26 : #include "Parallel/CreateFromOptions.hpp"
27 : #include "Parallel/ExitCode.hpp"
28 : #include "Parallel/GlobalCache.hpp"
29 : #include "Parallel/Local.hpp"
30 : #include "Parallel/ParallelComponentHelpers.hpp"
31 : #include "Parallel/Phase.hpp"
32 : #include "Parallel/PhaseControl/ExecutePhaseChange.hpp"
33 : #include "Parallel/PhaseControl/InitializePhaseChangeDecisionData.hpp"
34 : #include "Parallel/PhaseControl/PhaseControlTags.hpp"
35 : #include "Parallel/PhaseControlReductionHelpers.hpp"
36 : #include "Parallel/Printf/Printf.hpp"
37 : #include "Parallel/Reduction.hpp"
38 : #include "Parallel/ResourceInfo.hpp"
39 : #include "Parallel/Tags/ResourceInfo.hpp"
40 : #include "Parallel/TypeTraits.hpp"
41 : #include "Utilities/Algorithm.hpp"
42 : #include "Utilities/ErrorHandling/Error.hpp"
43 : #include "Utilities/FileSystem.hpp"
44 : #include "Utilities/Formaline.hpp"
45 : #include "Utilities/Kokkos/KokkosCore.hpp"
46 : #include "Utilities/MakeString.hpp"
47 : #include "Utilities/Overloader.hpp"
48 : #include "Utilities/StdHelpers.hpp"
49 : #include "Utilities/System/Exit.hpp"
50 : #include "Utilities/System/ParallelInfo.hpp"
51 : #include "Utilities/TMPL.hpp"
52 : #include "Utilities/TypeTraits/CreateGetTypeAliasOrDefault.hpp"
53 : #include "Utilities/TypeTraits/CreateIsCallable.hpp"
54 : #include "Utilities/UtcTime.hpp"
55 :
56 : #include "Parallel/Main.decl.h"
57 :
58 : namespace Parallel {
59 : namespace detail {
60 : CREATE_IS_CALLABLE(run_deadlock_analysis_simple_actions)
61 : CREATE_IS_CALLABLE_V(run_deadlock_analysis_simple_actions)
62 :
63 : // Return the dir name for the Charm++ checkpoints as well as the prefix for
64 : // checkpoint names and their padding. This is a "detail" function so that
65 : // these pieces can be defined in one place only.
66 : std::tuple<std::string, std::string, size_t> checkpoints_dir_prefix_pad();
67 :
68 : // Return the dir name for the next Charm++ checkpoint; check and error if
69 : // this name already exists and writing the checkpoint would be unsafe.
70 : std::string next_checkpoint_dir(size_t checkpoint_dir_counter);
71 :
72 : // Check if future checkpoint dirs are available; error if any already exist.
73 : void check_future_checkpoint_dirs_available(size_t checkpoint_dir_counter);
74 : } // namespace detail
75 :
76 : /// \ingroup ParallelGroup
77 : /// The main function of a Charm++ executable.
78 : /// See [the Parallelization documentation](group__ParallelGroup.html#details)
79 : /// for an overview of Metavariables, Phases, and parallel components.
80 : template <typename Metavariables>
81 1 : class Main : public CBase_Main<Metavariables> {
82 : public:
83 0 : using component_list = typename Metavariables::component_list;
84 0 : using const_global_cache_tags = get_const_global_cache_tags<Metavariables>;
85 0 : using mutable_global_cache_tags =
86 : get_mutable_global_cache_tags<Metavariables>;
87 :
88 0 : using phase_change_tags_and_combines_list =
89 : PhaseControl::get_phase_change_tags<Metavariables>;
90 : /// \cond HIDDEN_SYMBOLS
91 : /// The constructor used to register the class
92 : explicit Main(const Parallel::charmxx::
93 : MainChareRegistrationConstructor& /*used_for_reg*/) {}
94 : ~Main() override {
95 : (void)Parallel::charmxx::RegisterChare<
96 : Main<Metavariables>, CkIndex_Main<Metavariables>>::registrar;
97 : }
98 : Main(const Main&) = default;
99 : Main& operator=(const Main&) = default;
100 : Main(Main&&) = default;
101 : Main& operator=(Main&&) = default;
102 : /// \endcond
103 :
104 0 : explicit Main(CkArgMsg* msg);
105 0 : explicit Main(CkMigrateMessage* msg);
106 :
107 : // NOLINTNEXTLINE(google-runtime-references)
108 0 : void pup(PUP::er& p) override;
109 :
110 : /// Allocate singleton components and the initial elements of array
111 : /// components, then execute the initialization phase on each component
112 1 : void allocate_remaining_components_and_execute_initialization_phase();
113 :
114 : /// Determine the next phase of the simulation and execute it.
115 1 : void execute_next_phase();
116 :
117 : /// Place the Charm++ call that starts load balancing
118 : ///
119 : /// \details This call is wrapped within an entry method so that it may be
120 : /// used as the callback after a quiescence detection.
121 1 : void start_load_balance();
122 :
123 : /// Place the Charm++ call that starts writing a checkpoint
124 : /// Reset the checkpoint counter to zero if the checkpoints directory does not
125 : /// exist (this happens when the simulation continues in a new segment).
126 : ///
127 : /// \details This call is wrapped within an entry method so that it may be
128 : /// used as the callback after a quiescence detection.
129 1 : void start_write_checkpoint();
130 :
131 : /// Reduction target for data used in phase change decisions.
132 : ///
133 : /// It is required that the `Parallel::ReductionData` holds a single
134 : /// `tuples::TaggedTuple`.
135 : template <typename InvokeCombine, typename... Tags>
136 1 : void phase_change_reduction(
137 : ReductionData<ReductionDatum<tuples::TaggedTuple<Tags...>, InvokeCombine,
138 : funcl::Identity, std::index_sequence<>>>
139 : reduction_data);
140 :
141 : /// Add an exception to the list of exceptions. Upon a phase change we print
142 : /// all the received exceptions and exit.
143 : ///
144 : /// Upon receiving an exception all algorithms are terminated to guarantee
145 : /// quiescence occurs soon after the exception is reported.
146 1 : void add_exception_message(std::string exception_message);
147 :
148 : /// A reduction target used to determine if all the elements of the array,
149 : /// group, nodegroup, or singleton parallel components terminated
150 : /// successfully.
151 : ///
152 : /// This allows detecting deadlocks in iterable actions, but not simple or
153 : /// reduction action.
154 1 : void did_all_elements_terminate(bool all_elements_terminated);
155 :
156 : /// Prints exit info and stops the executable with failure if a deadlock was
157 : /// detected.
158 1 : void post_deadlock_analysis_termination();
159 :
160 : private:
161 : // Starts a reduction on the component specified by
162 : // the current_termination_check_index_ member variable, then increment
163 : // current_termination_check_index_
164 0 : void check_if_component_terminated_correctly();
165 :
166 : // After a restart, update the const global cache with new values constructed
167 : // from parsing an overlay input file.
168 0 : void update_const_global_cache_from_input_file();
169 :
170 : // Lists of all parallel component types
171 0 : using group_component_list =
172 : tmpl::filter<component_list, tmpl::or_<Parallel::is_group<tmpl::_1>,
173 : Parallel::is_nodegroup<tmpl::_1>>>;
174 0 : using all_array_component_list =
175 : tmpl::filter<component_list, Parallel::is_array<tmpl::_1>>;
176 0 : using non_bound_array_component_list =
177 : tmpl::filter<component_list,
178 : tmpl::and_<Parallel::is_array<tmpl::_1>,
179 : tmpl::not_<Parallel::is_bound_array<tmpl::_1>>>>;
180 0 : using bound_array_component_list =
181 : tmpl::filter<component_list,
182 : tmpl::and_<Parallel::is_array<tmpl::_1>,
183 : Parallel::is_bound_array<tmpl::_1>>>;
184 0 : using singleton_component_list =
185 : tmpl::filter<component_list, Parallel::is_singleton<tmpl::_1>>;
186 :
187 0 : using option_list = tmpl::push_front<
188 : Parallel::get_option_tags<
189 : Parallel::all_initialization_tags_t<Metavariables>, Metavariables>,
190 : Parallel::OptionTags::ResourceInfo<Metavariables>>;
191 0 : using overlayable_option_list =
192 : Parallel::get_overlayable_option_list<Metavariables>;
193 :
194 0 : Parallel::Phase current_phase_{Parallel::Phase::Initialization};
195 0 : CProxy_GlobalCache<Metavariables> global_cache_proxy_;
196 0 : detail::CProxy_AtSyncIndicator<Metavariables> at_sync_indicator_proxy_;
197 0 : std::string input_file_{};
198 : Options::Parser<tmpl::remove<option_list, Options::Tags::InputSource>>
199 0 : parser_{Metavariables::help};
200 : // This is only used during startup, and will be cleared after all
201 : // the chares are created. It is a member variable because passing
202 : // local state through charm callbacks is painful.
203 0 : tuples::tagged_tuple_from_typelist<option_list> options_{};
204 : // type to be determined by the collection of available phase changers in the
205 : // Metavariables
206 : tuples::tagged_tuple_from_typelist<phase_change_tags_and_combines_list>
207 0 : phase_change_decision_data_;
208 0 : size_t checkpoint_dir_counter_ = 0_st;
209 0 : Parallel::ResourceInfo<Metavariables> resource_info_{};
210 : // All exception errors we've received so far.
211 0 : std::vector<std::string> exception_messages_{};
212 : // Used to keep track of which parallel component we are checking has
213 : // successfully terminated.
214 0 : size_t current_termination_check_index_{0};
215 0 : std::vector<std::string> components_that_did_not_terminate_{};
216 0 : bool just_restored_from_checkpoint_ = false;
217 : };
218 :
219 : namespace detail {
220 :
221 : // Charm++ AtSync effectively requires an additional global sync to the
222 : // quiescence detection we do for switching phases. However, AtSync only needs
223 : // to be called for one array to trigger the sync-based load balancing, so the
224 : // AtSyncIndicator is a silly hack to have a centralized indication to start
225 : // load-balancing. It participates in the `AtSync` barrier, but is not
226 : // migratable, and should only be constructed by the `Main` chare on the same
227 : // processor as the `Main` chare. When load-balancing occurs, main invokes the
228 : // member function `IndicateAtSync()`, and when `ResumeFromSync()` is called
229 : // from charm, `AtSyncIndicator` simply passes control back to the Main chare
230 : // via `execute_next_phase()`.
231 : template <typename Metavariables>
232 : class AtSyncIndicator : public CBase_AtSyncIndicator<Metavariables> {
233 : public:
234 : AtSyncIndicator(CProxy_Main<Metavariables> main_proxy);
235 : AtSyncIndicator(const AtSyncIndicator&) = default;
236 : AtSyncIndicator& operator=(const AtSyncIndicator&) = default;
237 : AtSyncIndicator(AtSyncIndicator&&) = default;
238 : AtSyncIndicator& operator=(AtSyncIndicator&&) = default;
239 : ~AtSyncIndicator() override {
240 : (void)Parallel::charmxx::RegisterChare<
241 : AtSyncIndicator<Metavariables>,
242 : CkIndex_AtSyncIndicator<Metavariables>>::registrar;
243 : }
244 :
245 : void IndicateAtSync();
246 :
247 : void ResumeFromSync() override;
248 :
249 : explicit AtSyncIndicator(CkMigrateMessage* msg)
250 : : CBase_AtSyncIndicator<Metavariables>(msg) {}
251 :
252 : void pup(PUP::er& p) override { p | main_proxy_; }
253 :
254 : private:
255 : CProxy_Main<Metavariables> main_proxy_;
256 : };
257 :
258 : template <typename Metavariables>
259 : AtSyncIndicator<Metavariables>::AtSyncIndicator(
260 : CProxy_Main<Metavariables> main_proxy)
261 : : main_proxy_{main_proxy} {
262 : this->usesAtSync = true;
263 : this->setMigratable(false);
264 : }
265 :
266 : template <typename Metavariables>
267 : void AtSyncIndicator<Metavariables>::IndicateAtSync() {
268 : this->AtSync();
269 : }
270 :
271 : template <typename Metavariables>
272 : void AtSyncIndicator<Metavariables>::ResumeFromSync() {
273 : main_proxy_.execute_next_phase();
274 : }
275 : } // namespace detail
276 :
277 : // ================================================================
278 :
279 : template <typename Metavariables>
280 : Main<Metavariables>::Main(CkArgMsg* msg) {
281 : #ifdef SPECTRE_KOKKOS
282 : Kokkos::initialize(msg->argc, msg->argv);
283 : #endif // SPECTRE_KOKKOS
284 :
285 : Informer::print_startup_info(msg);
286 :
287 : /// \todo detail::register_events_to_trace();
288 :
289 : namespace bpo = boost::program_options;
290 : try {
291 : bpo::options_description command_line_options;
292 : // disable clang-format because it combines the repeated call operator
293 : // invocations making the code more difficult to parse.
294 : // clang-format off
295 : command_line_options.add_options()
296 : ("help,h", "Describe program options")
297 : ("check-options", "Check input file options")
298 : ("dump-source-tree-as", bpo::value<std::string>(),
299 : "If specified, then a gzip archive of the source tree is dumped "
300 : "with the specified name. The archive can be expanded using "
301 : "'tar -xzf ARCHIVE.tar.gz'")
302 : ("dump-paths",
303 : "Dump the PATH, CPATH, LD_LIBRARY_PATH, LIBRARY_PATH, and "
304 : "CMAKE_PREFIX_PATH at compile time.")
305 : ("dump-environment",
306 : "Dump the result of printenv at compile time.")
307 : ("dump-build-info",
308 : "Dump the contents of SpECTRE's BuildInfo.txt")
309 : ("dump-only",
310 : "Exit after dumping requested information.")
311 : ("copyright-and-licenses",
312 : "Returns all of the copyright and license info for SpECTRE and "
313 : "its dependencies.")
314 : ;
315 : // clang-format on
316 :
317 : // False if there are no other options besides the explicitly added
318 : // Parallel::OptionTags::ResourceInfo<Metavariables>,
319 : constexpr bool has_options = tmpl::size<option_list>::value > 1;
320 : // Add input-file option if it makes sense
321 : Overloader{
322 : [&command_line_options](std::true_type /*meta*/, auto mv,
323 : int /*gcc_bug*/)
324 : -> std::void_t<
325 : decltype(tmpl::type_from<decltype(mv)>::input_file)> {
326 : // Metavariables has options and default input file name
327 : command_line_options.add_options()(
328 : "input-file",
329 : bpo::value<std::string>()->default_value(
330 : tmpl::type_from<decltype(mv)>::input_file),
331 : "Input file name");
332 : },
333 : [&command_line_options](std::true_type /*meta*/, auto /*mv*/,
334 : auto... /*unused*/) {
335 : // Metavariables has options and no default input file name
336 : command_line_options.add_options()(
337 : "input-file", bpo::value<std::string>(), "Input file name");
338 : },
339 : [](std::false_type /*meta*/, auto mv, int /*gcc_bug*/)
340 : -> std::void_t<
341 : decltype(tmpl::type_from<decltype(mv)>::input_file)> {
342 : // Metavariables has no options and default input file name
343 :
344 : // always false, but must depend on mv
345 : static_assert(std::is_same_v<decltype(mv), void>,
346 : "Metavariables supplies input file name, "
347 : "but there are no options");
348 : ERROR("This should have failed at compile time");
349 : },
350 : [](std::false_type /*meta*/, auto... /*unused*/) {
351 : // Metavariables has no options and no default input file name
352 : }}(std::bool_constant<has_options>{}, tmpl::type_<Metavariables>{}, 0);
353 :
354 : bpo::command_line_parser command_line_parser(msg->argc, msg->argv);
355 : command_line_parser.options(command_line_options);
356 :
357 : const bool ignore_unrecognized_command_line_options = Overloader{
358 : [](auto mv, int /*gcc_bug*/)
359 : -> decltype(tmpl::type_from<decltype(mv)>::
360 : ignore_unrecognized_command_line_options) {
361 : return tmpl::type_from<
362 : decltype(mv)>::ignore_unrecognized_command_line_options;
363 : },
364 : [](auto /*mv*/, auto... /*meta*/) { return false; }}(
365 : tmpl::type_<Metavariables>{}, 0);
366 : if (ignore_unrecognized_command_line_options) {
367 : // Allow unknown --options
368 : command_line_parser.allow_unregistered();
369 : } else {
370 : // Forbid positional parameters
371 : command_line_parser.positional({});
372 : }
373 :
374 : bpo::variables_map parsed_command_line_options;
375 : bpo::store(command_line_parser.run(), parsed_command_line_options);
376 : bpo::notify(parsed_command_line_options);
377 :
378 : if (parsed_command_line_options.count("help") != 0) {
379 : Parallel::printf("%s\n%s", command_line_options, parser_.help());
380 : sys::exit();
381 : }
382 :
383 : if (parsed_command_line_options.count("dump-source-tree-as") != 0) {
384 : formaline::write_to_file(
385 : parsed_command_line_options["dump-source-tree-as"].as<std::string>());
386 : Parallel::printf("Dumping archive of source tree at link time.\n");
387 : }
388 : if (parsed_command_line_options.count("dump-paths") != 0) {
389 : Parallel::printf("Paths at link time were:\n%s\n",
390 : formaline::get_paths());
391 : }
392 : if (parsed_command_line_options.count("dump-environment") != 0) {
393 : Parallel::printf("Environment variables at link time were:\n%s\n",
394 : formaline::get_environment_variables());
395 : }
396 : if (parsed_command_line_options.count("dump-build-info") != 0) {
397 : Parallel::printf("BuildInfo.txt at link time was:\n%s\n",
398 : formaline::get_build_info());
399 : }
400 : if (parsed_command_line_options.count("copyright-and-licenses") != 0) {
401 : Parallel::printf("%s\n", copyright_and_license_info());
402 : }
403 : if (parsed_command_line_options.count("dump-only") != 0) {
404 : sys::exit();
405 : }
406 :
407 : if (has_options) {
408 : if (parsed_command_line_options.count("input-file") == 0) {
409 : ERROR_NO_TRACE("No default input file name. Pass --input-file.");
410 : }
411 : input_file_ = parsed_command_line_options["input-file"].as<std::string>();
412 : parser_.parse_file(input_file_);
413 : } else {
414 : if constexpr (tmpl::size<singleton_component_list>::value > 0) {
415 : parser_.parse(
416 : "ResourceInfo:\n"
417 : " AvoidGlobalProc0: false\n"
418 : " Singletons: Auto\n");
419 : } else {
420 : parser_.parse(
421 : "ResourceInfo:\n"
422 : " AvoidGlobalProc0: false\n");
423 : }
424 : }
425 :
426 : if (parsed_command_line_options.count("check-options") != 0) {
427 : // Force all the options to be created.
428 : parser_.template apply<option_list, Metavariables>([](auto... args) {
429 : (void)std::initializer_list<char>{((void)args, '0')...};
430 : });
431 : if (has_options) {
432 : Parallel::printf("\n%s parsed successfully!\n", input_file_);
433 : } else {
434 : // This is still considered successful, since it means the
435 : // program would have started.
436 : Parallel::printf("\nNo options to check!\n");
437 : }
438 :
439 : // Include a check that the checkpoint dirs are available for writing as
440 : // part of checking the option parsing. Doing these checks together helps
441 : // catch more user errors before running the executable 'for real'.
442 : //
443 : // Note we don't do this check at the beginning of the Main chare
444 : // constructor because we don't _always_ want to error if checkpoint dirs
445 : // already exist. For example, running the executable with flags like
446 : // `--help` or `--dump-source-tree-as` should succeed even if checkpoints
447 : // were previously written.
448 : detail::check_future_checkpoint_dirs_available(checkpoint_dir_counter_);
449 :
450 : sys::exit();
451 : }
452 :
453 : options_ =
454 : parser_.template apply<option_list, Metavariables>([](auto... args) {
455 : return tuples::tagged_tuple_from_typelist<option_list>(
456 : std::move(args)...);
457 : });
458 :
459 : resource_info_ =
460 : tuples::get<Parallel::OptionTags::ResourceInfo<Metavariables>>(
461 : options_);
462 :
463 : Parallel::printf("\nOption parsing completed.\n");
464 : } catch (const bpo::error& e) {
465 : ERROR(e.what());
466 : }
467 :
468 : detail::check_future_checkpoint_dirs_available(checkpoint_dir_counter_);
469 :
470 : global_cache_proxy_ = CProxy_GlobalCache<Metavariables>::ckNew(
471 : Parallel::create_from_options<Metavariables>(options_,
472 : const_global_cache_tags{}),
473 : Parallel::create_from_options<Metavariables>(options_,
474 : mutable_global_cache_tags{}),
475 : this->thisProxy);
476 :
477 : // Now that the GlobalCache has been built, create the singleton map which
478 : // will be used to allocate all the singletons. We need to be careful here
479 : // because the parallel components have not been set at this point, so if we
480 : // try to Parallel::get_parallel_component here, an error will occur. This
481 : // call is OK though because build_singleton_map() only uses the parallel info
482 : // functions from the GlobalCache (like cache.number_of_procs()).
483 : resource_info_.build_singleton_map(
484 : *Parallel::local_branch(global_cache_proxy_));
485 :
486 : // Now that the singleton map has been built, set the resource info in the
487 : // GlobalCache (if the tags exist). Since this info will be constant
488 : // throughout a simulation, we opt for directly editing a const tag in the
489 : // GlobalCache before we pass it to any other parallel component rather than
490 : // having a mutable tag and using a mutate call to set it.
491 : global_cache_proxy_.set_resource_info(resource_info_);
492 :
493 : // Now that the singleton map has been built, we have to replace the
494 : // ResourceInfo that was created from options with the one that has all the
495 : // correct singleton assignments so simple tags can be created from options
496 : // with a valid ResourceInfo.
497 : get<Parallel::OptionTags::ResourceInfo<Metavariables>>(options_) =
498 : resource_info_;
499 :
500 : at_sync_indicator_proxy_ =
501 : detail::CProxy_AtSyncIndicator<Metavariables>::ckNew();
502 : at_sync_indicator_proxy_[0].insert(this->thisProxy, sys::my_proc());
503 : at_sync_indicator_proxy_.doneInserting();
504 :
505 : using parallel_component_tag_list = tmpl::transform<
506 : component_list,
507 : tmpl::bind<
508 : tmpl::type_,
509 : tmpl::bind<Parallel::proxy_from_parallel_component, tmpl::_1>>>;
510 : tuples::tagged_tuple_from_typelist<parallel_component_tag_list>
511 : the_parallel_components;
512 :
513 : // Print info on DataBox variants
514 : #ifdef SPECTRE_DEBUG
515 : Parallel::printf("\nParallel components:\n");
516 : tmpl::for_each<component_list>([](auto parallel_component_v) {
517 : using parallel_component = tmpl::type_from<decltype(parallel_component_v)>;
518 : using chare_type = typename parallel_component::chare_type;
519 : using charm_type = Parallel::charm_types_with_parameters<
520 : parallel_component, typename Parallel::get_array_index<
521 : chare_type>::template f<parallel_component>>;
522 : Parallel::printf(
523 : " %s (%s) has a DataBox with %u items.\n",
524 : pretty_type::name<parallel_component>(),
525 : pretty_type::name<chare_type>(),
526 : tmpl::size<
527 : typename charm_type::algorithm::databox_type::tags_list>::value);
528 : });
529 : Parallel::printf("\n");
530 : #endif // SPECTRE_DEBUG
531 :
532 : // Construct the group proxies with a dependency on the GlobalCache proxy
533 : CkEntryOptions global_cache_dependency;
534 : global_cache_dependency.setGroupDepID(global_cache_proxy_.ckGetGroupID());
535 :
536 : tmpl::for_each<group_component_list>([this, &the_parallel_components,
537 : &global_cache_dependency](
538 : auto parallel_component_v) {
539 : using parallel_component = tmpl::type_from<decltype(parallel_component_v)>;
540 : using ParallelComponentProxy =
541 : Parallel::proxy_from_parallel_component<parallel_component>;
542 : tuples::get<tmpl::type_<ParallelComponentProxy>>(the_parallel_components) =
543 : ParallelComponentProxy::ckNew(
544 : global_cache_proxy_,
545 : Parallel::create_from_options<Metavariables>(
546 : options_,
547 : typename parallel_component::simple_tags_from_options{}),
548 : &global_cache_dependency);
549 : });
550 :
551 : // Create proxies for empty array chares (whose elements will be created by
552 : // the allocate functions of the array components during
553 : // execute_initialization_phase)
554 : tmpl::for_each<non_bound_array_component_list>([&the_parallel_components](
555 : auto parallel_component) {
556 : using ParallelComponentProxy = Parallel::proxy_from_parallel_component<
557 : tmpl::type_from<decltype(parallel_component)>>;
558 : tuples::get<tmpl::type_<ParallelComponentProxy>>(the_parallel_components) =
559 : ParallelComponentProxy::ckNew();
560 : });
561 :
562 : // Create proxies for empty bound array chares
563 : tmpl::for_each<bound_array_component_list>([&the_parallel_components](
564 : auto parallel_component) {
565 : using ParallelComponentProxy = Parallel::proxy_from_parallel_component<
566 : tmpl::type_from<decltype(parallel_component)>>;
567 : CkArrayOptions opts;
568 : opts.bindTo(
569 : tuples::get<tmpl::type_<Parallel::proxy_from_parallel_component<
570 : typename tmpl::type_from<decltype(parallel_component)>::bind_to>>>(
571 : the_parallel_components));
572 : tuples::get<tmpl::type_<ParallelComponentProxy>>(the_parallel_components) =
573 : ParallelComponentProxy::ckNew(opts);
574 : });
575 :
576 : // Create proxies for singletons (which are single-element charm++ arrays)
577 : tmpl::for_each<singleton_component_list>([&the_parallel_components](
578 : auto parallel_component) {
579 : using ParallelComponentProxy = Parallel::proxy_from_parallel_component<
580 : tmpl::type_from<decltype(parallel_component)>>;
581 : tuples::get<tmpl::type_<ParallelComponentProxy>>(the_parallel_components) =
582 : ParallelComponentProxy::ckNew();
583 : });
584 :
585 : // Send the complete list of parallel_components to the GlobalCache on
586 : // each Charm++ node. After all nodes have finished, the callback is
587 : // executed.
588 : CkCallback callback(
589 : CkIndex_Main<Metavariables>::
590 : allocate_remaining_components_and_execute_initialization_phase(),
591 : this->thisProxy);
592 : global_cache_proxy_.set_parallel_components(the_parallel_components,
593 : callback);
594 :
595 : get<Tags::ExitCode>(phase_change_decision_data_) =
596 : Parallel::ExitCode::Complete;
597 : PhaseControl::initialize_phase_change_decision_data(
598 : make_not_null(&phase_change_decision_data_),
599 : *Parallel::local_branch(global_cache_proxy_));
600 :
601 : printer_chare = CProxy_PrinterChare::ckNew(1);
602 : printer_chare_is_set = true;
603 : }
604 :
605 : template <typename Metavariables>
606 : Main<Metavariables>::Main(CkMigrateMessage* msg)
607 : : CBase_Main<Metavariables>(msg) {}
608 :
609 : template <typename Metavariables>
610 : void Main<Metavariables>::pup(PUP::er& p) { // NOLINT
611 : p | current_phase_;
612 : p | global_cache_proxy_;
613 : p | at_sync_indicator_proxy_;
614 : p | input_file_;
615 : p | parser_;
616 : // Note: we don't serialize options_, because it's used as a temporary in
617 : // startup only (see comment in class definition).
618 : p | phase_change_decision_data_;
619 :
620 : p | checkpoint_dir_counter_;
621 : p | resource_info_;
622 : p | exception_messages_;
623 : p | current_termination_check_index_;
624 : p | components_that_did_not_terminate_;
625 : if (p.isUnpacking()) {
626 : detail::check_future_checkpoint_dirs_available(checkpoint_dir_counter_);
627 : // Main doesn't migrate unless checkpointing or restarting, so we can
628 : // indicate here that we've just restored from checkpoint.
629 : just_restored_from_checkpoint_ = true;
630 : // Initialize Kokkos on restart
631 : #ifdef SPECTRE_KOKKOS
632 : Kokkos::initialize();
633 : #endif // SPECTRE_KOKKOS
634 : }
635 :
636 : // For now we only support restarts on the same hardware configuration (same
637 : // number of nodes and same procs per node) used when writing the checkpoint.
638 : // We check this by adding counters to the pup stream.
639 : if (p.isUnpacking()) {
640 : int previous_nodes = 0;
641 : int previous_procs = 0;
642 : p | previous_nodes;
643 : p | previous_procs;
644 : if (previous_nodes != sys::number_of_nodes() or
645 : previous_procs != sys::number_of_procs()) {
646 : ERROR(
647 : "Must restart on the same hardware configuration used when writing "
648 : "the checkpoint.\n"
649 : "Checkpoint written with "
650 : << previous_nodes << " nodes, " << previous_procs
651 : << " procs.\n"
652 : "Restarted with "
653 : << sys::number_of_nodes() << " nodes, " << sys::number_of_procs()
654 : << " procs.");
655 : }
656 : } else {
657 : int current_nodes = sys::number_of_nodes();
658 : int current_procs = sys::number_of_procs();
659 : p | current_nodes;
660 : p | current_procs;
661 : }
662 : }
663 :
664 : template <typename Metavariables>
665 : void Main<Metavariables>::
666 : allocate_remaining_components_and_execute_initialization_phase() {
667 : if (current_phase_ != Parallel::Phase::Initialization) {
668 : ERROR("Must be in the Initialization phase.");
669 : }
670 : // Since singletons are actually single-element Charm++ arrays, we have to
671 : // allocate them here along with the other Charm++ arrays.
672 : tmpl::for_each<singleton_component_list>([this](auto singleton_component_v) {
673 : using singleton_component =
674 : tmpl::type_from<decltype(singleton_component_v)>;
675 : auto& local_cache = *Parallel::local_branch(global_cache_proxy_);
676 : auto& singleton_proxy =
677 : Parallel::get_parallel_component<singleton_component>(local_cache);
678 : auto options = Parallel::create_from_options<Metavariables>(
679 : options_, typename singleton_component::simple_tags_from_options{});
680 :
681 : const size_t proc = resource_info_.template proc_for<singleton_component>();
682 : singleton_proxy[0].insert(global_cache_proxy_, std::move(options), proc);
683 : singleton_proxy.doneInserting();
684 : });
685 :
686 : // These are Spectre array components built on Charm++ array chares. Each
687 : // component is in charge of allocating and distributing its elements over the
688 : // computing system.
689 : tmpl::for_each<all_array_component_list>([this](auto parallel_component_v) {
690 : using parallel_component = tmpl::type_from<decltype(parallel_component_v)>;
691 : parallel_component::allocate_array(
692 : global_cache_proxy_,
693 : Parallel::create_from_options<Metavariables>(
694 : options_, typename parallel_component::simple_tags_from_options{}),
695 : Parallel::create_from_options<Metavariables>(
696 : options_, typename parallel_component::array_allocation_tags{}),
697 : resource_info_.procs_to_ignore());
698 : });
699 :
700 : // Free any resources from the initial option parsing.
701 : options_ = decltype(options_){};
702 :
703 : tmpl::for_each<component_list>([this](auto parallel_component_v) {
704 : using parallel_component = tmpl::type_from<decltype(parallel_component_v)>;
705 : Parallel::get_parallel_component<parallel_component>(
706 : *Parallel::local_branch(global_cache_proxy_))
707 : .start_phase(current_phase_);
708 : });
709 : CkStartQD(CkCallback(CkIndex_Main<Metavariables>::execute_next_phase(),
710 : this->thisProxy));
711 : }
712 :
713 : template <typename Metavariables>
714 : void Main<Metavariables>::execute_next_phase() {
715 : if (not exception_messages_.empty()) {
716 : // Print exceptions whether we errored during execution or cleanup
717 : Parallel::printf(
718 : "\n\n###############################\n"
719 : "The following exceptions were reported during the phase: %s\n",
720 : current_phase_);
721 : for (const std::string& exception_message : exception_messages_) {
722 : Parallel::printf("%s\n\n", exception_message);
723 : }
724 : exception_messages_.clear();
725 : Parallel::printf(
726 : "To determine where an exception is thrown, run gdb and do\n"
727 : "catch throw EXCEPTION_TYPE\n"
728 : "run\n"
729 : "where EXCEPTION_TYPE is the Type of the exception above.\n"
730 : "You may have to type `continue` to skip some option parser\n"
731 : "exceptions until you get to the one you care about\n"
732 : "You may also have to type `up` or `down` to go up and down\n"
733 : "the function calls in order to find a useful line number.\n\n");
734 :
735 : // Errored during cleanup. Can't have this, so just abort
736 : if (current_phase_ == Parallel::Phase::PostFailureCleanup) {
737 : Parallel::printf(
738 : "Received termination while cleaning up a previous termination. This "
739 : "is cyclic behavior and cannot be supported. Cleanup must exit "
740 : "cleanly without errors.");
741 : sys::abort("");
742 : }
743 :
744 : // Errored during execution. Go to cleanup
745 : current_phase_ = Parallel::Phase::PostFailureCleanup;
746 : Parallel::printf("Entering phase: %s at time %s\n", current_phase_,
747 : sys::pretty_wall_time());
748 : } else {
749 : if (Parallel::Phase::Exit == current_phase_) {
750 : ERROR("Current phase is Exit, but program did not exit!");
751 : }
752 :
753 : if (current_phase_ == Parallel::Phase::PostFailureCleanup) {
754 : Parallel::printf("PostFailureCleanup phase complete. Aborting.\n");
755 : Informer::print_exit_info();
756 : sys::abort("");
757 : }
758 :
759 : std::optional<Parallel::Phase> next_phase{};
760 : if (just_restored_from_checkpoint_) {
761 : just_restored_from_checkpoint_ = false;
762 : Parallel::printf("Restarting from checkpoint. Date and time: %s\n",
763 : utc_time());
764 : next_phase.emplace(Parallel::Phase::UpdateOptionsAtRestartFromCheckpoint);
765 : } else if (current_phase_ ==
766 : Parallel::Phase::UpdateOptionsAtRestartFromCheckpoint) {
767 : next_phase.emplace(Parallel::Phase::Restart);
768 : } else {
769 : next_phase = PhaseControl::arbitrate_phase_change(
770 : make_not_null(&phase_change_decision_data_), current_phase_,
771 : *Parallel::local_branch(global_cache_proxy_));
772 : }
773 : if (next_phase.has_value()) {
774 : // Only print info if there was an actual phase change.
775 : if (current_phase_ != next_phase.value()) {
776 : Parallel::printf("Entering phase from phase control: %s at time %s\n",
777 : next_phase.value(), sys::pretty_wall_time());
778 : current_phase_ = next_phase.value();
779 : }
780 : } else {
781 : if (current_phase_ ==
782 : Parallel::Phase::UpdateOptionsAtRestartFromCheckpoint) {
783 : ERROR(
784 : "Failed to find the next phase after overlaying options during a "
785 : "restart. This is caused by removing the phase change that caused "
786 : "the restart.");
787 : }
788 : const auto& default_order = Metavariables::default_phase_order;
789 : auto it = alg::find(default_order, current_phase_);
790 : using ::operator<<;
791 : if (it == std::end(default_order)) {
792 : ERROR("Cannot determine next phase as '"
793 : << current_phase_
794 : << "' is not in Metavariables::default_phase_order "
795 : << default_order << "\n");
796 : }
797 : if (std::next(it) == std::end(default_order)) {
798 : ERROR("Cannot determine next phase as '"
799 : << current_phase_
800 : << "' is last in Metavariables::default_phase_order "
801 : << default_order << "\n");
802 : }
803 : current_phase_ = *std::next(it);
804 :
805 : Parallel::printf("Entering phase: %s at time %s\n", current_phase_,
806 : sys::pretty_wall_time());
807 : }
808 : }
809 :
810 : if (Parallel::Phase::Exit == current_phase_) {
811 : check_if_component_terminated_correctly();
812 : #ifdef SPECTRE_KOKKOS
813 : Kokkos::finalize();
814 : #endif // SPECTRE_KOKKOS
815 : return;
816 : }
817 : tmpl::for_each<component_list>([this](auto parallel_component) {
818 : tmpl::type_from<decltype(parallel_component)>::execute_next_phase(
819 : current_phase_, global_cache_proxy_);
820 : });
821 :
822 : // Here we handle phases with direct Charm++ calls. By handling these phases
823 : // after calling each component's execute_next_phase entry method, we ensure
824 : // that each component knows what phase it is in. This is useful for pup
825 : // functions that need special handling that depends on the phase.
826 : //
827 : // Note that in future versions of Charm++ it may become possible for pup
828 : // functions to have knowledge of the migration type. At that point, it
829 : // should no longer be necessary to wait until after
830 : // component::execute_next_phase to make the direct charm calls. Instead, the
831 : // load balance or checkpoint work could be initiated *before* the call to
832 : // component::execute_next_phase and *without* the need for a quiescence
833 : // detection. This may be a slight optimization.
834 : if (current_phase_ == Parallel::Phase::LoadBalancing) {
835 : CkStartQD(CkCallback(CkIndex_Main<Metavariables>::start_load_balance(),
836 : this->thisProxy));
837 : return;
838 : }
839 : if (current_phase_ == Parallel::Phase::WriteCheckpoint) {
840 : CkStartQD(CkCallback(CkIndex_Main<Metavariables>::start_write_checkpoint(),
841 : this->thisProxy));
842 : return;
843 : }
844 :
845 : // We skip the reparsing and overlaying if there are no eligible tags
846 : if constexpr (tmpl::size<overlayable_option_list>::value > 0) {
847 : if (current_phase_ ==
848 : Parallel::Phase::UpdateOptionsAtRestartFromCheckpoint) {
849 : update_const_global_cache_from_input_file();
850 : }
851 : }
852 :
853 : // The general case simply returns to execute_next_phase
854 : CkStartQD(CkCallback(CkIndex_Main<Metavariables>::execute_next_phase(),
855 : this->thisProxy));
856 : }
857 :
858 : template <typename Metavariables>
859 : void Main<Metavariables>::start_load_balance() {
860 : at_sync_indicator_proxy_.IndicateAtSync();
861 : // No need for a callback to return to execute_next_phase: this is done by
862 : // ResumeFromSync instead.
863 : }
864 :
865 : template <typename Metavariables>
866 : void Main<Metavariables>::start_write_checkpoint() {
867 : // Reset the counter if the checkpoints directory does not exist.
868 : // This happens when the simulation continues in a new segment.
869 : const auto [checkpoints_dir, prefix, pad] =
870 : detail::checkpoints_dir_prefix_pad();
871 : if (not file_system::check_if_dir_exists(checkpoints_dir)) {
872 : checkpoint_dir_counter_ = 0;
873 : }
874 : const std::string dir = detail::next_checkpoint_dir(checkpoint_dir_counter_);
875 : checkpoint_dir_counter_++;
876 : file_system::create_directory(dir);
877 : CkStartCheckpoint(
878 : dir.c_str(), CkCallback(CkIndex_Main<Metavariables>::execute_next_phase(),
879 : this->thisProxy));
880 : }
881 :
882 : template <typename Metavariables>
883 : template <typename InvokeCombine, typename... Tags>
884 : void Main<Metavariables>::phase_change_reduction(
885 : ReductionData<ReductionDatum<tuples::TaggedTuple<Tags...>, InvokeCombine,
886 : funcl::Identity, std::index_sequence<>>>
887 : reduction_data) {
888 : using tagged_tuple_type = std::decay_t<
889 : std::tuple_element_t<0, std::decay_t<decltype(reduction_data.data())>>>;
890 : (void)Parallel::charmxx::RegisterPhaseChangeReduction<
891 : Metavariables, InvokeCombine, Tags...>::registrar;
892 : static_assert(tt::is_a_v<tuples::TaggedTuple, tagged_tuple_type>,
893 : "The main chare expects a tagged tuple in the phase change "
894 : "reduction target.");
895 : reduction_data.finalize();
896 : PhaseControl::TaggedTupleMainCombine::apply(
897 : make_not_null(&phase_change_decision_data_),
898 : get<0>(reduction_data.data()));
899 : }
900 :
901 : template <typename Metavariables>
902 : void Main<Metavariables>::add_exception_message(std::string exception_message) {
903 : exception_messages_.push_back(std::move(exception_message));
904 : auto* global_cache = Parallel::local_branch(global_cache_proxy_);
905 : ASSERT(global_cache != nullptr, "Could not retrieve the local global cache.");
906 : // Set terminate_=true on all components to cause them to stop the current
907 : // phase.
908 : tmpl::for_each<component_list>([global_cache](auto component_tag_v) {
909 : using component_tag = tmpl::type_from<decltype(component_tag_v)>;
910 : Parallel::get_parallel_component<component_tag>(*global_cache)
911 : .set_terminate(true);
912 : });
913 : }
914 :
915 : template <typename Metavariables>
916 : void Main<Metavariables>::did_all_elements_terminate(
917 : const bool all_elements_terminated) {
918 : if (not all_elements_terminated) {
919 : tmpl::for_each<component_list>([this](auto component_tag_v) {
920 : using component_tag = tmpl::type_from<decltype(component_tag_v)>;
921 : if (tmpl::index_of<component_list, component_tag>::value ==
922 : current_termination_check_index_ - 1) {
923 : components_that_did_not_terminate_.push_back(
924 : pretty_type::name<component_tag>());
925 : }
926 : });
927 : }
928 : if (current_termination_check_index_ == tmpl::size<component_list>::value) {
929 : if (not components_that_did_not_terminate_.empty()) {
930 : using ::operator<<;
931 : // Need the MakeString to avoid GCC compilation failure that it can't
932 : // print out the vector...
933 : Parallel::printf(
934 : "\n############ ERROR ############\n"
935 : "The following components did not terminate cleanly:\n"
936 : "%s\n\n"
937 : "This means the executable stopped because of a hang/deadlock.\n"
938 : "############ ERROR ############\n\n",
939 : std::string{MakeString{} << components_that_did_not_terminate_});
940 : if constexpr (detail::is_run_deadlock_analysis_simple_actions_callable_v<
941 : Metavariables, Parallel::GlobalCache<Metavariables>&,
942 : const std::vector<std::string>&>) {
943 : Parallel::printf("Starting deadlock analysis.\n");
944 : Metavariables::run_deadlock_analysis_simple_actions(
945 : *Parallel::local_branch(global_cache_proxy_),
946 : components_that_did_not_terminate_);
947 : CkStartQD(CkCallback(
948 : CkIndex_Main<Metavariables>::post_deadlock_analysis_termination(),
949 : this->thisProxy));
950 : return;
951 : } else {
952 : Parallel::printf(
953 : "No deadlock analysis function found in metavariables. To enable "
954 : "deadlock analysis via simple actions add a function:\n"
955 : " static void run_deadlock_analysis_simple_actions(\n"
956 : " Parallel::GlobalCache<metavariables>& cache,\n"
957 : " const std::vector<std::string>& deadlocked_components);\n"
958 : "to your metavariables.\n");
959 : }
960 : }
961 : post_deadlock_analysis_termination();
962 : }
963 :
964 : check_if_component_terminated_correctly();
965 : }
966 :
967 : template <typename Metavariables>
968 : void Main<Metavariables>::check_if_component_terminated_correctly() {
969 : if constexpr (tmpl::size<component_list>::value == 0) {
970 : post_deadlock_analysis_termination();
971 : }
972 : auto* global_cache = Parallel::local_branch(global_cache_proxy_);
973 : ASSERT(global_cache != nullptr, "Could not retrieve the local global cache.");
974 :
975 : tmpl::for_each<component_list>([global_cache, this](auto component_tag_v) {
976 : using component_tag = tmpl::type_from<decltype(component_tag_v)>;
977 : if (tmpl::index_of<component_list, component_tag>::value ==
978 : current_termination_check_index_) {
979 : Parallel::get_parallel_component<component_tag>(*global_cache)
980 : .contribute_termination_status_to_main();
981 : }
982 : });
983 : current_termination_check_index_++;
984 : }
985 :
986 : namespace detail {
987 : CREATE_IS_CALLABLE(execute_next_phase)
988 : CREATE_IS_CALLABLE_V(execute_next_phase)
989 : } // namespace detail
990 :
991 : template <typename Metavariables>
992 : void Main<Metavariables>::post_deadlock_analysis_termination() {
993 : // If no components deadlocked, then we just exit.
994 : if (components_that_did_not_terminate_.empty()) {
995 : Informer::print_exit_info();
996 : const Parallel::ExitCode exit_code =
997 : get<Tags::ExitCode>(phase_change_decision_data_);
998 : sys::exit(static_cast<int>(exit_code));
999 : } else {
1000 : // Even though we didn't raise an exception, a deadlock is still a
1001 : // failure and some components may want to run things during cleanup so
1002 : // we switch phases here and start the PostFailureCleanup phase on each
1003 : // component.
1004 : current_phase_ = Parallel::Phase::PostFailureCleanup;
1005 : Parallel::printf("Entering phase: %s at time %s\n", current_phase_,
1006 : sys::pretty_wall_time());
1007 : tmpl::for_each<component_list>([this](auto component_tag_v) {
1008 : using component_tag = tmpl::type_from<decltype(component_tag_v)>;
1009 : // If a component deadlocked, then the terminate_ flag in the
1010 : // DistributedObject isn't correct and we can't start a new phase.
1011 : // Therefore, we have to manually set it so that it thinks it terminated
1012 : // cleanly.
1013 : if (alg::found(components_that_did_not_terminate_,
1014 : pretty_type::name<component_tag>())) {
1015 : Parallel::printf("Setting terminate to true on %s\n",
1016 : pretty_type::name<component_tag>());
1017 : }
1018 : if constexpr (detail::is_execute_next_phase_callable_v<
1019 : component_tag, Parallel::Phase,
1020 : CProxy_GlobalCache<Metavariables>, bool>) {
1021 : component_tag::execute_next_phase(current_phase_, global_cache_proxy_,
1022 : true);
1023 : } else {
1024 : component_tag::execute_next_phase(current_phase_, global_cache_proxy_);
1025 : }
1026 : });
1027 : // As a callback, we set execute_next_phase() because it will just
1028 : // immediately exit after the PostFailureCleanup phase is complete which is
1029 : // what we want
1030 : CkStartQD(CkCallback(CkIndex_Main<Metavariables>::execute_next_phase(),
1031 : this->thisProxy));
1032 : }
1033 : }
1034 :
1035 : template <typename Metavariables>
1036 : void Main<Metavariables>::update_const_global_cache_from_input_file() {
1037 : if (checkpoint_dir_counter_ < 1) {
1038 : ERROR("Executable is unaware of previous checkpoints, so can't reparse.");
1039 : }
1040 :
1041 : // Get the padded counter (e.g., 000XYZ) of the checkpoint we restarted from
1042 : const size_t restart_checkpoint = checkpoint_dir_counter_ - 1;
1043 : const std::string counter = std::to_string(restart_checkpoint);
1044 : const auto [checkpoints_dir, prefix, pad] =
1045 : detail::checkpoints_dir_prefix_pad();
1046 : const std::string padded_counter =
1047 : std::string(pad - counter.size(), '0').append(counter);
1048 : (void)checkpoints_dir;
1049 : (void)prefix;
1050 :
1051 : // Given input file "Input.yaml", overlay file is "Input.overlay_WXYZ.yaml"
1052 : const std::string dot_yaml = ".yaml";
1053 : const std::string overlay_counter = ".overlay_" + padded_counter;
1054 : const auto found = input_file_.find(dot_yaml);
1055 : std::string input_file_for_reparse = input_file_;
1056 : if (found != std::string::npos) {
1057 : input_file_for_reparse.insert(found, overlay_counter);
1058 : } else {
1059 : ERROR("Overlaying assumes the input file has a .yaml extension");
1060 : }
1061 :
1062 : Parallel::printf("Attempting to overlay input file: %s\n",
1063 : input_file_for_reparse);
1064 : if (file_system::check_if_file_exists(input_file_for_reparse)) {
1065 : parser_.template overlay_file<overlayable_option_list>(
1066 : input_file_for_reparse);
1067 : const auto updated_options =
1068 : parser_.template apply<option_list, Metavariables>([](auto... args) {
1069 : return tuples::tagged_tuple_from_typelist<option_list>(
1070 : std::move(args)...);
1071 : });
1072 : auto new_global_cache_values = Parallel::create_from_options<Metavariables>(
1073 : updated_options, Parallel::get_overlayable_tag_list<Metavariables>{});
1074 : global_cache_proxy_.overlay_cache_data(std::move(new_global_cache_values));
1075 : Parallel::printf("... success!\n");
1076 : } else {
1077 : Parallel::printf(
1078 : "... file not found. Continuing with values from previous run.\n");
1079 : }
1080 : }
1081 :
1082 : } // namespace Parallel
1083 :
1084 0 : #define CK_TEMPLATES_ONLY
1085 : #include "Parallel/Main.def.h"
1086 : #undef CK_TEMPLATES_ONLY
|
