Line data Source code
1 0 : // Distributed under the MIT License.
2 : // See LICENSE.txt for details.
3 :
4 : #pragma once
5 :
6 : #include <limits>
7 : #include <optional>
8 : #include <pup.h>
9 : #include <string>
10 : #include <type_traits>
11 : #include <utility>
12 :
13 : #include "DataStructures/TaggedTuple.hpp"
14 : #include "Options/Auto.hpp"
15 : #include "Options/Options.hpp"
16 : #include "Parallel/AlgorithmMetafunctions.hpp"
17 : #include "Parallel/ExitCode.hpp"
18 : #include "Parallel/GlobalCache.hpp"
19 : #include "Parallel/Phase.hpp"
20 : #include "Parallel/PhaseControl/ContributeToPhaseChangeReduction.hpp"
21 : #include "Parallel/PhaseControl/PhaseChange.hpp"
22 : #include "Utilities/ErrorHandling/Assert.hpp"
23 : #include "Utilities/Functional.hpp"
24 : #include "Utilities/Serialization/CharmPupable.hpp"
25 : #include "Utilities/System/ParallelInfo.hpp"
26 : #include "Utilities/TMPL.hpp"
27 :
28 1 : namespace PhaseControl {
29 :
30 0 : namespace Tags {
31 : /// Storage in the phase change decision tuple so that the Main chare can record
32 : /// the phase to go to when restarting the run from a checkpoint file.
33 : ///
34 : /// \note This tag is not intended to participate in any of the reduction
35 : /// procedures, so will error if the combine method is called.
36 1 : struct RestartPhase {
37 0 : using type = std::optional<Parallel::Phase>;
38 :
39 0 : struct combine_method {
40 0 : [[noreturn]] std::optional<Parallel::Phase> operator()(
41 : const std::optional<Parallel::Phase> /*first_phase*/,
42 : const std::optional<Parallel::Phase>& /*second_phase*/);
43 : };
44 :
45 0 : using main_combine_method = combine_method;
46 : };
47 :
48 : /// Stores whether the checkpoint and exit has been requested.
49 : ///
50 : /// Combinations are performed via `funcl::Or`, as the phase in question should
51 : /// be chosen if any component requests the jump.
52 1 : struct CheckpointAndExitRequested {
53 0 : using type = bool;
54 :
55 0 : using combine_method = funcl::Or<>;
56 0 : using main_combine_method = funcl::Or<>;
57 : };
58 :
59 : } // namespace Tags
60 :
61 : /*!
62 : * \brief Phase control object that runs the WriteCheckpoint and Exit phases
63 : * after a specified amount of wallclock time has elapsed.
64 : *
65 : * When the executable exits from here, it does so with
66 : * `Parallel::ExitCode::ContinueFromCheckpoint`.
67 : *
68 : * This phase control is useful for running SpECTRE executables performing
69 : * lengthy computations that may exceed a supercomputer's wallclock limits.
70 : * Writing a single checkpoint at the end of the job's allocated time allows
71 : * the computation to be continued, while minimizing the disc space taken up by
72 : * checkpoint files.
73 : *
74 : * Note that this phase control is not a trigger on wallclock time. Rather,
75 : * it checks the elapsed wallclock time when called, likely from a global sync
76 : * point triggered by some other mechanism, e.g., at some slab boundary.
77 : * Therefore, the WriteCheckpoint and Exit phases will run the first time
78 : * this phase control is called after the specified wallclock time has been
79 : * reached.
80 : *
81 : * \warning the global sync points _must_ be triggered often enough to ensure
82 : * there will be at least one sync point (i.e., one call to this phase control)
83 : * in the window between the requested checkpoint-and-exit time and the time at
84 : * which the batch system will kill the executable. To make this more concrete,
85 : * consider this example: when running on a 12-hour queue with a
86 : * checkpoint-and-exit requested after 11.5 hours, there is a 0.5-hour window
87 : * for a global sync to occur, the checkpoint files to be written to disc, and
88 : * the executable to clean up. In this case, triggering a global sync every
89 : * 2-10 minutes might be desirable. Matching the global sync frequency with the
90 : * time window for checkpoint and exit is the responsibility of the user!
91 : *
92 : * \parblock
93 : * \warning If modifying the phase-change logic on a
94 : * checkpoint-restart, this PhaseChange must remain in the list after
95 : * modification so that the end of the restart logic will run. The
96 : * WallclockHours can be changed to None to disable further restarts.
97 : * \endparblock
98 : */
99 1 : struct CheckpointAndExitAfterWallclock : public PhaseChange {
100 0 : CheckpointAndExitAfterWallclock(const std::optional<double> wallclock_hours,
101 : const Options::Context& context = {});
102 :
103 0 : explicit CheckpointAndExitAfterWallclock(CkMigrateMessage* msg);
104 :
105 : /// \cond
106 : CheckpointAndExitAfterWallclock() = default;
107 : using PUP::able::register_constructor;
108 : WRAPPED_PUPable_decl_template(CheckpointAndExitAfterWallclock); // NOLINT
109 : /// \endcond
110 :
111 0 : struct WallclockHours {
112 0 : using type = Options::Auto<double, Options::AutoLabel::None>;
113 0 : static constexpr Options::String help = {
114 : "Time in hours after which to write the checkpoint and exit. "
115 : "If 'None' is specified, no action will be taken."};
116 : };
117 :
118 0 : using options = tmpl::list<WallclockHours>;
119 0 : static constexpr Options::String help{
120 : "Once the wallclock time has exceeded the specified amount, trigger "
121 : "writing a checkpoint and then exit with the 'ContinueFromCheckpoint' "
122 : "exit code."};
123 :
124 0 : using argument_tags = tmpl::list<>;
125 0 : using return_tags = tmpl::list<>;
126 :
127 0 : using phase_change_tags_and_combines =
128 : tmpl::list<Tags::RestartPhase, Tags::CheckpointAndExitRequested>;
129 :
130 : template <typename Metavariables>
131 0 : using participating_components = typename Metavariables::component_list;
132 :
133 : template <typename... DecisionTags>
134 0 : void initialize_phase_data_impl(
135 : const gsl::not_null<tuples::TaggedTuple<DecisionTags...>*>
136 : phase_change_decision_data) const;
137 :
138 : template <typename ParallelComponent, typename ArrayIndex,
139 : typename Metavariables>
140 0 : void contribute_phase_data_impl(Parallel::GlobalCache<Metavariables>& cache,
141 : const ArrayIndex& array_index) const;
142 :
143 : template <typename... DecisionTags, typename Metavariables>
144 : typename std::optional<std::pair<Parallel::Phase, ArbitrationStrategy>>
145 0 : arbitrate_phase_change_impl(
146 : const gsl::not_null<tuples::TaggedTuple<DecisionTags...>*>
147 : phase_change_decision_data,
148 : const Parallel::Phase current_phase,
149 : const Parallel::GlobalCache<Metavariables>& /*cache*/) const;
150 :
151 0 : void pup(PUP::er& p) override;
152 :
153 : private:
154 0 : std::optional<double> wallclock_hours_for_checkpoint_and_exit_ = std::nullopt;
155 : // This flag is set during arbitration when the class decides to
156 : // halt the run. As it is not checkpointed, this distinguishes the
157 : // state immediately after writing the checkpoint from that
158 : // immediately after reading it during the restart.
159 : //
160 : // Phase arbitration is only run from Main, so there are no
161 : // threading issues here.
162 : // NOLINTNEXTLINE(spectre-mutable)
163 0 : mutable bool halting_ = false;
164 : };
165 :
166 : template <typename... DecisionTags>
167 : void CheckpointAndExitAfterWallclock::initialize_phase_data_impl(
168 : const gsl::not_null<tuples::TaggedTuple<DecisionTags...>*>
169 : phase_change_decision_data) const {
170 : tuples::get<Tags::RestartPhase>(*phase_change_decision_data) = std::nullopt;
171 : tuples::get<Tags::CheckpointAndExitRequested>(*phase_change_decision_data) =
172 : false;
173 : }
174 :
175 : template <typename ParallelComponent, typename ArrayIndex,
176 : typename Metavariables>
177 : void CheckpointAndExitAfterWallclock::contribute_phase_data_impl(
178 : Parallel::GlobalCache<Metavariables>& cache,
179 : const ArrayIndex& array_index) const {
180 : if constexpr (std::is_same_v<typename ParallelComponent::chare_type,
181 : Parallel::Algorithms::Array>) {
182 : Parallel::contribute_to_phase_change_reduction<ParallelComponent>(
183 : tuples::TaggedTuple<Tags::CheckpointAndExitRequested>{true}, cache,
184 : array_index);
185 : } else {
186 : Parallel::contribute_to_phase_change_reduction<ParallelComponent>(
187 : tuples::TaggedTuple<Tags::CheckpointAndExitRequested>{true}, cache);
188 : }
189 : }
190 :
191 : template <typename... DecisionTags, typename Metavariables>
192 : typename std::optional<std::pair<Parallel::Phase, ArbitrationStrategy>>
193 : CheckpointAndExitAfterWallclock::arbitrate_phase_change_impl(
194 : const gsl::not_null<tuples::TaggedTuple<DecisionTags...>*>
195 : phase_change_decision_data,
196 : const Parallel::Phase current_phase,
197 : const Parallel::GlobalCache<Metavariables>& /*cache*/) const {
198 : const double elapsed_hours = sys::wall_time() / 3600.0;
199 :
200 : auto& restart_phase =
201 : tuples::get<Tags::RestartPhase>(*phase_change_decision_data);
202 : auto& exit_code =
203 : tuples::get<Parallel::Tags::ExitCode>(*phase_change_decision_data);
204 : if (restart_phase.has_value()) {
205 : // This `if` branch, where restart_phase has a value, is the
206 : // post-checkpoint call to arbitrate_phase_change.
207 : if (halting_) {
208 : // Preserve restart_phase for use after restarting from the checkpoint
209 : exit_code = Parallel::ExitCode::ContinueFromCheckpoint;
210 : return std::make_pair(Parallel::Phase::Exit,
211 : ArbitrationStrategy::RunPhaseImmediately);
212 : } else {
213 : // Reset restart_phase until it is needed for the next checkpoint
214 : const auto result = restart_phase;
215 : restart_phase.reset();
216 : return std::make_pair(result.value(),
217 : ArbitrationStrategy::PermitAdditionalJumps);
218 : }
219 : }
220 :
221 : auto& checkpoint_and_exit_requested =
222 : tuples::get<Tags::CheckpointAndExitRequested>(
223 : *phase_change_decision_data);
224 : if (checkpoint_and_exit_requested) {
225 : checkpoint_and_exit_requested = false;
226 : if (elapsed_hours >= wallclock_hours_for_checkpoint_and_exit_.value_or(
227 : std::numeric_limits<double>::infinity())) {
228 : // Record phase and actual elapsed time for determining following phase
229 : restart_phase = current_phase;
230 : ASSERT(not halting_, "Halting for checkpoint recursively");
231 : halting_ = true;
232 : return std::make_pair(Parallel::Phase::WriteCheckpoint,
233 : ArbitrationStrategy::RunPhaseImmediately);
234 : }
235 : }
236 : return std::nullopt;
237 : }
238 : } // namespace PhaseControl
|
