TestingFramework.hpp
Go to the documentation of this file.
1 // Distributed under the MIT License.
2 // See LICENSE.txt for details.
3 
4 /// \file
5 /// Code to wrap or improve the Catch testing framework used for unit tests.
6 
7 #pragma once
8 
9 #include <catch.hpp>
10 #include <csignal>
11 #include <cstddef>
12 #include <iomanip>
13 #include <limits>
14 #include <sstream>
15 #include <stdexcept>
16 #include <string>
17 
18 #include "ErrorHandling/Error.hpp"
19 #include "Parallel/Abort.hpp"
20 #include "Parallel/Exit.hpp"
21 #include "Utilities/Requires.hpp"
22 #include "Utilities/TypeTraits.hpp"
23 
24 /// \cond
25 // The macro SPECTRE_TEST_REGISTER_FUNCTION is defined inside the
26 // add_test_library CMake function. It is used to make a call into a translation
27 // unit so that static variables for Catch are properly initialized.
28 #ifdef SPECTRE_TEST_REGISTER_FUNCTION
29 void SPECTRE_TEST_REGISTER_FUNCTION() noexcept {} // NOLINT
30 #endif // SPECTRE_TEST_REGISTER_FUNCTION
31 /// \endcond
32 
33 namespace TestHelpers_detail {
34 template <typename T>
35 std::string format_capture_precise(const T& t) noexcept {
37  os << std::scientific << std::setprecision(18) << t;
38  return os.str();
39 }
40 } // namespace TestHelpers_detail
41 
42 /*!
43  * \ingroup TestingFrameworkGroup
44  * \brief Alternative to Catch's CAPTURE that prints more digits.
45  */
46 #define CAPTURE_PRECISE(variable) \
47  INFO(#variable << ": " \
48  << TestHelpers_detail::format_capture_precise(variable))
49 
50 /*!
51  * \ingroup TestingFrameworkGroup
52  * \brief A replacement for Catch's TEST_CASE that silences clang-tidy warnings
53  */
54 #define SPECTRE_TEST_CASE(m, n) TEST_CASE(m, n) // NOLINT
55 
56 /*!
57  * \ingroup TestingFrameworkGroup
58  * \brief A similar to Catch's REQUIRE statement, but can be used in tests that
59  * spawn several chares with possibly complex interaction between the chares.
60  */
61 #define SPECTRE_PARALLEL_REQUIRE(expr) \
62  do { \
63  if (not(expr)) { \
64  ERROR("\nFailed comparison: " << #expr << "\nLine: " << __LINE__ \
65  << "\nFile: " << __FILE__ << "\n"); \
66  } \
67  } while (false)
68 
69 /*!
70  * \ingroup TestingFrameworkGroup
71  * \brief A similar to Catch's REQUIRE_FALSE statement, but can be used in tests
72  * that spawn several chares with possibly complex interaction between the
73  * chares.
74  */
75 #define SPECTRE_PARALLEL_REQUIRE_FALSE(expr) \
76  do { \
77  if ((expr)) { \
78  ERROR("\nFailed comparison: " << #expr << "\nLine: " << __LINE__ \
79  << "\nFile: " << __FILE__ << "\n"); \
80  } \
81  } while (false)
82 
83 /*!
84  * \ingroup TestingFrameworkGroup
85  * \brief Set a default tolerance for floating-point number comparison
86  *
87  * \details
88  * Catch's default (relative) tolerance for comparing floating-point numbers is
89  * `std::numeric_limits<float>::%epsilon() * 100`, or roughly \f$10^{-5}\f$.
90  * This tolerance is too loose for checking many scientific algorithms that
91  * rely on double precision floating-point accuracy, so we provide a tighter
92  * tighter tolerance through the `approx` static object.
93  *
94  * \example
95  * \snippet Test_TestingFramework.cpp approx_test
96  */
97 // clang-tidy: static object creation may throw exception
98 static Approx approx = // NOLINT
99  Approx::custom() // NOLINT
100  .epsilon(std::numeric_limits<double>::epsilon() * 100) // NOLINT
101  .scale(1.0); // NOLINT
102 
103 /*!
104  * \ingroup TestingFrameworkGroup
105  * \brief A wrapper around Catch's CHECK macro that checks approximate
106  * equality of entries in iterable containers. For maplike
107  * containers, keys are checked for strict equality and values are
108  * checked for approximate equality.
109  */
110 #define CHECK_ITERABLE_APPROX(a, b) \
111  do { \
112  INFO(__FILE__ ":" + std::to_string(__LINE__) + ": " #a " == " #b); \
113  check_iterable_approx<std::common_type_t< \
114  std::decay_t<decltype(a)>, std::decay_t<decltype(b)>>>::apply(a, b); \
115  } while (false)
116 
117 /*!
118  * \ingroup TestingFrameworkGroup
119  * \brief Same as `CHECK_ITERABLE_APPROX` with user-defined Approx.
120  * The third argument should be of type `Approx`.
121  */
122 #define CHECK_ITERABLE_CUSTOM_APPROX(a, b, appx) \
123  do { \
124  INFO(__FILE__ ":" + std::to_string(__LINE__) + ": " #a " == " #b); \
125  check_iterable_approx<std::common_type_t< \
126  std::decay_t<decltype(a)>, std::decay_t<decltype(b)>>>::apply(a, b, \
127  appx); \
128  } while (false)
129 
130 /// \cond HIDDEN_SYMBOLS
131 template <typename T, typename = std::nullptr_t>
132 struct check_iterable_approx {
133  // clang-tidy: non-const reference
134  static void apply(const T& a, const T& b, Approx& appx = approx) { // NOLINT
135  CAPTURE_PRECISE(a);
136  CAPTURE_PRECISE(b);
137  CHECK(a == appx(b));
138  }
139 };
140 
141 template <typename T>
142 struct check_iterable_approx<std::complex<T>, std::nullptr_t> {
143  // clang-tidy: non-const reference
144  static void apply(const std::complex<T>& a, const std::complex<T>& b,
145  Approx& appx = approx) { // NOLINT
146  check_iterable_approx<T>::apply(real(a), real(b), appx);
147  check_iterable_approx<T>::apply(imag(a), imag(b), appx);
148  }
149 };
150 
151 template <typename T>
152 struct check_iterable_approx<
153  T, Requires<not tt::is_maplike_v<T> and tt::is_iterable_v<T> and
154  not tt::is_a_v<std::unordered_set, T>>> {
155  // clang-tidy: non-const reference
156  static void apply(const T& a, const T& b, Approx& appx = approx) { // NOLINT
157  CAPTURE_PRECISE(a);
158  CAPTURE_PRECISE(b);
159  auto a_it = a.begin();
160  auto b_it = b.begin();
161  CHECK(a_it != a.end());
162  CHECK(b_it != b.end());
163  while (a_it != a.end() and b_it != b.end()) {
164  check_iterable_approx<std::decay_t<decltype(*a_it)>>::apply(*a_it, *b_it,
165  appx);
166  ++a_it;
167  ++b_it;
168  }
169  {
170  INFO("Iterable is longer in first argument than in second argument");
171  CHECK(a_it == a.end());
172  }
173  {
174  INFO("Iterable is shorter in first argument than in second argument");
175  CHECK(b_it == b.end());
176  }
177  }
178 };
179 
180 template <typename T>
181 struct check_iterable_approx<T, Requires<tt::is_a_v<std::unordered_set, T>>> {
182  // clang-tidy: non-const reference
183  static void apply(const T& a, const T& b,
184  Approx& /*appx*/ = approx) { // NOLINT
185  CAPTURE_PRECISE(a);
186  CAPTURE_PRECISE(b);
187  // Approximate comparison of unordered sets is difficult
188  CHECK(a == b);
189  }
190 };
191 
192 template <typename T>
193 struct check_iterable_approx<
194  T, Requires<tt::is_maplike_v<T> and tt::is_iterable_v<T>>> {
195  // clang-tidy: non-const reference
196  static void apply(const T& a, const T& b, Approx& appx = approx) { // NOLINT
197  CAPTURE_PRECISE(a);
198  CAPTURE_PRECISE(b);
199  for (const auto& kv : a) {
200  const auto& key = kv.first;
201  try {
202  const auto& a_value = kv.second;
203  const auto& b_value = b.at(key);
204  CAPTURE(key);
205  check_iterable_approx<std::decay_t<decltype(a_value)>>::apply(
206  a_value, b_value, appx);
207  } catch (const std::out_of_range&) {
208  INFO("Missing key in second container: " << key);
209  CHECK(false);
210  }
211  }
212 
213  for (const auto& kv : b) {
214  const auto& key = kv.first;
215  try {
216  a.at(key);
217  // We've checked that the values match above.
218  } catch (const std::out_of_range&) {
219  INFO("Missing key in first container: " << key);
220  CHECK(false);
221  }
222  }
223  }
224 };
225 
226 #define CHECK_MATRIX_APPROX(a, b) \
227  do { \
228  INFO(__FILE__ ":" + std::to_string(__LINE__) + ": " #a " == " #b); \
229  check_matrix_approx<std::common_type_t< \
230  std::decay_t<decltype(a)>, std::decay_t<decltype(b)>>>::apply(a, b); \
231  } while (false)
232 
233 #define CHECK_MATRIX_CUSTOM_APPROX(a, b, appx) \
234  do { \
235  INFO(__FILE__ ":" + std::to_string(__LINE__) + ": " #a " == " #b); \
236  check_matrix_approx<std::common_type_t< \
237  std::decay_t<decltype(a)>, std::decay_t<decltype(b)>>>::apply(a, b, \
238  appx); \
239  } while (false)
240 
241 template <typename M>
242 struct check_matrix_approx {
243  // clang-tidy: non-const reference
244  static void apply(const M& a, const M& b,
245  Approx& appx = approx) { // NOLINT
246  // This implementation is for a column-major matrix. It does not trivially
247  // generalize to a row-major matrix because the iterator
248  // `blaze::DynamicMatrix<T, SO>::cbegin(i)` traverses either a row or a
249  // column, and takes its index as argument.
250  CHECK(a.columns() == b.columns());
251  for (size_t j = 0; j < a.columns(); j++) {
252  CAPTURE_PRECISE(a);
253  CAPTURE_PRECISE(b);
254  auto a_it = a.cbegin(j);
255  auto b_it = b.cbegin(j);
256  CHECK(a_it != a.cend(j));
257  CHECK(b_it != b.cend(j));
258  while (a_it != a.cend(j) and b_it != b.cend(j)) {
259  check_iterable_approx<std::decay_t<decltype(*a_it)>>::apply(
260  *a_it, *b_it, appx);
261  ++a_it;
262  ++b_it;
263  }
264  {
265  INFO("Column " << j
266  << " of the first matrix is longer than that of the "
267  "second matrix.");
268  CHECK(a_it == a.end(j));
269  }
270  {
271  INFO("Column " << j
272  << " of the first matrix is shorter than that of the "
273  "second matrix.");
274  CHECK(b_it == b.end(j));
275  }
276  }
277  }
278 };
279 /// \endcond
280 
281 /// \cond HIDDEN_SYMBOLS
282 [[noreturn]] inline void spectre_testing_signal_handler(int /*signal*/) {
283  Parallel::exit();
284 }
285 /// \endcond
286 
287 /*!
288  * \ingroup TestingFrameworkGroup
289  * \brief Mark a test as checking a call to ERROR
290  *
291  * \details
292  * In order to properly handle aborting with Catch versions newer than 1.6.1
293  * we must install a signal handler after Catch does, which means inside the
294  * SPECTRE_TEST_CASE itself. The ERROR_TEST() macro should be the first line in
295  * the SPECTRE_TEST_CASE.
296  *
297  * \example
298  * \snippet Test_TestingFramework.cpp error_test
299  */
300 #define ERROR_TEST() \
301  do { \
302  std::signal(SIGABRT, spectre_testing_signal_handler); \
303  } while (false)
304 
305 /*!
306  * \ingroup TestingFrameworkGroup
307  * \brief Mark a test to be checking an ASSERT
308  *
309  * \details
310  * Testing error handling is just as important as testing functionality. Tests
311  * that are supposed to exit with an error must be annotated with the attribute
312  * \code
313  * // [[OutputRegex, The regex that should be found in the output]]
314  * \endcode
315  * Note that the regex only needs to be a sub-expression of the error message,
316  * that is, there are implicit wildcards before and after the string.
317  *
318  * In order to test ASSERT's properly the test must also fail for release
319  * builds. This is done by adding this macro at the beginning for the test.
320  *
321  * \example
322  * \snippet Test_Time.cpp example_of_error_test
323  */
324 #ifdef SPECTRE_DEBUG
325 #define ASSERTION_TEST() \
326  do { \
327  ERROR_TEST(); \
328  } while (false)
329 #else
330 #include "Parallel/Abort.hpp"
331 #define ASSERTION_TEST() \
332  do { \
333  ERROR_TEST(); \
334  Parallel::abort("### No ASSERT tests in release mode ###"); \
335  } while (false)
336 #endif
constexpr bool is_iterable_v
Definition: TypeTraits.hpp:591
void exit()
Exit the program normally. This should only be called once over all processors.
Definition: Exit.hpp:18
#define CAPTURE_PRECISE(variable)
Alternative to Catch&#39;s CAPTURE that prints more digits.
Definition: TestingFramework.hpp:46
Defines the type alias Requires.
constexpr auto apply(F &&f, const DataBox< BoxTags > &box, Args &&... args)
Apply the function f with argument Tags TagsList from DataBox box
Definition: DataBox.hpp:1595
constexpr bool is_a_v
Definition: TypeTraits.hpp:543
Defines function Parallel::exit.
constexpr bool is_maplike_v
Definition: TypeTraits.hpp:988
Defines function Parallel::abort.
Defines macro ERROR.
Defines type traits, some of which are future STL type_traits header.
Definition: MakeWithRandomValues.hpp:21