CheckWithRandomValues.hpp
1 // Distributed under the MIT License.
2 // See LICENSE.txt for details.
3 
4 #pragma once
5 
7 
8 // IWYU pragma: begin_exports
9 #include <boost/preprocessor/arithmetic/inc.hpp>
10 #include <boost/preprocessor/comparison/equal.hpp>
11 #include <boost/preprocessor/comparison/not_equal.hpp>
12 #include <boost/preprocessor/control/expr_iif.hpp>
13 #include <boost/preprocessor/control/if.hpp>
14 #include <boost/preprocessor/debug/assert.hpp>
15 #include <boost/preprocessor/list/adt.hpp>
16 #include <boost/preprocessor/list/fold_right.hpp>
17 #include <boost/preprocessor/list/for_each.hpp>
18 #include <boost/preprocessor/list/for_each_product.hpp>
19 #include <boost/preprocessor/list/to_tuple.hpp>
20 #include <boost/preprocessor/list/transform.hpp>
21 #include <boost/preprocessor/logical/compl.hpp>
22 #include <boost/preprocessor/logical/not.hpp>
23 #include <boost/preprocessor/punctuation/is_begin_parens.hpp>
24 #include <boost/preprocessor/repetition/for.hpp>
25 #include <boost/preprocessor/repetition/repeat.hpp>
26 #include <boost/preprocessor/tuple/elem.hpp>
27 #include <boost/preprocessor/tuple/enum.hpp>
28 #include <boost/preprocessor/tuple/pop_front.hpp>
29 #include <boost/preprocessor/tuple/push_back.hpp>
30 #include <boost/preprocessor/tuple/push_front.hpp>
31 #include <boost/preprocessor/tuple/rem.hpp>
32 #include <boost/preprocessor/tuple/size.hpp>
33 #include <boost/preprocessor/tuple/to_array.hpp>
34 #include <boost/preprocessor/tuple/to_list.hpp>
35 #include <boost/preprocessor/variadic/elem.hpp>
36 #include <boost/preprocessor/variadic/to_list.hpp>
37 #include <boost/preprocessor/variadic/to_tuple.hpp>
38 #include <boost/vmd/is_empty.hpp>
39 // IWYU pragma: end_exports
40 
41 #include <initializer_list>
42 #include <limits>
43 #include <random>
44 #include <string>
45 #include <tuple>
46 #include <utility>
47 #include <vector>
48 
49 #include "DataStructures/DataVector.hpp"
51 #include "ErrorHandling/Error.hpp"
52 #include "Utilities/Gsl.hpp"
54 #include "Utilities/TMPL.hpp"
55 #include "Utilities/TypeTraits.hpp"
56 #include "tests/Unit/Pypp/Pypp.hpp"
58 
59 namespace pypp {
60 namespace TestHelpers_detail {
61 template <typename T>
62 using is_not_null = tt::is_a<gsl::not_null, T>;
63 
64 template <typename T>
65 struct RemoveNotNull {
66  using type = T;
67 };
68 
69 template <typename T>
70 struct RemoveNotNull<gsl::not_null<T>> {
71  using type = std::remove_pointer_t<T>;
72 };
73 
74 template <typename MemberArg, typename UsedForSize, typename = std::nullptr_t>
75 struct ConvertToTensorImpl;
76 
77 template <typename UsedForSize>
78 struct ConvertToTensorImpl<double, UsedForSize> {
79  static auto apply(const double& value,
80  const UsedForSize& used_for_size) noexcept {
81  return make_with_value<Scalar<DataVector>>(used_for_size, value);
82  }
83 };
84 
85 template <size_t Dim, typename UsedForSize>
86 struct ConvertToTensorImpl<std::array<double, Dim>, UsedForSize> {
87  static auto apply(const std::array<double, Dim>& arr,
88  const UsedForSize& used_for_size) noexcept {
89  auto array_as_tensor =
90  make_with_value<tnsr::i<DataVector, Dim>>(used_for_size, 0.);
91  for (size_t i = 0; i < Dim; ++i) {
92  array_as_tensor.get(i) = gsl::at(arr, i);
93  }
94  return array_as_tensor;
95  }
96 };
97 
98 template <typename ReturnType, typename = std::nullptr_t>
99 struct ForwardToPyppImpl {
100  template <typename MemberArg, typename UsedForSize>
101  static decltype(auto) apply(const MemberArg& member_arg,
102  const UsedForSize& /*used_for_size*/) noexcept {
103  return member_arg;
104  }
105 };
106 
107 template <typename ReturnType>
108 struct ForwardToPyppImpl<
109  ReturnType,
110  Requires<(tt::is_a_v<Tensor, ReturnType> or
111  tt::is_std_array_v<ReturnType>)and cpp17::
112  is_same_v<typename ReturnType::value_type, DataVector>>> {
113  template <typename MemberArg, typename UsedForSize>
114  static decltype(auto) apply(const MemberArg& member_arg,
115  const UsedForSize& used_for_size) noexcept {
117  used_for_size);
118  }
119 };
120 
121 // Given the member variable of type MemberArg (either a double or array of
122 // doubles), performs a conversion so that it can be correctly forwarded to
123 // Pypp. If ReturnType is not a Tensor or array of DataVectors,
124 // member_arg is simply forwarded, otherwise it is converted to a Tensor of
125 // DataVectors.
126 template <typename PyppReturn, typename MemberArg, typename UsedForSize>
127 decltype(auto) forward_to_pypp(const MemberArg& member_arg,
128  const UsedForSize& used_for_size) noexcept {
129  return ForwardToPyppImpl<PyppReturn>::apply(member_arg, used_for_size);
130 }
131 
132 template <class F, class T, class TagsList, class Klass, class... ReturnTypes,
133  class... ArgumentTypes, class... MemberArgs, size_t... ResultIs,
134  size_t... ArgumentIs, size_t... MemberArgsIs>
135 void check_with_random_values_impl(
136  F&& f, const Klass& klass, const std::string& module_name,
137  const std::vector<std::string>& function_names, std::mt19937 generator,
138  std::array<std::uniform_real_distribution<>, sizeof...(ArgumentTypes)>
139  distributions,
140  const std::tuple<MemberArgs...>& member_args, const T& used_for_size,
141  tmpl::list<ReturnTypes...> /*return_types*/,
142  tmpl::list<ArgumentTypes...> /*argument_types*/,
143  std::index_sequence<ResultIs...> /*index_return_types*/,
144  std::index_sequence<ArgumentIs...> /*index_argument_types*/,
145  std::index_sequence<MemberArgsIs...> /*index_member_args*/,
146  TagsList /*meta*/, const double epsilon = 1.0e-12) {
147  // Note: generator and distributions cannot be const.
148  std::tuple<ArgumentTypes...> args{
149  make_with_value<ArgumentTypes>(used_for_size, 0.0)...};
150  // We fill with random values after initialization because the order of
151  // evaluation is not guaranteed for a constructor call and so then knowing
152  // the seed would not lead to reproducible results.
155  make_not_null(&std::get<ArgumentIs>(args)), make_not_null(&generator),
156  make_not_null(&(distributions[ArgumentIs]))),
157  '0')...};
158 
159  size_t count = 0;
160  tmpl::for_each<TagsList>([&f, &klass, &args, &used_for_size, &member_args,
161  &epsilon, &module_name, &function_names,
162  &count](auto tag) {
163  (void)member_args; // Avoid compiler warning
164  (void)used_for_size; // Avoid compiler warning
165  using Tag = tmpl::type_from<decltype(tag)>;
166  const auto result =
167  tuples::get<Tag>((klass.*f)(std::get<ArgumentIs>(args)...));
168  INFO("function: " << function_names[count]);
170  result,
171  (pypp::call<std::decay_t<decltype(result)>>(
172  module_name, function_names[count], std::get<ArgumentIs>(args)...,
173  forward_to_pypp<std::decay_t<decltype(result)>>(
174  std::get<MemberArgsIs>(member_args), used_for_size)...)),
175  Approx::custom().epsilon(epsilon).scale(1.0));
176  count++;
177  });
178 }
179 
180 template <class F, class T, class Klass, class... ArgumentTypes,
181  class... MemberArgs, size_t... ArgumentIs, size_t... MemberArgsIs>
182 void check_with_random_values_impl(
183  F&& f, const Klass& klass, const std::string& module_name,
184  const std::string& function_name, std::mt19937 generator,
185  std::array<std::uniform_real_distribution<>, sizeof...(ArgumentTypes)>
186  distributions,
187  const std::tuple<MemberArgs...>& member_args, const T& used_for_size,
188  tmpl::list<ArgumentTypes...> /*argument_types*/,
189  std::index_sequence<ArgumentIs...> /*index_argument_types*/,
190  std::index_sequence<MemberArgsIs...> /*index_member_args*/,
191  NoSuchType /*meta*/, const double epsilon = 1.0e-12) {
192  // Note: generator and distributions cannot be const.
194  using ResultType = typename f_info::return_type;
195  std::tuple<ArgumentTypes...> args{
196  make_with_value<ArgumentTypes>(used_for_size, 0.0)...};
197  // We fill with random values after initialization because the order of
198  // evaluation is not guaranteed for a constructor call and so then knowing the
199  // seed would not lead to reproducible results.
202  make_not_null(&std::get<ArgumentIs>(args)), make_not_null(&generator),
203  make_not_null(&(distributions[ArgumentIs]))),
204  '0')...};
205  const auto result = make_overloader(
206  [&](std::true_type /*is_class*/, auto&& local_f) {
207  return (klass.*local_f)(std::get<ArgumentIs>(args)...);
208  },
209  [&](std::false_type /*is_class*/, auto&& local_f) {
210  return local_f(std::get<ArgumentIs>(args)...);
211  })(
214  std::forward<F>(f));
215  INFO("function: " << function_name);
217  result,
218  pypp::call<ResultType>(
219  module_name, function_name, std::get<ArgumentIs>(args)...,
220  forward_to_pypp<ResultType>(std::get<MemberArgsIs>(member_args),
221  used_for_size)...),
222  Approx::custom().epsilon(epsilon).scale(1.0));
223 }
224 
225 template <class F, class T, class Klass, class... ReturnTypes,
226  class... ArgumentTypes, class... MemberArgs, size_t... ResultIs,
227  size_t... ArgumentIs, size_t... MemberArgsIs>
228 void check_with_random_values_impl(
229  F&& f, const Klass& klass, const std::string& module_name,
230  const std::vector<std::string>& function_names, std::mt19937 generator,
231  std::array<std::uniform_real_distribution<>, sizeof...(ArgumentTypes)>
232  distributions,
233  const std::tuple<MemberArgs...>& member_args, const T& used_for_size,
234  tmpl::list<ReturnTypes...> /*return_types*/,
235  tmpl::list<ArgumentTypes...> /*argument_types*/,
236  std::index_sequence<ResultIs...> /*index_return_types*/,
237  std::index_sequence<ArgumentIs...> /*index_argument_types*/,
238  std::index_sequence<MemberArgsIs...> /*index_member_args*/,
239  NoSuchType /* meta */, const double epsilon = 1.0e-12) {
240  // Note: generator and distributions cannot be const.
241  std::tuple<ReturnTypes...> results{
242  make_with_value<ReturnTypes>(used_for_size, 0.0)...};
243  std::tuple<ArgumentTypes...> args{
244  make_with_value<ArgumentTypes>(used_for_size, 0.0)...};
245  // We fill with random values after initialization because the order of
246  // evaluation is not guaranteed for a constructor call and so then knowing the
247  // seed would not lead to reproducible results.
250  make_not_null(&std::get<ArgumentIs>(args)), make_not_null(&generator),
251  make_not_null(&(distributions[ArgumentIs]))),
252  '0')...};
253  // We intentionally do not set the return value to signaling NaN so that not
254  // all of our functions need to be able to handle the cases where they
255  // receive a NaN. Instead, we fill the return value with random numbers.
257  (void)fill_with_random_values(make_not_null(&std::get<ResultIs>(results)),
258  make_not_null(&generator),
259  make_not_null(&(distributions[0]))),
260  '0')...};
262  [&](std::true_type /*is_class*/, auto&& local_f) {
263  (klass.*local_f)(make_not_null(&std::get<ResultIs>(results))...,
264  std::get<ArgumentIs>(args)...);
265  },
266  [&](std::false_type /*is_class*/, auto&& local_f) {
267  local_f(make_not_null(&std::get<ResultIs>(results))...,
268  std::get<ArgumentIs>(args)...);
269  })(
272  std::forward<F>(f));
273  const auto helper = [&module_name, &function_names, &args, &results, &epsilon,
274  &member_args, &used_for_size](auto result_i) {
275  (void)member_args; // avoid compiler warning
276  (void)used_for_size; // avoid compiler warning
277  constexpr size_t iter = decltype(result_i)::value;
278  INFO("function: " << function_names[iter]);
280  std::get<iter>(results),
281  (pypp::call<std::tuple_element_t<iter, std::tuple<ReturnTypes...>>>(
282  module_name, function_names[iter], std::get<ArgumentIs>(args)...,
283  forward_to_pypp<
284  std::tuple_element_t<iter, std::tuple<ReturnTypes...>>>(
285  std::get<MemberArgsIs>(member_args), used_for_size)...)),
286  Approx::custom().epsilon(epsilon).scale(1.0));
287  return '0';
288  };
291 }
292 } // namespace TestHelpers_detail
293 
294 /*!
295  * \brief Tests a C++ function returning by value by comparing the result to a
296  * python function
297  *
298  * Tests the function `f` by comparing the result to that of the python function
299  * `function_name` in the file `module_name`. The function is tested by
300  * generated random values in the half-open range [`lower_bound`,
301  * `upper_bound`). The argument `used_for_size` is used for constructing the
302  * arguments of `f` by calling `make_with_value<ArgumentType>(used_for_size,
303  * 0.0)`.
304  *
305  * \note You must explicitly pass the number of bounds you will be passing as
306  * the first template parameter, the rest will be inferred.
307  *
308  * \note If you have a test fail you can replay the scenario by feeding in the
309  * seed that was printed out in the failed test as the last argument.
310  *
311  * \param f The C++ function to test
312  * \param module_name The python file relative to the directory used in
313  * `SetupLocalPythonEnvironment`
314  * \param function_name The name of the python function inside `module_name`
315  * \param lower_and_upper_bounds The lower and upper bounds for the randomly
316  * generated numbers. Must be either an array of a single pair, or of as many
317  * pairs as there are arguments to `f` that are not a `gsl::not_null`
318  * \param used_for_size The type `X` for the arguments of `f` of type
319  *`Tensor<X>`
320  * \param epsilon A double specifying the comparison tolerance
321  * (default 1.0e-12)
322  * \param seed The seed for the random number generator. This should only be
323  * specified when debugging a failure with a particular set of random numbers,
324  * in general it should be left to the default value.
325  */
326 template <size_t NumberOfBounds, class F, class T,
328  typename tt::function_info<cpp20::remove_cvref_t<F>>::return_type,
329  void>> = nullptr>
330 // The Requires is used so that we can call the std::vector<std::string> with
331 // braces and not have it be ambiguous.
333  F&& f, const std::string& module_name, const std::string& function_name,
334  const std::array<std::pair<double, double>, NumberOfBounds>&
335  lower_and_upper_bounds,
336  const T& used_for_size, const double epsilon = 1.0e-12,
337  const typename std::random_device::result_type seed =
338  std::random_device{}()) {
339  INFO("seed: " << seed);
340  std::mt19937 generator(seed);
342  using number_of_not_null =
343  tmpl::count_if<typename f_info::argument_types,
344  tmpl::bind<TestHelpers_detail::is_not_null, tmpl::_1>>;
345  using argument_types = tmpl::transform<
346  tmpl::pop_front<typename f_info::argument_types, number_of_not_null>,
348 
349  static_assert(number_of_not_null::value == 0,
350  "Cannot return arguments by gsl::not_null if the python "
351  "function name is passed as a string. If the function only "
352  "returns one gsl::not_null then you must pass in a one element "
353  "vector<string>.");
354  static_assert(tmpl::size<argument_types>::value != 0,
355  "The function 'f' must take at least one argument.");
356  static_assert(NumberOfBounds == 1 or
357  NumberOfBounds == tmpl::size<argument_types>::value,
358  "The number of lower and upper bound pairs must be either 1 or "
359  "equal to the number of arguments taken by f that are not "
360  "gsl::not_null.");
362  tmpl::size<argument_types>::value>
363  distributions;
364  for (size_t i = 0; i < tmpl::size<argument_types>::value; ++i) {
365  gsl::at(distributions, i) = std::uniform_real_distribution<>{
366  gsl::at(lower_and_upper_bounds, NumberOfBounds == 1 ? 0 : i).first,
367  gsl::at(lower_and_upper_bounds, NumberOfBounds == 1 ? 0 : i).second};
368  }
369  TestHelpers_detail::check_with_random_values_impl(
370  std::forward<F>(f), NoSuchType{}, module_name, function_name, generator,
371  std::move(distributions), std::tuple<>{}, used_for_size, argument_types{},
374 }
375 
376 /*!
377  * \brief Tests a C++ function returning by `gsl::not_null` by comparing the
378  * result to a python function
379  *
380  * Tests the function `f` by comparing the result to that of the python function
381  * `function_name` in the file `module_name`. The function is tested by
382  * generated random values in the half-open range [`lower_bound`, `upper_bound`)
383  * for each argument. The argument `used_for_size` is used for constructing the
384  * arguments of `f` by calling `make_with_value<ArgumentType>(used_for_size,
385  * 0.0)`. For functions that return by `gsl::not_null`, the result will be
386  * initialized with random values rather than to signaling `NaN`s. This means
387  * functions do not need to support receiving a signaling `NaN` in their return
388  * argument to be tested using this function.
389  *
390  * \note You must explicitly pass the number of bounds you will be passing as
391  * the first template parameter, the rest will be inferred.
392  *
393  * \note If you have a test fail you can replay the scenario by feeding in the
394  * seed that was printed out in the failed test as the last argument.
395  *
396  * \param f The C++ function to test
397  * \param module_name The python file relative to the directory used in
398  * `SetupLocalPythonEnvironment`
399  * \param function_names The names of the python functions inside `module_name`
400  * in the order that they return the `gsl::not_null` results
401  * \param lower_and_upper_bounds The lower and upper bounds for the randomly
402  * generated numbers. Must be either an array of a single pair, or of as many
403  * pairs as there are arguments to `f` that are not a `gsl::not_null`
404  * \param used_for_size The type `X` for the arguments of `f` of type
405  * `Tensor<X>`
406  * \param epsilon A double specifying the comparison tolerance
407  * (default 1.0e-12)
408  * \param seed The seed for the random number generator. This should only be
409  * specified when debugging a failure with a particular set of random numbers,
410  * in general it should be left to the default value.
411  */
412 template <size_t NumberOfBounds, class F, class T>
414  F&& f, const std::string& module_name,
415  const std::vector<std::string>& function_names,
416  const std::array<std::pair<double, double>, NumberOfBounds>&
417  lower_and_upper_bounds,
418  const T& used_for_size, const double epsilon = 1.0e-12,
419  const typename std::random_device::result_type seed =
420  std::random_device{}()) {
421  INFO("seed: " << seed);
422  std::mt19937 generator(seed);
424  using number_of_not_null =
425  tmpl::count_if<typename f_info::argument_types,
426  tmpl::bind<TestHelpers_detail::is_not_null, tmpl::_1>>;
427  using argument_types = tmpl::transform<
428  tmpl::pop_front<typename f_info::argument_types, number_of_not_null>,
430  using return_types =
431  tmpl::transform<tmpl::pop_back<typename f_info::argument_types,
432  tmpl::size<argument_types>>,
433  TestHelpers_detail::RemoveNotNull<tmpl::_1>>;
434 
435  static_assert(number_of_not_null::value != 0,
436  "You must return at least one argument by gsl::not_null when "
437  "passing the python function names as a vector<string>. If "
438  "your function returns by value do not pass the function name "
439  "as a vector<string> but just a string.");
440  static_assert(
441  cpp17::is_same_v<typename f_info::return_type, void>,
442  "A function returning by gsl::not_null must have a void return type.");
443  static_assert(tmpl::size<argument_types>::value != 0,
444  "The function 'f' must take at least one argument.");
445  static_assert(NumberOfBounds == 1 or
446  NumberOfBounds == tmpl::size<argument_types>::value,
447  "The number of lower and upper bound pairs must be either 1 or "
448  "equal to the number of arguments taken by f that are not "
449  "gsl::not_null.");
450  if (function_names.size() != number_of_not_null::value) {
451  ERROR(
452  "The number of python functions passed must be the same as the number "
453  "of gsl::not_null arguments in the C++ function. The order of the "
454  "python functions must also be the same as the order of the "
455  "gsl::not_null arguments.");
456  }
458  tmpl::size<argument_types>::value>
459  distributions;
460  for (size_t i = 0; i < tmpl::size<argument_types>::value; ++i) {
461  gsl::at(distributions, i) = std::uniform_real_distribution<>{
462  gsl::at(lower_and_upper_bounds, NumberOfBounds == 1 ? 0 : i).first,
463  gsl::at(lower_and_upper_bounds, NumberOfBounds == 1 ? 0 : i).second};
464  }
465  TestHelpers_detail::check_with_random_values_impl(
466  std::forward<F>(f), NoSuchType{}, module_name, function_names, generator,
467  std::move(distributions), std::tuple<>{}, used_for_size, return_types{},
468  argument_types{},
472 }
473 
474 /*!
475  * \brief Tests a member function of a class returning by value by comparing the
476  * result to a python function
477  *
478  * Tests the function `f` by comparing the result to that of the python function
479  * `function_name` in the file `module_name`. An instance of the class is passed
480  * in as the second argument and is the object on which the member function `f`
481  * will be invoked. The member function is invoked as `klass.function`, so
482  * passing in pointers is not supported. The function is tested by generated
483  * random values in the half-open range [`lower_bound`, `upper_bound`). The
484  * argument `used_for_size` is used for constructing the arguments of `f` by
485  * calling `make_with_value<ArgumentType>(used_for_size, 0.0)`.
486  *
487  * \note You must explicitly pass the number of bounds you will be passing
488  * as the first template parameter, the rest will be inferred.
489  *
490  * \note If you have a test fail you can replay the scenario by feeding in
491  * the seed that was printed out in the failed test as the last argument.
492  *
493  * \param f The member function to test
494  * \param klass the object on which to invoke `f`
495  * \param module_name The python file relative to the directory used in
496  * `SetupLocalPythonEnvironment`
497  * \param function_name The name of the python function inside `module_name`
498  * \param lower_and_upper_bounds The lower and upper bounds for the randomly
499  * generated numbers. Must be either an array of a single pair, or of as many
500  * pairs as there are arguments to `f` that are not a `gsl::not_null`
501  * \param member_args a tuple of the member variables of the object `klass` that
502  * the python function will need in order to perform the computation. These
503  * should have the same types as the normal arguments passed to the member
504  * function, e.g. `Tensor<X>`.
505  * \param used_for_size The type `X` for the arguments of `f` of type
506  * `Tensor<X>`
507  * \param epsilon A double specifying the comparison tolerance
508  * (default 1.0e-12)
509  * \param seed The seed for the random number generator. This should only be
510  * specified when debugging a failure with a particular set of random numbers,
511  * in general it should be left to the default value.
512  */
513 template <size_t NumberOfBounds, class F, class T, class... MemberArgs,
515  typename tt::function_info<cpp20::remove_cvref_t<F>>::return_type,
516  void>> = nullptr>
517 // The Requires is used so that we can call the std::vector<std::string> with
518 // braces and not have it be ambiguous.
520  F&& f,
521  const typename tt::function_info<cpp20::remove_cvref_t<F>>::class_type&
522  klass,
523  const std::string& module_name, const std::string& function_name,
524  const std::array<std::pair<double, double>, NumberOfBounds>&
525  lower_and_upper_bounds,
526  const std::tuple<MemberArgs...>& member_args, const T& used_for_size,
527  const double epsilon = 1.0e-12,
528  const typename std::random_device::result_type seed =
529  std::random_device{}()) {
530  INFO("seed: " << seed);
531  std::mt19937 generator(seed);
533  using number_of_not_null =
534  tmpl::count_if<typename f_info::argument_types,
535  tmpl::bind<TestHelpers_detail::is_not_null, tmpl::_1>>;
536  using argument_types = tmpl::transform<
537  tmpl::pop_front<typename f_info::argument_types, number_of_not_null>,
539 
540  static_assert(number_of_not_null::value == 0,
541  "Cannot return arguments by gsl::not_null if the python "
542  "function name is passed as a string. If the function only "
543  "returns one gsl::not_null then you must pass in a one element "
544  "vector<string>.");
545  static_assert(tmpl::size<argument_types>::value != 0,
546  "The function 'f' must take at least one argument.");
547  static_assert(NumberOfBounds == 1 or
548  NumberOfBounds == tmpl::size<argument_types>::value,
549  "The number of lower and upper bound pairs must be either 1 or "
550  "equal to the number of arguments taken by f that are not "
551  "gsl::not_null.");
553  tmpl::size<argument_types>::value>
554  distributions;
555  for (size_t i = 0; i < tmpl::size<argument_types>::value; ++i) {
556  gsl::at(distributions, i) = std::uniform_real_distribution<>{
557  gsl::at(lower_and_upper_bounds, NumberOfBounds == 1 ? 0 : i).first,
558  gsl::at(lower_and_upper_bounds, NumberOfBounds == 1 ? 0 : i).second};
559  }
560  TestHelpers_detail::check_with_random_values_impl(
561  std::forward<F>(f), klass, module_name, function_name, generator,
562  std::move(distributions), member_args, used_for_size, argument_types{},
564  std::make_index_sequence<sizeof...(MemberArgs)>{}, NoSuchType{}, epsilon);
565 }
566 
567 /*!
568  * \brief Tests a member function of a class returning by either `gsl::not_null`
569  * or `TaggedTuple` by comparing the result to a python function
570  *
571  * Tests the function `f` by comparing the result to that of the python
572  * functions `function_names` in the file `module_name`. An instance of the
573  * class is passed in as the second argument and is the object on which the
574  * member function `f` will be invoked. The member function is invoked as
575  * `klass.function`, so passing in pointers is not supported. The function is
576  * tested by generated random values in the half-open range [`lower_bound`,
577  * `upper_bound`). The argument `used_for_size` is used for constructing the
578  * arguments of `f` by calling `make_with_value<ArgumentType>(used_for_size,
579  * 0.0)`. For functions that return by `gsl::not_null`, the result will be
580  * initialized with random values rather than to signaling `NaN`s. This means
581  * functions do not need to support receiving a signaling `NaN` in their return
582  * argument to be tested using this function.
583  *
584  * If `TagsList` is passed as a `tmpl::list`, then `f` is expected to
585  * return a TaggedTuple. The result of each python function will be
586  * compared with calling `tuples::get` on the result of `f`. The order of the
587  * tags within `TagsList` should match the order of the functions in
588  * `function_names`
589  *
590  * \note You must explicitly pass the number of bounds you will be passing as
591  * the first template parameter, the rest will be inferred.
592  *
593  * \note If you have a test fail you can replay the scenario by feeding in the
594  * seed that was printed out in the failed test as the last argument.
595  *
596  * \param f The member function to test
597  * \param klass the object on which to invoke `f`
598  * \param module_name The python file relative to the directory used in
599  * `SetupLocalPythonEnvironment`
600  * \param function_names The names of the python functions inside `module_name`
601  * in the order that they return the `gsl::not_null` results
602  * \param lower_and_upper_bounds The lower and upper bounds for the randomly
603  * generated numbers. Must be either an array of a single pair, or of as many
604  * pairs as there are arguments to `f` that are not a `gsl::not_null`
605  * \param member_args a tuple of the member variables of the object `klass` that
606  * the python function will need in order to perform the computation. These
607  * should have the same types as the normal arguments passed to the member
608  * function, e.g. `Tensor<X>`.
609  * \param used_for_size The type `X` for the arguments of `f` of type
610  * `Tensor<X>`
611  * \param epsilon A double specifying the comparison tolerance
612  * (default 1.0e-12)
613  * \param seed The seed for the random number generator. This should only be
614  * specified when debugging a failure with a particular set of random numbers,
615  * in general it should be left to the default value.
616  */
617 template <size_t NumberOfBounds, typename TagsList = NoSuchType, class F,
618  class T, class... MemberArgs>
620  F&& f,
621  const typename tt::function_info<cpp20::remove_cvref_t<F>>::class_type&
622  klass,
623  const std::string& module_name,
624  const std::vector<std::string>& function_names,
625  const std::array<std::pair<double, double>, NumberOfBounds>&
626  lower_and_upper_bounds,
627  const std::tuple<MemberArgs...>& member_args, const T& used_for_size,
628  const double epsilon = 1.0e-12,
629  const typename std::random_device::result_type seed =
630  std::random_device{}()) {
631  INFO("seed: " << seed);
632  std::mt19937 generator(seed);
634  using number_of_not_null =
635  tmpl::count_if<typename f_info::argument_types,
636  tmpl::bind<TestHelpers_detail::is_not_null, tmpl::_1>>;
637  using argument_types = tmpl::transform<
638  tmpl::pop_front<typename f_info::argument_types, number_of_not_null>,
640  using return_types =
641  tmpl::transform<tmpl::pop_back<typename f_info::argument_types,
642  tmpl::size<argument_types>>,
643  TestHelpers_detail::RemoveNotNull<tmpl::_1>>;
644 
645  static_assert(
646  number_of_not_null::value != 0 or tt::is_a_v<tmpl::list, TagsList>,
647  "You must either return at least one argument by gsl::not_null when"
648  "passing the python function names as a vector<string>, or return by "
649  "value using a TaggedTuple. If your function returns by value with a "
650  "type that is not a TaggedTuple do not pass the function name as a "
651  "vector<string> but just a string.");
652  static_assert(cpp17::is_same_v<typename f_info::return_type, void> or
653  tt::is_a_v<tmpl::list, TagsList>,
654  "The function must either return by gsl::not_null and have a "
655  "void return type, or return by TaggedTuple");
656  static_assert(tmpl::size<argument_types>::value != 0,
657  "The function 'f' must take at least one argument.");
658  static_assert(NumberOfBounds == 1 or
659  NumberOfBounds == tmpl::size<argument_types>::value,
660  "The number of lower and upper bound pairs must be either 1 or "
661  "equal to the number of arguments taken by f that are not "
662  "gsl::not_null.");
663  if (function_names.size() != number_of_not_null::value and
664  not tt::is_a_v<tmpl::list, TagsList>) {
665  ERROR(
666  "If testing a function that returns by gsl::not_null, the number of "
667  "python functions passed must be the same as the number of "
668  "gsl::not_null arguments in the C++ function. The order of the "
669  "python functions must also be the same as the order of the "
670  "gsl::not_null arguments.");
671  }
673  tmpl::size<argument_types>::value>
674  distributions;
675  for (size_t i = 0; i < tmpl::size<argument_types>::value; ++i) {
676  gsl::at(distributions, i) = std::uniform_real_distribution<>{
677  gsl::at(lower_and_upper_bounds, NumberOfBounds == 1 ? 0 : i).first,
678  gsl::at(lower_and_upper_bounds, NumberOfBounds == 1 ? 0 : i).second};
679  }
680  TestHelpers_detail::check_with_random_values_impl(
681  std::forward<F>(f), klass, module_name, function_names, generator,
682  std::move(distributions), member_args, used_for_size, return_types{},
683  argument_types{},
686  std::make_index_sequence<sizeof...(MemberArgs)>{}, TagsList{}, epsilon);
687 }
688 
689 /// \cond
690 #define INVOKE_FUNCTION_TUPLE_PUSH_BACK(r, DATA, ELEM) \
691  BOOST_PP_TUPLE_PUSH_BACK(DATA, ELEM)
692 
693 #define INVOKE_FUNCTION_WITH_MANY_TEMPLATE_PARAMS_IMPL(r, DATA) \
694  BOOST_PP_TUPLE_ELEM( \
695  0, BOOST_PP_TUPLE_ELEM( \
696  0, DATA))<BOOST_PP_TUPLE_ELEM(2, BOOST_PP_TUPLE_ELEM(0, DATA)), \
697  BOOST_PP_TUPLE_ENUM(BOOST_PP_TUPLE_POP_FRONT(DATA))> \
698  BOOST_PP_TUPLE_ELEM(1, BOOST_PP_TUPLE_ELEM(0, DATA));
699 
700 // The case where there is more than one tuple of template parameters to tensor
701 // product together. The first tuple of template parameters is transformed to a
702 // tuple of (FUNCTION_NAME, TUPLE_ARGS, TEMPLATE_PARAM). Then the macro will
703 // extract the values. The reason for needing to do this is that
704 // BOOST_PP_LIST_FOR_EACH_PRODUCT does not allow for passing extra data along
705 // to the macro.
706 #define INVOKE_FUNCTION_WITH_MANY_TEMPLATE_PARAMS(TUPLE_OF_TEMPLATE_PARAMS) \
707  BOOST_PP_LIST_FOR_EACH_PRODUCT( \
708  INVOKE_FUNCTION_WITH_MANY_TEMPLATE_PARAMS_IMPL, \
709  BOOST_PP_TUPLE_SIZE(TUPLE_OF_TEMPLATE_PARAMS), TUPLE_OF_TEMPLATE_PARAMS)
710 
711 // The case where there is one tuple of template parameters to iterate over.
712 #define INVOKE_FUNCTION_WITH_SINGLE_TEMPLATE_PARAM(_, DATA, ELEM) \
713  BOOST_PP_TUPLE_ELEM(0, DATA)<ELEM> BOOST_PP_TUPLE_ELEM(1, DATA);
714 
715 #define INVOKE_FUNCTION_WITH_MANY_TEMPLATE_PARAMS_TUPLE_TO_LIST(r, _, ELEM) \
716  BOOST_PP_TUPLE_TO_LIST(ELEM)
717 /// \endcond
718 
719 /*!
720  * \ingroup TestingFrameworkGroup
721  * \brief Macro used to invoke a test function of multiple template arguments.
722  *
723  * This macro allows to generate calls to multiple instances of
724  * a test function template, all of which will receive the same parameters.
725  * The first argument to this macro is the name of the function. The second
726  * argument is a macro-tuple containing the parameters passed to each instance,
727  * e.g. `(x, y)`. The remaining arguments are macro-tuples of the values for
728  * each template parameter one wants to loop over, e.g.
729  * `(1, 2, 3), (Frame::Inertial, Frame::Grid)`. For example, a function template
730  *
731  * \code
732  * template <class Arg1, size_t Arg2, class Arg3>
733  * my_function(const double& var_1, const int& var_2) noexcept { ... }
734  * \endcode
735  *
736  * can be invoked by writing
737  *
738  * \code
739  * INVOKE_TEST_FUNCTION(my_function, (d, i), (a, b, c), (1, 2, 3), (A, B, C))
740  * \endcode
741  *
742  * which will generate
743  *
744  * \code
745  * my_function<a, 1, A>(d, i);
746  * my_function<a, 1, B>(d, i);
747  * my_function<a, 1, C>(d, i);
748  * my_function<a, 2, A>(d, i);
749  * my_function<a, 2, B>(d, i);
750  * my_function<a, 2, C>(d, i);
751  * my_function<a, 3, A>(d, i);
752  * my_function<a, 3, B>(d, i);
753  * my_function<a, 3, C>(d, i);
754  * my_function<b, 1, A>(d, i);
755  * my_function<b, 1, B>(d, i);
756  * my_function<b, 1, C>(d, i);
757  * my_function<b, 2, A>(d, i);
758  * my_function<b, 2, B>(d, i);
759  * my_function<b, 2, C>(d, i);
760  * my_function<b, 3, A>(d, i);
761  * my_function<b, 3, B>(d, i);
762  * my_function<b, 3, C>(d, i);
763  * my_function<c, 1, A>(d, i);
764  * my_function<c, 1, B>(d, i);
765  * my_function<c, 1, C>(d, i);
766  * my_function<c, 2, A>(d, i);
767  * my_function<c, 2, B>(d, i);
768  * my_function<c, 2, C>(d, i);
769  * my_function<c, 3, A>(d, i);
770  * my_function<c, 3, B>(d, i);
771  * my_function<c, 3, C>(d, i);
772  * \endcode
773  *
774  * \note The order of the macro-tuples of values must match the order of the
775  * template parameters of the function.
776  *
777  * \note The function to be called must at least have one template argument,
778  * so passing an empty set of template parameters will not work.
779  */
780 #define INVOKE_TEST_FUNCTION(FUNCTION_NAME, TUPLE_ARGS, ...) \
781  BOOST_PP_ASSERT_MSG(BOOST_PP_NOT(BOOST_VMD_IS_EMPTY(__VA_ARGS__)), \
782  "You cannot pass an empty set of template parameters " \
783  "to INVOKE_TEST_FUNCTION") \
784  BOOST_PP_TUPLE_ENUM( \
785  0, \
786  BOOST_PP_IF( \
787  BOOST_PP_EQUAL( \
788  BOOST_PP_TUPLE_SIZE(BOOST_PP_VARIADIC_TO_TUPLE(__VA_ARGS__)), \
789  1), \
790  (BOOST_PP_LIST_FOR_EACH( \
791  INVOKE_FUNCTION_WITH_SINGLE_TEMPLATE_PARAM, \
792  (FUNCTION_NAME, TUPLE_ARGS), \
793  BOOST_PP_TUPLE_TO_LIST( \
794  BOOST_PP_VARIADIC_ELEM(0, __VA_ARGS__)))), \
795  (INVOKE_FUNCTION_WITH_MANY_TEMPLATE_PARAMS( \
796  BOOST_PP_TUPLE_PUSH_FRONT( \
797  BOOST_PP_LIST_TO_TUPLE(BOOST_PP_LIST_TRANSFORM( \
798  INVOKE_FUNCTION_WITH_MANY_TEMPLATE_PARAMS_TUPLE_TO_LIST, \
799  _, \
800  BOOST_PP_LIST_REST( \
801  BOOST_PP_VARIADIC_TO_LIST(__VA_ARGS__)))), \
802  BOOST_PP_LIST_TRANSFORM( \
803  INVOKE_FUNCTION_TUPLE_PUSH_BACK, \
804  (FUNCTION_NAME, TUPLE_ARGS), \
805  BOOST_PP_TUPLE_TO_LIST( \
806  BOOST_PP_VARIADIC_ELEM(0, __VA_ARGS__))))))))
807 
808 /// \cond
809 #define GENERATE_UNINITIALIZED_DOUBLE \
810  const double d(std::numeric_limits<double>::signaling_NaN())
811 
812 #define GENERATE_UNINITIALIZED_DATAVECTOR const DataVector dv(5)
813 
814 #define GENERATE_UNINITIALIZED_DOUBLE_AND_DATAVECTOR \
815  GENERATE_UNINITIALIZED_DOUBLE; \
816  GENERATE_UNINITIALIZED_DATAVECTOR
817 
818 #define CHECK_FOR_DOUBLES(FUNCTION_NAME, ...) \
819  INVOKE_TEST_FUNCTION(FUNCTION_NAME, (d), __VA_ARGS__)
820 
821 #define CHECK_FOR_DATAVECTORS(FUNCTION_NAME, ...) \
822  INVOKE_TEST_FUNCTION(FUNCTION_NAME, (dv), __VA_ARGS__)
823 /// \endcond
824 
825 /*!
826  * \ingroup TestingFrameworkGroup
827  * \brief Macro used to test functions whose parameter can be a `double` or a
828  * `DataVector`.
829  *
830  * In testing multiple instances of a function template using random values, it
831  * often proves useful to write a wrapper around
832  * `pypp::check_with_random_values`. This way, one can easily loop over several
833  * values of one or multiple template parameters (e.g. when testing a
834  * function templated in the number of spacetime dimensions.) The template
835  * parameters of the wrapper will then correspond to the template parameters of
836  * the function, which will be used by `pypp::check_with_random_values`
837  * to invoke and test each instance. Each of these wrappers will generally
838  * need only one parameter, namely a variable `used_for_size` passed to
839  * `pypp::check_with_random_values` that can be a `double`, a `DataVector`, or
840  * both (provided that the function being tested is templated in the type of
841  * `used_for_size`.) Since this is applied in multiple test files, all of these
842  * files will share the same way to generate the required calls to the wrapper.
843  *
844  * This macro, along with
845  *
846  * \code
847  * CHECK_FOR_DOUBLES(FUNCTION_NAME, ...)
848  * \endcode
849  * \code
850  * CHECK_FOR_DATAVECTORS(FUNCTION_NAME, ...)
851  * \endcode
852  *
853  * allow to generate calls to multiple instances of a test function template in
854  * the same way as done by `INVOKE_TEST_FUNCTION(FUNCTION_NAME, ARGS_TUPLE,
855  * ...)`
856  * (to which these macros call), except that the tuple of arguments is not
857  * passed, as these macros will assume that a `double` `d`
858  * and/or a `DataVector` `dv` will be previously defined. Although any `d`s and
859  * `dv`s will work, one can (and it is recommended to) generate signaling `NaN`
860  * values for `d` and `dv`. This can be done by invoking one of the three
861  * provided macros: `GENERATE_UNINIATILIZED_DOUBLE`,
862  * `GENERATE_UNINITIALIZED_DATAVECTOR`, or
863  * `GENERATE_UNINITIALIZED_DOUBLE_AND_DATAVECTOR`. For example,
864  *
865  * \code
866  * GENERATE_UNINITIALIZED_DATAVECTOR;
867  * CHECK_FOR_DATAVECTORS(test_fluxes, (1, 2, 3))
868  * \endcode
869  *
870  * will generate a test case for 1, 2 and 3 dimensions:
871  *
872  * \code
873  * const DataVector dv(5);
874  * test_fluxes<1>(dv);
875  * test_fluxes<2>(dv);
876  * test_fluxes<3>(dv);
877  * \endcode
878  *
879  * Analogously, the wrapper
880  *
881  * \code
882  * template <size_t Dim, IndexType TypeOfIndex, typename DataType>
883  * test_ricci(const DataType& used_for_size) noexcept { ... }
884  * \endcode
885  *
886  * can be invoked by writing
887  *
888  * \code
889  * GENERATE_UNINITIALIZED_DOUBLE_AND_DATAVECTOR;
890  *
891  * CHECK_FOR_DOUBLES_AND_DATAVECTORS(test_ricci, (1, 2, 3),
892  * (IndexType::Spatial, IndexType::Spacetime))
893  * \endcode
894  *
895  * which will generate
896  *
897  * \code
898  * const double d(std::numeric_limits<double>::signaling_NaN());
899  * const DataVector dv(5);
900  *
901  * test_ricci<1, IndexType::Spatial>(d);
902  * test_ricci<1, IndexType::Spacetime>(d);
903  * test_ricci<2, IndexType::Spatial>(d);
904  * test_ricci<2, IndexType::Spacetime>(d);
905  * test_ricci<3, IndexType::Spatial>(d);
906  * test_ricci<3, IndexType::Spacetime>(d);
907  * test_ricci<1, IndexType::Spatial>(dv);
908  * test_ricci<1, IndexType::Spacetime>(dv);
909  * test_ricci<2, IndexType::Spatial>(dv);
910  * test_ricci<2, IndexType::Spacetime>(dv);
911  * test_ricci<3, IndexType::Spatial>(dv);
912  * test_ricci<3, IndexType::Spacetime>(dv);
913  * \endcode
914  *
915  * Note that it is not necessary to pass values for `DataType`, as they are
916  * deduced from `used_for_size`.
917  *
918  * \note The order of the macro-tuples of values must match the order of the
919  * template parameters of the function.
920  *
921  * \note The function to be called must at least have one template argument,
922  * so passing an empty set of template parameters will not work.
923  */
924 #define CHECK_FOR_DOUBLES_AND_DATAVECTORS(FUNCTION_NAME, ...) \
925  CHECK_FOR_DOUBLES(FUNCTION_NAME, __VA_ARGS__) \
926  CHECK_FOR_DATAVECTORS(FUNCTION_NAME, __VA_ARGS__)
927 } // namespace pypp
#define ERROR(m)
prints an error message to the standard error stream and aborts the program.
Definition: Error.hpp:35
void check_with_random_values(F &&f, const std::string &module_name, const std::string &function_name, const std::array< std::pair< double, double >, NumberOfBounds > &lower_and_upper_bounds, const T &used_for_size, const double epsilon=1.0e-12, const typename std::random_device::result_type seed=std::random_device{}())
Tests a C++ function returning by value by comparing the result to a python function.
Definition: CheckWithRandomValues.hpp:332
Overloader< Fs... > make_overloader(Fs... fs)
Create Overloader<Fs...>, see Overloader for details.
Definition: Overloader.hpp:109
Contains all functions for calling python from C++.
Definition: CheckWithRandomValues.hpp:59
Defines function pypp::call<R,Args...>
Helper functions for data structures used in unit tests.
void fill_with_random_values(const gsl::not_null< T *> data, const gsl::not_null< UniformRandomBitGenerator *> generator, const gsl::not_null< RandomNumberDistribution *> distribution) noexcept
Fill an existing data structure with random values.
Definition: MakeWithRandomValues.hpp:140
constexpr bool is_std_array_v
Definition: TypeTraits.hpp:451
constexpr auto apply(F &&f, const DataBox< BoxTags > &box, Args &&... args)
Apply the function f with argument Tags TagsList from DataBox box
Definition: DataBox.hpp:1595
#define CHECK_ITERABLE_CUSTOM_APPROX(a, b, appx)
Same as CHECK_ITERABLE_APPROX with user-defined Approx. The third argument should be of type Approx...
Definition: TestingFramework.hpp:122
Used to mark "no type" or "bad state" for metaprogramming.
Definition: NoSuchType.hpp:10
constexpr bool is_same_v
Variable template for is_same.
Definition: TypeTraits.hpp:221
constexpr bool is_a_v
Definition: TypeTraits.hpp:543
Defines classes for Tensor.
Check if type T is a template specialization of U
Definition: TypeTraits.hpp:536
Wraps the template metaprogramming library used (brigand)
R call(const std::string &module_name, const std::string &function_name, const Args &... t)
Calls a Python function from a module/file with given parameters.
Definition: Pypp.hpp:233
Defines functions and classes from the GSL.
gsl::not_null< T * > make_not_null(T *ptr) noexcept
Construct a not_null from a pointer. Often this will be done as an implicit conversion, but it may be necessary to perform the conversion explicitly when type deduction is desired.
Definition: Gsl.hpp:863
Code to wrap or improve the Catch testing framework used for unit tests.
typename Requires_detail::requires_impl< B >::template_error_type_failed_to_meet_requirements_on_template_parameters Requires
Express requirements on the template parameters of a function or class, replaces std::enable_if_t ...
Definition: Requires.hpp:67
C++ STL code present in C++17.
Definition: Array.hpp:16
Defines macro ERROR.
Defines type traits, some of which are future STL type_traits header.
Definition: MakeWithRandomValues.hpp:21
tt_detail::function_info_impl< F > function_info
Returns a struct that contains the return type, argument types, and the class type if the F is a non-...
Definition: TypeTraits.hpp:1508
Defines make_with_value.
constexpr T & at(std::array< T, N > &arr, Size index)
Retrieve a entry from a container, with checks in Debug mode that the index being retrieved is valid...
Definition: Gsl.hpp:124