ReadVolumeData.hpp
1 // Distributed under the MIT License.
2 // See LICENSE.txt for details.
3 
4 #pragma once
5 
6 #include <cstddef>
7 #include <tuple>
8 
10 #include "DataStructures/DataVector.hpp"
13 #include "IO/H5/AccessType.hpp"
14 #include "IO/H5/File.hpp"
15 #include "IO/H5/VolumeData.hpp"
16 #include "IO/Importers/Tags.hpp"
17 #include "IO/Observer/ArrayComponentId.hpp"
18 #include "Parallel/ArrayIndex.hpp"
19 #include "Parallel/GlobalCache.hpp"
20 #include "Parallel/Invoke.hpp"
21 #include "Utilities/Gsl.hpp"
22 #include "Utilities/Literals.hpp"
23 #include "Utilities/Requires.hpp"
24 #include "Utilities/TMPL.hpp"
25 #include "Utilities/TaggedTuple.hpp"
26 
27 /// \cond
28 namespace importers {
29 template <typename Metavariables>
30 struct ElementDataReader;
31 namespace Actions {
32 template <typename ImporterOptionsGroup, typename FieldTagsList,
33  typename ReceiveComponent>
34 struct ReadAllVolumeDataAndDistribute;
35 } // namespace Actions
36 } // namespace importers
37 /// \endcond
38 
39 namespace importers::Actions {
40 
41 /*!
42  * \brief Read a volume data file and distribute the data to all registered
43  * elements.
44  *
45  * Invoke this action on the elements of an array parallel component to dispatch
46  * reading the volume data file specified by the options in
47  * `ImporterOptionsGroup`. The tensors in `FieldTagsList` will be loaded from
48  * the file and distributed to all elements that have previously registered. Use
49  * `importers::Actions::RegisterWithElementDataReader` to register the elements
50  * of the array parallel component in a previous phase.
51  *
52  * Note that the volume data file will only be read once per node, triggered by
53  * the first element that invokes this action. All subsequent invocations of
54  * this action on the node will do nothing. See
55  * `importers::Actions::ReadAllVolumeDataAndDistribute` for details.
56  *
57  * The data is distributed to the elements using `Parallel::receive_data`. The
58  * elements can monitor `importers::Tags::VolumeData` in their inbox to wait for
59  * the data and process it once it's available. We provide the action
60  * `importers::Actions::ReceiveVolumeData` that waits for the data and moves it
61  * directly into the DataBox. You can also implement a specialized action that
62  * might verify and post-process the data before populating the DataBox.
63  *
64  * \see Dev guide on \ref dev_guide_importing
65  */
66 template <typename ImporterOptionsGroup, typename FieldTagsList>
68  using const_global_cache_tags =
69  tmpl::list<Tags::FileName<ImporterOptionsGroup>,
72 
73  template <typename DbTagsList, typename... InboxTags, typename Metavariables,
74  typename ArrayIndex, typename ActionList,
75  typename ParallelComponent>
76  static std::tuple<db::DataBox<DbTagsList>&&> apply(
77  db::DataBox<DbTagsList>& box,
78  const tuples::TaggedTuple<InboxTags...>& /*inboxes*/,
80  const ArrayIndex& /*array_index*/, const ActionList /*meta*/,
81  const ParallelComponent* const /*meta*/) noexcept {
82  // Not using `ckLocalBranch` here to make sure the simple action invocation
83  // is asynchronous.
84  auto& reader_component = Parallel::get_parallel_component<
87  ImporterOptionsGroup, FieldTagsList, ParallelComponent>>(
88  reader_component);
89  return {std::move(box)};
90  }
91 };
92 
93 /*!
94  * \brief Read a volume data file and distribute the data to all registered
95  * elements.
96  *
97  * This action can be invoked on the `importers::ElementDataReader` component
98  * once all elements have been registered with it. It opens the data file, reads
99  * the data for each registered element and uses `Parallel::receive_data` to
100  * distribute the data to the elements. The elements can monitor
101  * `importers::Tags::VolumeData` in their inbox to wait for the data and process
102  * it once it's available. You can use `importers::Actions::ReceiveVolumeData`
103  * to wait for the data and move it directly into the DataBox, or implement a
104  * specialized action that might verify and post-process the data.
105  *
106  * Note that instead of invoking this action directly on the
107  * `importers::ElementDataReader` component you can invoke the iterable action
108  * `importers::Actions::ReadVolumeData` on the elements of an array parallel
109  * component.
110  *
111  * - The `ImporterOptionsGroup` parameter specifies the \ref OptionGroupsGroup
112  * "options group" in the input file that provides the following run-time
113  * options:
114  * - `importers::OptionTags::FileName`
115  * - `importers::OptionTags::Subgroup`
116  * - `importers::OptionTags::ObservationValue`
117  * - The `FieldTagsList` parameter specifies a typelist of tensor tags that
118  * are read from the file and provided to each element. It is assumed that the
119  * tensor data is stored in datasets named `db::tag_name<Tag>() + suffix`, where
120  * the `suffix` is empty for scalars or `"_"` followed by the
121  * `Tensor::component_name` for each independent tensor component.
122  * - `Parallel::receive_data` is invoked on each registered element of the
123  * `ReceiveComponent` to populate `importers::Tags::VolumeData` in the element's
124  * inbox with a `tuples::tagged_tuple_from_typelist<FieldTagsList>` containing
125  * the tensor data for that element. The `ReceiveComponent` must the the same
126  * that was encoded into the `observers::ArrayComponentId` used to register the
127  * elements.
128  *
129  * \see Dev guide on \ref dev_guide_importing
130  */
131 template <typename ImporterOptionsGroup, typename FieldTagsList,
132  typename ReceiveComponent>
134  template <
135  typename ParallelComponent, typename DataBox, typename Metavariables,
136  typename ArrayIndex,
138  db::tag_is_retrievable_v<Tags::ElementDataAlreadyRead,
139  DataBox>> = nullptr>
140  static void apply(DataBox& box, Parallel::GlobalCache<Metavariables>& cache,
141  const ArrayIndex& /*array_index*/) noexcept {
142  // Only read and distribute the volume data once
143  // This action will be invoked by `importers::Actions::ReadVolumeData` from
144  // every element on the node, but only the first invocation reads the file
145  // and distributes the data to all elements. Subsequent invocations do
146  // nothing. We use the `ImporterOptionsGroup` that specifies the data file
147  // to read in as the identifier for whether or not we have already read the
148  // requested data. Doing this at runtime avoids having to collect all
149  // data files that will be read in at compile-time to initialize a flag in
150  // the DataBox for each of them.
151  const auto& has_read_volume_data =
152  db::get<Tags::ElementDataAlreadyRead>(box);
153  const auto volume_data_id = pretty_type::get_name<ImporterOptionsGroup>();
154  if (has_read_volume_data.find(volume_data_id) !=
155  has_read_volume_data.end()) {
156  return;
157  }
158  db::mutate<Tags::ElementDataAlreadyRead>(
159  make_not_null(&box),
160  [&volume_data_id](const gsl::not_null<std::unordered_set<std::string>*>
161  local_has_read_volume_data) noexcept {
162  local_has_read_volume_data->insert(std::move(volume_data_id));
163  });
164 
165  // Open the volume data file
168  constexpr size_t version_number = 0;
169  const auto& volume_file = h5file.get<h5::VolumeData>(
170  "/" + Parallel::get<Tags::Subgroup<ImporterOptionsGroup>>(cache),
171  version_number);
172  const auto observation_id = volume_file.find_observation_id(
174  // Read the tensor data for all elements at once, since that's how it's
175  // stored in the file
176  tuples::tagged_tuple_from_typelist<FieldTagsList> all_tensor_data{};
177  tmpl::for_each<FieldTagsList>([&all_tensor_data, &volume_file,
178  &observation_id](auto field_tag_v) noexcept {
179  using field_tag = tmpl::type_from<decltype(field_tag_v)>;
180  auto& tensor_data = get<field_tag>(all_tensor_data);
181  for (size_t i = 0; i < tensor_data.size(); i++) {
182  tensor_data[i] = volume_file.get_tensor_component(
183  observation_id,
184  db::tag_name<field_tag>() +
185  tensor_data.component_suffix(tensor_data.get_tensor_index(i)));
186  }
187  });
188  // Retrieve the information needed to reconstruct which element the data
189  // belongs to
190  const auto all_grid_names = volume_file.get_grid_names(observation_id);
191  const auto all_extents = volume_file.get_extents(observation_id);
192  // Distribute the tensor data to the registered elements
193  for (auto& element_and_name : get<Tags::RegisteredElements>(box)) {
194  const CkArrayIndex& raw_element_index =
195  element_and_name.first.array_index();
196  // Check if the parallel component of the registered element matches the
197  // callback, because it's possible that elements from other components
198  // with the same index are also registered.
199  // Since the way the component is encoded in `ArrayComponentId` is
200  // private to that class, we construct one and compare.
201  if (element_and_name.first !=
204  raw_element_index)) {
205  continue;
206  }
207  // Find the data offset that corresponds to this element
208  const auto element_data_offset_and_length =
209  h5::offset_and_length_for_grid(element_and_name.second,
210  all_grid_names, all_extents);
211  // Extract this element's data from the read-in dataset
212  tuples::tagged_tuple_from_typelist<FieldTagsList> element_data{};
213  tmpl::for_each<FieldTagsList>([&element_data,
214  &element_data_offset_and_length,
215  &all_tensor_data](
216  auto field_tag_v) noexcept {
217  using field_tag = tmpl::type_from<decltype(field_tag_v)>;
218  auto& element_tensor_data = get<field_tag>(element_data);
219  // Iterate independent components of the tensor
220  for (size_t i = 0; i < element_tensor_data.size(); i++) {
221  const DataVector& data_tensor_component =
222  get<field_tag>(all_tensor_data)[i];
223  DataVector element_tensor_component{
224  element_data_offset_and_length.second};
225  // Retrieve data from slice of the contigious dataset
226  for (size_t j = 0; j < element_tensor_component.size(); j++) {
227  element_tensor_component[j] =
228  data_tensor_component[element_data_offset_and_length.first + j];
229  }
230  element_tensor_data[i] = element_tensor_component;
231  }
232  });
233  // Pass the data to the element
234  const auto element_index =
236  raw_element_index)
237  .get_index();
240  Parallel::get_parallel_component<ReceiveComponent>(
241  cache)[element_index],
242  // Using `0` for the temporal ID since we only read the volume data
243  // once, so there's no need to keep track of the temporal ID.
244  0_st, std::move(element_data));
245  }
246  }
247 };
248 
249 } // namespace importers::Actions
AccessType.hpp
importers::Tags::Subgroup
The subgroup within the file to read data from.
Definition: Tags.hpp:105
Parallel::GlobalCache
Definition: ElementReceiveInterpPoints.hpp:15
std::unordered_set< std::string >
Literals.hpp
GlobalCache.hpp
Parallel::get_parallel_component
auto get_parallel_component(GlobalCache< Metavariables > &cache) noexcept -> Parallel::proxy_from_parallel_component< GlobalCache_detail::get_component_if_mocked< typename Metavariables::component_list, ParallelComponentTag >> &
Access the Charm++ proxy associated with a ParallelComponent.
Definition: GlobalCache.hpp:535
importers
Items related to loading data from files.
Definition: ReadVolumeData.hpp:39
tuple
importers::Tags::VolumeData
Inbox tag that carries the data read from a volume data file.
Definition: Tags.hpp:160
File.hpp
ElementId.hpp
h5::H5File< h5::AccessType::ReadOnly >
std::add_pointer_t
DataBox.hpp
importers::ElementDataReader
A nodegroup parallel component that reads in a volume data file and distributes its data to elements ...
Definition: ElementDataReader.hpp:38
h5::offset_and_length_for_grid
std::pair< size_t, size_t > offset_and_length_for_grid(const std::string &grid_name, const std::vector< std::string > &all_grid_names, const std::vector< std::vector< size_t >> &all_extents) noexcept
Find the interval within the contiguous dataset stored in h5::VolumeData that holds data for a partic...
cstddef
importers::Tags::ObservationValue
The observation value at which to read data from the file.
Definition: Tags.hpp:122
tuples::TaggedTuple
An associative container that is indexed by structs.
Definition: TaggedTuple.hpp:271
DataVector
Stores a collection of function values.
Definition: DataVector.hpp:46
Parallel::receive_data
void receive_data(Proxy &&proxy, typename ReceiveTag::temporal_id temporal_id, ReceiveDataType &&receive_data, const bool enable_if_disabled=false) noexcept
Send the data args... to the algorithm running on proxy, and tag the message with the identifier temp...
Definition: Invoke.hpp:32
ActionTesting::cache
Parallel::GlobalCache< Metavariables > & cache(MockRuntimeSystem< Metavariables > &runner, const ArrayIndex &array_index) noexcept
Returns the GlobalCache of Component with index array_index.
Definition: MockRuntimeSystemFreeFunctions.hpp:382
Gsl.hpp
Tensor.hpp
Requires.hpp
observers::ArrayComponentId
An ID type that identifies both the parallel component and the index in the parallel component.
Definition: ArrayComponentId.hpp:27
Parallel::simple_action
void simple_action(Proxy &&proxy) noexcept
Invoke a simple action on proxy
Definition: Invoke.hpp:62
make_not_null
gsl::not_null< T * > make_not_null(T *ptr) noexcept
Construct a not_null from a pointer. Often this will be done as an implicit conversion,...
Definition: Gsl.hpp:880
Parallel::ArrayIndex
The array index used for indexing Chare Arrays, mostly an implementation detail.
Definition: ArrayIndex.hpp:28
Requires
typename Requires_detail::requires_impl< B >::template_error_type_failed_to_meet_requirements_on_template_parameters Requires
Express requirements on the template parameters of a function or class, replaces std::enable_if_t
Definition: Requires.hpp:67
importers::Tags::FileName
The file to read data from.
Definition: Tags.hpp:86
importers::Actions::ReadAllVolumeDataAndDistribute
Read a volume data file and distribute the data to all registered elements.
Definition: ReadVolumeData.hpp:133
importers::Actions::ReadVolumeData
Read a volume data file and distribute the data to all registered elements.
Definition: ReadVolumeData.hpp:67
Parallel::get
auto get(const GlobalCache< Metavariables > &cache) noexcept -> const GlobalCache_detail::type_for_get< GlobalCacheTag, Metavariables > &
Access data in the cache.
Definition: GlobalCache.hpp:566
TMPL.hpp
h5::VolumeData
A volume data subfile written inside an H5 file.
Definition: VolumeData.hpp:68
gsl::not_null
Require a pointer to not be a nullptr
Definition: ReadSpecThirdOrderPiecewisePolynomial.hpp:13
importers::Tags::ElementDataAlreadyRead
Indicates which volume data files have already been read.
Definition: Tags.hpp:149