Line data    Source code

       1           0 : // Distributed under the MIT License.
       2             : // See LICENSE.txt for details.
       3             : 
       4             : #pragma once
       5             : 
       6             : #include <atomic>
       7             : #include <converse.h>
       8             : #include <cstddef>
       9             : #include <memory>
      10             : #include <set>
      11             : #include <string>
      12             : #include <unordered_map>
      13             : #include <unordered_set>
      14             : #include <vector>
      15             : 
      16             : #include "DataStructures/DataBox/Tag.hpp"
      17             : #include "DataStructures/DataVector.hpp"
      18             : #include "IO/H5/TensorData.hpp"
      19             : #include "IO/Observer/ObservationId.hpp"
      20             : #include "Options/String.hpp"
      21             : #include "Parallel/ArrayComponentId.hpp"
      22             : #include "Parallel/NodeLock.hpp"
      23             : #include "Parallel/Reduction.hpp"
      24             : #include "Utilities/PrettyType.hpp"
      25             : 
      26             : namespace observers {
      27             : /// \ingroup ObserversGroup
      28             : /// %Tags used on the observer parallel component
      29           1 : namespace Tags {
      30             : /// \brief The set of `ArrayComponentId`s that will contribute to each
      31             : /// `ObservationId` for reduction.
      32             : ///
      33             : /// This is set during registration is only read from later on in the
      34             : /// simulation, except for possibly during migration of an `ArrayComponentId`
      35             : /// component.
      36           1 : struct ExpectedContributorsForObservations : db::SimpleTag {
      37           0 :   using type =
      38             :       std::unordered_map<ObservationKey,
      39             :                          std::unordered_set<Parallel::ArrayComponentId>>;
      40             : };
      41             : 
      42             : /// \brief The set of `ArrayComponentId` that have contributed to each
      43             : /// `ObservationId` for reductions
      44             : ///
      45             : /// The tag is used both on the `Observer` and on the `ObserverWriter`
      46             : /// components since all we need to do is keep track of array component IDs in
      47             : /// both cases.
      48           1 : struct ContributorsOfReductionData : db::SimpleTag {
      49           0 :   using type =
      50             :       std::unordered_map<ObservationId,
      51             :                          std::unordered_set<Parallel::ArrayComponentId>>;
      52             : };
      53             : 
      54             : /// \brief The set of nodes that have contributed to each `ObservationId` for
      55             : /// writing reduction data
      56             : ///
      57             : /// This is used on node 0 (or whichever node has been designated as the one to
      58             : /// write the reduction files). The `unordered_set` is the node IDs that have
      59             : /// contributed so far.
      60           1 : struct NodesThatContributedReductions : db::SimpleTag {
      61           0 :   using type = std::unordered_map<ObservationId, std::unordered_set<size_t>>;
      62             : };
      63             : 
      64             : /// \brief The set of nodes that are registered with each
      65             : /// `ObservationIdRegistrationKey` for writing reduction data
      66             : ///
      67             : /// The set contains all the nodes that have been registered.
      68             : ///
      69             : /// We need to keep track of this separately from the local reductions on the
      70             : /// node that are contributing so we need a separate tag. Since nodes are easily
      71             : /// indexed by an unsigned integer, we use `size_t` to keep track of them.
      72           1 : struct NodesExpectedToContributeReductions : db::SimpleTag {
      73           0 :   using type = std::unordered_map<ObservationKey, std::set<size_t>>;
      74             : };
      75             : 
      76             : /// \brief Lock used when contributing reduction data.
      77             : ///
      78             : /// A separate lock from the node lock of the nodegroup is used in order to
      79             : /// allow other cores to contribute volume data, write to disk, etc.
      80           1 : struct ReductionDataLock : db::SimpleTag {
      81           0 :   using type = Parallel::NodeLock;
      82             : };
      83             : 
      84             : /// \brief The set of `ArrayComponentId` that have contributed to each
      85             : /// `ObservationId` for volume observation
      86             : ///
      87             : /// The tag is used both on the `Observer` and on the `ObserverWriter`
      88             : /// components since all we need to do is keep track of array component IDs in
      89             : /// both cases.
      90           1 : struct ContributorsOfTensorData : db::SimpleTag {
      91           0 :   using type =
      92             :       std::unordered_map<ObservationId,
      93             :                          std::unordered_set<Parallel::ArrayComponentId>>;
      94             : };
      95             : 
      96             : /// Volume tensor data to be written to disk.
      97           1 : struct TensorData : db::SimpleTag {
      98           0 :   using type = std::unordered_map<
      99             :       observers::ObservationId,
     100             :       std::unordered_map<Parallel::ArrayComponentId, ElementVolumeData>>;
     101             : };
     102             : 
     103             : /// Volume tensor data to be written to disk from the Interpolator
     104           1 : struct InterpolatorTensorData : db::SimpleTag {
     105           0 :   using type =
     106             :       std::unordered_map<observers::ObservationId,
     107             :                          std::unordered_map<Parallel::ArrayComponentId,
     108             :                                             std::vector<ElementVolumeData>>>;
     109             : };
     110             : 
     111             : /// Map of serialized FunctionOfTime data for volume data to be written.
     112           1 : struct SerializedFunctionsOfTime : db::SimpleTag {
     113           0 :   using type =
     114             :       std::unordered_map<ObservationId, std::optional<std::vector<char>>>;
     115             : };
     116             : 
     117             : /// Map of ObservationIds that have dependencies for volume data to be written
     118             : /// and whether or not the dependency has the volume data being written or
     119             : /// discarded.
     120           1 : struct Dependencies : db::SimpleTag {
     121           0 :   using type = std::unordered_map<ObservationId, std::pair<std::string, bool>>;
     122             : };
     123             : 
     124             : /// \cond
     125             : template <class... ReductionDatums>
     126             : struct ReductionDataNames;
     127             : /// \endcond
     128             : 
     129             : /// Reduction data to be written to disk.
     130             : ///
     131             : /// If we have `M` `ArrayComponentIds` registered with a given `Observer`
     132             : /// for a given `ObservationId`, then we expect the `Observer` to receive
     133             : /// `M` contributions with the given `ObservationId`. We combine the reduction
     134             : /// data as we receive it, so we only need one copy for each `ObservationId`.
     135             : template <class... ReductionDatums>
     136           1 : struct ReductionData : db::SimpleTag {
     137           0 :   using type = std::unordered_map<observers::ObservationId,
     138             :                                   Parallel::ReductionData<ReductionDatums...>>;
     139           0 :   using names_tag = ReductionDataNames<ReductionDatums...>;
     140             : };
     141             : 
     142             : /// Names of the reduction data to be written to disk.
     143             : template <class... ReductionDatums>
     144           1 : struct ReductionDataNames : db::SimpleTag {
     145           0 :   using type =
     146             :       std::unordered_map<observers::ObservationId, std::vector<std::string>>;
     147           0 :   using data_tag = ReductionData<ReductionDatums...>;
     148             : };
     149             : 
     150             : /// Node lock used when needing to read/write to H5 files on disk.
     151             : ///
     152             : /// The reason for only having one lock for all files is that we currently don't
     153             : /// require a thread-safe HDF5 installation. In the future we will need to
     154             : /// experiment with different HDF5 configurations.
     155           1 : struct H5FileLock : db::SimpleTag {
     156           0 :   using type = Parallel::NodeLock;
     157             : };
     158             : 
     159             : /*!
     160             :  * \brief A string identifying observations related to the `Tag`.
     161             :  *
     162             :  * For example, the `Tag` could refer to the block in the computational domain
     163             :  * and the `ObservationKey` holds the name of the block, so reduction
     164             :  * observations can be registered per-block. A value of `std::nullopt` means
     165             :  * that the element doesn't participate in observations related to the `Tag`.
     166             :  *
     167             :  * This tag only provides a way to store and share the observation key related
     168             :  * to the `Tag`. How it is used to construct composite observation keys or
     169             :  * subfile paths is up to the code that performs the observations.
     170             :  */
     171             : template <typename Tag>
     172           1 : struct ObservationKey : db::SimpleTag {
     173           0 :   using type = std::optional<std::string>;
     174           0 :   static std::string name() {
     175             :     return "ObservationKey(" + pretty_type::name<Tag>() + ")";
     176             :   }
     177             : };
     178             : 
     179             : }  // namespace Tags
     180             : 
     181             : /// \ingroup ObserversGroup
     182             : /// Option tags related to recording data
     183           1 : namespace OptionTags {
     184             : /// Groups option tags related to recording data, e.g. file names.
     185           1 : struct Group {
     186           0 :   static std::string name() { return "Observers"; }
     187           0 :   static constexpr Options::String help = {"Options for recording data"};
     188             : };
     189             : 
     190             : /// The name of the H5 file on disk to which all volume data is written.
     191           1 : struct VolumeFileName {
     192           0 :   using type = std::string;
     193           0 :   static constexpr Options::String help = {
     194             :       "Name of the volume data file without extension"};
     195           0 :   using group = Group;
     196             : };
     197             : 
     198             : /// The name of the H5 file on disk to which all reduction data is written.
     199           1 : struct ReductionFileName {
     200           0 :   using type = std::string;
     201           0 :   static constexpr Options::String help = {
     202             :       "Name of the reduction data file without extension"};
     203           0 :   using group = Group;
     204             : };
     205             : 
     206             : /// The name of the H5 file on disk to which all surface data is written.
     207           1 : struct SurfaceFileName {
     208           0 :   using type = std::string;
     209           0 :   static constexpr Options::String help = {
     210             :       "Name of the surface data file without extension"};
     211           0 :   using group = Group;
     212             : };
     213             : }  // namespace OptionTags
     214             : 
     215             : namespace Tags {
     216             : /// \brief The name of the HDF5 file on disk into which volume data is
     217             : /// written.
     218             : ///
     219             : /// By volume data we mean any data that is not written once across all nodes.
     220             : /// For example, data on a 2d surface written from a 2d simulation is
     221             : /// considered volume data, while an integral over the entire (or a subset of
     222             : /// the) domain is considered reduction data. Data for a 2d surface in
     223             : /// a 3d simulation, such as a horizon, is considered surface data and
     224             : /// is written to a file specified by SurfaceFileName.
     225           1 : struct VolumeFileName : db::SimpleTag {
     226           0 :   using type = std::string;
     227           0 :   using option_tags = tmpl::list<::observers::OptionTags::VolumeFileName>;
     228             : 
     229           0 :   static constexpr bool pass_metavariables = false;
     230           0 :   static std::string create_from_options(const std::string& volume_file_name) {
     231             :     return volume_file_name;
     232             :   }
     233             : };
     234             : 
     235             : /// \brief The name of the HDF5 file on disk into which reduction data is
     236             : /// written.
     237             : ///
     238             : /// By reduction data we mean any data that is written once across all nodes.
     239             : /// For example, an integral over the entire (or a subset of the) domain
     240             : /// is considered reduction data, while data on a 3d surface written from a 3d
     241             : /// simulation is considered volume data. Data for a 2d surface in
     242             : /// a 3d simulation, such as a horizon, is considered surface data and
     243             : /// is written to a file specified by SurfaceFileName.
     244           1 : struct ReductionFileName : db::SimpleTag {
     245           0 :   using type = std::string;
     246           0 :   using option_tags = tmpl::list<::observers::OptionTags::ReductionFileName>;
     247             : 
     248           0 :   static constexpr bool pass_metavariables = false;
     249           0 :   static std::string create_from_options(
     250             :       const std::string& reduction_file_name) {
     251             :     return reduction_file_name;
     252             :   }
     253             : };
     254             : 
     255             : /// \brief The name of the HDF5 file on disk into which surface data is
     256             : /// written.
     257             : ///
     258             : /// By surface data we mean data for a 2d surface in
     259             : /// a 3d simulation, such as a horizon. Surface data is written once across
     260             : /// all nodes. For example, data on a 2d surface written from a 2d simulation is
     261             : /// considered volume data, while an integral over the entire (or a subset of
     262             : /// the) domain is considered reduction data.
     263           1 : struct SurfaceFileName : db::SimpleTag {
     264           0 :   using type = std::string;
     265           0 :   using option_tags = tmpl::list<::observers::OptionTags::SurfaceFileName>;
     266             : 
     267           0 :   static constexpr bool pass_metavariables = false;
     268           0 :   static std::string create_from_options(const std::string& surface_file_name) {
     269             :     return surface_file_name;
     270             :   }
     271             : };
     272             : }  // namespace Tags
     273             : }  // namespace observers