Line data    Source code

       1           0 : // Distributed under the MIT License.
       2             : // See LICENSE.txt for details.
       3             : 
       4             : #pragma once
       5             : 
       6             : #include <atomic>
       7             : #include <converse.h>
       8             : #include <cstddef>
       9             : #include <memory>
      10             : #include <set>
      11             : #include <string>
      12             : #include <unordered_map>
      13             : #include <unordered_set>
      14             : #include <vector>
      15             : 
      16             : #include "DataStructures/DataBox/Tag.hpp"
      17             : #include "DataStructures/DataVector.hpp"
      18             : #include "IO/H5/TensorData.hpp"
      19             : #include "IO/Observer/ObservationId.hpp"
      20             : #include "Options/String.hpp"
      21             : #include "Parallel/ArrayComponentId.hpp"
      22             : #include "Parallel/NodeLock.hpp"
      23             : #include "Parallel/Reduction.hpp"
      24             : #include "Utilities/PrettyType.hpp"
      25             : 
      26             : namespace observers {
      27             : /// \ingroup ObserversGroup
      28             : /// %Tags used on the observer parallel component
      29           1 : namespace Tags {
      30             : /// \brief The set of `ArrayComponentId`s that will contribute to each
      31             : /// `ObservationId` for reduction.
      32             : ///
      33             : /// This is set during registration is only read from later on in the
      34             : /// simulation, except for possibly during migration of an `ArrayComponentId`
      35             : /// component.
      36           1 : struct ExpectedContributorsForObservations : db::SimpleTag {
      37           0 :   using type =
      38             :       std::unordered_map<ObservationKey,
      39             :                          std::unordered_set<Parallel::ArrayComponentId>>;
      40             : };
      41             : 
      42             : /// \brief The set of `ArrayComponentId` that have contributed to each
      43             : /// `ObservationId` for reductions
      44             : ///
      45             : /// The tag is used both on the `Observer` and on the `ObserverWriter`
      46             : /// components since all we need to do is keep track of array component IDs in
      47             : /// both cases.
      48           1 : struct ContributorsOfReductionData : db::SimpleTag {
      49           0 :   using type =
      50             :       std::unordered_map<ObservationId,
      51             :                          std::unordered_set<Parallel::ArrayComponentId>>;
      52             : };
      53             : 
      54             : /// \brief The set of nodes that have contributed to each `ObservationId` for
      55             : /// writing reduction data
      56             : ///
      57             : /// This is used on node 0 (or whichever node has been designated as the one to
      58             : /// write the reduction files). The `unordered_set` is the node IDs that have
      59             : /// contributed so far.
      60           1 : struct NodesThatContributedReductions : db::SimpleTag {
      61           0 :   using type = std::unordered_map<ObservationId, std::unordered_set<size_t>>;
      62             : };
      63             : 
      64             : /// \brief The set of nodes that are registered with each
      65             : /// `ObservationIdRegistrationKey` for writing reduction data
      66             : ///
      67             : /// The set contains all the nodes that have been registered.
      68             : ///
      69             : /// We need to keep track of this separately from the local reductions on the
      70             : /// node that are contributing so we need a separate tag. Since nodes are easily
      71             : /// indexed by an unsigned integer, we use `size_t` to keep track of them.
      72           1 : struct NodesExpectedToContributeReductions : db::SimpleTag {
      73           0 :   using type = std::unordered_map<ObservationKey, std::set<size_t>>;
      74             : };
      75             : 
      76             : /// \brief Lock used when contributing reduction data.
      77             : ///
      78             : /// A separate lock from the node lock of the nodegroup is used in order to
      79             : /// allow other cores to contribute volume data, write to disk, etc.
      80           1 : struct ReductionDataLock : db::SimpleTag {
      81           0 :   using type = Parallel::NodeLock;
      82             : };
      83             : 
      84             : /// \brief The set of `ArrayComponentId` that have contributed to each
      85             : /// `ObservationId` for volume observation
      86             : ///
      87             : /// The tag is used both on the `Observer` and on the `ObserverWriter`
      88             : /// components since all we need to do is keep track of array component IDs in
      89             : /// both cases.
      90           1 : struct ContributorsOfTensorData : db::SimpleTag {
      91           0 :   using type =
      92             :       std::unordered_map<ObservationId,
      93             :                          std::unordered_set<Parallel::ArrayComponentId>>;
      94             : };
      95             : 
      96             : /// Volume tensor data to be written to disk.
      97           1 : struct TensorData : db::SimpleTag {
      98           0 :   using type = std::unordered_map<
      99             :       observers::ObservationId,
     100             :       std::unordered_map<Parallel::ArrayComponentId, ElementVolumeData>>;
     101             : };
     102             : 
     103             : /// Volume tensor data to be written to disk from the Interpolator
     104           1 : struct InterpolatorTensorData : db::SimpleTag {
     105           0 :   using type =
     106             :       std::unordered_map<observers::ObservationId,
     107             :                          std::unordered_map<Parallel::ArrayComponentId,
     108             :                                             std::vector<ElementVolumeData>>>;
     109             : };
     110             : 
     111             : /// Map of ObservationIds that have dependencies for volume data to be written
     112             : /// and whether or not the dependency has the volume data being written or
     113             : /// discarded.
     114           1 : struct Dependencies : db::SimpleTag {
     115           0 :   using type = std::unordered_map<ObservationId, std::pair<std::string, bool>>;
     116             : };
     117             : 
     118             : /// \cond
     119             : template <class... ReductionDatums>
     120             : struct ReductionDataNames;
     121             : /// \endcond
     122             : 
     123             : /// Reduction data to be written to disk.
     124             : ///
     125             : /// If we have `M` `ArrayComponentIds` registered with a given `Observer`
     126             : /// for a given `ObservationId`, then we expect the `Observer` to receive
     127             : /// `M` contributions with the given `ObservationId`. We combine the reduction
     128             : /// data as we receive it, so we only need one copy for each `ObservationId`.
     129             : template <class... ReductionDatums>
     130           1 : struct ReductionData : db::SimpleTag {
     131           0 :   using type = std::unordered_map<observers::ObservationId,
     132             :                                   Parallel::ReductionData<ReductionDatums...>>;
     133           0 :   using names_tag = ReductionDataNames<ReductionDatums...>;
     134             : };
     135             : 
     136             : /// Names of the reduction data to be written to disk.
     137             : template <class... ReductionDatums>
     138           1 : struct ReductionDataNames : db::SimpleTag {
     139           0 :   using type =
     140             :       std::unordered_map<observers::ObservationId, std::vector<std::string>>;
     141           0 :   using data_tag = ReductionData<ReductionDatums...>;
     142             : };
     143             : 
     144             : /// Node lock used when needing to read/write to H5 files on disk.
     145             : ///
     146             : /// The reason for only having one lock for all files is that we currently don't
     147             : /// require a thread-safe HDF5 installation. In the future we will need to
     148             : /// experiment with different HDF5 configurations.
     149           1 : struct H5FileLock : db::SimpleTag {
     150           0 :   using type = Parallel::NodeLock;
     151             : };
     152             : 
     153             : /*!
     154             :  * \brief A string identifying observations related to the `Tag`.
     155             :  *
     156             :  * For example, the `Tag` could refer to the block in the computational domain
     157             :  * and the `ObservationKey` holds the name of the block, so reduction
     158             :  * observations can be registered per-block. A value of `std::nullopt` means
     159             :  * that the element doesn't participate in observations related to the `Tag`.
     160             :  *
     161             :  * This tag only provides a way to store and share the observation key related
     162             :  * to the `Tag`. How it is used to construct composite observation keys or
     163             :  * subfile paths is up to the code that performs the observations.
     164             :  */
     165             : template <typename Tag>
     166           1 : struct ObservationKey : db::SimpleTag {
     167           0 :   using type = std::optional<std::string>;
     168           0 :   static std::string name() {
     169             :     return "ObservationKey(" + pretty_type::name<Tag>() + ")";
     170             :   }
     171             : };
     172             : 
     173             : }  // namespace Tags
     174             : 
     175             : /// \ingroup ObserversGroup
     176             : /// Option tags related to recording data
     177           1 : namespace OptionTags {
     178             : /// Groups option tags related to recording data, e.g. file names.
     179           1 : struct Group {
     180           0 :   static std::string name() { return "Observers"; }
     181           0 :   static constexpr Options::String help = {"Options for recording data"};
     182             : };
     183             : 
     184             : /// The name of the H5 file on disk to which all volume data is written.
     185           1 : struct VolumeFileName {
     186           0 :   using type = std::string;
     187           0 :   static constexpr Options::String help = {
     188             :       "Name of the volume data file without extension"};
     189           0 :   using group = Group;
     190             : };
     191             : 
     192             : /// The name of the H5 file on disk to which all reduction data is written.
     193           1 : struct ReductionFileName {
     194           0 :   using type = std::string;
     195           0 :   static constexpr Options::String help = {
     196             :       "Name of the reduction data file without extension"};
     197           0 :   using group = Group;
     198             : };
     199             : 
     200             : /// The name of the H5 file on disk to which all surface data is written.
     201           1 : struct SurfaceFileName {
     202           0 :   using type = std::string;
     203           0 :   static constexpr Options::String help = {
     204             :       "Name of the surface data file without extension"};
     205           0 :   using group = Group;
     206             : };
     207             : }  // namespace OptionTags
     208             : 
     209             : namespace Tags {
     210             : /// \brief The name of the HDF5 file on disk into which volume data is
     211             : /// written.
     212             : ///
     213             : /// By volume data we mean any data that is not written once across all nodes.
     214             : /// For example, data on a 2d surface written from a 2d simulation is
     215             : /// considered volume data, while an integral over the entire (or a subset of
     216             : /// the) domain is considered reduction data. Data for a 2d surface in
     217             : /// a 3d simulation, such as a horizon, is considered surface data and
     218             : /// is written to a file specified by SurfaceFileName.
     219           1 : struct VolumeFileName : db::SimpleTag {
     220           0 :   using type = std::string;
     221           0 :   using option_tags = tmpl::list<::observers::OptionTags::VolumeFileName>;
     222             : 
     223           0 :   static constexpr bool pass_metavariables = false;
     224           0 :   static std::string create_from_options(const std::string& volume_file_name) {
     225             :     return volume_file_name;
     226             :   }
     227             : };
     228             : 
     229             : /// \brief The name of the HDF5 file on disk into which reduction data is
     230             : /// written.
     231             : ///
     232             : /// By reduction data we mean any data that is written once across all nodes.
     233             : /// For example, an integral over the entire (or a subset of the) domain
     234             : /// is considered reduction data, while data on a 3d surface written from a 3d
     235             : /// simulation is considered volume data. Data for a 2d surface in
     236             : /// a 3d simulation, such as a horizon, is considered surface data and
     237             : /// is written to a file specified by SurfaceFileName.
     238           1 : struct ReductionFileName : db::SimpleTag {
     239           0 :   using type = std::string;
     240           0 :   using option_tags = tmpl::list<::observers::OptionTags::ReductionFileName>;
     241             : 
     242           0 :   static constexpr bool pass_metavariables = false;
     243           0 :   static std::string create_from_options(
     244             :       const std::string& reduction_file_name) {
     245             :     return reduction_file_name;
     246             :   }
     247             : };
     248             : 
     249             : /// \brief The name of the HDF5 file on disk into which surface data is
     250             : /// written.
     251             : ///
     252             : /// By surface data we mean data for a 2d surface in
     253             : /// a 3d simulation, such as a horizon. Surface data is written once across
     254             : /// all nodes. For example, data on a 2d surface written from a 2d simulation is
     255             : /// considered volume data, while an integral over the entire (or a subset of
     256             : /// the) domain is considered reduction data.
     257           1 : struct SurfaceFileName : db::SimpleTag {
     258           0 :   using type = std::string;
     259           0 :   using option_tags = tmpl::list<::observers::OptionTags::SurfaceFileName>;
     260             : 
     261           0 :   static constexpr bool pass_metavariables = false;
     262           0 :   static std::string create_from_options(const std::string& surface_file_name) {
     263             :     return surface_file_name;
     264             :   }
     265             : };
     266             : }  // namespace Tags
     267             : }  // namespace observers