SpECTRE Documentation Coverage Report
Current view: top level - Parallel/ArrayCollection - CreateElementCollection.hpp Hit Total Coverage
Commit: 6e1258ccd353220e12442198913007fb6c170b6b Lines: 1 8 12.5 %
Date: 2024-10-23 19:54:13
Legend: Lines: hit not hit

          Line data    Source code
       1           0 : // Distributed under the MIT License.
       2             : // See LICENSE.txt for details.
       3             : 
       4             : #pragma once
       5             : 
       6             : #include <cstddef>
       7             : #include <optional>
       8             : #include <tuple>
       9             : #include <unordered_set>
      10             : #include <utility>
      11             : #include <vector>
      12             : 
      13             : #include "DataStructures/DataBox/DataBox.hpp"
      14             : #include "Domain/Block.hpp"
      15             : #include "Domain/Creators/Tags/Domain.hpp"
      16             : #include "Domain/Creators/Tags/InitialExtents.hpp"
      17             : #include "Domain/Creators/Tags/InitialRefinementLevels.hpp"
      18             : #include "Domain/Domain.hpp"
      19             : #include "Domain/Structure/ElementId.hpp"
      20             : #include "Domain/Tags/ElementDistribution.hpp"
      21             : #include "Evolution/DiscontinuousGalerkin/Initialization/QuadratureTag.hpp"
      22             : #include "Parallel/AlgorithmExecution.hpp"
      23             : #include "Parallel/ArrayCollection/SpawnInitializeElementsInCollection.hpp"
      24             : #include "Parallel/ArrayCollection/Tags/ElementCollection.hpp"
      25             : #include "Parallel/ArrayCollection/Tags/ElementLocations.hpp"
      26             : #include "Parallel/ArrayCollection/Tags/NumberOfElementsTerminated.hpp"
      27             : #include "Parallel/CreateElementsUsingDistribution.hpp"
      28             : #include "Parallel/GlobalCache.hpp"
      29             : #include "Parallel/Info.hpp"
      30             : #include "Parallel/Local.hpp"
      31             : #include "Parallel/NodeLock.hpp"
      32             : #include "Parallel/Reduction.hpp"
      33             : #include "Utilities/ErrorHandling/Assert.hpp"
      34             : #include "Utilities/ErrorHandling/Error.hpp"
      35             : #include "Utilities/Functional.hpp"
      36             : #include "Utilities/Gsl.hpp"
      37             : #include "Utilities/TaggedTuple.hpp"
      38             : 
      39           0 : namespace Parallel::Actions {
      40             : /*!
      41             :  * \brief Creates the `DgElementArrayMember`s on the (node)group component.
      42             :  *
      43             :  * First the distribution of elements is computed using
      44             :  * `Parallel::create_elements_using_distribution()`, they are inserted on
      45             :  * each (node)group element. A reduction is done over the (node)group before
      46             :  * initializing the `DgElementArrayMember`s themselves, since they are allowed
      47             :  * to communicate with each other during their initialization. The reduction
      48             :  * target is `Parallel::Actions::SpawnInitializeElementsInCollection`.
      49             :  *
      50             :  * Uses:
      51             :  * - DataBox:
      52             :  *   - `domain::Tags::Domain<Dim>`
      53             :  *   - `domain::Tags::InitialRefinementLevels<Dim>`
      54             :  *   - `domain::Tags::InitialExtents<Dim>`
      55             :  *   - `evolution::dg::Tags::Quadrature`
      56             :  *   - `domain::Tags::ElementDistribution`
      57             :  *
      58             :  * DataBox changes:
      59             :  * - Adds:
      60             :  *   - `Parallel::Tags::ElementCollection`
      61             :  *   - `Parallel::Tags::ElementLocations<Dim>`
      62             :  *   - `Parallel::Tags::NumberOfElementsTerminated`
      63             :  * - Removes: nothing
      64             :  * - Modifies:
      65             :  *   - `Parallel::Tags::ElementCollection`
      66             :  *   - `Parallel::Tags::ElementLocations<Dim>`
      67             :  *   - `Parallel::Tags::NumberOfElementsTerminated`
      68             :  */
      69             : template <size_t Dim, class Metavariables, class PhaseDepActionList,
      70             :           typename SimpleTagsFromOptions>
      71           1 : struct CreateElementCollection {
      72           0 :   using simple_tags = tmpl::list<
      73             :       Parallel::Tags::ElementCollection<Dim, Metavariables, PhaseDepActionList,
      74             :                                         SimpleTagsFromOptions>,
      75             :       Parallel::Tags::ElementLocations<Dim>, Tags::NumberOfElementsTerminated>;
      76           0 :   using compute_tags = tmpl::list<>;
      77           0 :   using const_global_cache_tags =
      78             :       tmpl::list<::domain::Tags::Domain<Dim>,
      79             :                  ::domain::Tags::ElementDistribution>;
      80             : 
      81           0 :   using return_tag_list = tmpl::append<simple_tags, compute_tags>;
      82             : 
      83             :   template <typename DbTagsList, typename... InboxTags, typename ArrayIndex,
      84             :             typename ActionList, typename ParallelComponent>
      85           0 :   static Parallel::iterable_action_return_t apply(
      86             :       db::DataBox<DbTagsList>& box,
      87             :       const tuples::TaggedTuple<InboxTags...>& /*inboxes*/,
      88             :       Parallel::GlobalCache<Metavariables>& local_cache,
      89             :       const ArrayIndex& /*array_index*/, const ActionList /*meta*/,
      90             :       const ParallelComponent* const /*meta*/) {
      91             :     const std::unordered_set<size_t> procs_to_ignore{};
      92             : 
      93             :     const auto& domain = Parallel::get<domain::Tags::Domain<Dim>>(local_cache);
      94             :     const auto& initial_refinement_levels =
      95             :         get<domain::Tags::InitialRefinementLevels<Dim>>(box);
      96             :     const auto& initial_extents = get<domain::Tags::InitialExtents<Dim>>(box);
      97             :     const auto& quadrature = get<evolution::dg::Tags::Quadrature>(box);
      98             :     const std::optional<domain::ElementWeight>& element_weight =
      99             :         Parallel::get<domain::Tags::ElementDistribution>(local_cache);
     100             : 
     101             :     const size_t number_of_procs =
     102             :         Parallel::number_of_procs<size_t>(local_cache);
     103             :     const size_t number_of_nodes =
     104             :         Parallel::number_of_nodes<size_t>(local_cache);
     105             :     const size_t num_of_procs_to_use = number_of_procs - procs_to_ignore.size();
     106             : 
     107             :     const auto& blocks = domain.blocks();
     108             : 
     109             :     const size_t total_num_elements = [&blocks, &initial_refinement_levels]() {
     110             :       size_t result = 0;
     111             :       for (const auto& block : blocks) {
     112             :         const auto& initial_ref_levs = initial_refinement_levels[block.id()];
     113             :         for (const size_t ref_lev : initial_ref_levs) {
     114             :           result += two_to_the(ref_lev);
     115             :         }
     116             :       }
     117             :       return result;
     118             :     }();
     119             :     std::vector<std::pair<ElementId<Dim>, size_t>> my_elements_and_cores{};
     120             :     my_elements_and_cores.reserve(total_num_elements / number_of_nodes + 1);
     121             :     std::unordered_map<ElementId<Dim>, size_t> node_of_elements{};
     122             :     const size_t my_node = Parallel::my_node<size_t>(local_cache);
     123             : 
     124             :     Parallel::create_elements_using_distribution(
     125             :         [&my_elements_and_cores, my_node, &node_of_elements](
     126             :             const ElementId<Dim>& element_id, const size_t target_proc,
     127             :             const size_t target_node) {
     128             :           node_of_elements.insert(std::pair{element_id, target_node});
     129             :           if (target_node == my_node) {
     130             :             my_elements_and_cores.push_back(std::pair{element_id, target_proc});
     131             :           }
     132             :         },
     133             :         element_weight, blocks, initial_extents, initial_refinement_levels,
     134             :         quadrature,
     135             :         // The below arguments control how the elements are mapped to the
     136             :         // hardware.
     137             :         procs_to_ignore, number_of_procs, number_of_nodes, num_of_procs_to_use,
     138             :         local_cache, my_node == 0);
     139             : 
     140             :     tuples::tagged_tuple_from_typelist<SimpleTagsFromOptions>
     141             :         initialization_items = db::copy_items<SimpleTagsFromOptions>(box);
     142             : 
     143             :     const gsl::not_null<Parallel::NodeLock*> node_lock = make_not_null(
     144             :         &Parallel::local_branch(
     145             :              Parallel::get_parallel_component<ParallelComponent>(local_cache))
     146             :              ->get_node_lock());
     147             :     db::mutate<Tags::ElementLocations<Dim>,
     148             :                Tags::ElementCollection<Dim, Metavariables, PhaseDepActionList,
     149             :                                        SimpleTagsFromOptions>,
     150             :                Tags::NumberOfElementsTerminated>(
     151             :         [&local_cache, &initialization_items, &my_elements_and_cores,
     152             :          &node_of_elements](
     153             :             const auto element_locations_ptr, const auto collection_ptr,
     154             :             const gsl::not_null<size_t*> number_of_elements_terminated) {
     155             :           *number_of_elements_terminated = 0;
     156             :           const auto serialized_initialization_items =
     157             :               serialize(initialization_items);
     158             :           *element_locations_ptr = std::move(node_of_elements);
     159             :           for (const auto& element_id_and_core : my_elements_and_cores) {
     160             :             const auto& element_id = element_id_and_core.first;
     161             :             const auto core = element_id_and_core.second;
     162             :             if (not collection_ptr
     163             :                         ->emplace(
     164             :                             std::piecewise_construct,
     165             :                             std::forward_as_tuple(element_id),
     166             :                             std::forward_as_tuple(
     167             :                                 local_cache.get_this_proxy(),
     168             :                                 deserialize<tuples::tagged_tuple_from_typelist<
     169             :                                     SimpleTagsFromOptions>>(
     170             :                                     serialized_initialization_items.data()),
     171             :                                 element_id))
     172             :                         .second) {
     173             :               ERROR("Failed to insert element with ID: " << element_id);
     174             :             }
     175             :             if (collection_ptr->at(element_id).get_terminate() == true) {
     176             :               ++(*number_of_elements_terminated);
     177             :             } else {
     178             :               ERROR("Inserted element with ID "
     179             :                     << element_id
     180             :                     << " was not initialized in a terminated state. This is a "
     181             :                        "bug.");
     182             :             }
     183             :             collection_ptr->at(element_id).set_core(core);
     184             :           }
     185             :           ASSERT(*number_of_elements_terminated == collection_ptr->size(),
     186             :                  "The number of elements inserted must match the number of "
     187             :                  "elements set to terminate since the default state of an "
     188             :                  "inserted element is terminated. This is a bug.");
     189             :         },
     190             :         make_not_null(&box));
     191             : 
     192             :     Parallel::contribute_to_reduction<
     193             :         Parallel::Actions::SpawnInitializeElementsInCollection>(
     194             :         Parallel::ReductionData<
     195             :             Parallel::ReductionDatum<int, funcl::AssertEqual<>>>{0},
     196             :         Parallel::get_parallel_component<ParallelComponent>(
     197             :             local_cache)[my_node],
     198             :         Parallel::get_parallel_component<ParallelComponent>(local_cache));
     199             : 
     200             :     return {Parallel::AlgorithmExecution::Continue, std::nullopt};
     201             :   }
     202             : };
     203             : }  // namespace Parallel::Actions

Generated by: LCOV version 1.14