Line data Source code
1 0 : // Distributed under the MIT License. 2 : // See LICENSE.txt for details. 3 : 4 : #pragma once 5 : 6 : #include <boost/functional/hash.hpp> 7 : #include <cstddef> 8 : #include <iomanip> 9 : #include <map> 10 : #include <memory> 11 : #include <optional> 12 : #include <sstream> 13 : #include <string> 14 : #include <tuple> 15 : #include <type_traits> 16 : #include <utility> 17 : 18 : #include "Domain/Structure/Direction.hpp" 19 : #include "Domain/Structure/DirectionalId.hpp" 20 : #include "Domain/Structure/DirectionalIdMap.hpp" 21 : #include "Domain/Structure/ElementId.hpp" 22 : #include "Evolution/DiscontinuousGalerkin/Messages/BoundaryMessage.hpp" 23 : #include "NumericalAlgorithms/Spectral/Mesh.hpp" 24 : #include "Parallel/InboxInserters.hpp" 25 : #include "Time/TimeStepId.hpp" 26 : #include "Utilities/TMPL.hpp" 27 : 28 : /// \cond 29 : class DataVector; 30 : /// \endcond 31 : 32 1 : namespace evolution::dg::Tags { 33 : /*! 34 : * \brief The inbox tag for boundary correction communication and DG-subcell 35 : * ghost zone cells. 36 : * 37 : * The stored data consists of the following (in argument order of the tuple): 38 : * 39 : * 1. the mesh of the ghost cell data we received. This allows eliding 40 : * projection when all neighboring elements are doing DG. 41 : * 2. the mesh of the neighboring element's face (not the mortar mesh!) 42 : * 3. the variables at the ghost zone cells for finite difference/volume 43 : * reconstruction 44 : * 4. the data on the mortar needed for computing the boundary corrections (e.g. 45 : * fluxes, characteristic speeds, conserved variables) 46 : * 5. the TimeStepId beyond which the boundary terms are no longer valid, when 47 : * using local time stepping. 48 : * 49 : * The TimeStepId is the neighboring element's next time step. When using local 50 : * time stepping, the neighbor's boundary data is valid up until this time, 51 : * which may include multiple local time steps. By receiving and storing the 52 : * neighbor time step, the local element knows whether or not it should remove 53 : * boundary data and expect new data to be sent from the neighbor. 54 : * 55 : * The ghost cell data (second element of the tuple) will be valid whenever a 56 : * DG-subcell scheme is being used. Whenever a DG-subcell scheme is being used, 57 : * elements using DG and not FD/FV always send both the ghost cells and boundary 58 : * correction data together. Elements using FD/FV send the ghost cells first 59 : * followed by the boundary correction data once the element has received all 60 : * neighbor ghost cell data. Note that the second send/receive only modifies the 61 : * flux and the TimeStepId used for the flux validity range. 62 : * 63 : * When only a DG scheme (not a DG-subcell scheme) is used the ghost cell data 64 : * will never be valid. 65 : * 66 : * In the DG-subcell scheme this tag is used both for communicating the ghost 67 : * cell data needed for the FD/FV reconstruction step and the data needed for 68 : * the boundary corrections. 69 : * - For an element using DG, both ghost cells and boundary corrections are 70 : * sent using a single communication. After receiving all neighbor 71 : * boundary corrections the element can finish computing the time step. 72 : * The ghost cell data from neighbors is unused. 73 : * - For an element using FD/FV, first the ghost cells are sent. Once all 74 : * neighboring ghost cell data is received, reconstruction is done and the 75 : * boundary terms are computed and sent to the neighbors. After receiving all 76 : * neighbor boundary corrections the element can finish computing the time 77 : * step. 78 : * - Whether or not an extra communication is needed when an element switches 79 : * from DG to FD/FV depends on how exactly the decision to switch is 80 : * implemented. If the volume terms are integrated and verified to be 81 : * valid before a DG element sends ghost cell and boundary data then no 82 : * additional communication is needed when switching from DG to FD/FV. In this 83 : * case a second check of the data that includes the boundary correction needs 84 : * to be done. If the second check determines a switch from DG to FD/FV is 85 : * needed, we can continue to use the DG fluxes since the evolution in the 86 : * small was valid, thus avoiding an additional communication. However, to 87 : * fully guarantee physical realizability a second communication or evolving 88 : * the neighboring ghost cells needs to be done. We have not yet decided how 89 : * to deal with the possible need for an additional communication since it 90 : * also depends on whether or not we decide to couple to Voronoi instead of 91 : * just Neumann neighbors. 92 : * - The data for the inbox tags is erased after the boundary correction is 93 : * complete and the solution has been verified to be valid at the new time 94 : * step. The ghost cells could be invalidated immediately after 95 : * reconstruction, thus using the ghost cell data after reconstruction is 96 : * complete is considered undefined behavior. That is, we make no guarantee as 97 : * to whether or not it will work. 98 : * - The reason for minimizing the number of communications rather than having a 99 : * more uniform implementation between DG and FD/FV is that it is the number 100 : * of communications that adds the most overhead, not the size of each 101 : * communication. Thus, one large communication is cheaper than several small 102 : * communications. 103 : */ 104 : template <size_t Dim> 105 1 : struct BoundaryCorrectionAndGhostCellsInbox { 106 0 : using stored_type = 107 : std::tuple<Mesh<Dim>, Mesh<Dim - 1>, std::optional<DataVector>, 108 : std::optional<DataVector>, ::TimeStepId, int>; 109 : 110 : public: 111 0 : using temporal_id = TimeStepId; 112 0 : using type = std::map<TimeStepId, DirectionalIdMap<Dim, stored_type>>; 113 : 114 : template <typename Inbox, typename ReceiveDataType> 115 0 : static void insert_into_inbox(const gsl::not_null<Inbox*> inbox, 116 : const temporal_id& time_step_id, 117 : ReceiveDataType&& data) { 118 : auto& current_inbox = (*inbox)[time_step_id]; 119 : auto& [volume_mesh_of_ghost_cell_data, face_mesh, ghost_cell_data, 120 : boundary_data, boundary_data_validity_range, boundary_tci_status] = 121 : data.second; 122 : (void)ghost_cell_data; 123 : 124 : if (auto it = current_inbox.find(data.first); it != current_inbox.end()) { 125 : auto& [current_volume_mesh_of_ghost_cell_data, current_face_mesh, 126 : current_ghost_cell_data, current_boundary_data, 127 : current_boundary_data_validity_range, current_tci_status] = 128 : it->second; 129 : (void)current_volume_mesh_of_ghost_cell_data; // Need to use when 130 : // optimizing subcell 131 : // We have already received some data at this time. Receiving data twice 132 : // at the same time should only occur when receiving fluxes after having 133 : // previously received ghost cells. We sanity check that the data we 134 : // already have is the ghost cells and that we have not yet received flux 135 : // data. 136 : // 137 : // This is used if a 2-send implementation is used (which we don't right 138 : // now!). We generally find that the number of communications is more 139 : // important than the size of each communication, and so a single 140 : // communication per time/sub step is preferred. 141 : ASSERT(current_ghost_cell_data.has_value(), 142 : "Have not yet received ghost cells at time step " 143 : << time_step_id 144 : << " but the inbox entry already exists. This is a bug in the " 145 : "ordering of the actions."); 146 : ASSERT(not current_boundary_data.has_value(), 147 : "The fluxes have already been received at time step " 148 : << time_step_id 149 : << ". They are either being received for a second time, there " 150 : "is a bug in the ordering of the actions (though a " 151 : "different ASSERT should've caught that), or the incorrect " 152 : "temporal ID is being sent."); 153 : 154 : ASSERT(current_face_mesh == face_mesh, 155 : "The mesh being received for the fluxes is different than the " 156 : "mesh received for the ghost cells. Mesh for fluxes: " 157 : << face_mesh << " mesh for ghost cells " << current_face_mesh); 158 : ASSERT(current_volume_mesh_of_ghost_cell_data == 159 : volume_mesh_of_ghost_cell_data, 160 : "The mesh being received for the ghost cell data is different " 161 : "than the mesh received previously. Mesh for received when we got " 162 : "fluxes: " 163 : << volume_mesh_of_ghost_cell_data 164 : << " mesh received when we got ghost cells " 165 : << current_volume_mesh_of_ghost_cell_data); 166 : 167 : // We always move here since we take ownership of the data and moves 168 : // implicitly decay to copies 169 : current_boundary_data = std::move(boundary_data); 170 : current_boundary_data_validity_range = boundary_data_validity_range; 171 : current_tci_status = boundary_tci_status; 172 : } else { 173 : // We have not received ghost cells or fluxes at this time. 174 : if (not current_inbox.insert(std::forward<ReceiveDataType>(data)) 175 : .second) { 176 : ERROR("Failed to insert data to receive at instance '" 177 : << time_step_id 178 : << "' with tag 'BoundaryCorrectionAndGhostCellsInbox'.\n"); 179 : } 180 : } 181 : } 182 : 183 0 : static std::string output_inbox(const type& inbox, 184 : const size_t padding_size) { 185 : std::stringstream ss{}; 186 : const std::string pad(padding_size, ' '); 187 : ss << std::scientific << std::setprecision(16); 188 : ss << pad << "BoundaryCorrectionAndGhostCellInbox:\n"; 189 : 190 : for (const auto& [current_time_step_id, hash_map] : inbox) { 191 : ss << pad << " Current time: " << current_time_step_id << "\n"; 192 : // We only care about the next time because that's important for deadlock 193 : // detection. The data itself isn't super important 194 : for (const auto& [key, tuple_data] : hash_map) { 195 : ss << pad << " Key: " << key 196 : << ", next time: " << std::get<4>(tuple_data) << "\n"; 197 : } 198 : } 199 : 200 : return ss.str(); 201 : } 202 : 203 0 : void pup(PUP::er& /*p*/) {} 204 : }; 205 : 206 : /*! 207 : * \brief The inbox tag for boundary correction communication and DG-subcell 208 : * ghost zone cells using a `BoundaryMessage` object 209 : * 210 : * To see what is stored within a `BoundaryMessage`, see its documentation. 211 : * 212 : * This inbox tag is very similar to `BoundaryCorrectionAndGhostCellsInbox` in 213 : * that it stores subcell/DG data sent from neighboring elements. To see exactly 214 : * when data is stored and how it's used, see the docs for 215 : * `BoundaryCorrectionAndGhostCellsInbox`. This inbox tag is different than 216 : * `BoundaryCorrectionAndGhostCellsInbox` in that it only takes a pointer to a 217 : * `BoundaryMessage` as an argument to `insert_into_inbox` and stores a 218 : * `std::unique_ptr<BoundaryMessage>` inside the inbox. 219 : * 220 : * This inbox tag is meant to be used to avoid unnecessary copies between 221 : * elements on the same node which share a block of memory. If two elements 222 : * aren't on the same node, then a copy/send is done regardless. 223 : * 224 : * \warning The `boundary_message` argument to `insert_into_inbox()` will be 225 : * invalid after the function is called because a `std::unique_ptr` now controls 226 : * the memory. Calling a method on the `boundary_message` pointer after the 227 : * `insert_into_inbox()` function is called can result in undefined behaviour. 228 : */ 229 : template <size_t Dim> 230 1 : struct BoundaryMessageInbox { 231 0 : using stored_type = std::unique_ptr<BoundaryMessage<Dim>>; 232 : 233 : public: 234 0 : using temporal_id = TimeStepId; 235 0 : using type = std::map<TimeStepId, DirectionalIdMap<Dim, stored_type>>; 236 0 : using message_type = BoundaryMessage<Dim>; 237 : 238 : template <typename Inbox> 239 0 : static void insert_into_inbox(const gsl::not_null<Inbox*> inbox, 240 : BoundaryMessage<Dim>* boundary_message) { 241 : const auto& time_step_id = boundary_message->current_time_step_id; 242 : auto& current_inbox = (*inbox)[time_step_id]; 243 : 244 : const auto key = DirectionalId<Dim>{boundary_message->neighbor_direction, 245 : boundary_message->element_id}; 246 : 247 : if (auto it = current_inbox.find(key); it != current_inbox.end()) { 248 : auto& current_boundary_data = it->second; 249 : // We have already received some data at this time. Receiving data twice 250 : // at the same time should only occur when receiving fluxes after having 251 : // previously received ghost cells. We sanity check that the data we 252 : // already have is the ghost cells and that we have not yet received flux 253 : // data. 254 : // 255 : // This is used if a 2-send implementation is used (which we don't right 256 : // now!). We generally find that the number of communications is more 257 : // important than the size of each communication, and so a single 258 : // communication per time/sub step is preferred. 259 : ASSERT(current_boundary_data->subcell_ghost_data != nullptr, 260 : "Have not yet received ghost cells at time step " 261 : << time_step_id 262 : << " but the inbox entry already exists. This is a bug in the " 263 : "ordering of the actions."); 264 : ASSERT(current_boundary_data->dg_flux_data == nullptr, 265 : "The fluxes have already been received at time step " 266 : << time_step_id 267 : << ". They are either being received for a second time, there " 268 : "is a bug in the ordering of the actions (though a " 269 : "different ASSERT should've caught that), or the incorrect " 270 : "temporal ID is being sent."); 271 : ASSERT(boundary_message->subcell_ghost_data == nullptr, 272 : "Have already received ghost cells at time step " 273 : << time_step_id 274 : << ", but we are being sent ghost cells again. This data will " 275 : "not be cleaned up while the dg flux data is being " 276 : "inserted into the inbox which will cause a memory leak."); 277 : 278 : ASSERT(current_boundary_data->interface_mesh == 279 : boundary_message->interface_mesh, 280 : "The mesh being received for the fluxes is different than the " 281 : "mesh received for the ghost cells. Mesh for fluxes: " 282 : << boundary_message->interface_mesh << " mesh for ghost cells " 283 : << current_boundary_data->interface_mesh); 284 : ASSERT(current_boundary_data->volume_or_ghost_mesh == 285 : boundary_message->volume_or_ghost_mesh, 286 : "The mesh being received for the ghost cell data is different " 287 : "than the mesh received previously. Mesh for received when we got " 288 : "fluxes: " 289 : << boundary_message->volume_or_ghost_mesh 290 : << " mesh received when we got ghost cells " 291 : << current_boundary_data->volume_or_ghost_mesh); 292 : 293 : // Just need to set the pointer. No need to worry about the data being 294 : // deleted upon destruction of boundary_message because the destructor of 295 : // a BoundaryMessage will only delete the pointer stored inside the 296 : // BoundaryMessage. It won't delete the actual data it points to. Now the 297 : // unique_ptr owns the data that was previously pointed to by 298 : // boundary_message 299 : current_boundary_data->dg_flux_data_size = 300 : boundary_message->dg_flux_data_size; 301 : current_boundary_data->dg_flux_data = boundary_message->dg_flux_data; 302 : current_boundary_data->next_time_step_id = 303 : boundary_message->next_time_step_id; 304 : current_boundary_data->tci_status = boundary_message->tci_status; 305 : } else { 306 : // We have not received ghost cells or fluxes at this time. 307 : // Once we insert boundary_message into the unique_ptr we cannot use 308 : // boundary_message anymore because it is invalidated. The unique_ptr now 309 : // owns the memory. 310 : if (not current_inbox 311 : .insert(std::pair{key, std::unique_ptr<BoundaryMessage<Dim>>( 312 : boundary_message)}) 313 : .second) { 314 : ERROR("Failed to insert data to receive at instance '" 315 : << time_step_id << "' with tag 'BoundaryMessageInbox'.\n"); 316 : } 317 : } 318 : } 319 : }; 320 : } // namespace evolution::dg::Tags