Send positions of indices for communication after f to c to neighboring process (MPI)

src/gpu/VirtualFluids_GPU/Communication/Communicator.cpp

@@ -219,3 +219,16 @@ std::vector<double> Communicator::gatherNUPS(double processNups)
 
 } // namespace GPU
 } // namespace VF
+
+void vf::gpu::Communicator::exchangeIndices(uint *rbuf, int count_r, int nb_rank_r, uint *sbuf, int count_s,
+                                            int nb_rank_s)
+{
+    MPI_Request recv_request;
+    MPI_Irecv(rbuf, count_r, MPI_UNSIGNED, nb_rank_r, 0, commGPU, &recv_request);
+    //printf("exchangeIndices PID: %i,   nbRev: nb_rank_recv: %i", this->getPID(), nb_rank_r);
+    //fflush(stdout);
+    MPI_Send(sbuf, count_s, MPI_UNSIGNED, nb_rank_s, 0, commGPU);
+    //printf("exchangeIndices PID: %i,   sendUintGPU: nb_rank_send: %i", this->getPID(), nb_rank_s);
+    //fflush(stdout);
+    MPI_Wait(&recv_request, MPI_STATUSES_IGNORE);
+}
\ No newline at end of file

src/gpu/VirtualFluids_GPU/Communication/Communicator.h

@@ -61,6 +61,8 @@ public:
     double getTime();
     int mapCudaDevice(const int &rank, const int &size, const std::vector<unsigned int> &devices, const int &maxdev);
     std::vector<double> gatherNUPS(double processNups);
+    //////////////////////////////////////////////////////////////////////////
+    void exchangeIndices(uint *rbuf, int count_r, int nb_rank_r, uint *sbuf, int count_s, int nb_rank_s);
 
 protected:
 private:

src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp

@@ -12,6 +12,7 @@
 
 #include "utilities/communication.h"
 
+#include "Communication/Communicator.h"
 
 
 GridGenerator::GridGenerator(std::shared_ptr<GridBuilder> builder, std::shared_ptr<Parameter> para, std::shared_ptr<CudaMemoryManager> cudaManager)
@@ -261,8 +262,7 @@ void GridGenerator::allocArrays_BoundaryValues()
         }
     }//ende geo
 
-initalValuesDomainDecompostion();
-
+    initalValuesDomainDecompostion();
 }
 
 void GridGenerator::initalValuesDomainDecompostion()
@@ -715,6 +715,7 @@ void GridGenerator::initCommunicationArraysForCommAfterFinetoCoarseZ(const uint
 
 void GridGenerator::initCommunicationArraysForCommAfterFinetoCoarseY(const uint &level, int j, int direction)
 {
+    // init send indices
     para->initNumberOfProcessNeighborsAfterFtoCY(level);
     std::vector<uint> sendIndicesForCommAfterFtoCPositions;
     builder->reorderSendIndicesForCommAfterFtoC(
@@ -723,11 +724,26 @@ void GridGenerator::initCommunicationArraysForCommAfterFinetoCoarseY(const uint
         para->getParH(level)->K_CF, para->getParH(level)->intCF.ICellCFC, para->getParH(level)->K_FC,
         para->getParH(level)->neighborX_SP, para->getParH(level)->neighborY_SP, para->getParH(level)->neighborZ_SP,
         direction, level, sendIndicesForCommAfterFtoCPositions);
-    builder->reorderRecvIndicesForCommAfterFtoC(para->getParH(level)->recvProcessNeighborY[j].index,
-                                                para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCY[j],
-                                                sendIndicesForCommAfterFtoCPositions, direction, level);
     para->getParD(level)->numberOfSendProcessNeighborsAfterFtoCY[j] =
         para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCY[j];
+
+
+    // send sendIndicesForCommAfterFtoCPositions to receiving process
+    std::vector<uint> recvIndicesForCommAfterFtoCPositions; 
+    recvIndicesForCommAfterFtoCPositions.resize(para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCY[j] * 2); // give vector an arbitraty size (larger than needed)
+    auto comm = vf::gpu::Communicator::getInstanz();
+    comm->exchangeIndices(recvIndicesForCommAfterFtoCPositions.data(), recvIndicesForCommAfterFtoCPositions.size(),
+                          para->getParH(level)->recvProcessNeighborY[j].rankNeighbor,
+                          sendIndicesForCommAfterFtoCPositions.data(), sendIndicesForCommAfterFtoCPositions.size(),
+                          para->getParH(level)->sendProcessNeighborY[j].rankNeighbor);
+    // resize vector to correct size
+    auto it = std::unique(recvIndicesForCommAfterFtoCPositions.begin(), recvIndicesForCommAfterFtoCPositions.end());
+    recvIndicesForCommAfterFtoCPositions.erase(std::prev(it, 1), recvIndicesForCommAfterFtoCPositions.end());
+
+    // init receive indices
+    builder->reorderRecvIndicesForCommAfterFtoC(para->getParH(level)->recvProcessNeighborY[j].index,
+                                                para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCY[j],
+                                                recvIndicesForCommAfterFtoCPositions, direction, level);
     para->getParD(level)->numberOfRecvProcessNeighborsAfterFtoCY[j] =
         para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCY[j];
 }

src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h

@@ -11,6 +11,13 @@
 
 class Parameter;
 class GridBuilder;
+namespace vf
+{
+	namespace gpu
+	{
+		class Communicator;
+	}
+} // namespace vf
 
 class GridGenerator
 	: public GridProvider