diff --git a/src/gpu/VirtualFluids_GPU/Communication/Communicator.cpp b/src/gpu/VirtualFluids_GPU/Communication/Communicator.cpp index 80823e8c14bc54c7db75f36012b539c6d3f68a77..2364070757ff5fbf65ffef6ad59365e669633b36 100644 --- a/src/gpu/VirtualFluids_GPU/Communication/Communicator.cpp +++ b/src/gpu/VirtualFluids_GPU/Communication/Communicator.cpp @@ -219,3 +219,16 @@ std::vector<double> Communicator::gatherNUPS(double processNups) } // namespace GPU } // namespace VF + +void vf::gpu::Communicator::exchangeIndices(uint *rbuf, int count_r, int nb_rank_r, uint *sbuf, int count_s, + int nb_rank_s) +{ + MPI_Request recv_request; + MPI_Irecv(rbuf, count_r, MPI_UNSIGNED, nb_rank_r, 0, commGPU, &recv_request); + //printf("exchangeIndices PID: %i, nbRev: nb_rank_recv: %i", this->getPID(), nb_rank_r); + //fflush(stdout); + MPI_Send(sbuf, count_s, MPI_UNSIGNED, nb_rank_s, 0, commGPU); + //printf("exchangeIndices PID: %i, sendUintGPU: nb_rank_send: %i", this->getPID(), nb_rank_s); + //fflush(stdout); + MPI_Wait(&recv_request, MPI_STATUSES_IGNORE); +} \ No newline at end of file diff --git a/src/gpu/VirtualFluids_GPU/Communication/Communicator.h b/src/gpu/VirtualFluids_GPU/Communication/Communicator.h index 22e6612629f4bc81bdb38434b50fc2c2de4a8e21..afa954cd7fc1993c543c2b4536c95c7e3ee25ca7 100644 --- a/src/gpu/VirtualFluids_GPU/Communication/Communicator.h +++ b/src/gpu/VirtualFluids_GPU/Communication/Communicator.h @@ -61,6 +61,8 @@ public: double getTime(); int mapCudaDevice(const int &rank, const int &size, const std::vector<unsigned int> &devices, const int &maxdev); std::vector<double> gatherNUPS(double processNups); + ////////////////////////////////////////////////////////////////////////// + void exchangeIndices(uint *rbuf, int count_r, int nb_rank_r, uint *sbuf, int count_s, int nb_rank_s); protected: private: diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp index 879c81596f68a6664d85c4fe6a612d0e5dc1e172..c7b7a983c6e6076c165b887825ac2ff45f755bc2 100644 --- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp +++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp @@ -12,6 +12,7 @@ #include "utilities/communication.h" +#include "Communication/Communicator.h" GridGenerator::GridGenerator(std::shared_ptr<GridBuilder> builder, std::shared_ptr<Parameter> para, std::shared_ptr<CudaMemoryManager> cudaManager) @@ -261,8 +262,7 @@ void GridGenerator::allocArrays_BoundaryValues() } }//ende geo -initalValuesDomainDecompostion(); - + initalValuesDomainDecompostion(); } void GridGenerator::initalValuesDomainDecompostion() @@ -715,6 +715,7 @@ void GridGenerator::initCommunicationArraysForCommAfterFinetoCoarseZ(const uint void GridGenerator::initCommunicationArraysForCommAfterFinetoCoarseY(const uint &level, int j, int direction) { + // init send indices para->initNumberOfProcessNeighborsAfterFtoCY(level); std::vector<uint> sendIndicesForCommAfterFtoCPositions; builder->reorderSendIndicesForCommAfterFtoC( @@ -723,11 +724,26 @@ void GridGenerator::initCommunicationArraysForCommAfterFinetoCoarseY(const uint para->getParH(level)->K_CF, para->getParH(level)->intCF.ICellCFC, para->getParH(level)->K_FC, para->getParH(level)->neighborX_SP, para->getParH(level)->neighborY_SP, para->getParH(level)->neighborZ_SP, direction, level, sendIndicesForCommAfterFtoCPositions); - builder->reorderRecvIndicesForCommAfterFtoC(para->getParH(level)->recvProcessNeighborY[j].index, - para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCY[j], - sendIndicesForCommAfterFtoCPositions, direction, level); para->getParD(level)->numberOfSendProcessNeighborsAfterFtoCY[j] = para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCY[j]; + + + // send sendIndicesForCommAfterFtoCPositions to receiving process + std::vector<uint> recvIndicesForCommAfterFtoCPositions; + recvIndicesForCommAfterFtoCPositions.resize(para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCY[j] * 2); // give vector an arbitraty size (larger than needed) + auto comm = vf::gpu::Communicator::getInstanz(); + comm->exchangeIndices(recvIndicesForCommAfterFtoCPositions.data(), recvIndicesForCommAfterFtoCPositions.size(), + para->getParH(level)->recvProcessNeighborY[j].rankNeighbor, + sendIndicesForCommAfterFtoCPositions.data(), sendIndicesForCommAfterFtoCPositions.size(), + para->getParH(level)->sendProcessNeighborY[j].rankNeighbor); + // resize vector to correct size + auto it = std::unique(recvIndicesForCommAfterFtoCPositions.begin(), recvIndicesForCommAfterFtoCPositions.end()); + recvIndicesForCommAfterFtoCPositions.erase(std::prev(it, 1), recvIndicesForCommAfterFtoCPositions.end()); + + // init receive indices + builder->reorderRecvIndicesForCommAfterFtoC(para->getParH(level)->recvProcessNeighborY[j].index, + para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCY[j], + recvIndicesForCommAfterFtoCPositions, direction, level); para->getParD(level)->numberOfRecvProcessNeighborsAfterFtoCY[j] = para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCY[j]; } diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h index 0845dafeb4ce635df537603e6e8f83d83971f399..bfcd8d4ad5c04a2c497c76c85192f03febd2297f 100644 --- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h +++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h @@ -11,6 +11,13 @@ class Parameter; class GridBuilder; +namespace vf +{ + namespace gpu + { + class Communicator; + } +} // namespace vf class GridGenerator : public GridProvider