From a7823a106db2fafa453de56e204e52917c4bce72 Mon Sep 17 00:00:00 2001
From: Anna Wellmann <a.wellmann@tu-braunschweig.de>
Date: Fri, 1 Oct 2021 13:42:13 +0200
Subject: [PATCH] Send positions of indices for communication after f to c to
neighboring process (MPI)
---
.../Communication/Communicator.cpp | 13 ++++++++++
.../Communication/Communicator.h | 2 ++
.../GridReaderGenerator/GridGenerator.cpp | 26 +++++++++++++++----
.../GridReaderGenerator/GridGenerator.h | 7 +++++
4 files changed, 43 insertions(+), 5 deletions(-)
diff --git a/src/gpu/VirtualFluids_GPU/Communication/Communicator.cpp b/src/gpu/VirtualFluids_GPU/Communication/Communicator.cpp
index 80823e8c1..236407075 100644
--- a/src/gpu/VirtualFluids_GPU/Communication/Communicator.cpp
+++ b/src/gpu/VirtualFluids_GPU/Communication/Communicator.cpp
@@ -219,3 +219,16 @@ std::vector<double> Communicator::gatherNUPS(double processNups)
} // namespace GPU
} // namespace VF
+
+void vf::gpu::Communicator::exchangeIndices(uint *rbuf, int count_r, int nb_rank_r, uint *sbuf, int count_s,
+ int nb_rank_s)
+{
+ MPI_Request recv_request;
+ MPI_Irecv(rbuf, count_r, MPI_UNSIGNED, nb_rank_r, 0, commGPU, &recv_request);
+ //printf("exchangeIndices PID: %i, nbRev: nb_rank_recv: %i", this->getPID(), nb_rank_r);
+ //fflush(stdout);
+ MPI_Send(sbuf, count_s, MPI_UNSIGNED, nb_rank_s, 0, commGPU);
+ //printf("exchangeIndices PID: %i, sendUintGPU: nb_rank_send: %i", this->getPID(), nb_rank_s);
+ //fflush(stdout);
+ MPI_Wait(&recv_request, MPI_STATUSES_IGNORE);
+}
\ No newline at end of file
diff --git a/src/gpu/VirtualFluids_GPU/Communication/Communicator.h b/src/gpu/VirtualFluids_GPU/Communication/Communicator.h
index 22e661262..afa954cd7 100644
--- a/src/gpu/VirtualFluids_GPU/Communication/Communicator.h
+++ b/src/gpu/VirtualFluids_GPU/Communication/Communicator.h
@@ -61,6 +61,8 @@ public:
double getTime();
int mapCudaDevice(const int &rank, const int &size, const std::vector<unsigned int> &devices, const int &maxdev);
std::vector<double> gatherNUPS(double processNups);
+ //////////////////////////////////////////////////////////////////////////
+ void exchangeIndices(uint *rbuf, int count_r, int nb_rank_r, uint *sbuf, int count_s, int nb_rank_s);
protected:
private:
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
index 879c81596..c7b7a983c 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
@@ -12,6 +12,7 @@
#include "utilities/communication.h"
+#include "Communication/Communicator.h"
GridGenerator::GridGenerator(std::shared_ptr<GridBuilder> builder, std::shared_ptr<Parameter> para, std::shared_ptr<CudaMemoryManager> cudaManager)
@@ -261,8 +262,7 @@ void GridGenerator::allocArrays_BoundaryValues()
}
}//ende geo
-initalValuesDomainDecompostion();
-
+ initalValuesDomainDecompostion();
}
void GridGenerator::initalValuesDomainDecompostion()
@@ -715,6 +715,7 @@ void GridGenerator::initCommunicationArraysForCommAfterFinetoCoarseZ(const uint
void GridGenerator::initCommunicationArraysForCommAfterFinetoCoarseY(const uint &level, int j, int direction)
{
+ // init send indices
para->initNumberOfProcessNeighborsAfterFtoCY(level);
std::vector<uint> sendIndicesForCommAfterFtoCPositions;
builder->reorderSendIndicesForCommAfterFtoC(
@@ -723,11 +724,26 @@ void GridGenerator::initCommunicationArraysForCommAfterFinetoCoarseY(const uint
para->getParH(level)->K_CF, para->getParH(level)->intCF.ICellCFC, para->getParH(level)->K_FC,
para->getParH(level)->neighborX_SP, para->getParH(level)->neighborY_SP, para->getParH(level)->neighborZ_SP,
direction, level, sendIndicesForCommAfterFtoCPositions);
- builder->reorderRecvIndicesForCommAfterFtoC(para->getParH(level)->recvProcessNeighborY[j].index,
- para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCY[j],
- sendIndicesForCommAfterFtoCPositions, direction, level);
para->getParD(level)->numberOfSendProcessNeighborsAfterFtoCY[j] =
para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCY[j];
+
+
+ // send sendIndicesForCommAfterFtoCPositions to receiving process
+ std::vector<uint> recvIndicesForCommAfterFtoCPositions;
+ recvIndicesForCommAfterFtoCPositions.resize(para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCY[j] * 2); // give vector an arbitraty size (larger than needed)
+ auto comm = vf::gpu::Communicator::getInstanz();
+ comm->exchangeIndices(recvIndicesForCommAfterFtoCPositions.data(), recvIndicesForCommAfterFtoCPositions.size(),
+ para->getParH(level)->recvProcessNeighborY[j].rankNeighbor,
+ sendIndicesForCommAfterFtoCPositions.data(), sendIndicesForCommAfterFtoCPositions.size(),
+ para->getParH(level)->sendProcessNeighborY[j].rankNeighbor);
+ // resize vector to correct size
+ auto it = std::unique(recvIndicesForCommAfterFtoCPositions.begin(), recvIndicesForCommAfterFtoCPositions.end());
+ recvIndicesForCommAfterFtoCPositions.erase(std::prev(it, 1), recvIndicesForCommAfterFtoCPositions.end());
+
+ // init receive indices
+ builder->reorderRecvIndicesForCommAfterFtoC(para->getParH(level)->recvProcessNeighborY[j].index,
+ para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCY[j],
+ recvIndicesForCommAfterFtoCPositions, direction, level);
para->getParD(level)->numberOfRecvProcessNeighborsAfterFtoCY[j] =
para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCY[j];
}
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h
index 0845dafeb..bfcd8d4ad 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h
@@ -11,6 +11,13 @@
class Parameter;
class GridBuilder;
+namespace vf
+{
+ namespace gpu
+ {
+ class Communicator;
+ }
+} // namespace vf
class GridGenerator
: public GridProvider
--
GitLab