diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
index c5b841cab845fb91c99056da0322a243f33a3280..2c1636769a6d408fb9394504d8477495c4b71919 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
@@ -697,44 +697,46 @@ void GridGenerator::initalValuesDomainDecompostion()
void GridGenerator::initCommunicationArraysForCommAfterFinetoCoarseX(const uint &level, int j, int direction)
{
+ // init send indices for communication after coarse to fine
+ std::cout << "communication: reorder send indices X ";
para->initNumberOfProcessNeighborsAfterFtoCX(level);
std::vector<uint> sendIndicesForCommAfterFtoCPositions;
- std::vector<uint> recvIndicesForCommAfterFtoCPositions;
reorderSendIndicesForCommAfterFtoCX(direction, level, j, sendIndicesForCommAfterFtoCPositions);
- reorderRecvIndicesForCommAfterFtoCX(direction, level, j, recvIndicesForCommAfterFtoCPositions);
para->setSendProcessNeighborsAfterFtoCX(para->getParH(level)->sendProcessNeighborsAfterFtoCX[j].numberOfNodes,
level, j);
+
+ // send sendIndicesForCommAfterFtoCPositions to receiving process and receive recvIndicesForCommAfterFtoCPositions from sending process
+ std::cout << "mpi send and receive ";
+ std::vector<uint> recvIndicesForCommAfterFtoCPositions;
+ recvIndicesForCommAfterFtoCPositions.resize(
+ (size_t)para->getParH(level)->sendProcessNeighborsAfterFtoCX[j].numberOfNodes *
+ 2); // give vector an arbitraty size (larger than needed) // TODO: This is stupid! Find a better way
+ auto comm = vf::gpu::Communicator::getInstanz();
+ comm->exchangeIndices(recvIndicesForCommAfterFtoCPositions.data(), recvIndicesForCommAfterFtoCPositions.size(),
+ para->getParH(level)->recvProcessNeighborX[j].rankNeighbor,
+ sendIndicesForCommAfterFtoCPositions.data(), sendIndicesForCommAfterFtoCPositions.size(),
+ para->getParH(level)->sendProcessNeighborX[j].rankNeighbor);
+ // resize receiving vector to correct size
+ auto it = std::unique(recvIndicesForCommAfterFtoCPositions.begin(), recvIndicesForCommAfterFtoCPositions.end());
+ recvIndicesForCommAfterFtoCPositions.erase(std::prev(it, 1), recvIndicesForCommAfterFtoCPositions.end());
+
+ // init receive indices for communication after coarse to fine
+ std::cout << "reorder receive indices ";
+ reorderRecvIndicesForCommAfterFtoCX(direction, level, j, recvIndicesForCommAfterFtoCPositions);
para->setRecvProcessNeighborsAfterFtoCX(para->getParH(level)->recvProcessNeighborsAfterFtoCX[j].numberOfNodes,
level, j);
copyProcessNeighborToAfterFtoCX(level, j);
-}
-
-void GridGenerator::copyProcessNeighborToAfterFtoCX(const uint &level, int j)
-{
- // init f[0]*
- para->getParD(level)->sendProcessNeighborsAfterFtoCX[j].f[0] = para->getParD(level)->sendProcessNeighborX[j].f[0];
- para->getParH(level)->sendProcessNeighborsAfterFtoCX[j].f[0] = para->getParH(level)->sendProcessNeighborX[j].f[0];
- para->getParD(level)->recvProcessNeighborsAfterFtoCX[j].f[0] = para->getParD(level)->recvProcessNeighborX[j].f[0];
- para->getParH(level)->recvProcessNeighborsAfterFtoCX[j].f[0] = para->getParH(level)->recvProcessNeighborX[j].f[0];
-
- // init index*
- para->getParD(level)->sendProcessNeighborsAfterFtoCX[j].index = para->getParD(level)->sendProcessNeighborX[j].index;
- para->getParH(level)->sendProcessNeighborsAfterFtoCX[j].index = para->getParH(level)->sendProcessNeighborX[j].index;
- para->getParD(level)->recvProcessNeighborsAfterFtoCX[j].index = para->getParD(level)->recvProcessNeighborX[j].index;
- para->getParH(level)->recvProcessNeighborsAfterFtoCX[j].index = para->getParH(level)->recvProcessNeighborX[j].index;
- // rank neighbor
- para->getParH(level)->sendProcessNeighborsAfterFtoCX[j].rankNeighbor = para->getParH(level)->sendProcessNeighborX[j].rankNeighbor;
- para->getParH(level)->recvProcessNeighborsAfterFtoCX[j].rankNeighbor = para->getParH(level)->recvProcessNeighborX[j].rankNeighbor;
+ std::cout << "done." << std::endl;
}
void GridGenerator::initCommunicationArraysForCommAfterFinetoCoarseY(const uint &level, int j, int direction)
{
// init send indices for communication after coarse to fine
- std::cout << "communication: reorder send indices ";
+ std::cout << "communication: reorder send indices Y ";
para->initNumberOfProcessNeighborsAfterFtoCY(level);
std::vector<uint> sendIndicesForCommAfterFtoCPositions;
- this->reorderSendIndicesForCommAfterFtoCY(direction, level, j, sendIndicesForCommAfterFtoCPositions);
+ reorderSendIndicesForCommAfterFtoCY(direction, level, j, sendIndicesForCommAfterFtoCPositions);
para->setSendProcessNeighborsAfterFtoCY(para->getParH(level)->sendProcessNeighborsAfterFtoCY[j].numberOfNodes,
level, j);
@@ -763,6 +765,60 @@ void GridGenerator::initCommunicationArraysForCommAfterFinetoCoarseY(const uint
std::cout << "done." << std::endl;
}
+void GridGenerator::initCommunicationArraysForCommAfterFinetoCoarseZ(const uint &level, int j, int direction)
+{
+ // init send indices for communication after coarse to fine
+ std::cout << "communication: reorder send indices Z ";
+ para->initNumberOfProcessNeighborsAfterFtoCZ(level);
+ std::vector<uint> sendIndicesForCommAfterFtoCPositions;
+ reorderSendIndicesForCommAfterFtoCZ(direction, level, j, sendIndicesForCommAfterFtoCPositions);
+ para->setSendProcessNeighborsAfterFtoCZ(para->getParH(level)->sendProcessNeighborsAfterFtoCZ[j].numberOfNodes,
+ level, j);
+
+ // send sendIndicesForCommAfterFtoCPositions to receiving process and receive recvIndicesForCommAfterFtoCPositions from sending process
+ std::cout << "mpi send and receive ";
+ std::vector<uint> recvIndicesForCommAfterFtoCPositions;
+ recvIndicesForCommAfterFtoCPositions.resize((size_t) para->getParH(level)->sendProcessNeighborsAfterFtoCZ[j].numberOfNodes *
+ 2); // give vector an arbitraty size (larger than needed) // TODO: This is stupid! Find a better way
+ auto comm = vf::gpu::Communicator::getInstanz();
+ comm->exchangeIndices(recvIndicesForCommAfterFtoCPositions.data(), recvIndicesForCommAfterFtoCPositions.size(),
+ para->getParH(level)->recvProcessNeighborZ[j].rankNeighbor,
+ sendIndicesForCommAfterFtoCPositions.data(), sendIndicesForCommAfterFtoCPositions.size(),
+ para->getParH(level)->sendProcessNeighborZ[j].rankNeighbor);
+ // resize receiving vector to correct size
+ auto it = std::unique(recvIndicesForCommAfterFtoCPositions.begin(), recvIndicesForCommAfterFtoCPositions.end());
+ recvIndicesForCommAfterFtoCPositions.erase(std::prev(it, 1), recvIndicesForCommAfterFtoCPositions.end());
+
+ // init receive indices for communication after coarse to fine
+ std::cout << "reorder receive indices ";
+ reorderRecvIndicesForCommAfterFtoCZ(direction, level, j, recvIndicesForCommAfterFtoCPositions);
+ para->setRecvProcessNeighborsAfterFtoCZ(para->getParH(level)->recvProcessNeighborsAfterFtoCZ[j].numberOfNodes,
+ level, j);
+
+ copyProcessNeighborToAfterFtoCZ(level, j);
+
+ std::cout << "done." << std::endl;
+}
+
+void GridGenerator::copyProcessNeighborToAfterFtoCX(const uint &level, int j)
+{
+ // init f[0]*
+ para->getParD(level)->sendProcessNeighborsAfterFtoCX[j].f[0] = para->getParD(level)->sendProcessNeighborX[j].f[0];
+ para->getParH(level)->sendProcessNeighborsAfterFtoCX[j].f[0] = para->getParH(level)->sendProcessNeighborX[j].f[0];
+ para->getParD(level)->recvProcessNeighborsAfterFtoCX[j].f[0] = para->getParD(level)->recvProcessNeighborX[j].f[0];
+ para->getParH(level)->recvProcessNeighborsAfterFtoCX[j].f[0] = para->getParH(level)->recvProcessNeighborX[j].f[0];
+
+ // init index*
+ para->getParD(level)->sendProcessNeighborsAfterFtoCX[j].index = para->getParD(level)->sendProcessNeighborX[j].index;
+ para->getParH(level)->sendProcessNeighborsAfterFtoCX[j].index = para->getParH(level)->sendProcessNeighborX[j].index;
+ para->getParD(level)->recvProcessNeighborsAfterFtoCX[j].index = para->getParD(level)->recvProcessNeighborX[j].index;
+ para->getParH(level)->recvProcessNeighborsAfterFtoCX[j].index = para->getParH(level)->recvProcessNeighborX[j].index;
+
+ // rank neighbor
+ para->getParH(level)->sendProcessNeighborsAfterFtoCX[j].rankNeighbor = para->getParH(level)->sendProcessNeighborX[j].rankNeighbor;
+ para->getParH(level)->recvProcessNeighborsAfterFtoCX[j].rankNeighbor = para->getParH(level)->recvProcessNeighborX[j].rankNeighbor;
+}
+
void GridGenerator::copyProcessNeighborToAfterFtoCY(const uint &level, int j)
{
// init f[0]*
@@ -782,21 +838,6 @@ void GridGenerator::copyProcessNeighborToAfterFtoCY(const uint &level, int j)
para->getParH(level)->recvProcessNeighborsAfterFtoCY[j].rankNeighbor = para->getParH(level)->recvProcessNeighborY[j].rankNeighbor;
}
-void GridGenerator::initCommunicationArraysForCommAfterFinetoCoarseZ(const uint &level, int j, int direction)
-{
- para->initNumberOfProcessNeighborsAfterFtoCZ(level);
- std::vector<uint> sendIndicesForCommAfterFtoCPositions;
- std::vector<uint> recvIndicesForCommAfterFtoCPositions;
- reorderSendIndicesForCommAfterFtoCZ(direction, level, j, sendIndicesForCommAfterFtoCPositions);
- reorderRecvIndicesForCommAfterFtoCZ(direction, level, j, recvIndicesForCommAfterFtoCPositions);
- para->setSendProcessNeighborsAfterFtoCZ(para->getParH(level)->sendProcessNeighborsAfterFtoCZ[j].numberOfNodes,
- level, j);
- para->setRecvProcessNeighborsAfterFtoCZ(para->getParH(level)->recvProcessNeighborsAfterFtoCZ[j].numberOfNodes,
- level, j);
-
- copyProcessNeighborToAfterFtoCZ(level, j);
-}
-
void GridGenerator::copyProcessNeighborToAfterFtoCZ(const uint &level, int j)
{
// init f[0]*
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h
index 4a9b9c774f59af771f2f9bdb9b57378b7d2e6221..d53f083af0bc8edca6c6b37c44a8ed2130970ac0 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h
@@ -36,40 +36,6 @@ public:
void allocArrays_BoundaryValues() override;
void initalValuesDomainDecompostion();
- // communication after coarse to fine
- void initCommunicationArraysForCommAfterFinetoCoarseX(const uint &level, int j, int direction);
- void copyProcessNeighborToAfterFtoCX(const uint &level, int j);
- void initCommunicationArraysForCommAfterFinetoCoarseY(const uint &level, int j, int direction);
- void copyProcessNeighborToAfterFtoCY(const uint &level, int j);
- void initCommunicationArraysForCommAfterFinetoCoarseZ(const uint &level, int j, int direction);
- void copyProcessNeighborToAfterFtoCZ(const uint &level, int j);
- void reorderSendIndicesForCommAfterFtoCX(int direction, int level, int j,
- std::vector<uint> &sendIndicesForCommAfterFtoCPositions);
- void reorderSendIndicesForCommAfterFtoCY(int direction, int level, int j,
- std::vector<uint> &sendIndicesForCommAfterFtoCPositions);
- void reorderSendIndicesForCommAfterFtoCZ(int direction, int level, int j,
- std::vector<uint> &sendIndicesForCommAfterFtoCPositions);
- void reorderSendIndicesForCommAfterFtoC(int *sendIndices, int &numberOfSendNeighborsAfterFtoC, int direction,
- int level, int j, std::vector<uint> &sendIndicesForCommAfterFtoCPositions);
- bool isSparseIndexInICellFCC(uint sizeOfICellFCC, int sparseIndexSend, int level);
- void aggregateNodesInICellCFC(int level, std::vector<uint> &nodesCFC);
- void addUniqueIndexToCommunicationVectors(std::vector<int> &sendIndicesAfterFtoC, int &sparseIndexSend,
- std::vector<unsigned int> &sendIndicesForCommAfterFtoCPositions,
- uint &posInSendIndices) const;
- void findIfSparseIndexIsInSendIndicesAndAddToCommVectors(int sparseIndex, int *sendIndices, uint numberOfSendIndices,
- std::vector<int> &sendIndicesAfterFtoC,
- std::vector<uint> &sendIndicesForCommAfterFtoCPositions) const;
- void findIndicesNotInCommAfterFtoC(const uint &numberOfSendIndices, int *sendIndices,
- std::vector<int> &sendIndicesAfterFtoC, std::vector<int> &sendIndicesOther);
- void reorderRecvIndicesForCommAfterFtoCX(int direction, int level, int j,
- std::vector<uint> &sendIndicesForCommAfterFtoCPositions);
- void reorderRecvIndicesForCommAfterFtoCY(int direction, int level, int j,
- std::vector<uint> &sendIndicesForCommAfterFtoCPositions);
- void reorderRecvIndicesForCommAfterFtoCZ(int direction, int level, int j,
- std::vector<uint> &sendIndicesForCommAfterFtoCPositions);
- void reorderRecvIndicesForCommAfterFtoC(int *recvIndices, int &numberOfRecvNeighborsAfterFtoC, int direction,
- int level, int j, std::vector<uint> &sendIndicesForCommAfterFtoCPositions);
-
void allocArrays_BoundaryQs() override;
void allocArrays_OffsetScale() override;
void allocArrays_fluidNodeIndices() override;
@@ -104,6 +70,41 @@ private:
void setQ27Size(QforBoundaryConditions &Q, real* QQ, unsigned int sizeQ) const;
bool hasQs(int channelSide, unsigned int level) const;
+ // communication after coarse to fine
+ void initCommunicationArraysForCommAfterFinetoCoarseX(const uint &level, int j, int direction);
+ void initCommunicationArraysForCommAfterFinetoCoarseY(const uint &level, int j, int direction);
+ void initCommunicationArraysForCommAfterFinetoCoarseZ(const uint &level, int j, int direction);
+
+ void copyProcessNeighborToAfterFtoCX(const uint &level, int j);
+ void copyProcessNeighborToAfterFtoCY(const uint &level, int j);
+ void copyProcessNeighborToAfterFtoCZ(const uint &level, int j);
+ void reorderSendIndicesForCommAfterFtoCX(int direction, int level, int j,
+ std::vector<uint> &sendIndicesForCommAfterFtoCPositions);
+ void reorderSendIndicesForCommAfterFtoCY(int direction, int level, int j,
+ std::vector<uint> &sendIndicesForCommAfterFtoCPositions);
+ void reorderSendIndicesForCommAfterFtoCZ(int direction, int level, int j,
+ std::vector<uint> &sendIndicesForCommAfterFtoCPositions);
+ void reorderSendIndicesForCommAfterFtoC(int *sendIndices, int &numberOfSendNeighborsAfterFtoC, int direction,
+ int level, int j, std::vector<uint> &sendIndicesForCommAfterFtoCPositions);
+ bool isSparseIndexInICellFCC(uint sizeOfICellFCC, int sparseIndexSend, int level);
+ void aggregateNodesInICellCFC(int level, std::vector<uint> &nodesCFC);
+ void addUniqueIndexToCommunicationVectors(std::vector<int> &sendIndicesAfterFtoC, int &sparseIndexSend,
+ std::vector<unsigned int> &sendIndicesForCommAfterFtoCPositions,
+ uint &posInSendIndices) const;
+ void findIfSparseIndexIsInSendIndicesAndAddToCommVectors(int sparseIndex, int *sendIndices, uint numberOfSendIndices,
+ std::vector<int> &sendIndicesAfterFtoC,
+ std::vector<uint> &sendIndicesForCommAfterFtoCPositions) const;
+ void findIndicesNotInCommAfterFtoC(const uint &numberOfSendIndices, int *sendIndices,
+ std::vector<int> &sendIndicesAfterFtoC, std::vector<int> &sendIndicesOther);
+ void reorderRecvIndicesForCommAfterFtoCX(int direction, int level, int j,
+ std::vector<uint> &sendIndicesForCommAfterFtoCPositions);
+ void reorderRecvIndicesForCommAfterFtoCY(int direction, int level, int j,
+ std::vector<uint> &sendIndicesForCommAfterFtoCPositions);
+ void reorderRecvIndicesForCommAfterFtoCZ(int direction, int level, int j,
+ std::vector<uint> &sendIndicesForCommAfterFtoCPositions);
+ void reorderRecvIndicesForCommAfterFtoC(int *recvIndices, int &numberOfRecvNeighborsAfterFtoC, int direction,
+ int level, int j, std::vector<uint> &sendIndicesForCommAfterFtoCPositions);
+
public:
void initalGridInformations() override;