diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.cpp b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.cpp
index 3520703b133748a753e476ee3299b74417beab43..1bdf32f281dc4f2d22cf6bf3b3ff43ca62cd592c 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.cpp
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.cpp
@@ -233,9 +233,9 @@ void IndexRearrangementForStreams::reorderSendIndicesForCommAfterFtoCX(
int direction, int level, int indexOfProcessNeighbor, std::vector<uint> &sendIndicesForCommAfterFtoCPositions)
{
int *sendIndices = para->getParH(level)->sendProcessNeighborX[indexOfProcessNeighbor].index;
- int &numberOfSendNeighborsAfterFtoC =
+ int &numberOfSendNodesAfterFtoC =
para->getParH(level)->sendProcessNeighborsAfterFtoCX[indexOfProcessNeighbor].numberOfNodes;
- reorderSendIndicesForCommAfterFtoC(sendIndices, numberOfSendNeighborsAfterFtoC, direction, level,
+ reorderSendIndicesForCommAfterFtoC(sendIndices, numberOfSendNodesAfterFtoC, direction, level,
sendIndicesForCommAfterFtoCPositions);
}
@@ -243,9 +243,9 @@ void IndexRearrangementForStreams::reorderSendIndicesForCommAfterFtoCY(
int direction, int level, int indexOfProcessNeighbor, std::vector<uint> &sendIndicesForCommAfterFtoCPositions)
{
int *sendIndices = para->getParH(level)->sendProcessNeighborY[indexOfProcessNeighbor].index;
- int &numberOfSendNeighborsAfterFtoC =
+ int &numberOfSendNodesAfterFtoC =
para->getParH(level)->sendProcessNeighborsAfterFtoCY[indexOfProcessNeighbor].numberOfNodes;
- reorderSendIndicesForCommAfterFtoC(sendIndices, numberOfSendNeighborsAfterFtoC, direction, level,
+ reorderSendIndicesForCommAfterFtoC(sendIndices, numberOfSendNodesAfterFtoC, direction, level,
sendIndicesForCommAfterFtoCPositions);
}
@@ -253,14 +253,14 @@ void IndexRearrangementForStreams::reorderSendIndicesForCommAfterFtoCZ(
int direction, int level, int indexOfProcessNeighbor, std::vector<uint> &sendIndicesForCommAfterFtoCPositions)
{
int *sendIndices = para->getParH(level)->sendProcessNeighborZ[indexOfProcessNeighbor].index;
- int &numberOfSendNeighborsAfterFtoC =
+ int &numberOfSendNodesAfterFtoC =
para->getParH(level)->sendProcessNeighborsAfterFtoCZ[indexOfProcessNeighbor].numberOfNodes;
- reorderSendIndicesForCommAfterFtoC(sendIndices, numberOfSendNeighborsAfterFtoC, direction, level,
+ reorderSendIndicesForCommAfterFtoC(sendIndices, numberOfSendNodesAfterFtoC, direction, level,
sendIndicesForCommAfterFtoCPositions);
}
void IndexRearrangementForStreams::reorderSendIndicesForCommAfterFtoC(
- int *sendIndices, int &numberOfSendNeighborsAfterFtoC, int direction, int level,
+ int *sendIndices, int &numberOfSendNodesAfterFtoC, int direction, int level,
std::vector<uint> &sendIndicesForCommAfterFtoCPositions)
{
*logging::out << logging::Logger::INFO_INTERMEDIATE
@@ -268,7 +268,7 @@ void IndexRearrangementForStreams::reorderSendIndicesForCommAfterFtoC(
<< " direction: " << direction;
if (para->getParH(level)->intCF.kCF == 0 || para->getParH(level)->intFC.kFC == 0)
*logging::out << logging::Logger::LOGGER_ERROR
- << "reorderSendIndicesForCommAfterFtoC(): iCellFCC needs to be inititalized before calling "
+ << "reorderSendIndicesForCommAfterFtoC(): para->getParH(level)->intCF needs to be inititalized before calling "
"this function "
<< "\n";
@@ -293,25 +293,25 @@ void IndexRearrangementForStreams::reorderSendIndicesForCommAfterFtoC(
findIfSparseIndexIsInSendIndicesAndAddToCommVectors(sparseIndex, sendIndices, numberOfSendIndices,
sendIndicesAfterFtoC, sendIndicesForCommAfterFtoCPositions);
- numberOfSendNeighborsAfterFtoC = (int)sendIndicesAfterFtoC.size();
+ numberOfSendNodesAfterFtoC = (int)sendIndicesAfterFtoC.size();
findIndicesNotInCommAfterFtoC(numberOfSendIndices, sendIndices, sendIndicesAfterFtoC, sendIndicesOther);
// copy new vectors back to sendIndices array
- for (int i = 0; i < numberOfSendNeighborsAfterFtoC; i++)
+ for (int i = 0; i < numberOfSendNodesAfterFtoC; i++)
sendIndices[i] = sendIndicesAfterFtoC[i];
for (uint i = 0; i < (uint)sendIndicesOther.size(); i++)
- sendIndices[i + numberOfSendNeighborsAfterFtoC] = sendIndicesOther[i];
+ sendIndices[i + numberOfSendNodesAfterFtoC] = sendIndicesOther[i];
*logging::out << logging::Logger::INFO_INTERMEDIATE << "... Process "
<< " " << vf::gpu::Communicator::getInstanz()->getPID()
- << " numberOfSendNeighborsAfterFtoC: " << numberOfSendNeighborsAfterFtoC << "\n ";
+ << " numberOfSendNodesAfterFtoC: " << numberOfSendNodesAfterFtoC << "\n ";
- if (numberOfSendNeighborsAfterFtoC + sendIndicesOther.size() != numberOfSendIndices) {
+ if (numberOfSendNodesAfterFtoC + sendIndicesOther.size() != numberOfSendIndices) {
*logging::out << logging::Logger::LOGGER_ERROR
<< "reorderSendIndicesForCommAfterFtoC(): incorrect number of nodes"
<< "\n";
- std::cout << "numberOfSendNeighborsAfterFtoC = " << numberOfSendNeighborsAfterFtoC
+ std::cout << "numberOfSendNodesAfterFtoC = " << numberOfSendNodesAfterFtoC
<< ", sendOrIndicesOther.size() = " << sendIndicesOther.size()
<< ", numberOfSendOrRecvIndices = " << numberOfSendIndices << std::endl;
}
@@ -358,7 +358,7 @@ void IndexRearrangementForStreams::addUniqueIndexToCommunicationVectors(
std::vector<int> &sendIndicesAfterFtoC, int &sparseIndexSend,
std::vector<unsigned int> &sendIndicesForCommAfterFtoCPositions, uint &posInSendIndices) const
{
- // add index to corresponding vectors but omit indices which are already in sendIndicesAfterFtoC
+ // add index to corresponding vectors, but omit indices which are already in sendIndicesAfterFtoC
if (std::find(sendIndicesAfterFtoC.begin(), sendIndicesAfterFtoC.end(), sparseIndexSend) ==
sendIndicesAfterFtoC.end()) {
sendIndicesAfterFtoC.push_back(sparseIndexSend);
@@ -399,8 +399,8 @@ void IndexRearrangementForStreams::reorderRecvIndicesForCommAfterFtoCX(
int direction, int level, int indexOfProcessNeighbor, std::vector<uint> &sendIndicesForCommAfterFtoCPositions)
{
int *recvIndices = para->getParH(level)->recvProcessNeighborX[indexOfProcessNeighbor].index;
- int &numberOfRecvNeighborsAfterFtoC = para->getParH(level)->recvProcessNeighborsAfterFtoCX[indexOfProcessNeighbor].numberOfNodes;
- reorderRecvIndicesForCommAfterFtoC(recvIndices, numberOfRecvNeighborsAfterFtoC, direction, level,
+ int &numberOfRecvNodesAfterFtoC = para->getParH(level)->recvProcessNeighborsAfterFtoCX[indexOfProcessNeighbor].numberOfNodes;
+ reorderRecvIndicesForCommAfterFtoC(recvIndices, numberOfRecvNodesAfterFtoC, direction, level,
sendIndicesForCommAfterFtoCPositions);
}
@@ -408,8 +408,8 @@ void IndexRearrangementForStreams::reorderRecvIndicesForCommAfterFtoCY(
int direction, int level, int indexOfProcessNeighbor, std::vector<uint> &sendIndicesForCommAfterFtoCPositions)
{
int *recvIndices = para->getParH(level)->recvProcessNeighborY[indexOfProcessNeighbor].index;
- int &numberOfRecvNeighborsAfterFtoC = para->getParH(level)->recvProcessNeighborsAfterFtoCY[indexOfProcessNeighbor].numberOfNodes;
- reorderRecvIndicesForCommAfterFtoC(recvIndices, numberOfRecvNeighborsAfterFtoC, direction, level,
+ int &numberOfRecvNodesAfterFtoC = para->getParH(level)->recvProcessNeighborsAfterFtoCY[indexOfProcessNeighbor].numberOfNodes;
+ reorderRecvIndicesForCommAfterFtoC(recvIndices, numberOfRecvNodesAfterFtoC, direction, level,
sendIndicesForCommAfterFtoCPositions);
}
@@ -417,14 +417,14 @@ void IndexRearrangementForStreams::reorderRecvIndicesForCommAfterFtoCZ(
int direction, int level, int indexOfProcessNeighbor, std::vector<uint> &sendIndicesForCommAfterFtoCPositions)
{
int *recvIndices = para->getParH(level)->recvProcessNeighborZ[indexOfProcessNeighbor].index;
- int &numberOfRecvNeighborsAfterFtoC =
+ int &numberOfRecvNodesAfterFtoC =
para->getParH(level)->recvProcessNeighborsAfterFtoCZ[indexOfProcessNeighbor].numberOfNodes;
- reorderRecvIndicesForCommAfterFtoC(recvIndices, numberOfRecvNeighborsAfterFtoC, direction, level,
+ reorderRecvIndicesForCommAfterFtoC(recvIndices, numberOfRecvNodesAfterFtoC, direction, level,
sendIndicesForCommAfterFtoCPositions);
}
void IndexRearrangementForStreams::reorderRecvIndicesForCommAfterFtoC(
- int *recvIndices, int &numberOfRecvNeighborsAfterFtoC, int direction, int level,
+ int *recvIndices, int &numberOfRecvNodesAfterFtoC, int direction, int level,
std::vector<uint> &sendIndicesForCommAfterFtoCPositions)
{
*logging::out << logging::Logger::INFO_INTERMEDIATE
@@ -445,23 +445,23 @@ void IndexRearrangementForStreams::reorderRecvIndicesForCommAfterFtoC(
findIndicesNotInCommAfterFtoC(numberOfRecvIndices, recvIndices, recvIndicesAfterFtoC, recvIndicesOther);
- numberOfRecvNeighborsAfterFtoC = (int)recvIndicesAfterFtoC.size();
+ numberOfRecvNodesAfterFtoC = (int)recvIndicesAfterFtoC.size();
// copy new vectors back to sendIndices array
- for (int i = 0; i < numberOfRecvNeighborsAfterFtoC; i++)
+ for (int i = 0; i < numberOfRecvNodesAfterFtoC; i++)
recvIndices[i] = recvIndicesAfterFtoC[i];
for (uint i = 0; i < (uint)recvIndicesOther.size(); i++)
- recvIndices[i + numberOfRecvNeighborsAfterFtoC] = recvIndicesOther[i];
+ recvIndices[i + numberOfRecvNodesAfterFtoC] = recvIndicesOther[i];
*logging::out << logging::Logger::INFO_INTERMEDIATE << "... Process "
<< " " << vf::gpu::Communicator::getInstanz()->getPID()
- << " numberOfRecvNeighborsAfterFtoC: " << numberOfRecvNeighborsAfterFtoC << "\n ";
+ << " numberOfRecvNodesAfterFtoC: " << numberOfRecvNodesAfterFtoC << "\n ";
- if (numberOfRecvNeighborsAfterFtoC + recvIndicesOther.size() != numberOfRecvIndices) {
+ if (numberOfRecvNodesAfterFtoC + recvIndicesOther.size() != numberOfRecvIndices) {
*logging::out << logging::Logger::LOGGER_ERROR
<< "reorderRecvIndicesForCommAfterFtoC(): incorrect number of nodes"
<< "\n";
- std::cout << "numberOfRecvNeighborsAfterFtoC = " << numberOfRecvNeighborsAfterFtoC
+ std::cout << "numberOfRecvNodesAfterFtoC = " << numberOfRecvNodesAfterFtoC
<< ", recvIndicesOther.size() = " << recvIndicesOther.size()
<< ", numberOfRecvIndices = " << numberOfRecvIndices << std::endl;
}
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.h b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.h
index 802526ef863206ae372864fff8caea7ca4dd2965..65ee08666247308c3cdf1e533106189b441a325e 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.h
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.h
@@ -37,8 +37,19 @@ public:
//////////////////////////////////////////////////////////////////////////
// communication after coarse to fine
//////////////////////////////////////////////////////////////////////////
+
+ //! \brief initialize the arrays for the communication after the interpolation from fine to coarse in x direction
+ //! \details Only the nodes involved in the interpolation need to be exchanged. Therefore in this method all nodes,
+ //! which are part of the interpolation as well as the communication, are identified.
+ //!
+ //! \ref see master thesis of Anna
+ //! Wellmann (p. 59-62: "Reduzieren der auszutauschenden Knoten")
void initCommunicationArraysForCommAfterFinetoCoarseX(const uint &level, int j, int direction);
+ //! \brief initialize the arrays for the communication after the interpolation from fine to coarse in y direction
+ //! \details --> see x direction
void initCommunicationArraysForCommAfterFinetoCoarseY(const uint &level, int j, int direction);
+ //! \brief initialize the arrays for the communication after the interpolation from fine to coarse in z direction
+ //! \details --> see x direction
void initCommunicationArraysForCommAfterFinetoCoarseZ(const uint &level, int j, int direction);
public:
@@ -52,8 +63,9 @@ public:
//!
//! - cells which are at the border between two gpus --> "border"
//!
- //! - the other cells which are not directly related to the communication betweeen the two gpus --> "bulk"
- //! \ref see master thesis of Anna Wellmann (p. 62-68)
+ //! - the other cells which are not directly related to the communication between the two gpus --> "bulk"
+ //!
+ //! \ref see master thesis of Anna Wellmann (p. 62-68: "Ãœberdeckung der reduzierten Kommunikation")
void splitCoarseToFineIntoBorderAndBulk(const uint &level);
//! \brief split the interpolation cells from fine to coarse into border an bulk
@@ -62,15 +74,18 @@ public:
//!
//! - cells which are at the border between two gpus --> "border"
//!
- //! - the other cells which are not directly related to the communication betweeen the two gpus --> "bulk"
+ //! - the other cells which are not directly related to the communication between the two gpus --> "bulk"
//!
- //! \ref see master thesis of Anna Wellmann (p. 62-68)
+ //! \ref see master thesis of Anna Wellmann (p. 62-68: "Ãœberdeckung der reduzierten Kommunikation")
void splitFineToCoarseIntoBorderAndBulk(const uint &level);
private:
//////////////////////////////////////////////////////////////////////////
// communication after coarse to fine
//////////////////////////////////////////////////////////////////////////
+
+ //! \brief inits pointers for reduced communication after interpolation fine to coarse by copying them from "normal"
+ //! communication
void copyProcessNeighborToCommAfterFtoCX(const uint &level, int indexOfProcessNeighbor);
void copyProcessNeighborToCommAfterFtoCY(const uint &level, int indexOfProcessNeighbor);
void copyProcessNeighborToCommAfterFtoCZ(const uint &level, int indexOfProcessNeighbor);
@@ -81,18 +96,32 @@ private:
std::vector<uint> &sendIndicesForCommAfterFtoCPositions);
void reorderSendIndicesForCommAfterFtoCZ(int direction, int level, int indexOfProcessNeighbor,
std::vector<uint> &sendIndicesForCommAfterFtoCPositions);
- void reorderSendIndicesForCommAfterFtoC(int *sendIndices, int &numberOfSendNeighborsAfterFtoC, int direction,
- int level, std::vector<uint> &sendIndicesForCommAfterFtoCPositions);
+ //! \brief the send indices are reordered for the communication after the interpolation from fine to coarse
+ //! \details The indices of nodes which are part of the interpolation are moved to the front of vector with the send
+ //! indices.
+ //! \pre para->getParH(level)->intCF needs to be inititalized
+ //! \param sendIndices is the pointer to the vector with the send indices, which will be reordered in this function
+ //! \param numberOfSendNodesAfterFtoC will be set in this method
+ //! \param sendIndicesForCommAfterFtoCPositions stores each sendIndex's positions before reordering
+ void reorderSendIndicesForCommAfterFtoC(int *sendIndices, int &numberOfSendNodesAfterFtoC, int direction,
+ int level, std::vector<uint> &sendIndicesForCommAfterFtoCPositions);
+ //! \brief check if a sparse index occurs in the ICellFCC
bool isSparseIndexInICellFCC(uint sizeOfICellFCC, int sparseIndexSend, int level);
+ //! \brief aggregate all nodes in the coarse cells for the interpolation in coarse to fine
+ //! \details For the coarse cells in the interpolation from coarse to fine only one node is stored. This methods
+ //! looks for the other nodes of each cell and puts them into vector. Duplicate nodes are only stored once.
void aggregateNodesInICellCFC(int level, std::vector<uint> &nodesCFC);
+ //! \brief add index to sendIndicesAfterFtoC and sendIndicesForCommAfterFtoCPositions, but omit indices which are already in sendIndicesAfterFtoC
void addUniqueIndexToCommunicationVectors(std::vector<int> &sendIndicesAfterFtoC, int &sparseIndexSend,
std::vector<unsigned int> &sendIndicesForCommAfterFtoCPositions,
uint &posInSendIndices) const;
+ //! \brief find if a sparse index is a send index. If true, call addUniqueIndexToCommunicationVectors()
void
findIfSparseIndexIsInSendIndicesAndAddToCommVectors(int sparseIndex, int *sendIndices, uint numberOfSendIndices,
std::vector<int> &sendIndicesAfterFtoC,
std::vector<uint> &sendIndicesForCommAfterFtoCPositions) const;
+ //! \brief find all indices which are not part of the communication after the interpolation from fine to coarse
void findIndicesNotInCommAfterFtoC(const uint &numberOfSendOrRecvIndices, int *sendOrReceiveIndices,
std::vector<int> &sendOrReceiveIndicesAfterFtoC,
std::vector<int> &sendOrIndicesOther);
@@ -103,7 +132,14 @@ private:
std::vector<uint> &sendIndicesForCommAfterFtoCPositions);
void reorderRecvIndicesForCommAfterFtoCZ(int direction, int level, int indexOfProcessNeighbor,
std::vector<uint> &sendIndicesForCommAfterFtoCPositions);
- void reorderRecvIndicesForCommAfterFtoC(int *recvIndices, int &numberOfRecvNeighborsAfterFtoC, int direction,
+
+ //! \brief reorder the receive indices in the same way that the send indices were reordered.
+ //! \details When the send indices are reordered, the receive indices need to be reordered accordingly.
+ //! \pre sendIndicesForCommAfterFtoCPositions should not be empty
+ //! \param recvIndices is the pointer to the vector with the receive indices, which will be reordered in this function
+ //! \param numberOfRecvNodesAfterFtoC will be set in this function
+ //! \param sendIndicesForCommAfterFtoCPositions stores each sendIndex's positions before reordering and is used to reorder the receive indices in the same way
+ void reorderRecvIndicesForCommAfterFtoC(int *recvIndices, int &numberOfRecvNodesAfterFtoC, int direction,
int level, std::vector<uint> &sendIndicesForCommAfterFtoCPositions);
private: