diff --git a/src/gpu/GridGenerator/grid/GridBuilder/GridBuilder.h b/src/gpu/GridGenerator/grid/GridBuilder/GridBuilder.h
index 52918af73cfcd6ba9674cb8788e95cb9fa3d2dd4..78a4fa174e07df992f8e8566711fbc9e69c4c17f 100644
--- a/src/gpu/GridGenerator/grid/GridBuilder/GridBuilder.h
+++ b/src/gpu/GridGenerator/grid/GridBuilder/GridBuilder.h
@@ -88,10 +88,17 @@ public:
virtual void getReceiveIndices( int* sendIndices, int direction, int level ) = 0;
virtual void reorderSendIndexForCommAfterFtoC(int *sendIndices, int &numberOfSendNeighborsAfterFtoC,
uint *iCellFCCBorder, uint sizeOfICellFCCBorder, int direction,
- int level) = 0;
+ int level) = 0;
virtual void reorderRecvIndexForCommAfterFtoC(int *recvIndices, int &numberOfRecvNeighborsAfterFtoC,
uint *iCellFCCBorder, uint sizeOfICellFCCBorder, int direction,
- int level) = 0;
+ int level) = 0;
+ virtual void getAndReorderSendIndices(int *sendIndices, int &numberOfSendNeighborsAfterFtoC, uint *iCellFCCBorder,
+ uint sizeOfICellFCCBorder, int direction, int level,
+ bool sendIndicesNeedToBeReordered) = 0;
+ virtual void getAndReorderReceiveIndices(int *recvIndices, int &numberOfRecvNeighborsAfterFtoC,
+ uint *iCellFCCBorder, uint sizeOfICellFCCBorder, int direction, int level,
+ bool receiveIndicesNeedToBeReordered) = 0;
+
virtual uint getNumberOfFluidNodes(unsigned int level) const = 0;
virtual void getFluidNodeIndices(uint *fluidNodeIndices, const int level) const = 0;
diff --git a/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.cpp b/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.cpp
index 59b304d5a1623fafccadfb3ebfdd9550b50ed4c2..acc2f872bcd40e1b7ea16d6e0541589f59b61831 100644
--- a/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.cpp
+++ b/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.cpp
@@ -265,19 +265,47 @@ GRIDGENERATOR_EXPORT void LevelGridBuilder::getReceiveIndices(int * receiveIndic
receiveIndices[i] = grid->getSparseIndex( grid->getReceiveIndex(direction, i) ) + 1;
}
}
+GRIDGENERATOR_EXPORT void LevelGridBuilder::getAndReorderSendIndices(int *sendIndices,
+ int &numberOfSendNeighborsAfterFtoC,
+ uint *iCellFCCBorder, uint sizeOfICellFCCBorder,
+ int direction, int level,
+ bool sendIndicesNeedToBeReordered)
+{
+ getSendIndices(sendIndices, direction, level);
+ if (sendIndicesNeedToBeReordered)
+ reorderSendIndexForCommAfterFtoC(sendIndices, numberOfSendNeighborsAfterFtoC, iCellFCCBorder,
+ sizeOfICellFCCBorder, direction, level);
+}
+GRIDGENERATOR_EXPORT void LevelGridBuilder::getAndReorderReceiveIndices(int *recvIndices,
+ int &numberOfRecvNeighborsAfterFtoC,
+ uint *iCellFCCBorder, uint sizeOfICellFCCBorder,
+ int direction, int level,
+ bool receiveIndicesNeedToBeReordered)
+{
+ getReceiveIndices(recvIndices, direction, level);
+ if (receiveIndicesNeedToBeReordered)
+ reorderRecvIndexForCommAfterFtoC(recvIndices, numberOfRecvNeighborsAfterFtoC, iCellFCCBorder,
+ sizeOfICellFCCBorder, direction, level);
+}
GRIDGENERATOR_EXPORT void LevelGridBuilder::reorderSendIndexForCommAfterFtoC(int *sendIndices, int &numberOfSendNeighborsAfterFtoC,
uint* iCellFCCBorder, uint sizeOfICellFCCBorder,
int direction, int level)
{
- *logging::out << logging::Logger::INFO_INTERMEDIATE << "reorder send indices for communication after fine to coarse " << "\n";
+ *logging::out << logging::Logger::INFO_INTERMEDIATE
+ << "reorder send indices for communication after fine to coarse: level: " << level
+ << " direction: " << direction;
if (sizeOfICellFCCBorder == 0)
*logging::out << logging::Logger::LOGGER_ERROR
<< "reorderSendIndexForCommAfterFtoC(): iCellFCCBorder needs to be inititalized before calling "
"this function "
<< "\n";
-
uint numberOfIndices = getNumberOfSendIndices(direction, level);
+ if (numberOfIndices == 0) {
+ numberOfSendNeighborsAfterFtoC = 0;
+ return;
+ }
+
int sparseIndexSend;
bool isInICellFCCBorder;
std::vector<int> sendIndicesAfterFtoC;
@@ -288,12 +316,11 @@ GRIDGENERATOR_EXPORT void LevelGridBuilder::reorderSendIndexForCommAfterFtoC(int
// check if sparse index is in ICellFCC border
isInICellFCCBorder = false;
- for (uint j = 0; j < sizeOfICellFCCBorder; j++) {
+ for (uint j = 0; j < sizeOfICellFCCBorder; j++)
if (iCellFCCBorder[j] == sparseIndexSend) {
isInICellFCCBorder = true;
break;
}
- }
// add index to corresponding vector
if (isInICellFCCBorder)
@@ -302,21 +329,29 @@ GRIDGENERATOR_EXPORT void LevelGridBuilder::reorderSendIndexForCommAfterFtoC(int
sendIndicesOther.push_back(sparseIndexSend);
}
- numberOfSendNeighborsAfterFtoC = sendIndicesAfterFtoC.size();
+ numberOfSendNeighborsAfterFtoC = (uint) sendIndicesAfterFtoC.size();
// copy new vectors back to sendIndices array
for (uint i = 0; i < numberOfSendNeighborsAfterFtoC; i++)
sendIndices[i] = sendIndicesAfterFtoC[i];
- for (uint i = 0; i < sendIndicesOther.size(); i++) {
+ for (uint i = 0; i < sendIndicesOther.size(); i++)
sendIndices[i + numberOfSendNeighborsAfterFtoC] = sendIndicesOther[i];
- }
+
+ *logging::out << logging::Logger::INFO_INTERMEDIATE
+ << "... numberOfSendNeighborsAfterFtoC: " << numberOfSendNeighborsAfterFtoC << "\n";
+
+ bool numberOfNodesIsCorrect = numberOfSendNeighborsAfterFtoC + sendIndicesOther.size() == numberOfIndices;
+ std::cout << "correct number of nodes?: " << numberOfNodesIsCorrect << std::endl;
}
GRIDGENERATOR_EXPORT void LevelGridBuilder::reorderRecvIndexForCommAfterFtoC(int *recvIndices, int &numberOfRecvNeighborsAfterFtoC,
uint *iCellFCCBorder, uint sizeOfICellFCCBorder,
int direction, int level)
{
- *logging::out << logging::Logger::INFO_INTERMEDIATE << "reorder receive indices for communication after fine to coarse " << "\n";
+ *logging::out << logging::Logger::INFO_INTERMEDIATE
+ << "reorder receive indices for communication after fine to coarse: level: " << level
+ << " direction: " << direction;
+ *logging::out << logging::Logger::INFO_INTERMEDIATE << " sizeOfICellFCCBorder: " << sizeOfICellFCCBorder;
if (sizeOfICellFCCBorder == 0)
*logging::out << logging::Logger::LOGGER_ERROR
<< "reorderRecvIndexForCommAfterFtoC(): iCellFCCBorder needs to be inititalized before calling "
@@ -324,6 +359,11 @@ GRIDGENERATOR_EXPORT void LevelGridBuilder::reorderRecvIndexForCommAfterFtoC(int
<< "\n";
uint numberOfIndices = getNumberOfReceiveIndices(direction, level);
+ if (numberOfIndices == 0) {
+ numberOfRecvNeighborsAfterFtoC = 0;
+ return;
+ }
+
int sparseIndexRecv;
bool isInICellFCCBorder;
std::vector<int> recvIndicesAfterFtoC;
@@ -334,12 +374,11 @@ GRIDGENERATOR_EXPORT void LevelGridBuilder::reorderRecvIndexForCommAfterFtoC(int
// check if sparse index is in ICellFCC border
isInICellFCCBorder = false;
- for (uint j = 0; j < sizeOfICellFCCBorder; j++) {
+ for (uint j = 0; j < sizeOfICellFCCBorder; j++)
if (iCellFCCBorder[j] == sparseIndexRecv) {
isInICellFCCBorder = true;
break;
}
- }
// add index to corresponding vector
if (isInICellFCCBorder)
@@ -348,13 +387,19 @@ GRIDGENERATOR_EXPORT void LevelGridBuilder::reorderRecvIndexForCommAfterFtoC(int
recvIndicesOther.push_back(sparseIndexRecv);
}
- numberOfRecvNeighborsAfterFtoC = recvIndicesAfterFtoC.size();
+ numberOfRecvNeighborsAfterFtoC = (uint) recvIndicesAfterFtoC.size();
// copy new vectors back to receiveIndices array
for (uint i = 0; i < numberOfRecvNeighborsAfterFtoC; i++)
recvIndices[i] = recvIndicesAfterFtoC[i];
for (uint i = 0; i < recvIndicesOther.size(); i++)
recvIndices[i + numberOfRecvNeighborsAfterFtoC] = recvIndicesOther[i];
+
+ *logging::out << logging::Logger::INFO_INTERMEDIATE
+ << "... numberOfRecvNeighborsAfterFtoC: " << numberOfRecvNeighborsAfterFtoC << "\n";
+
+ bool numberOfNodesIsCorrect = numberOfRecvNeighborsAfterFtoC + recvIndicesOther.size() == numberOfIndices;
+ std::cout << "correct number of nodes?: " << numberOfNodesIsCorrect << std::endl;
}
uint LevelGridBuilder::getNumberOfNodes(unsigned int level) const
diff --git a/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h b/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h
index 704067e2f685050215d65d800fb47308db9e7c4f..8c3fe9256ee6a4ca24450431fbd0869f605010ce 100644
--- a/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h
+++ b/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h
@@ -149,6 +149,13 @@ public:
GRIDGENERATOR_EXPORT uint getNumberOfReceiveIndices( int direction, uint level ) override;
GRIDGENERATOR_EXPORT void getSendIndices( int* sendIndices, int direction, int level ) override;
GRIDGENERATOR_EXPORT void getReceiveIndices( int* sendIndices, int direction, int level ) override;
+ GRIDGENERATOR_EXPORT void getAndReorderSendIndices(int *sendIndices, int &numberOfSendNeighborsAfterFtoC,
+ uint *iCellFCCBorder, uint sizeOfICellFCCBorder, int direction,
+ int level, bool sendIndicesNeedToBeReordered) override;
+ GRIDGENERATOR_EXPORT void getAndReorderReceiveIndices(int *recvIndices, int &numberOfRecvNeighborsAfterFtoC,
+ uint *iCellFCCBorder, uint sizeOfICellFCCBorder,
+ int direction, int level,
+ bool receiveIndicesNeedToBeReordered) override;
GRIDGENERATOR_EXPORT void reorderSendIndexForCommAfterFtoC(int *sendIndices, int &numberOfSendNeighborsAfterFtoC,
uint *iCellFCCBorder, uint sizeOfICellFCCBorder,
int direction, int level) override;
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
index 0debc795de1f51e6289d05e73d241bca86a89602..edd9d4eb63085d915136132412b57e7c78aefe89 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
@@ -321,8 +321,21 @@ void GridGenerator::allocArrays_BoundaryValues()
cudaMemoryManager->cudaAllocProcessNeighborX(level, j);
////////////////////////////////////////////////////////////////////////////////////////
//init index arrays
- builder->getSendIndices (para->getParH(level)->sendProcessNeighborX[j].index, direction, level);
- builder->getReceiveIndices(para->getParH(level)->recvProcessNeighborX[j].index, direction, level);
+ para->initNumberOfProcessNeighborsAfterFtoCX(level);
+ builder->getAndReorderSendIndices(
+ para->getParH(level)->sendProcessNeighborX[j].index,
+ para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCX[j],
+ para->getParH(level)->intFCBorder.ICellFCC, para->getParH(level)->intFCBorder.kFC,
+ direction, level, level != builder->getNumberOfGridLevels() - 1);
+ builder->getAndReorderReceiveIndices(
+ para->getParH(level)->recvProcessNeighborX[j].index,
+ para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCX[j],
+ para->getParH(level)->intFC.ICellFCC, para->getParH(level)->intFC.kFC, direction, level,
+ level!=builder->getNumberOfGridLevels()-1);
+ para->getParD(level)->numberOfSendProcessNeighborsAfterFtoCX[j] =
+ para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCX[j];
+ para->getParD(level)->numberOfRecvProcessNeighborsAfterFtoCX[j] =
+ para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCX[j];
////////////////////////////////////////////////////////////////////////////////////////
cudaMemoryManager->cudaCopyProcessNeighborXIndex(level, j);
////////////////////////////////////////////////////////////////////////////////////////
@@ -381,8 +394,21 @@ void GridGenerator::allocArrays_BoundaryValues()
cudaMemoryManager->cudaAllocProcessNeighborY(level, j);
////////////////////////////////////////////////////////////////////////////////////////
//init index arrays
- builder->getSendIndices (para->getParH(level)->sendProcessNeighborY[j].index, direction, level);
- builder->getReceiveIndices(para->getParH(level)->recvProcessNeighborY[j].index, direction, level);
+ para->initNumberOfProcessNeighborsAfterFtoCY(level);
+ builder->getAndReorderSendIndices(
+ para->getParH(level)->sendProcessNeighborY[j].index,
+ para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCY[j],
+ para->getParH(level)->intFCBorder.ICellFCC, para->getParH(level)->intFCBorder.kFC,
+ direction, level, level != builder->getNumberOfGridLevels() - 1);
+ builder->getAndReorderReceiveIndices(
+ para->getParH(level)->recvProcessNeighborY[j].index,
+ para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCY[j],
+ para->getParH(level)->intFC.ICellFCC, para->getParH(level)->K_CF, direction, level,
+ level != builder->getNumberOfGridLevels() - 1);
+ para->getParD(level)->numberOfSendProcessNeighborsAfterFtoCY[j] =
+ para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCY[j];
+ para->getParD(level)->numberOfRecvProcessNeighborsAfterFtoCY[j] =
+ para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCY[j];
////////////////////////////////////////////////////////////////////////////////////////
cudaMemoryManager->cudaCopyProcessNeighborYIndex(level, j);
////////////////////////////////////////////////////////////////////////////////////////
@@ -441,8 +467,21 @@ void GridGenerator::allocArrays_BoundaryValues()
cudaMemoryManager->cudaAllocProcessNeighborZ(level, j);
////////////////////////////////////////////////////////////////////////////////////////
//init index arrays
- builder->getSendIndices (para->getParH(level)->sendProcessNeighborZ[j].index, direction, level);
- builder->getReceiveIndices(para->getParH(level)->recvProcessNeighborZ[j].index, direction, level);
+ para->initNumberOfProcessNeighborsAfterFtoCZ(level);
+ builder->getAndReorderSendIndices(
+ para->getParH(level)->sendProcessNeighborZ[j].index,
+ para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCZ[j],
+ para->getParH(level)->intFCBorder.ICellFCC, para->getParH(level)->intFCBorder.kFC,
+ direction, level, level != builder->getNumberOfGridLevels() - 1);
+ builder->getAndReorderReceiveIndices(
+ para->getParH(level)->recvProcessNeighborZ[j].index,
+ para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCZ[j],
+ para->getParH(level)->intFC.ICellFCC, para->getParH(level)->intFC.kFC, direction, level,
+ level != builder->getNumberOfGridLevels() - 1);
+ para->getParD(level)->numberOfSendProcessNeighborsAfterFtoCZ[j] =
+ para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCZ[j];
+ para->getParD(level)->numberOfRecvProcessNeighborsAfterFtoCZ[j] =
+ para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCZ[j];
////////////////////////////////////////////////////////////////////////////////////////
cudaMemoryManager->cudaCopyProcessNeighborZIndex(level, j);
////////////////////////////////////////////////////////////////////////////////////////