From dbde36ce6df87ea17fc5aee6f54ed31a708eb65e Mon Sep 17 00:00:00 2001
From: Anna Wellmann <a.wellmann@tu-braunschweig.de>
Date: Tue, 28 Sep 2021 16:34:55 +0200
Subject: [PATCH] Restructure calls to getAndReorderSendIndices() in
GridGenerator
---
.../grid/GridBuilder/GridBuilder.h | 3 +-
.../grid/GridBuilder/LevelGridBuilder.cpp | 14 +-
.../grid/GridBuilder/LevelGridBuilder.h | 2 +-
.../GridReaderGenerator/GridGenerator.cpp | 128 ++++++++++--------
4 files changed, 81 insertions(+), 66 deletions(-)
diff --git a/src/gpu/GridGenerator/grid/GridBuilder/GridBuilder.h b/src/gpu/GridGenerator/grid/GridBuilder/GridBuilder.h
index d5d629e34..e1fb6e5cd 100644
--- a/src/gpu/GridGenerator/grid/GridBuilder/GridBuilder.h
+++ b/src/gpu/GridGenerator/grid/GridBuilder/GridBuilder.h
@@ -89,8 +89,7 @@ public:
virtual std::vector<uint> getAndReorderSendIndices(int *sendIndices, uint &numberOfSendNeighborsAfterFtoC,
uint *iCellFCC, uint sizeOfICellFCCBorder, uint *iCellCFC,
uint sizeOfICellCFC, uint *neighborX, uint *neighborY,
- uint *neighborZ, int direction, int level,
- bool sendIndicesNeedToBeReordered) = 0;
+ uint *neighborZ, int direction, int level) = 0;
virtual void getAndReorderReceiveIndices(int *recvIndices, uint &numberOfRecvNeighborsAfterFtoC,
uint *iCellFCCBorder, uint sizeOfICellFCCBorder, int direction, int level,
bool receiveIndicesNeedToBeReordered) = 0;
diff --git a/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.cpp b/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.cpp
index 190ea9edf..2ac33ff0d 100644
--- a/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.cpp
+++ b/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.cpp
@@ -270,15 +270,13 @@ GRIDGENERATOR_EXPORT void LevelGridBuilder::getReceiveIndices(int * receiveIndic
GRIDGENERATOR_EXPORT std::vector<uint>
LevelGridBuilder::getAndReorderSendIndices(int *sendIndices, uint &numberOfSendNeighborsAfterFtoC, uint *iCellFCC,
uint sizeOfICellFCCBorder, uint *iCellCFC, uint sizeOfICellCFC,
- uint *neighborX, uint *neighborY, uint *neighborZ, int direction, int level,
- bool sendIndicesNeedToBeReordered)
+ uint *neighborX, uint *neighborY, uint *neighborZ, int direction, int level)
{
std::vector<uint> sendIndicesForCommAfterFtoCPositions;
getSendIndices(sendIndices, direction, level);
- if (sendIndicesNeedToBeReordered)
- reorderSendIndecesForCommAfterFtoC(sendIndices, numberOfSendNeighborsAfterFtoC, iCellFCC, sizeOfICellCFC,
- iCellCFC, sizeOfICellCFC, neighborX, neighborY, neighborZ, direction, level,
- sendIndicesForCommAfterFtoCPositions);
+ reorderSendIndecesForCommAfterFtoC(sendIndices, numberOfSendNeighborsAfterFtoC, iCellFCC, sizeOfICellCFC, iCellCFC,
+ sizeOfICellCFC, neighborX, neighborY, neighborZ, direction, level,
+ sendIndicesForCommAfterFtoCPositions);
return sendIndicesForCommAfterFtoCPositions;
}
@@ -340,8 +338,7 @@ GRIDGENERATOR_EXPORT void LevelGridBuilder::reorderSendIndecesForCommAfterFtoC(
if (sparseIndexSend < 0)
continue;
if (iCellCFC[j] == (uint)sparseIndexSend) {
- isInICells = true;
- std::cout << "found sparse index in ICellCF" << std::endl;
+ isInICells = true;
// also find neighbors
if (direction != 1)
neighborToAddX = neighborX[sparseIndexSend];
@@ -417,7 +414,6 @@ GRIDGENERATOR_EXPORT void LevelGridBuilder::reorderRecvIndexForCommAfterFtoC(int
*logging::out << logging::Logger::INFO_INTERMEDIATE
<< "reorder receive indices for communication after fine to coarse: level: " << level
<< " direction: " << direction;
- *logging::out << logging::Logger::INFO_INTERMEDIATE << " sizeOfICellFCC: " << sizeOfICellFCCBorder;
if (sizeOfICellFCCBorder == 0)
*logging::out << logging::Logger::LOGGER_ERROR
<< "reorderRecvIndexForCommAfterFtoC(): iCellFCC needs to be inititalized before calling "
diff --git a/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h b/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h
index 4e71e6487..d3ff87999 100644
--- a/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h
+++ b/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h
@@ -152,7 +152,7 @@ public:
GRIDGENERATOR_EXPORT std::vector<uint> GRIDGENERATOR_EXPORT getAndReorderSendIndices(
int *sendIndices, uint &numberOfSendNeighborsAfterFtoC, uint *iCellFCC, uint sizeOfICellFCCBorder,
uint *iCellCFC, uint sizeOfICellCFC, uint *neighborX, uint *neighborY, uint *neighborZ, int direction,
- int level, bool sendIndicesNeedToBeReordered) override;
+ int level) override;
GRIDGENERATOR_EXPORT void getAndReorderReceiveIndices(int *recvIndices, uint &numberOfRecvNeighborsAfterFtoC,
uint *iCellFCCBorder, uint sizeOfICellFCCBorder,
int direction, int level,
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
index 1640cf1d6..ed7ec278e 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
@@ -321,24 +321,30 @@ void GridGenerator::allocArrays_BoundaryValues()
cudaMemoryManager->cudaAllocProcessNeighborX(level, j);
////////////////////////////////////////////////////////////////////////////////////////
//init index arrays
- para->initNumberOfProcessNeighborsAfterFtoCX(level);
- std::vector<uint> sendIndicesForCommAfterFtoCPositions = builder->getAndReorderSendIndices(
- para->getParH(level)->sendProcessNeighborX[j].index,
- para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCX[j],
- para->getParH(level)->intFC.ICellFCC, para->getParH(level)->K_CF,
- para->getParH(level)->intCF.ICellCFC, para->getParH(level)->K_FC,
- para->getParH(level)->neighborX_SP, para->getParH(level)->neighborY_SP,
- para->getParH(level)->neighborZ_SP,
- direction, level, level != builder->getNumberOfGridLevels() - 1);
- builder->getAndReorderReceiveIndices(
- para->getParH(level)->recvProcessNeighborX[j].index,
- para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCX[j],
- para->getParH(level)->intFC.ICellFCC, para->getParH(level)->intFC.kFC, direction, level,
- level != builder->getNumberOfGridLevels() - 1);
- para->getParD(level)->numberOfSendProcessNeighborsAfterFtoCX[j] =
- para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCX[j];
- para->getParD(level)->numberOfRecvProcessNeighborsAfterFtoCX[j] =
- para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCX[j];
+ if (level == builder->getNumberOfGridLevels() - 1) {
+ builder->getSendIndices(para->getParH(level)->sendProcessNeighborX[j].index, direction,
+ level);
+ builder->getReceiveIndices(para->getParH(level)->recvProcessNeighborX[j].index, direction,
+ level);
+ } else {
+ para->initNumberOfProcessNeighborsAfterFtoCX(level);
+ std::vector<uint> sendIndicesForCommAfterFtoCPositions = builder->getAndReorderSendIndices(
+ para->getParH(level)->sendProcessNeighborX[j].index,
+ para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCX[j],
+ para->getParH(level)->intFC.ICellFCC, para->getParH(level)->K_CF,
+ para->getParH(level)->intCF.ICellCFC, para->getParH(level)->K_FC,
+ para->getParH(level)->neighborX_SP, para->getParH(level)->neighborY_SP,
+ para->getParH(level)->neighborZ_SP, direction, level);
+ builder->getAndReorderReceiveIndices(
+ para->getParH(level)->recvProcessNeighborX[j].index,
+ para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCX[j],
+ para->getParH(level)->intFC.ICellFCC, para->getParH(level)->intFC.kFC, direction, level,
+ level != builder->getNumberOfGridLevels() - 1);
+ para->getParD(level)->numberOfSendProcessNeighborsAfterFtoCX[j] =
+ para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCX[j];
+ para->getParD(level)->numberOfRecvProcessNeighborsAfterFtoCX[j] =
+ para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCX[j];
+ }
////////////////////////////////////////////////////////////////////////////////////////
cudaMemoryManager->cudaCopyProcessNeighborXIndex(level, j);
////////////////////////////////////////////////////////////////////////////////////////
@@ -397,24 +403,31 @@ void GridGenerator::allocArrays_BoundaryValues()
cudaMemoryManager->cudaAllocProcessNeighborY(level, j);
////////////////////////////////////////////////////////////////////////////////////////
//init index arrays
- para->initNumberOfProcessNeighborsAfterFtoCY(level);
- std::vector<uint> sendIndicesForCommAfterFtoCPositions = builder->getAndReorderSendIndices(
- para->getParH(level)->sendProcessNeighborY[j].index,
- para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCY[j],
- para->getParH(level)->intFC.ICellFCC, para->getParH(level)->K_CF,
- para->getParH(level)->intCF.ICellCFC, para->getParH(level)->K_FC,
- para->getParH(level)->neighborX_SP, para->getParH(level)->neighborY_SP,
- para->getParH(level)->neighborZ_SP,
- direction, level, level != builder->getNumberOfGridLevels() - 1);
- builder->getAndReorderReceiveIndices(
- para->getParH(level)->recvProcessNeighborY[j].index,
- para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCY[j],
- para->getParH(level)->intFC.ICellFCC, para->getParH(level)->K_CF, direction, level,
- level != builder->getNumberOfGridLevels() - 1);
- para->getParD(level)->numberOfSendProcessNeighborsAfterFtoCY[j] =
- para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCY[j];
- para->getParD(level)->numberOfRecvProcessNeighborsAfterFtoCY[j] =
- para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCY[j];
+ if (level == builder->getNumberOfGridLevels() - 1) {
+ builder->getSendIndices(para->getParH(level)->sendProcessNeighborY[j].index, direction,
+ level);
+ builder->getReceiveIndices(para->getParH(level)->recvProcessNeighborY[j].index, direction,
+ level);
+ } else {
+ para->initNumberOfProcessNeighborsAfterFtoCY(level);
+ std::vector<uint> sendIndicesForCommAfterFtoCPositions = builder->getAndReorderSendIndices(
+ para->getParH(level)->sendProcessNeighborY[j].index,
+ para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCY[j],
+ para->getParH(level)->intFC.ICellFCC, para->getParH(level)->K_CF,
+ para->getParH(level)->intCF.ICellCFC, para->getParH(level)->K_FC,
+ para->getParH(level)->neighborX_SP, para->getParH(level)->neighborY_SP,
+ para->getParH(level)->neighborZ_SP,
+ direction, level);
+ builder->getAndReorderReceiveIndices(
+ para->getParH(level)->recvProcessNeighborY[j].index,
+ para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCY[j],
+ para->getParH(level)->intFC.ICellFCC, para->getParH(level)->K_CF, direction, level,
+ level != builder->getNumberOfGridLevels() - 1);
+ para->getParD(level)->numberOfSendProcessNeighborsAfterFtoCY[j] =
+ para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCY[j];
+ para->getParD(level)->numberOfRecvProcessNeighborsAfterFtoCY[j] =
+ para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCY[j];
+ }
////////////////////////////////////////////////////////////////////////////////////////
cudaMemoryManager->cudaCopyProcessNeighborYIndex(level, j);
////////////////////////////////////////////////////////////////////////////////////////
@@ -473,24 +486,31 @@ void GridGenerator::allocArrays_BoundaryValues()
cudaMemoryManager->cudaAllocProcessNeighborZ(level, j);
////////////////////////////////////////////////////////////////////////////////////////
//init index arrays
- para->initNumberOfProcessNeighborsAfterFtoCZ(level);
- std::vector<uint> sendIndicesForCommAfterFtoCPositions = builder->getAndReorderSendIndices(
- para->getParH(level)->sendProcessNeighborZ[j].index,
- para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCZ[j],
- para->getParH(level)->intFC.ICellFCC, para->getParH(level)->K_CF,
- para->getParH(level)->intCF.ICellCFC, para->getParH(level)->K_FC,
- para->getParH(level)->neighborX_SP, para->getParH(level)->neighborY_SP,
- para->getParH(level)->neighborZ_SP,
- direction, level, level != builder->getNumberOfGridLevels() - 1);
- builder->getAndReorderReceiveIndices(
- para->getParH(level)->recvProcessNeighborZ[j].index,
- para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCZ[j],
- para->getParH(level)->intFC.ICellFCC, para->getParH(level)->intFC.kFC, direction, level,
- level != builder->getNumberOfGridLevels() - 1);
- para->getParD(level)->numberOfSendProcessNeighborsAfterFtoCZ[j] =
- para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCZ[j];
- para->getParD(level)->numberOfRecvProcessNeighborsAfterFtoCZ[j] =
- para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCZ[j];
+ if (level == builder->getNumberOfGridLevels() - 1) {
+ builder->getSendIndices(para->getParH(level)->sendProcessNeighborZ[j].index, direction,
+ level);
+ builder->getReceiveIndices(para->getParH(level)->recvProcessNeighborZ[j].index, direction,
+ level);
+ } else {
+ para->initNumberOfProcessNeighborsAfterFtoCZ(level);
+ std::vector<uint> sendIndicesForCommAfterFtoCPositions = builder->getAndReorderSendIndices(
+ para->getParH(level)->sendProcessNeighborZ[j].index,
+ para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCZ[j],
+ para->getParH(level)->intFC.ICellFCC, para->getParH(level)->K_CF,
+ para->getParH(level)->intCF.ICellCFC, para->getParH(level)->K_FC,
+ para->getParH(level)->neighborX_SP, para->getParH(level)->neighborY_SP,
+ para->getParH(level)->neighborZ_SP,
+ direction, level);
+ builder->getAndReorderReceiveIndices(
+ para->getParH(level)->recvProcessNeighborZ[j].index,
+ para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCZ[j],
+ para->getParH(level)->intFC.ICellFCC, para->getParH(level)->intFC.kFC, direction, level,
+ level != builder->getNumberOfGridLevels() - 1);
+ para->getParD(level)->numberOfSendProcessNeighborsAfterFtoCZ[j] =
+ para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCZ[j];
+ para->getParD(level)->numberOfRecvProcessNeighborsAfterFtoCZ[j] =
+ para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCZ[j];
+ }
////////////////////////////////////////////////////////////////////////////////////////
cudaMemoryManager->cudaCopyProcessNeighborZIndex(level, j);
////////////////////////////////////////////////////////////////////////////////////////
--
GitLab