From 1d2ff604e3496300ba0bfe573d67024b826d5f4e Mon Sep 17 00:00:00 2001
From: Anna Wellmann <a.wellmann@tu-braunschweig.de>
Date: Wed, 6 Oct 2021 10:20:16 +0200
Subject: [PATCH] Use simmilar data structure to sendProcessNeighbor for
sendProcessNeighborsAfterFtoC
---
.../grid/GridBuilder/GridBuilder.h | 4 +-
.../grid/GridBuilder/LevelGridBuilder.cpp | 4 +-
.../grid/GridBuilder/LevelGridBuilder.h | 4 +-
.../GridReaderGenerator/GridGenerator.cpp | 39 ++++++++++---------
src/gpu/VirtualFluids_GPU/LBM/LB.h | 6 +++
.../VirtualFluids_GPU/Parameter/Parameter.cpp | 39 ++++++++++---------
.../VirtualFluids_GPU/Parameter/Parameter.h | 12 +++---
7 files changed, 58 insertions(+), 50 deletions(-)
diff --git a/src/gpu/GridGenerator/grid/GridBuilder/GridBuilder.h b/src/gpu/GridGenerator/grid/GridBuilder/GridBuilder.h
index da4ab7e5d..025fffaf7 100644
--- a/src/gpu/GridGenerator/grid/GridBuilder/GridBuilder.h
+++ b/src/gpu/GridGenerator/grid/GridBuilder/GridBuilder.h
@@ -86,10 +86,10 @@ public:
virtual uint getNumberOfReceiveIndices( int direction, uint level ) = 0;
virtual void getSendIndices( int* sendIndices, int direction, int level ) = 0;
virtual void getReceiveIndices( int* sendIndices, int direction, int level ) = 0;
- virtual void reorderRecvIndicesForCommAfterFtoC(int *recvIndices, uint &numberOfRecvNeighborsAfterFtoC,
+ virtual void reorderRecvIndicesForCommAfterFtoC(int *recvIndices, int &numberOfRecvNeighborsAfterFtoC,
std::vector<uint> &sendIndicesForCommAfterFtoCPositions,
int direction, int level) = 0;
- virtual void reorderSendIndicesForCommAfterFtoC(int *sendIndices, uint &numberOfSendNeighborsAfterFtoC,
+ virtual void reorderSendIndicesForCommAfterFtoC(int *sendIndices, int &numberOfSendNeighborsAfterFtoC,
uint *iCellFCC, uint sizeOfICellFCC, uint *iCellCFC,
uint sizeOfICellCFC, uint *neighborX, uint *neighborY,
uint *neighborZ, int direction, int level,
diff --git a/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.cpp b/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.cpp
index 026c4baad..401194531 100644
--- a/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.cpp
+++ b/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.cpp
@@ -268,7 +268,7 @@ GRIDGENERATOR_EXPORT void LevelGridBuilder::getReceiveIndices(int * receiveIndic
}
GRIDGENERATOR_EXPORT void LevelGridBuilder::reorderSendIndicesForCommAfterFtoC(
- int *sendIndices, uint &numberOfSendNeighborsAfterFtoC, uint *iCellFCC, uint sizeOfICellFCC, uint *iCellCFC,
+ int *sendIndices, int &numberOfSendNeighborsAfterFtoC, uint *iCellFCC, uint sizeOfICellFCC, uint *iCellCFC,
uint sizeOfICellCFC, uint *neighborX, uint *neighborY, uint *neighborZ, int direction, int level,
std::vector<uint> &sendIndicesForCommAfterFtoCPositions)
{
@@ -389,7 +389,7 @@ void LevelGridBuilder::findIfSparseIndexIsInSendIndicesAndAddToVectors(
}
}
-GRIDGENERATOR_EXPORT void LevelGridBuilder::reorderRecvIndicesForCommAfterFtoC(int *recvIndices, uint &numberOfRecvNeighborsAfterFtoC,
+GRIDGENERATOR_EXPORT void LevelGridBuilder::reorderRecvIndicesForCommAfterFtoC(int *recvIndices, int &numberOfRecvNeighborsAfterFtoC,
std::vector<uint> &sendIndicesForCommAfterFtoCPositions,
int direction, int level)
{
diff --git a/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h b/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h
index ebb96bf0b..2dcaf5b5b 100644
--- a/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h
+++ b/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h
@@ -149,7 +149,7 @@ public:
GRIDGENERATOR_EXPORT uint getNumberOfReceiveIndices( int direction, uint level ) override;
GRIDGENERATOR_EXPORT void getSendIndices( int* sendIndices, int direction, int level ) override;
GRIDGENERATOR_EXPORT void getReceiveIndices( int* sendIndices, int direction, int level ) override;
- GRIDGENERATOR_EXPORT void reorderSendIndicesForCommAfterFtoC(int *sendIndices, uint &numberOfSendNeighborsAfterFtoC,
+ GRIDGENERATOR_EXPORT void reorderSendIndicesForCommAfterFtoC(int *sendIndices, int &numberOfSendNeighborsAfterFtoC,
uint *iCellFCC, uint sizeOfICellFCC, uint *iCellCFC,
uint sizeOfICellCFC, uint *neighborX, uint *neighborY,
uint *neighborZ, int direction, int level,
@@ -157,7 +157,7 @@ public:
void findIfSparseIndexIsInSendIndicesAndAddToVectors(int sparseIndex, int *sendIndices, uint numberOfSendIndices,
std::vector<int> &sendIndicesAfterFtoC,
std::vector<uint> &sendIndicesForCommAfterFtoCPositions) const;
- GRIDGENERATOR_EXPORT void reorderRecvIndicesForCommAfterFtoC(int *recvIndices, uint &numberOfRecvNeighborsAfterFtoC,
+ GRIDGENERATOR_EXPORT void reorderRecvIndicesForCommAfterFtoC(int *recvIndices, int &numberOfRecvNeighborsAfterFtoC,
std::vector<uint> &sendIndicesForCommAfterFtoCPositions,
int direction, int level) override;
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
index c7b7a983c..ed1db24c5 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
@@ -700,17 +700,17 @@ void GridGenerator::initCommunicationArraysForCommAfterFinetoCoarseZ(const uint
std::vector<uint> sendIndicesForCommAfterFtoCPositions;
builder->reorderSendIndicesForCommAfterFtoC(
para->getParH(level)->sendProcessNeighborZ[j].index,
- para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCZ[j], para->getParH(level)->intFC.ICellFCC,
+ para->getParH(level)->sendProcessNeighborsAfterFtoCZ[j].numberOfNodes, para->getParH(level)->intFC.ICellFCC,
para->getParH(level)->K_CF, para->getParH(level)->intCF.ICellCFC, para->getParH(level)->K_FC,
para->getParH(level)->neighborX_SP, para->getParH(level)->neighborY_SP, para->getParH(level)->neighborZ_SP,
direction, level, sendIndicesForCommAfterFtoCPositions);
builder->reorderRecvIndicesForCommAfterFtoC(para->getParH(level)->recvProcessNeighborZ[j].index,
- para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCZ[j],
+ para->getParH(level)->recvProcessNeighborsAfterFtoCZ[j].numberOfNodes,
sendIndicesForCommAfterFtoCPositions, direction, level);
- para->getParD(level)->numberOfSendProcessNeighborsAfterFtoCZ[j] =
- para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCZ[j];
- para->getParD(level)->numberOfRecvProcessNeighborsAfterFtoCZ[j] =
- para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCZ[j];
+ para->getParD(level)->sendProcessNeighborsAfterFtoCZ[j].numberOfNodes =
+ para->getParH(level)->sendProcessNeighborsAfterFtoCZ[j].numberOfNodes;
+ para->getParD(level)->recvProcessNeighborsAfterFtoCZ[j].numberOfNodes =
+ para->getParH(level)->recvProcessNeighborsAfterFtoCZ[j].numberOfNodes;
}
void GridGenerator::initCommunicationArraysForCommAfterFinetoCoarseY(const uint &level, int j, int direction)
@@ -720,17 +720,18 @@ void GridGenerator::initCommunicationArraysForCommAfterFinetoCoarseY(const uint
std::vector<uint> sendIndicesForCommAfterFtoCPositions;
builder->reorderSendIndicesForCommAfterFtoC(
para->getParH(level)->sendProcessNeighborY[j].index,
- para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCY[j], para->getParH(level)->intFC.ICellFCC,
+ para->getParH(level)->sendProcessNeighborsAfterFtoCY[j].numberOfNodes, para->getParH(level)->intFC.ICellFCC,
para->getParH(level)->K_CF, para->getParH(level)->intCF.ICellCFC, para->getParH(level)->K_FC,
para->getParH(level)->neighborX_SP, para->getParH(level)->neighborY_SP, para->getParH(level)->neighborZ_SP,
direction, level, sendIndicesForCommAfterFtoCPositions);
- para->getParD(level)->numberOfSendProcessNeighborsAfterFtoCY[j] =
- para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCY[j];
+ para->getParD(level)->sendProcessNeighborsAfterFtoCY[j].numberOfNodes =
+ para->getParH(level)->sendProcessNeighborsAfterFtoCY[j].numberOfNodes;
// send sendIndicesForCommAfterFtoCPositions to receiving process
std::vector<uint> recvIndicesForCommAfterFtoCPositions;
- recvIndicesForCommAfterFtoCPositions.resize(para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCY[j] * 2); // give vector an arbitraty size (larger than needed)
+ recvIndicesForCommAfterFtoCPositions.resize(para->getParH(level)->sendProcessNeighborsAfterFtoCY[j].numberOfNodes *
+ 2); // give vector an arbitraty size (larger than needed)
auto comm = vf::gpu::Communicator::getInstanz();
comm->exchangeIndices(recvIndicesForCommAfterFtoCPositions.data(), recvIndicesForCommAfterFtoCPositions.size(),
para->getParH(level)->recvProcessNeighborY[j].rankNeighbor,
@@ -742,10 +743,10 @@ void GridGenerator::initCommunicationArraysForCommAfterFinetoCoarseY(const uint
// init receive indices
builder->reorderRecvIndicesForCommAfterFtoC(para->getParH(level)->recvProcessNeighborY[j].index,
- para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCY[j],
+ para->getParH(level)->recvProcessNeighborsAfterFtoCY[j].numberOfNodes,
recvIndicesForCommAfterFtoCPositions, direction, level);
- para->getParD(level)->numberOfRecvProcessNeighborsAfterFtoCY[j] =
- para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCY[j];
+ para->getParD(level)->recvProcessNeighborsAfterFtoCY[j].numberOfNodes =
+ para->getParH(level)->recvProcessNeighborsAfterFtoCY[j].numberOfNodes;
}
void GridGenerator::initCommunicationArraysForCommAfterFinetoCoarseX(const uint &level, int j, int direction)
@@ -754,17 +755,17 @@ void GridGenerator::initCommunicationArraysForCommAfterFinetoCoarseX(const uint
std::vector<uint> sendIndicesForCommAfterFtoCPositions;
builder->reorderSendIndicesForCommAfterFtoC(
para->getParH(level)->sendProcessNeighborX[j].index,
- para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCX[j], para->getParH(level)->intFC.ICellFCC,
+ para->getParH(level)->sendProcessNeighborsAfterFtoCX[j].numberOfNodes, para->getParH(level)->intFC.ICellFCC,
para->getParH(level)->K_CF, para->getParH(level)->intCF.ICellCFC, para->getParH(level)->K_FC,
para->getParH(level)->neighborX_SP, para->getParH(level)->neighborY_SP, para->getParH(level)->neighborZ_SP,
direction, level, sendIndicesForCommAfterFtoCPositions);
builder->reorderRecvIndicesForCommAfterFtoC(para->getParH(level)->recvProcessNeighborX[j].index,
- para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCX[j],
+ para->getParH(level)->recvProcessNeighborsAfterFtoCX[j].numberOfNodes,
sendIndicesForCommAfterFtoCPositions, direction, level);
- para->getParD(level)->numberOfSendProcessNeighborsAfterFtoCX[j] =
- para->getParH(level)->numberOfSendProcessNeighborsAfterFtoCX[j];
- para->getParD(level)->numberOfRecvProcessNeighborsAfterFtoCX[j] =
- para->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCX[j];
+ para->getParD(level)->sendProcessNeighborsAfterFtoCX[j].numberOfNodes =
+ para->getParH(level)->sendProcessNeighborsAfterFtoCX[j].numberOfNodes;
+ para->getParD(level)->recvProcessNeighborsAfterFtoCX[j].numberOfNodes =
+ para->getParH(level)->recvProcessNeighborsAfterFtoCX[j].numberOfNodes;
}
diff --git a/src/gpu/VirtualFluids_GPU/LBM/LB.h b/src/gpu/VirtualFluids_GPU/LBM/LB.h
index a33b3b792..b60de6a09 100644
--- a/src/gpu/VirtualFluids_GPU/LBM/LB.h
+++ b/src/gpu/VirtualFluids_GPU/LBM/LB.h
@@ -235,6 +235,12 @@ typedef struct PN27{
int numberOfFs;
}ProcessNeighbor27;
+typedef struct PN27AfterFtoC {
+ uint memsizeFs;
+ uint memsizeIndex;
+ int numberOfNodes;
+} ProcessNeighbor27AfterFtoC;
+
typedef struct PN_F3 {
real* g[6];
uint memsizeGs;
diff --git a/src/gpu/VirtualFluids_GPU/Parameter/Parameter.cpp b/src/gpu/VirtualFluids_GPU/Parameter/Parameter.cpp
index 7fceddba5..b7b8dd7ae 100644
--- a/src/gpu/VirtualFluids_GPU/Parameter/Parameter.cpp
+++ b/src/gpu/VirtualFluids_GPU/Parameter/Parameter.cpp
@@ -2505,7 +2505,8 @@ bool Parameter::getKernelNeedsFluidNodeIndicesToRun() { return this->kernelNeeds
void Parameter::findCornerNodesCommMultiGPU() {
for (uint level = 0; level < parH.size(); level++) {
findCornerNodesXY(level);
- findCornerNodesXY(level);
+ findCornerNodesXZ(level);
+ findCornerNodesYZ(level);
}
}
@@ -2577,37 +2578,37 @@ bool Parameter::findIndexInSendNodesYZ(int level, int index)
void Parameter::initNumberOfProcessNeighborsAfterFtoCX(int level)
{
- this->getParH(level)->numberOfSendProcessNeighborsAfterFtoCX.resize(
+ this->getParH(level)->sendProcessNeighborsAfterFtoCX.resize(
this->getParH(level)->sendProcessNeighborX.size());
- this->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCX.resize(
+ this->getParH(level)->recvProcessNeighborsAfterFtoCX.resize(
this->getParH(level)->recvProcessNeighborX.size());
- this->getParD(level)->numberOfSendProcessNeighborsAfterFtoCX.resize(
- this->getParH(level)->numberOfSendProcessNeighborsAfterFtoCX.size());
- this->getParD(level)->numberOfRecvProcessNeighborsAfterFtoCX.resize(
- this->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCX.size());
+ this->getParD(level)->sendProcessNeighborsAfterFtoCX.resize(
+ this->getParH(level)->sendProcessNeighborsAfterFtoCX.size());
+ this->getParD(level)->recvProcessNeighborsAfterFtoCX.resize(
+ this->getParH(level)->recvProcessNeighborsAfterFtoCX.size());
}
void Parameter::initNumberOfProcessNeighborsAfterFtoCY(int level)
{
- this->getParH(level)->numberOfSendProcessNeighborsAfterFtoCY.resize(
+ this->getParH(level)->sendProcessNeighborsAfterFtoCY.resize(
this->getParH(level)->sendProcessNeighborY.size());
- this->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCY.resize(
+ this->getParH(level)->recvProcessNeighborsAfterFtoCY.resize(
this->getParH(level)->recvProcessNeighborY.size());
- this->getParD(level)->numberOfSendProcessNeighborsAfterFtoCY.resize(
- this->getParH(level)->numberOfSendProcessNeighborsAfterFtoCY.size());
- this->getParD(level)->numberOfRecvProcessNeighborsAfterFtoCY.resize(
- this->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCY.size());
+ this->getParD(level)->sendProcessNeighborsAfterFtoCY.resize(
+ this->getParH(level)->sendProcessNeighborsAfterFtoCY.size());
+ this->getParD(level)->recvProcessNeighborsAfterFtoCY.resize(
+ this->getParH(level)->recvProcessNeighborsAfterFtoCY.size());
}
void Parameter::initNumberOfProcessNeighborsAfterFtoCZ(int level)
{
- this->getParH(level)->numberOfSendProcessNeighborsAfterFtoCZ.resize(
+ this->getParH(level)->sendProcessNeighborsAfterFtoCZ.resize(
this->getParH(level)->sendProcessNeighborZ.size());
- this->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCZ.resize(
+ this->getParH(level)->recvProcessNeighborsAfterFtoCZ.resize(
this->getParH(level)->recvProcessNeighborZ.size());
- this->getParD(level)->numberOfSendProcessNeighborsAfterFtoCZ.resize(
- this->getParH(level)->numberOfSendProcessNeighborsAfterFtoCZ.size());
- this->getParD(level)->numberOfRecvProcessNeighborsAfterFtoCZ.resize(
- this->getParH(level)->numberOfRecvProcessNeighborsAfterFtoCZ.size());
+ this->getParD(level)->sendProcessNeighborsAfterFtoCZ.resize(
+ this->getParH(level)->sendProcessNeighborsAfterFtoCZ.size());
+ this->getParD(level)->recvProcessNeighborsAfterFtoCZ.resize(
+ this->getParH(level)->recvProcessNeighborsAfterFtoCZ.size());
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/src/gpu/VirtualFluids_GPU/Parameter/Parameter.h b/src/gpu/VirtualFluids_GPU/Parameter/Parameter.h
index d5583df04..0592ba945 100644
--- a/src/gpu/VirtualFluids_GPU/Parameter/Parameter.h
+++ b/src/gpu/VirtualFluids_GPU/Parameter/Parameter.h
@@ -294,12 +294,12 @@ struct LBMSimulationParameter
std::vector<ProcessNeighbor27> recvProcessNeighborY;
std::vector<ProcessNeighbor27> recvProcessNeighborZ;
- std::vector<uint> numberOfSendProcessNeighborsAfterFtoCX;
- std::vector<uint> numberOfSendProcessNeighborsAfterFtoCY;
- std::vector<uint> numberOfSendProcessNeighborsAfterFtoCZ;
- std::vector<uint> numberOfRecvProcessNeighborsAfterFtoCX;
- std::vector<uint> numberOfRecvProcessNeighborsAfterFtoCY;
- std::vector<uint> numberOfRecvProcessNeighborsAfterFtoCZ;
+ std::vector<ProcessNeighbor27AfterFtoC> sendProcessNeighborsAfterFtoCX;
+ std::vector<ProcessNeighbor27AfterFtoC> sendProcessNeighborsAfterFtoCY;
+ std::vector<ProcessNeighbor27AfterFtoC> sendProcessNeighborsAfterFtoCZ;
+ std::vector<ProcessNeighbor27AfterFtoC> recvProcessNeighborsAfterFtoCX;
+ std::vector<ProcessNeighbor27AfterFtoC> recvProcessNeighborsAfterFtoCY;
+ std::vector<ProcessNeighbor27AfterFtoC> recvProcessNeighborsAfterFtoCZ;
///////////////////////////////////////////////////////
// 3D domain decomposition convection diffusion
std::vector<ProcessNeighbor27> sendProcessNeighborADX;
--
GitLab