diff --git a/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h b/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h
index c27c3c2769ec2c452fe4e62510e0b99741d6f274..afb027fc1665ab874523bf39ec2a05518d28f7a1 100644
--- a/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h
+++ b/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h
@@ -85,7 +85,7 @@ public:
GRIDGENERATOR_EXPORT void setCommunicationProcess(int direction, uint process);
- GRIDGENERATOR_EXPORT uint getCommunicationProcess(int direction) override;
+ GRIDGENERATOR_EXPORT virtual uint getCommunicationProcess(int direction) override;
GRIDGENERATOR_EXPORT std::shared_ptr<Grid> getGrid(int level, int box);
@@ -176,7 +176,7 @@ public:
GRIDGENERATOR_EXPORT void getGridInformations(std::vector<int>& gridX, std::vector<int>& gridY,
std::vector<int>& gridZ, std::vector<int>& distX, std::vector<int>& distY,
std::vector<int>& distZ) override;
- GRIDGENERATOR_EXPORT uint getNumberOfGridLevels() const override;
+ GRIDGENERATOR_EXPORT virtual uint getNumberOfGridLevels() const override;
GRIDGENERATOR_EXPORT uint getNumberOfNodesCF(int level) override;
GRIDGENERATOR_EXPORT uint getNumberOfNodesFC(int level) override;
@@ -186,10 +186,10 @@ public:
GRIDGENERATOR_EXPORT void getOffsetFC(real* xOffCf, real* yOffCf, real* zOffCf, int level) override;
GRIDGENERATOR_EXPORT void getOffsetCF(real* xOffFc, real* yOffFc, real* zOffFc, int level) override;
- GRIDGENERATOR_EXPORT uint getNumberOfSendIndices(int direction, uint level) override;
- GRIDGENERATOR_EXPORT uint getNumberOfReceiveIndices(int direction, uint level) override;
- GRIDGENERATOR_EXPORT void getSendIndices(int *sendIndices, int direction, int level) override;
- GRIDGENERATOR_EXPORT void getReceiveIndices(int *sendIndices, int direction, int level) override;
+ GRIDGENERATOR_EXPORT virtual uint getNumberOfSendIndices(int direction, uint level) override;
+ GRIDGENERATOR_EXPORT virtual uint getNumberOfReceiveIndices(int direction, uint level) override;
+ GRIDGENERATOR_EXPORT virtual void getSendIndices(int *sendIndices, int direction, int level) override;
+ GRIDGENERATOR_EXPORT virtual void getReceiveIndices(int *sendIndices, int direction, int level) override;
// needed for CUDA Streams MultiGPU (Communication Hiding)
diff --git a/src/gpu/GridGenerator/grid/GridImp.h b/src/gpu/GridGenerator/grid/GridImp.h
index 874202e898d4f78f3db782b65879bf87e6b170e8..edb5ca916bf68dcf992ea214dcddb2dc43810352 100644
--- a/src/gpu/GridGenerator/grid/GridImp.h
+++ b/src/gpu/GridGenerator/grid/GridImp.h
@@ -76,6 +76,7 @@ protected:
public:
static SPtr<GridImp> makeShared(Object* object, real startX, real startY, real startZ, real endX, real endY, real endZ, real delta, std::string d3Qxx, uint level);
+ virtual ~GridImp() = default;
private:
void initalNumberOfNodesAndSize();
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridProvider.h b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridProvider.h
index d2fa6e64edd408fdec3c5ad63a48fba016c3ead6..5fc5826735643ec748da169160e782004d7e5fb7 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridProvider.h
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridProvider.h
@@ -40,7 +40,7 @@ public:
virtual void freeMemoryOnHost();
virtual void cudaCopyDataToHost(int level);
- virtual ~GridProvider() {}
+ virtual ~GridProvider() = default;
virtual void initalGridInformations() = 0;
protected:
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
index e83169df3c1bc1c8a374231dc26ba951da828087..7f61b4357276f38d8fde71489dcf60348b402941 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
@@ -27,6 +27,11 @@ GridGenerator::GridGenerator(std::shared_ptr<GridBuilder> builder, std::shared_p
GridGenerator::~GridGenerator() = default;
+void GridGenerator::setIndexRearrangementForStreams(std::unique_ptr<IndexRearrangementForStreams> &&indexRearrangement)
+{
+ this->indexRearrangement = std::move(indexRearrangement);
+}
+
void GridGenerator::initalGridInformations()
{
if (para->getKernelNeedsFluidNodeIndicesToRun())
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h
index 5857724d27f1e9659bc44303b27ba4707e908d37..d2f56e1df4ee5658c61b8e8a3e94a820d1a4f2f1 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h
@@ -58,13 +58,16 @@ private:
std::shared_ptr<GridBuilder> builder;
std::unique_ptr<const IndexRearrangementForStreams> indexRearrangement;
- std::unique_ptr<InterpolationCellGrouper> interpolationGrouper;
+ std::unique_ptr<const InterpolationCellGrouper> interpolationGrouper;
const uint mpiProcessID;
public:
VIRTUALFLUIDS_GPU_EXPORT GridGenerator(std::shared_ptr<GridBuilder> builder, std::shared_ptr<Parameter> para, std::shared_ptr<CudaMemoryManager> cudaMemoryManager, vf::gpu::Communicator& communicator);
VIRTUALFLUIDS_GPU_EXPORT ~GridGenerator() override;
+ //! \brief overwrites the default IndexRearrangementForStreams
+ void setIndexRearrangementForStreams(std::unique_ptr<IndexRearrangementForStreams>&& indexRearrangement);
+
//! \brief allocates and initialized the data structures for Coordinates and node types
void allocArrays_CoordNeighborGeo() override;
//! \brief allocates and initialized the values at the boundary conditions
@@ -103,7 +106,7 @@ private:
void setSizeGeoQs(unsigned int level) const;
void setQ27Size(QforBoundaryConditions &Q, real* QQ, unsigned int sizeQ) const;
bool hasQs(int channelSide, unsigned int level) const;
-
+
void initalValuesDomainDecompostion();
public:
void initalGridInformations() override;
@@ -130,6 +133,9 @@ private:
//! \param subgridDistances is a pointer to an array containing the subgrid distances
//! \param numberOfBCnodes is the number of lattice nodes in the boundary condition
static void getPointersToBoundaryConditions(QforBoundaryConditions& boundaryConditionStruct, real* subgridDistances, const unsigned int numberOfBCnodes);
+
+private:
+ friend class GridGeneratorTests_initalValuesDomainDecompostion;
};
#endif
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGeneratorTest.cpp b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGeneratorTest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..17185ef21a9a3aa459cd7c8ca8eb34cb1360fe62
--- /dev/null
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGeneratorTest.cpp
@@ -0,0 +1,144 @@
+#include "GridGenerator.h"
+#include <gmock/gmock.h>
+
+#include "Communication/Communicator.h"
+#include "DataTypes.h"
+#include "GPU/CudaMemoryManager.h"
+#include "IndexRearrangementForStreams.h"
+#include "Parameter/Parameter.h"
+#include "gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h"
+#include "gpu/GridGenerator/grid/GridImp.h"
+#include "gpu/GridGenerator/utilities/communication.h"
+
+namespace GridGeneratorTest
+{
+
+class LevelGridBuilderStub : public LevelGridBuilder
+{
+private:
+ const SPtr<Grid> grid;
+ LevelGridBuilderStub() = default;
+
+public:
+ uint numberOfSendIndices = 0;
+
+ LevelGridBuilderStub(SPtr<Grid> grid) : LevelGridBuilder(), grid(grid){};
+
+ uint getCommunicationProcess(int direction) override
+ {
+ uint process = 0;
+ if (direction != CommunicationDirections::MX)
+ process = INVALID_INDEX;
+ return process;
+ }
+
+ uint getNumberOfGridLevels() const override
+ {
+ return 2;
+ }
+
+ uint getNumberOfSendIndices(int direction, uint level) override
+ {
+ return numberOfSendIndices;
+ }
+
+ uint getNumberOfReceiveIndices(int direction, uint level) override
+ {
+ return 0;
+ }
+
+ void getSendIndices(int *sendIndices, int direction, int level) override
+ {
+ }
+
+ void getReceiveIndices(int *sendIndices, int direction, int level) override
+ {
+ }
+};
+
+class CudaMemoryManagerDouble : public CudaMemoryManager
+{
+public:
+ CudaMemoryManagerDouble(std::shared_ptr<Parameter> parameter) : CudaMemoryManager(parameter){};
+
+ void cudaAllocProcessNeighborX(int lev, unsigned int processNeighbor) override{};
+ void cudaCopyProcessNeighborXIndex(int lev, unsigned int processNeighbor) override{};
+};
+
+class IndexRearrangementForStreamsDouble : public IndexRearrangementForStreams
+{
+public:
+ IndexRearrangementForStreamsDouble(std::shared_ptr<Parameter> para, std::shared_ptr<GridBuilder> builder,
+ vf::gpu::Communicator &communicator)
+ : IndexRearrangementForStreams(para, builder, communicator){};
+
+ void initCommunicationArraysForCommAfterFinetoCoarseX(uint level, int indexOfProcessNeighbor,
+ int direction) const override{};
+ void initCommunicationArraysForCommAfterFinetoCoarseY(uint level, int indexOfProcessNeighbor,
+ int direction) const override{};
+ void initCommunicationArraysForCommAfterFinetoCoarseZ(uint level, int indexOfProcessNeighbor,
+ int direction) const override{};
+};
+
+} // namespace GridGeneratorTest
+
+using namespace GridGeneratorTest;
+
+class GridGeneratorTests_initalValuesDomainDecompostion : public testing::Test
+{
+public:
+ void act() const
+ {
+ gridGenerator->initalValuesDomainDecompostion();
+ }
+
+protected:
+ SPtr<Parameter> para;
+ std::shared_ptr<LevelGridBuilderStub> builder;
+
+ const uint level = 1;
+ const uint direction = CommunicationDirections::MX;
+
+ SPtr<GridGenerator> gridGenerator;
+
+private:
+ void SetUp() override
+ {
+ logging::Logger::addStream(&std::cout);
+ logging::Logger::setDebugLevel(logging::Logger::WARNING);
+
+ para = std::make_shared<Parameter>();
+ para->setMaxLevel(level + 1); // setMaxLevel resizes parH and parD
+ for (uint i = 0; i <= level; i++) {
+ para->parH[i] = std::make_shared<LBMSimulationParameter>();
+ para->parD[i] = std::make_shared<LBMSimulationParameter>();
+ }
+ para->setNumprocs(2);
+
+ builder = std::make_shared<LevelGridBuilderStub>(nullptr);
+ vf::gpu::Communicator &communicator = vf::gpu::Communicator::getInstance();
+
+ gridGenerator = std::make_shared<GridGenerator>(builder, para, std::make_shared<CudaMemoryManagerDouble>(para),
+ communicator);
+ gridGenerator->setIndexRearrangementForStreams(
+ std::make_unique<IndexRearrangementForStreamsDouble>(para, builder, communicator));
+ }
+};
+
+TEST_F(GridGeneratorTests_initalValuesDomainDecompostion, whenNoCommunication_sendProcessNeighborShouldNotExist)
+{
+ act();
+ EXPECT_THAT(para->getParH(level)->sendProcessNeighborX.size(), testing::Eq(0));
+ EXPECT_THAT(para->getParH(level)->sendProcessNeighborY.size(), testing::Eq(0));
+ EXPECT_THAT(para->getParH(level)->sendProcessNeighborZ.size(), testing::Eq(0));
+}
+
+TEST_F(GridGeneratorTests_initalValuesDomainDecompostion, whenCommunicationInX_sendProcessNeighborShouldExistInX)
+{
+ builder->numberOfSendIndices = 1;
+ act();
+ EXPECT_THAT(para->getParH(level)->sendProcessNeighborX.size(),
+ testing::Eq(1)); // one entry for CommunicationDirections::MX
+ EXPECT_THAT(para->getParH(level)->sendProcessNeighborY.size(), testing::Eq(0));
+ EXPECT_THAT(para->getParH(level)->sendProcessNeighborZ.size(), testing::Eq(0));
+}
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.cpp b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.cpp
index 2ae473665cf77324e62e1fa50c2ed79b3c654ff6..f191c628dabbe9432c5930596e403a333c0593d4 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.cpp
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.cpp
@@ -1,6 +1,7 @@
#include "IndexRearrangementForStreams.h"
#include "Communication/Communicator.h"
+#include "Logger.h"
#include "Parameter/Parameter.h"
#include <GridGenerator/grid/Grid.h>
#include <GridGenerator/grid/GridBuilder/GridBuilder.h>
@@ -19,64 +20,52 @@ void IndexRearrangementForStreams::initCommunicationArraysForCommAfterFinetoCoar
int indexOfProcessNeighbor,
int direction) const
{
- std::cout << "communication: reorder send indices X ";
+ VF_LOG_INFO("Communication: reorder send indices in x direction");
std::vector<uint> sendIndicesForCommAfterFtoCPositions =
initSendIndicesForCommAfterFToCX(level, indexOfProcessNeighbor, direction);
- std::cout << "mpi send and receive ";
- std::vector<uint> recvIndicesForCommAfterFtoCPositions = exchangeIndicesForCommAfterFtoCX(
- level, indexOfProcessNeighbor, sendIndicesForCommAfterFtoCPositions);
+ std::vector<uint> recvIndicesForCommAfterFtoCPositions =
+ exchangeIndicesForCommAfterFtoCX(level, indexOfProcessNeighbor, sendIndicesForCommAfterFtoCPositions);
- std::cout << "reorder receive indices ";
initRecvIndicesForCommAfterFToCX(level, indexOfProcessNeighbor, direction, recvIndicesForCommAfterFtoCPositions);
copyProcessNeighborToCommAfterFtoCX(level, indexOfProcessNeighbor);
-
- std::cout << "done." << std::endl;
}
void IndexRearrangementForStreams::initCommunicationArraysForCommAfterFinetoCoarseY(uint level,
int indexOfProcessNeighbor,
int direction) const
{
- std::cout << "communication: reorder send indices Y ";
+ VF_LOG_INFO("Communication: reorder send indices in x direction");
std::vector<uint> sendIndicesForCommAfterFtoCPositions =
initSendIndicesForCommAfterFToCY(level, indexOfProcessNeighbor, direction);
- std::cout << "mpi send and receive ";
- std::vector<uint> recvIndicesForCommAfterFtoCPositions = exchangeIndicesForCommAfterFtoCY(
- level, indexOfProcessNeighbor, sendIndicesForCommAfterFtoCPositions);
+ std::vector<uint> recvIndicesForCommAfterFtoCPositions =
+ exchangeIndicesForCommAfterFtoCY(level, indexOfProcessNeighbor, sendIndicesForCommAfterFtoCPositions);
- std::cout << "reorder receive indices ";
initRecvIndicesForCommAfterFToCY(level, indexOfProcessNeighbor, direction, recvIndicesForCommAfterFtoCPositions);
copyProcessNeighborToCommAfterFtoCY(level, indexOfProcessNeighbor);
-
- std::cout << "done." << std::endl;
}
void IndexRearrangementForStreams::initCommunicationArraysForCommAfterFinetoCoarseZ(uint level,
int indexOfProcessNeighbor,
int direction) const
{
- std::cout << "communication: reorder send indices Z ";
+ VF_LOG_INFO("Communication: reorder send indices in z direction");
std::vector<uint> sendIndicesForCommAfterFtoCPositions =
initSendIndicesForCommAfterFToCZ(level, indexOfProcessNeighbor, direction);
- std::cout << "mpi send and receive ";
- std::vector<uint> recvIndicesForCommAfterFtoCPositions = exchangeIndicesForCommAfterFtoCZ(
- level, indexOfProcessNeighbor, sendIndicesForCommAfterFtoCPositions);
+ std::vector<uint> recvIndicesForCommAfterFtoCPositions =
+ exchangeIndicesForCommAfterFtoCZ(level, indexOfProcessNeighbor, sendIndicesForCommAfterFtoCPositions);
- std::cout << "reorder receive indices ";
initRecvIndicesForCommAfterFToCZ(level, indexOfProcessNeighbor, direction, recvIndicesForCommAfterFtoCPositions);
copyProcessNeighborToCommAfterFtoCZ(level, indexOfProcessNeighbor);
-
- std::cout << "done." << std::endl;
}
std::vector<uint> IndexRearrangementForStreams::initSendIndicesForCommAfterFToCX(uint level, int indexOfProcessNeighbor,
- int direction) const
+ int direction) const
{
para->initProcessNeighborsAfterFtoCX(level);
std::vector<uint> sendIndicesForCommAfterFtoCPositions;
@@ -88,7 +77,7 @@ std::vector<uint> IndexRearrangementForStreams::initSendIndicesForCommAfterFToCX
}
std::vector<uint> IndexRearrangementForStreams::initSendIndicesForCommAfterFToCY(uint level, int indexOfProcessNeighbor,
- int direction) const
+ int direction) const
{
para->initProcessNeighborsAfterFtoCY(level);
std::vector<uint> sendIndicesForCommAfterFtoCPositions;
@@ -100,7 +89,7 @@ std::vector<uint> IndexRearrangementForStreams::initSendIndicesForCommAfterFToCY
}
std::vector<uint> IndexRearrangementForStreams::initSendIndicesForCommAfterFToCZ(uint level, int indexOfProcessNeighbor,
- int direction) const
+ int direction) const
{
para->initProcessNeighborsAfterFtoCZ(level);
std::vector<uint> sendIndicesForCommAfterFtoCPositions;
@@ -172,7 +161,8 @@ std::vector<uint> IndexRearrangementForStreams::exchangeIndicesForCommAfterFtoCZ
}
void IndexRearrangementForStreams::initRecvIndicesForCommAfterFToCX(
- uint level, int indexOfProcessNeighbor, int direction, std::vector<uint> &recvIndicesForCommAfterFtoCPositions) const
+ uint level, int indexOfProcessNeighbor, int direction,
+ std::vector<uint> &recvIndicesForCommAfterFtoCPositions) const
{
reorderRecvIndicesForCommAfterFtoCX(direction, level, indexOfProcessNeighbor, recvIndicesForCommAfterFtoCPositions);
para->setRecvProcessNeighborsAfterFtoCX(
@@ -181,7 +171,8 @@ void IndexRearrangementForStreams::initRecvIndicesForCommAfterFToCX(
}
void IndexRearrangementForStreams::initRecvIndicesForCommAfterFToCY(
- uint level, int indexOfProcessNeighbor, int direction, std::vector<uint> &recvIndicesForCommAfterFtoCPositions) const
+ uint level, int indexOfProcessNeighbor, int direction,
+ std::vector<uint> &recvIndicesForCommAfterFtoCPositions) const
{
reorderRecvIndicesForCommAfterFtoCY(direction, level, indexOfProcessNeighbor, recvIndicesForCommAfterFtoCPositions);
para->setRecvProcessNeighborsAfterFtoCY(
@@ -189,7 +180,8 @@ void IndexRearrangementForStreams::initRecvIndicesForCommAfterFToCY(
indexOfProcessNeighbor);
}
void IndexRearrangementForStreams::initRecvIndicesForCommAfterFToCZ(
- uint level, int indexOfProcessNeighbor, int direction, std::vector<uint> &recvIndicesForCommAfterFtoCPositions) const
+ uint level, int indexOfProcessNeighbor, int direction,
+ std::vector<uint> &recvIndicesForCommAfterFtoCPositions) const
{
reorderRecvIndicesForCommAfterFtoCZ(direction, level, indexOfProcessNeighbor, recvIndicesForCommAfterFtoCPositions);
para->setRecvProcessNeighborsAfterFtoCZ(
@@ -197,7 +189,7 @@ void IndexRearrangementForStreams::initRecvIndicesForCommAfterFToCZ(
indexOfProcessNeighbor);
}
-void IndexRearrangementForStreams::copyProcessNeighborToCommAfterFtoCX(uint level, int indexOfProcessNeighbor) const
+void IndexRearrangementForStreams::copyProcessNeighborToCommAfterFtoCX(uint level, int indexOfProcessNeighbor) const
{
// init f[0]*
para->getParD(level)->sendProcessNeighborsAfterFtoCX[indexOfProcessNeighbor].f[0] =
@@ -226,7 +218,7 @@ void IndexRearrangementForStreams::copyProcessNeighborToCommAfterFtoCX(uint leve
para->getParH(level)->recvProcessNeighborX[indexOfProcessNeighbor].rankNeighbor;
}
-void IndexRearrangementForStreams::copyProcessNeighborToCommAfterFtoCY(uint level, int indexOfProcessNeighbor) const
+void IndexRearrangementForStreams::copyProcessNeighborToCommAfterFtoCY(uint level, int indexOfProcessNeighbor) const
{
// init f[0]*
para->getParD(level)->sendProcessNeighborsAfterFtoCY[indexOfProcessNeighbor].f[0] =
@@ -255,7 +247,7 @@ void IndexRearrangementForStreams::copyProcessNeighborToCommAfterFtoCY(uint leve
para->getParH(level)->recvProcessNeighborY[indexOfProcessNeighbor].rankNeighbor;
}
-void IndexRearrangementForStreams::copyProcessNeighborToCommAfterFtoCZ(uint level, int indexOfProcessNeighbor) const
+void IndexRearrangementForStreams::copyProcessNeighborToCommAfterFtoCZ(uint level, int indexOfProcessNeighbor) const
{
// init f[0]*
para->getParD(level)->sendProcessNeighborsAfterFtoCZ[indexOfProcessNeighbor].f[0] =
@@ -285,7 +277,7 @@ void IndexRearrangementForStreams::copyProcessNeighborToCommAfterFtoCZ(uint leve
}
void IndexRearrangementForStreams::reorderSendIndicesForCommAfterFtoCX(
- int direction, int level, int indexOfProcessNeighbor, std::vector<uint> &sendIndicesForCommAfterFtoCPositions) const
+ int direction, int level, int indexOfProcessNeighbor, std::vector<uint> &sendIndicesForCommAfterFtoCPositions) const
{
int *sendIndices = para->getParH(level)->sendProcessNeighborX[indexOfProcessNeighbor].index;
int &numberOfSendNodesAfterFtoC =
@@ -295,7 +287,7 @@ void IndexRearrangementForStreams::reorderSendIndicesForCommAfterFtoCX(
}
void IndexRearrangementForStreams::reorderSendIndicesForCommAfterFtoCY(
- int direction, int level, int indexOfProcessNeighbor, std::vector<uint> &sendIndicesForCommAfterFtoCPositions) const
+ int direction, int level, int indexOfProcessNeighbor, std::vector<uint> &sendIndicesForCommAfterFtoCPositions) const
{
int *sendIndices = para->getParH(level)->sendProcessNeighborY[indexOfProcessNeighbor].index;
int &numberOfSendNodesAfterFtoC =
@@ -305,7 +297,7 @@ void IndexRearrangementForStreams::reorderSendIndicesForCommAfterFtoCY(
}
void IndexRearrangementForStreams::reorderSendIndicesForCommAfterFtoCZ(
- int direction, int level, int indexOfProcessNeighbor, std::vector<uint> &sendIndicesForCommAfterFtoCPositions) const
+ int direction, int level, int indexOfProcessNeighbor, std::vector<uint> &sendIndicesForCommAfterFtoCPositions) const
{
int *sendIndices = para->getParH(level)->sendProcessNeighborZ[indexOfProcessNeighbor].index;
int &numberOfSendNodesAfterFtoC =
@@ -316,16 +308,13 @@ void IndexRearrangementForStreams::reorderSendIndicesForCommAfterFtoCZ(
void IndexRearrangementForStreams::reorderSendIndicesForCommAfterFtoC(
int *sendIndices, int &numberOfSendNodesAfterFtoC, int direction, int level,
- std::vector<uint> &sendIndicesForCommAfterFtoCPositions) const
+ std::vector<uint> &sendIndicesForCommAfterFtoCPositions) const
{
- *logging::out << logging::Logger::INFO_INTERMEDIATE
- << "reorder send indices for communication after fine to coarse: level: " << level
- << " direction: " << direction;
+ VF_LOG_INFO("Reorder send indices for communication after fine to coarse: level: {} direction: {}", level,
+ direction);
if (para->getParH(level)->intCF.kCF == 0 || para->getParH(level)->intFC.kFC == 0)
- *logging::out << logging::Logger::LOGGER_ERROR
- << "reorderSendIndicesForCommAfterFtoC(): para->getParH(level)->intCF needs to be initialized before calling "
- "this function "
- << "\n";
+ VF_LOG_CRITICAL("reorderSendIndicesForCommAfterFtoC(): para->getParH(level)->intCF needs to be initialized "
+ "before calling this function");
int sparseIndexSend;
std::vector<int> sendIndicesAfterFtoC;
@@ -358,21 +347,17 @@ void IndexRearrangementForStreams::reorderSendIndicesForCommAfterFtoC(
for (uint i = 0; i < (uint)sendIndicesOther.size(); i++)
sendIndices[i + numberOfSendNodesAfterFtoC] = sendIndicesOther[i];
- *logging::out << logging::Logger::INFO_INTERMEDIATE << "... Process "
- << " " << communicator.getPID()
- << " numberOfSendNodesAfterFtoC: " << numberOfSendNodesAfterFtoC << "\n ";
+ VF_LOG_INFO("Reorder send indices: process {}, numberOfSendNodesAfterFtoC {}", communicator.getPID(),
+ numberOfSendNodesAfterFtoC);
if (numberOfSendNodesAfterFtoC + sendIndicesOther.size() != numberOfSendIndices) {
- *logging::out << logging::Logger::LOGGER_ERROR
- << "reorderSendIndicesForCommAfterFtoC(): incorrect number of nodes"
- << "\n";
- std::cout << "numberOfSendNodesAfterFtoC = " << numberOfSendNodesAfterFtoC
- << ", sendOrIndicesOther.size() = " << sendIndicesOther.size()
- << ", numberOfSendOrRecvIndices = " << numberOfSendIndices << std::endl;
+ VF_LOG_CRITICAL("reorderSendIndicesForCommAfterFtoC(): incorrect number of nodes");
+ VF_LOG_CRITICAL("numberOfSendNodesAfterFtoC = {}, sendIndicesOther.size() = {}, numberOfSendIndices = {}",
+ numberOfSendNodesAfterFtoC, sendIndicesOther.size(), numberOfSendIndices);
}
}
-bool IndexRearrangementForStreams::isSparseIndexInICellFCC(uint sizeOfICellFCC, int sparseIndex, int level) const
+bool IndexRearrangementForStreams::isSparseIndexInICellFCC(uint sizeOfICellFCC, int sparseIndex, int level) const
{
for (uint j = 0; j < sizeOfICellFCC; j++) {
if (sparseIndex < 0)
@@ -384,7 +369,7 @@ bool IndexRearrangementForStreams::isSparseIndexInICellFCC(uint sizeOfICellFCC,
return false;
}
-void IndexRearrangementForStreams::aggregateNodesInICellCFC(int level, std::vector<uint> &nodesCFC) const
+void IndexRearrangementForStreams::aggregateNodesInICellCFC(int level, std::vector<uint> &nodesCFC) const
{
uint sparseIndex;
uint *neighborX = para->getParH(level)->neighborX;
@@ -439,7 +424,7 @@ void IndexRearrangementForStreams::findIfSparseIndexIsInSendIndicesAndAddToCommV
void IndexRearrangementForStreams::findIndicesNotInCommAfterFtoC(const uint &numberOfSendOrRecvIndices,
int *sendOrReceiveIndices,
std::vector<int> &sendOrReceiveIndicesAfterFtoC,
- std::vector<int> &sendOrIndicesOther) const
+ std::vector<int> &sendOrIndicesOther) const
{
int sparseIndexSend;
for (uint posInSendIndices = 0; posInSendIndices < numberOfSendOrRecvIndices; posInSendIndices++) {
@@ -451,25 +436,27 @@ void IndexRearrangementForStreams::findIndicesNotInCommAfterFtoC(const uint &num
}
void IndexRearrangementForStreams::reorderRecvIndicesForCommAfterFtoCX(
- int direction, int level, int indexOfProcessNeighbor, std::vector<uint> &sendIndicesForCommAfterFtoCPositions) const
+ int direction, int level, int indexOfProcessNeighbor, std::vector<uint> &sendIndicesForCommAfterFtoCPositions) const
{
- int *recvIndices = para->getParH(level)->recvProcessNeighborX[indexOfProcessNeighbor].index;
- int &numberOfRecvNodesAfterFtoC = para->getParH(level)->recvProcessNeighborsAfterFtoCX[indexOfProcessNeighbor].numberOfNodes;
+ int *recvIndices = para->getParH(level)->recvProcessNeighborX[indexOfProcessNeighbor].index;
+ int &numberOfRecvNodesAfterFtoC =
+ para->getParH(level)->recvProcessNeighborsAfterFtoCX[indexOfProcessNeighbor].numberOfNodes;
reorderRecvIndicesForCommAfterFtoC(recvIndices, numberOfRecvNodesAfterFtoC, direction, level,
sendIndicesForCommAfterFtoCPositions);
}
void IndexRearrangementForStreams::reorderRecvIndicesForCommAfterFtoCY(
- int direction, int level, int indexOfProcessNeighbor, std::vector<uint> &sendIndicesForCommAfterFtoCPositions) const
+ int direction, int level, int indexOfProcessNeighbor, std::vector<uint> &sendIndicesForCommAfterFtoCPositions) const
{
- int *recvIndices = para->getParH(level)->recvProcessNeighborY[indexOfProcessNeighbor].index;
- int &numberOfRecvNodesAfterFtoC = para->getParH(level)->recvProcessNeighborsAfterFtoCY[indexOfProcessNeighbor].numberOfNodes;
+ int *recvIndices = para->getParH(level)->recvProcessNeighborY[indexOfProcessNeighbor].index;
+ int &numberOfRecvNodesAfterFtoC =
+ para->getParH(level)->recvProcessNeighborsAfterFtoCY[indexOfProcessNeighbor].numberOfNodes;
reorderRecvIndicesForCommAfterFtoC(recvIndices, numberOfRecvNodesAfterFtoC, direction, level,
sendIndicesForCommAfterFtoCPositions);
}
void IndexRearrangementForStreams::reorderRecvIndicesForCommAfterFtoCZ(
- int direction, int level, int indexOfProcessNeighbor, std::vector<uint> &sendIndicesForCommAfterFtoCPositions) const
+ int direction, int level, int indexOfProcessNeighbor, std::vector<uint> &sendIndicesForCommAfterFtoCPositions) const
{
int *recvIndices = para->getParH(level)->recvProcessNeighborZ[indexOfProcessNeighbor].index;
int &numberOfRecvNodesAfterFtoC =
@@ -480,17 +467,13 @@ void IndexRearrangementForStreams::reorderRecvIndicesForCommAfterFtoCZ(
void IndexRearrangementForStreams::reorderRecvIndicesForCommAfterFtoC(
int *recvIndices, int &numberOfRecvNodesAfterFtoC, int direction, int level,
- std::vector<uint> &sendIndicesForCommAfterFtoCPositions) const
+ std::vector<uint> &sendIndicesForCommAfterFtoCPositions) const
{
- *logging::out << logging::Logger::INFO_INTERMEDIATE
- << "reorder receive indices for communication after fine to coarse: level: " << level
- << " direction: " << direction;
+ VF_LOG_INFO("Reorder recv indices for communication after fine to coarse: level: {} direction: {}", level,
+ direction);
- std::cout << "\n n send indices: " << (uint)sendIndicesForCommAfterFtoCPositions.size() << std::endl;
- if (sendIndicesForCommAfterFtoCPositions.size() == 0)
- *logging::out << logging::Logger::INFO_HIGH
- << "reorderRecvIndicesForCommAfterFtoC(): sendIndicesForCommAfterFtoCPositions is empty."
- << "\n";
+ if (sendIndicesForCommAfterFtoCPositions.size() <= 0)
+ VF_LOG_WARNING("ReorderRecvIndicesForCommAfterFtoC(): sendIndicesForCommAfterFtoCPositions is empty.");
uint numberOfRecvIndices = builder->getNumberOfReceiveIndices(direction, level);
std::vector<int> recvIndicesAfterFtoC;
@@ -510,16 +493,12 @@ void IndexRearrangementForStreams::reorderRecvIndicesForCommAfterFtoC(
for (uint i = 0; i < (uint)recvIndicesOther.size(); i++)
recvIndices[i + numberOfRecvNodesAfterFtoC] = recvIndicesOther[i];
- *logging::out << logging::Logger::INFO_INTERMEDIATE << "... Process "
- << " " << communicator.getPID()
- << " numberOfRecvNodesAfterFtoC: " << numberOfRecvNodesAfterFtoC << "\n ";
+ VF_LOG_INFO("Reorder send indices: process {}, numberOfRecvNodesAfterFtoC {}", communicator.getPID(),
+ numberOfRecvNodesAfterFtoC);
if (numberOfRecvNodesAfterFtoC + recvIndicesOther.size() != numberOfRecvIndices) {
- *logging::out << logging::Logger::LOGGER_ERROR
- << "reorderRecvIndicesForCommAfterFtoC(): incorrect number of nodes"
- << "\n";
- std::cout << "numberOfRecvNodesAfterFtoC = " << numberOfRecvNodesAfterFtoC
- << ", recvIndicesOther.size() = " << recvIndicesOther.size()
- << ", numberOfRecvIndices = " << numberOfRecvIndices << std::endl;
+ VF_LOG_CRITICAL("reorderRecvIndicesForCommAfterFtoC(): incorrect number of nodes");
+ VF_LOG_CRITICAL("numberOfRecvNodesAfterFtoC = {}, recvIndicesOther.size() = {}, numberOfRecvIndices = {}",
+ numberOfRecvNodesAfterFtoC, recvIndicesOther.size(), numberOfRecvIndices);
}
}
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.h b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.h
index c8abeaff932f72bbf074961e39127e3d8fa8d12c..e6753e21be43a4f004d812cca8b9adafa501ffe8 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.h
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.h
@@ -24,6 +24,8 @@ public:
//! \brief Construct IndexRearrangementForStreams object
IndexRearrangementForStreams(std::shared_ptr<Parameter> para, std::shared_ptr<GridBuilder> builder, vf::gpu::CommunicationRoutine& communicator);
+ virtual ~IndexRearrangementForStreams() = default;
+
//////////////////////////////////////////////////////////////////////////
// communication after fine to coarse
//////////////////////////////////////////////////////////////////////////
@@ -32,13 +34,13 @@ public:
//! \details Only the nodes involved in the interpolation need to be exchanged. Therefore in this method all nodes,
//! which are part of the interpolation as well as the communication, are identified.
//!See [master thesis of Anna Wellmann (p. 59-62: "Reduzieren der auszutauschenden Knoten")]
- void initCommunicationArraysForCommAfterFinetoCoarseX(uint level, int j, int direction) const;
+ virtual void initCommunicationArraysForCommAfterFinetoCoarseX(uint level, int j, int direction) const;
//! \brief Initialize the arrays for the communication after the interpolation from fine to coarse in y direction
//! \details --> see x direction
- void initCommunicationArraysForCommAfterFinetoCoarseY(uint level, int j, int direction) const;
+ virtual void initCommunicationArraysForCommAfterFinetoCoarseY(uint level, int j, int direction) const;
//! \brief Initialize the arrays for the communication after the interpolation from fine to coarse in z direction
//! \details --> see x direction
- void initCommunicationArraysForCommAfterFinetoCoarseZ(uint level, int j, int direction) const;
+ virtual void initCommunicationArraysForCommAfterFinetoCoarseZ(uint level, int j, int direction) const;
protected:
//////////////////////////////////////////////////////////////////////////
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/InterpolationCellGrouper.cpp b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/InterpolationCellGrouper.cpp
index ce1d74f631794a7c7ae9df52b810ceb34b82ba2a..347ab362d1f6b28a6c2b46f2e885085f955fb34e 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/InterpolationCellGrouper.cpp
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/InterpolationCellGrouper.cpp
@@ -10,7 +10,7 @@ InterpolationCellGrouper::InterpolationCellGrouper(const LBMSimulationParameters
{
}
-void InterpolationCellGrouper::splitFineToCoarseIntoBorderAndBulk(uint level)
+void InterpolationCellGrouper::splitFineToCoarseIntoBorderAndBulk(uint level) const
{
this->reorderFineToCoarseIntoBorderAndBulk(level);
@@ -25,12 +25,12 @@ void InterpolationCellGrouper::splitFineToCoarseIntoBorderAndBulk(uint level)
parDs[level]->offFCBulk.zOffFC = parDs[level]->offFC.zOffFC + parDs[level]->intFCBorder.kFC;
}
-void InterpolationCellGrouper::reorderFineToCoarseIntoBorderAndBulk(int level)
+void InterpolationCellGrouper::reorderFineToCoarseIntoBorderAndBulk(uint level) const
{
// create some local variables for better readability
uint *iCellFccAll = parHs[level]->intFC.ICellFCC;
uint *iCellFcfAll = parHs[level]->intFC.ICellFCF;
- auto grid = this->builder->getGrid((uint)level);
+ auto grid = this->builder->getGrid(level);
std::vector<uint> iCellFccBorderVector;
std::vector<uint> iCellFccBulkVector;
@@ -88,7 +88,7 @@ void InterpolationCellGrouper::reorderFineToCoarseIntoBorderAndBulk(int level)
}
}
-void InterpolationCellGrouper::splitCoarseToFineIntoBorderAndBulk(uint level)
+void InterpolationCellGrouper::splitCoarseToFineIntoBorderAndBulk(uint level) const
{
this->reorderCoarseToFineIntoBorderAndBulk(level);
@@ -103,7 +103,7 @@ void InterpolationCellGrouper::splitCoarseToFineIntoBorderAndBulk(uint level)
parDs[level]->offCFBulk.zOffCF = parDs[level]->offCF.zOffCF + parDs[level]->intCFBorder.kCF;
}
-void InterpolationCellGrouper::reorderCoarseToFineIntoBorderAndBulk(int level)
+void InterpolationCellGrouper::reorderCoarseToFineIntoBorderAndBulk(uint level) const
{
// create some local variables for better readability
uint *iCellCfcAll = parHs[level]->intCF.ICellCFC;
@@ -111,7 +111,7 @@ void InterpolationCellGrouper::reorderCoarseToFineIntoBorderAndBulk(int level)
uint *neighborX = this->parHs[level]->neighborX;
uint *neighborY = this->parHs[level]->neighborY;
uint *neighborZ = this->parHs[level]->neighborZ;
- auto grid = this->builder->getGrid((uint)level);
+ auto grid = this->builder->getGrid(level);
std::vector<uint> iCellCfcBorderVector;
std::vector<uint> iCellCfcBulkVector;
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/InterpolationCellGrouper.h b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/InterpolationCellGrouper.h
index 34ed9f0e6cbf991e7480aa3bfd1d78afc27d5940..5471194f643e044a7c2cdca1db45017b9d3a1022 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/InterpolationCellGrouper.h
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/InterpolationCellGrouper.h
@@ -35,7 +35,7 @@ public:
//! - the other cells which are not directly related to the communication between the two gpus --> "bulk"
//!
//! see [master thesis of Anna Wellmann (p. 62-68: "Ãœberdeckung der reduzierten Kommunikation")]
- void splitCoarseToFineIntoBorderAndBulk(uint level);
+ void splitCoarseToFineIntoBorderAndBulk(uint level) const;
//! \brief Split the interpolation cells from fine to coarse into border an bulk
//! \details For communication hiding, the interpolation cells from the fine to the coarse grid need to be split
@@ -46,7 +46,7 @@ public:
//! - the other cells which are not directly related to the communication between the two gpus --> "bulk"
//!
//! See [master thesis of Anna Wellmann (p. 62-68: "Ãœberdeckung der reduzierten Kommunikation")]
- void splitFineToCoarseIntoBorderAndBulk(uint level);
+ void splitFineToCoarseIntoBorderAndBulk(uint level) const;
protected:
//////////////////////////////////////////////////////////////////////////
@@ -57,13 +57,13 @@ protected:
//! subarrays: \details The coarse cells for interpolation from coarse to fine (iCellCFC) are divided into two
//! subgroups: border and bulk. The fine cells (iCellCFF) are reordered accordingly. The offset cells (xOffCF,
//! yOffCF, zOffCF) must be reordered in the same way.
- void reorderCoarseToFineIntoBorderAndBulk(int level);
+ void reorderCoarseToFineIntoBorderAndBulk(uint level) const;
//! \brief This function reorders the arrays of FCC/FCF indices and return pointers and sizes of the new subarrays:
//! \details The coarse cells for interpolation from fine to coarse (iCellFCC) are divided into two subgroups:
//! border and bulk. The fine cells (iCellFCF) are reordered accordingly. The offset cells (xOffFC,
//! yOffFC, zOffFC) must be reordered in the same way.
- void reorderFineToCoarseIntoBorderAndBulk(int level);
+ void reorderFineToCoarseIntoBorderAndBulk(uint level) const;
private:
SPtr<GridBuilder> builder;
diff --git a/src/gpu/VirtualFluids_GPU/GPU/CudaMemoryManager.h b/src/gpu/VirtualFluids_GPU/GPU/CudaMemoryManager.h
index 8df1b412c760ef3298f4426c206777d5c1db1b54..ca7a63244092f77ec398d830373197dfe385be26 100644
--- a/src/gpu/VirtualFluids_GPU/GPU/CudaMemoryManager.h
+++ b/src/gpu/VirtualFluids_GPU/GPU/CudaMemoryManager.h
@@ -25,6 +25,7 @@ class VIRTUALFLUIDS_GPU_EXPORT CudaMemoryManager
{
public:
CudaMemoryManager(std::shared_ptr<Parameter> parameter);
+ virtual ~CudaMemoryManager() = default;
void setMemsizeGPU(double admem, bool reset);
double getMemsizeGPU();
@@ -90,12 +91,12 @@ public:
//////////////////////////////////////////////////////////////////////////
//3D domain decomposition
- void cudaAllocProcessNeighborX(int lev, unsigned int processNeighbor);
+ virtual void cudaAllocProcessNeighborX(int lev, unsigned int processNeighbor);
void cudaCopyProcessNeighborXFsHD(int lev, unsigned int processNeighbor, const unsigned int &memsizeFsRecv,
int streamIndex);
void cudaCopyProcessNeighborXFsDH(int lev, unsigned int processNeighbor, const unsigned int &memsizeFsSend,
int streamIndex);
- void cudaCopyProcessNeighborXIndex(int lev, unsigned int processNeighbor);
+ virtual void cudaCopyProcessNeighborXIndex(int lev, unsigned int processNeighbor);
void cudaFreeProcessNeighborX(int lev, unsigned int processNeighbor);
//
void cudaAllocProcessNeighborY(int lev, unsigned int processNeighbor);