From 3124ef19e015b60ef76c8e447fa73a1ff60b98e4 Mon Sep 17 00:00:00 2001
From: Anna Wellmann <a.wellmann@tu-bs.de>
Date: Thu, 1 Dec 2022 08:47:05 +0000
Subject: [PATCH] Add test to GridGenerator::initialValuesDomainDecomposition
---
.../grid/GridBuilder/LevelGridBuilder.h | 12 +-
src/gpu/GridGenerator/grid/GridImp.h | 1 +
.../DataStructureInitializer/GridProvider.h | 2 +-
.../GridReaderGenerator/GridGenerator.cpp | 5 +
.../GridReaderGenerator/GridGenerator.h | 8 +-
.../GridReaderGenerator/GridGeneratorTest.cpp | 125 ++++++++++++++++++
.../IndexRearrangementForStreams.h | 2 +
.../VirtualFluids_GPU/GPU/CudaMemoryManager.h | 5 +-
8 files changed, 150 insertions(+), 10 deletions(-)
create mode 100644 src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGeneratorTest.cpp
diff --git a/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h b/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h
index c27c3c276..afb027fc1 100644
--- a/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h
+++ b/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h
@@ -85,7 +85,7 @@ public:
GRIDGENERATOR_EXPORT void setCommunicationProcess(int direction, uint process);
- GRIDGENERATOR_EXPORT uint getCommunicationProcess(int direction) override;
+ GRIDGENERATOR_EXPORT virtual uint getCommunicationProcess(int direction) override;
GRIDGENERATOR_EXPORT std::shared_ptr<Grid> getGrid(int level, int box);
@@ -176,7 +176,7 @@ public:
GRIDGENERATOR_EXPORT void getGridInformations(std::vector<int>& gridX, std::vector<int>& gridY,
std::vector<int>& gridZ, std::vector<int>& distX, std::vector<int>& distY,
std::vector<int>& distZ) override;
- GRIDGENERATOR_EXPORT uint getNumberOfGridLevels() const override;
+ GRIDGENERATOR_EXPORT virtual uint getNumberOfGridLevels() const override;
GRIDGENERATOR_EXPORT uint getNumberOfNodesCF(int level) override;
GRIDGENERATOR_EXPORT uint getNumberOfNodesFC(int level) override;
@@ -186,10 +186,10 @@ public:
GRIDGENERATOR_EXPORT void getOffsetFC(real* xOffCf, real* yOffCf, real* zOffCf, int level) override;
GRIDGENERATOR_EXPORT void getOffsetCF(real* xOffFc, real* yOffFc, real* zOffFc, int level) override;
- GRIDGENERATOR_EXPORT uint getNumberOfSendIndices(int direction, uint level) override;
- GRIDGENERATOR_EXPORT uint getNumberOfReceiveIndices(int direction, uint level) override;
- GRIDGENERATOR_EXPORT void getSendIndices(int *sendIndices, int direction, int level) override;
- GRIDGENERATOR_EXPORT void getReceiveIndices(int *sendIndices, int direction, int level) override;
+ GRIDGENERATOR_EXPORT virtual uint getNumberOfSendIndices(int direction, uint level) override;
+ GRIDGENERATOR_EXPORT virtual uint getNumberOfReceiveIndices(int direction, uint level) override;
+ GRIDGENERATOR_EXPORT virtual void getSendIndices(int *sendIndices, int direction, int level) override;
+ GRIDGENERATOR_EXPORT virtual void getReceiveIndices(int *sendIndices, int direction, int level) override;
// needed for CUDA Streams MultiGPU (Communication Hiding)
diff --git a/src/gpu/GridGenerator/grid/GridImp.h b/src/gpu/GridGenerator/grid/GridImp.h
index 874202e89..edb5ca916 100644
--- a/src/gpu/GridGenerator/grid/GridImp.h
+++ b/src/gpu/GridGenerator/grid/GridImp.h
@@ -76,6 +76,7 @@ protected:
public:
static SPtr<GridImp> makeShared(Object* object, real startX, real startY, real startZ, real endX, real endY, real endZ, real delta, std::string d3Qxx, uint level);
+ virtual ~GridImp() = default;
private:
void initalNumberOfNodesAndSize();
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridProvider.h b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridProvider.h
index d2fa6e64e..5fc582673 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridProvider.h
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridProvider.h
@@ -40,7 +40,7 @@ public:
virtual void freeMemoryOnHost();
virtual void cudaCopyDataToHost(int level);
- virtual ~GridProvider() {}
+ virtual ~GridProvider() = default;
virtual void initalGridInformations() = 0;
protected:
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
index e83169df3..7f61b4357 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
@@ -27,6 +27,11 @@ GridGenerator::GridGenerator(std::shared_ptr<GridBuilder> builder, std::shared_p
GridGenerator::~GridGenerator() = default;
+void GridGenerator::setIndexRearrangementForStreams(std::unique_ptr<IndexRearrangementForStreams> &&indexRearrangement)
+{
+ this->indexRearrangement = std::move(indexRearrangement);
+}
+
void GridGenerator::initalGridInformations()
{
if (para->getKernelNeedsFluidNodeIndicesToRun())
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h
index 5857724d2..ec0055b48 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h
@@ -65,6 +65,9 @@ public:
VIRTUALFLUIDS_GPU_EXPORT GridGenerator(std::shared_ptr<GridBuilder> builder, std::shared_ptr<Parameter> para, std::shared_ptr<CudaMemoryManager> cudaMemoryManager, vf::gpu::Communicator& communicator);
VIRTUALFLUIDS_GPU_EXPORT ~GridGenerator() override;
+ //! \brief overwrites the default IndexRearrangementForStreams
+ void setIndexRearrangementForStreams(std::unique_ptr<IndexRearrangementForStreams>&& indexRearrangement);
+
//! \brief allocates and initialized the data structures for Coordinates and node types
void allocArrays_CoordNeighborGeo() override;
//! \brief allocates and initialized the values at the boundary conditions
@@ -103,7 +106,7 @@ private:
void setSizeGeoQs(unsigned int level) const;
void setQ27Size(QforBoundaryConditions &Q, real* QQ, unsigned int sizeQ) const;
bool hasQs(int channelSide, unsigned int level) const;
-
+
void initalValuesDomainDecompostion();
public:
void initalGridInformations() override;
@@ -130,6 +133,9 @@ private:
//! \param subgridDistances is a pointer to an array containing the subgrid distances
//! \param numberOfBCnodes is the number of lattice nodes in the boundary condition
static void getPointersToBoundaryConditions(QforBoundaryConditions& boundaryConditionStruct, real* subgridDistances, const unsigned int numberOfBCnodes);
+
+private:
+ friend class GridGeneratorTests_initalValuesDomainDecompostion;
};
#endif
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGeneratorTest.cpp b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGeneratorTest.cpp
new file mode 100644
index 000000000..01d97f38c
--- /dev/null
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGeneratorTest.cpp
@@ -0,0 +1,125 @@
+#include "GridGenerator.h"
+#include <gmock/gmock.h>
+
+#include "Communication/Communicator.h"
+#include "GPU/CudaMemoryManager.h"
+#include "IndexRearrangementForStreams.h"
+#include "Parameter/Parameter.h"
+#include "gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h"
+#include "gpu/GridGenerator/grid/GridImp.h"
+#include "gpu/GridGenerator/utilities/communication.h"
+
+namespace GridGeneratorTest
+{
+
+class LevelGridBuilderStub : public LevelGridBuilder
+{
+private:
+ SPtr<Grid> grid;
+ LevelGridBuilderStub() = default;
+
+public:
+ LevelGridBuilderStub(SPtr<Grid> grid) : LevelGridBuilder(), grid(grid){};
+
+ uint getCommunicationProcess(int direction) override
+ {
+ return 0;
+ }
+
+ uint getNumberOfGridLevels() const override
+ {
+ return 2;
+ }
+
+ uint getNumberOfSendIndices(int direction, uint level) override
+ {
+ return 0;
+ }
+
+ uint getNumberOfReceiveIndices(int direction, uint level) override
+ {
+ return 0;
+ }
+
+ void getSendIndices(int *sendIndices, int direction, int level) override
+ {
+ }
+
+ void getReceiveIndices(int *sendIndices, int direction, int level) override
+ {
+ }
+};
+
+class CudaMemoryManagerDouble : public CudaMemoryManager
+{
+public:
+ CudaMemoryManagerDouble(std::shared_ptr<Parameter> parameter) : CudaMemoryManager(parameter){};
+
+ void cudaAllocProcessNeighborX(int lev, unsigned int processNeighbor) override{};
+ void cudaCopyProcessNeighborXIndex(int lev, unsigned int processNeighbor) override{};
+};
+
+class IndexRearrangementForStreamsDouble : public IndexRearrangementForStreams
+{
+public:
+ IndexRearrangementForStreamsDouble(std::shared_ptr<Parameter> para, std::shared_ptr<GridBuilder> builder,
+ vf::gpu::Communicator &communicator)
+ : IndexRearrangementForStreams(para, builder, communicator){};
+
+ void initCommunicationArraysForCommAfterFinetoCoarseX(uint level, int j, int direction) override {};
+ void initCommunicationArraysForCommAfterFinetoCoarseY(uint level, int j, int direction) override {};
+ void initCommunicationArraysForCommAfterFinetoCoarseZ(uint level, int j, int direction) override {};
+};
+
+} // namespace GridGeneratorTest
+
+using namespace GridGeneratorTest;
+
+class GridGeneratorTests_initalValuesDomainDecompostion : public testing::Test
+{
+public:
+ void act()
+ {
+ gridGenerator->initalValuesDomainDecompostion();
+ }
+
+protected:
+ SPtr<Parameter> para;
+ std::shared_ptr<LevelGridBuilderStub> builder;
+
+ uint level = 1;
+ uint direction = CommunicationDirections::MX;
+
+ SPtr<GridGenerator> gridGenerator;
+
+private:
+ void SetUp() override
+ {
+ logging::Logger::addStream(&std::cout);
+ logging::Logger::setDebugLevel(logging::Logger::WARNING);
+
+ para = std::make_shared<Parameter>();
+ para->setMaxLevel(level + 1); // setMaxLevel resizes parH and parD
+ for (uint i = 0; i <= level; i++) {
+ para->parH[i] = std::make_shared<LBMSimulationParameter>();
+ para->parD[i] = std::make_shared<LBMSimulationParameter>();
+ }
+ para->setNumprocs(2);
+
+ builder = std::make_shared<LevelGridBuilderStub>(nullptr);
+ vf::gpu::Communicator &communicator = vf::gpu::Communicator::getInstance();
+
+ gridGenerator = std::make_shared<GridGenerator>(builder, para, std::make_shared<CudaMemoryManagerDouble>(para),
+ communicator);
+ gridGenerator->setIndexRearrangementForStreams(
+ std::make_unique<IndexRearrangementForStreamsDouble>(para, builder, communicator));
+ }
+};
+
+TEST_F(GridGeneratorTests_initalValuesDomainDecompostion, whenNoCommunication_sendProcessNeighborShouldNotExist)
+{
+ act();
+ EXPECT_THAT(para->getParH(level)->sendProcessNeighborX.size(), testing::Eq(0));
+ EXPECT_THAT(para->getParH(level)->sendProcessNeighborY.size(), testing::Eq(0));
+ EXPECT_THAT(para->getParH(level)->sendProcessNeighborZ.size(), testing::Eq(0));
+}
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.h b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.h
index c8abeaff9..d38765dc9 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.h
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.h
@@ -24,6 +24,8 @@ public:
//! \brief Construct IndexRearrangementForStreams object
IndexRearrangementForStreams(std::shared_ptr<Parameter> para, std::shared_ptr<GridBuilder> builder, vf::gpu::CommunicationRoutine& communicator);
+ virtual ~IndexRearrangementForStreams() = default;
+
//////////////////////////////////////////////////////////////////////////
// communication after fine to coarse
//////////////////////////////////////////////////////////////////////////
diff --git a/src/gpu/VirtualFluids_GPU/GPU/CudaMemoryManager.h b/src/gpu/VirtualFluids_GPU/GPU/CudaMemoryManager.h
index 8df1b412c..ca7a63244 100644
--- a/src/gpu/VirtualFluids_GPU/GPU/CudaMemoryManager.h
+++ b/src/gpu/VirtualFluids_GPU/GPU/CudaMemoryManager.h
@@ -25,6 +25,7 @@ class VIRTUALFLUIDS_GPU_EXPORT CudaMemoryManager
{
public:
CudaMemoryManager(std::shared_ptr<Parameter> parameter);
+ virtual ~CudaMemoryManager() = default;
void setMemsizeGPU(double admem, bool reset);
double getMemsizeGPU();
@@ -90,12 +91,12 @@ public:
//////////////////////////////////////////////////////////////////////////
//3D domain decomposition
- void cudaAllocProcessNeighborX(int lev, unsigned int processNeighbor);
+ virtual void cudaAllocProcessNeighborX(int lev, unsigned int processNeighbor);
void cudaCopyProcessNeighborXFsHD(int lev, unsigned int processNeighbor, const unsigned int &memsizeFsRecv,
int streamIndex);
void cudaCopyProcessNeighborXFsDH(int lev, unsigned int processNeighbor, const unsigned int &memsizeFsSend,
int streamIndex);
- void cudaCopyProcessNeighborXIndex(int lev, unsigned int processNeighbor);
+ virtual void cudaCopyProcessNeighborXIndex(int lev, unsigned int processNeighbor);
void cudaFreeProcessNeighborX(int lev, unsigned int processNeighbor);
//
void cudaAllocProcessNeighborY(int lev, unsigned int processNeighbor);
--
GitLab