From b04b07221dee2cfe41a790092e7544c2a118755d Mon Sep 17 00:00:00 2001
From: Anna Wellmann <a.wellmann@tu-bs.de>
Date: Mon, 5 Dec 2022 15:17:37 +0000
Subject: [PATCH] Add interface for index exchange to Communicator
IndexRearrangementForStreams needs only a small subset of the Communicator's functionalities.
This change also makes testing easier. There is no need to derive from the concrete class Communicator to test IndexRearrangementForStreams anymore.
---
.../Communication/Communicator.cpp | 4 +-
.../Communication/Communicator.h | 11 +-
.../Communication/IndexExchange.h | 17 +++
.../IndexRearrangementForStreams.cpp | 2 +-
.../IndexRearrangementForStreams.h | 6 +-
.../IndexRearrangementForStreamsTest.cpp | 126 +++++++++---------
6 files changed, 91 insertions(+), 75 deletions(-)
create mode 100644 src/gpu/VirtualFluids_GPU/Communication/IndexExchange.h
diff --git a/src/gpu/VirtualFluids_GPU/Communication/Communicator.cpp b/src/gpu/VirtualFluids_GPU/Communication/Communicator.cpp
index 282de9d9d..9105ac98d 100644
--- a/src/gpu/VirtualFluids_GPU/Communication/Communicator.cpp
+++ b/src/gpu/VirtualFluids_GPU/Communication/Communicator.cpp
@@ -241,8 +241,8 @@ double Communicator::sumNups(double processNups)
return *buffer_recv;
}
-void vf::gpu::Communicator::exchangeIndices(uint *buffer_receive, int size_buffer_recv, int neighbor_rank_recv,
- uint *buffer_send, int size_buffer_send, int neighbor_rank_send)
+void Communicator::exchangeIndices(uint *buffer_receive, int size_buffer_recv, int neighbor_rank_recv, uint *buffer_send,
+ int size_buffer_send, int neighbor_rank_send) const
{
MPI_Request recv_request;
MPI_Irecv(buffer_receive, size_buffer_recv, MPI_UNSIGNED, neighbor_rank_recv, 0, commGPU, &recv_request);
diff --git a/src/gpu/VirtualFluids_GPU/Communication/Communicator.h b/src/gpu/VirtualFluids_GPU/Communication/Communicator.h
index 61d456cfd..9483c1abe 100644
--- a/src/gpu/VirtualFluids_GPU/Communication/Communicator.h
+++ b/src/gpu/VirtualFluids_GPU/Communication/Communicator.h
@@ -8,6 +8,7 @@
#include "VirtualFluids_GPU_export.h"
#include <basics/Core/DataTypes.h>
+#include "IndexExchange.h"
//////////////////////////////////
#ifdef VF_DOUBLE_ACCURACY
@@ -22,7 +23,7 @@ namespace vf::gpu
{
-class VIRTUALFLUIDS_GPU_EXPORT Communicator
+class VIRTUALFLUIDS_GPU_EXPORT Communicator : public IndexExchange
{
public:
static Communicator& getInstance();
@@ -33,7 +34,7 @@ public:
void exchngTopToBottom(float* sbuf, float* rbuf, int count);
void waitAll();
void distributeGeometry(unsigned int* dataRoot, unsigned int* dataNode, int dataSizePerNode);
- int getPID() const;
+ int getPID() const override;
int getNummberOfProcess() const;
int getNeighbourTop();
int getNeighbourBottom();
@@ -60,10 +61,10 @@ public:
std::vector<double> gatherNUPS(double processNups);
double sumNups(double processNups);
//////////////////////////////////////////////////////////////////////////
- virtual void exchangeIndices(uint *buffer_receive, int size_buffer_recv, int neighbor_rank_recv, uint *buffer_send,
- int size_buffer_send, int neighbor_rank_send);
+ void exchangeIndices(uint *buffer_receive, int size_buffer_recv, int neighbor_rank_recv, uint *buffer_send,
+ int size_buffer_send, int neighbor_rank_send) const override;
-protected:
+private:
int numprocs, PID;
int nbrbottom, nbrtop;
MPI_Comm comm1d, commGPU;
diff --git a/src/gpu/VirtualFluids_GPU/Communication/IndexExchange.h b/src/gpu/VirtualFluids_GPU/Communication/IndexExchange.h
new file mode 100644
index 000000000..67b87c741
--- /dev/null
+++ b/src/gpu/VirtualFluids_GPU/Communication/IndexExchange.h
@@ -0,0 +1,17 @@
+#ifndef INDEX_EXCHANGE
+#define INDEX_EXCHANGE
+
+#include <basics/Core/DataTypes.h>
+
+namespace vf::gpu
+{
+class IndexExchange
+{
+public:
+ virtual void exchangeIndices(uint *buffer_receive, int size_buffer_recv, int neighbor_rank_recv, uint *buffer_send,
+ int size_buffer_send, int neighbor_rank_send) const = 0;
+ virtual int getPID() const = 0;
+};
+} // namespace vf::gpu
+
+#endif
\ No newline at end of file
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.cpp b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.cpp
index affd3de26..4ed4afe7f 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.cpp
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.cpp
@@ -10,7 +10,7 @@
IndexRearrangementForStreams::IndexRearrangementForStreams(std::shared_ptr<Parameter> para,
std::shared_ptr<GridBuilder> builder,
- vf::gpu::Communicator &communicator)
+ vf::gpu::IndexExchange &communicator)
: para(para), builder(builder), communicator(communicator)
{
}
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.h b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.h
index 244889174..f78e6a52b 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.h
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreams.h
@@ -15,14 +15,14 @@ class Parameter;
class GridBuilder;
namespace vf::gpu
{
-class Communicator;
+class IndexExchange;
}
class IndexRearrangementForStreams
{
public:
//! \brief Construct IndexRearrangementForStreams object
- IndexRearrangementForStreams(std::shared_ptr<Parameter> para, std::shared_ptr<GridBuilder> builder, vf::gpu::Communicator& communicator);
+ IndexRearrangementForStreams(std::shared_ptr<Parameter> para, std::shared_ptr<GridBuilder> builder, vf::gpu::IndexExchange& communicator);
//////////////////////////////////////////////////////////////////////////
// communication after fine to coarse
@@ -127,7 +127,7 @@ protected:
private:
std::shared_ptr<GridBuilder> builder;
std::shared_ptr<Parameter> para;
- vf::gpu::Communicator& communicator;
+ vf::gpu::IndexExchange& communicator;
// used for tests
friend class IndexRearrangementForStreamsTest_reorderSendIndices;
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreamsTest.cpp b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreamsTest.cpp
index c74d6a9b9..04ed413e7 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreamsTest.cpp
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreamsTest.cpp
@@ -2,7 +2,6 @@
#include <algorithm>
#include <iostream>
-#include <mpi.h>
#include <vector>
#include "Utilities/testUtilitiesGPU.h"
@@ -16,9 +15,10 @@
#include "gpu/GridGenerator/utilities/communication.h"
#include "gpu/VirtualFluids_GPU/Communication/Communicator.cpp"
-namespace indexRearrangementTests {
+namespace indexRearrangementTests
+{
template <typename T>
-bool vectorsAreEqual(const T * vector1, const std::vector<T> vectorExpected)
+bool vectorsAreEqual(const T *vector1, const std::vector<T> vectorExpected)
{
for (uint i = 0; i < vectorExpected.size(); i++) {
if (vector1[i] != vectorExpected[i])
@@ -71,7 +71,7 @@ public:
}
};
-}
+} // namespace indexRearrangementTests
//////////////////////////////////////////////////////////////////////////
// Test reorderSendIndices
@@ -128,7 +128,6 @@ private:
para = testingVF::createParameterForLevel(si.level);
-
para->getParH(si.level)->intFC.kFC = si.kFC;
para->getParH(si.level)->intFC.ICellFCC = &(si.iCellFCC.front());
para->getParH(si.level)->intCF.ICellCFC = &(si.iCellCFC.front());
@@ -171,22 +170,21 @@ TEST_F(IndexRearrangementForStreamsTest_reorderSendIndices, reorderSendIndicesFo
// Test exchangeIndicesForCommAfterFtoC
//////////////////////////////////////////////////////////////////////////
-class CommunicatorDouble : public vf::gpu::Communicator
+class IndexExchangeDouble : public vf::gpu::IndexExchange
{
public:
- static CommunicatorDouble &getInstance()
- {
- static CommunicatorDouble comm;
- return comm;
- }
-
- void exchangeIndices(uint *buffer_receive, int, int, uint *, int, int) override
+ void exchangeIndices(uint *buffer_receive, int, int, uint *, int, int) const override
{
for (int i = 0; i < (int)receivedIndices.size(); ++i) {
*(buffer_receive + i) = receivedIndices[i];
}
}
+ int getPID() const override
+ {
+ return 0;
+ }
+
void setReceivedIndices(std::vector<uint> receivedIndices)
{
this->receivedIndices = receivedIndices;
@@ -200,9 +198,9 @@ class IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCX : public
{
public:
- void createTestSubject(vf::gpu::Communicator &communicator)
+ void createTestSubject(vf::gpu::IndexExchange &indexExchange)
{
- sut = std::make_unique<IndexRearrangementForStreams>(para, builder, communicator);
+ sut = std::make_unique<IndexRearrangementForStreams>(para, builder, indexExchange);
}
protected:
@@ -243,9 +241,9 @@ private:
TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCX, emptyRecvInX)
{
- CommunicatorDouble &comm = CommunicatorDouble::getInstance();
- comm.setReceivedIndices(std::vector<uint>());
- createTestSubject(comm);
+ IndexExchangeDouble communicator;
+ communicator.setReceivedIndices(std::vector<uint>());
+ createTestSubject(communicator);
std::vector<uint> recvIndicesForCommAfterFtoCPositions = act();
EXPECT_THAT(recvIndicesForCommAfterFtoCPositions.size(), testing::Eq(0));
@@ -253,9 +251,9 @@ TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCX, emptyR
TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCX, zeroRecvIndexX)
{
- CommunicatorDouble &comm = CommunicatorDouble::getInstance();
- comm.setReceivedIndices({0});
- createTestSubject(comm);
+ IndexExchangeDouble communicator;
+ communicator.setReceivedIndices({ 0 });
+ createTestSubject(communicator);
std::vector<uint> recvIndicesForCommAfterFtoCPositions = act();
EXPECT_THAT(recvIndicesForCommAfterFtoCPositions.size(), testing::Eq(0));
@@ -263,12 +261,12 @@ TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCX, zeroRe
TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCX, oneRecvIndexX)
{
- CommunicatorDouble &comm = CommunicatorDouble::getInstance();
+ IndexExchangeDouble communicator;
std::vector<uint> expected = { 10 };
std::vector<uint> receivedIndicesByComm(4, 0);
std::copy(expected.begin(), expected.end(), receivedIndicesByComm.begin());
- comm.setReceivedIndices(receivedIndicesByComm);
- createTestSubject(comm);
+ communicator.setReceivedIndices(receivedIndicesByComm);
+ createTestSubject(communicator);
std::vector<uint> recvIndicesForCommAfterFtoCPositions = act();
EXPECT_THAT(recvIndicesForCommAfterFtoCPositions.size(), testing::Eq(1));
@@ -277,12 +275,12 @@ TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCX, oneRec
TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCX, threeRecvIndicesX)
{
- CommunicatorDouble &comm = CommunicatorDouble::getInstance();
+ IndexExchangeDouble communicator;
std::vector<uint> expected = { 10, 20, 30 };
std::vector<uint> receivedIndicesByComm(5, 0);
std::copy(expected.begin(), expected.end(), receivedIndicesByComm.begin());
- comm.setReceivedIndices(receivedIndicesByComm);
- createTestSubject(comm);
+ communicator.setReceivedIndices(receivedIndicesByComm);
+ createTestSubject(communicator);
std::vector<uint> recvIndicesForCommAfterFtoCPositions = act();
EXPECT_THAT(recvIndicesForCommAfterFtoCPositions.size(), testing::Eq(3));
@@ -291,12 +289,12 @@ TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCX, threeR
TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCX, sixRecvIndicesX)
{
- CommunicatorDouble &comm = CommunicatorDouble::getInstance();
+ IndexExchangeDouble communicator;
std::vector<uint> expected = { 10, 20, 30, 40, 50, 60 };
std::vector<uint> receivedIndicesByComm(expected.size(), 0);
std::copy(expected.begin(), expected.end(), receivedIndicesByComm.begin());
- comm.setReceivedIndices(receivedIndicesByComm);
- createTestSubject(comm);
+ communicator.setReceivedIndices(receivedIndicesByComm);
+ createTestSubject(communicator);
std::vector<uint> recvIndicesForCommAfterFtoCPositions = act();
EXPECT_THAT(recvIndicesForCommAfterFtoCPositions.size(), testing::Eq(6));
@@ -307,9 +305,9 @@ class IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCY : public
{
public:
- void createTestSubject(vf::gpu::Communicator &communicator)
+ void createTestSubject(vf::gpu::IndexExchange &indexExchange)
{
- sut = std::make_unique<IndexRearrangementForStreams>(para, builder, communicator);
+ sut = std::make_unique<IndexRearrangementForStreams>(para, builder, indexExchange);
}
protected:
@@ -350,9 +348,9 @@ private:
TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCY, emptyRecvInY)
{
- CommunicatorDouble &comm = CommunicatorDouble::getInstance();
- comm.setReceivedIndices(std::vector<uint>());
- createTestSubject(comm);
+ IndexExchangeDouble communicator;
+ communicator.setReceivedIndices(std::vector<uint>());
+ createTestSubject(communicator);
std::vector<uint> recvIndicesForCommAfterFtoCPositions = act();
EXPECT_THAT(recvIndicesForCommAfterFtoCPositions.size(), testing::Eq(0));
@@ -360,9 +358,9 @@ TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCY, emptyR
TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCY, zeroRecvIndexY)
{
- CommunicatorDouble &comm = CommunicatorDouble::getInstance();
- comm.setReceivedIndices({0});
- createTestSubject(comm);
+ IndexExchangeDouble communicator;
+ communicator.setReceivedIndices({ 0 });
+ createTestSubject(communicator);
std::vector<uint> recvIndicesForCommAfterFtoCPositions = act();
EXPECT_THAT(recvIndicesForCommAfterFtoCPositions.size(), testing::Eq(0));
@@ -370,12 +368,12 @@ TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCY, zeroRe
TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCY, oneRecvIndexY)
{
- CommunicatorDouble &comm = CommunicatorDouble::getInstance();
+ IndexExchangeDouble communicator;
std::vector<uint> expected = { 10 };
std::vector<uint> receivedIndicesByComm(4, 0);
std::copy(expected.begin(), expected.end(), receivedIndicesByComm.begin());
- comm.setReceivedIndices(receivedIndicesByComm);
- createTestSubject(comm);
+ communicator.setReceivedIndices(receivedIndicesByComm);
+ createTestSubject(communicator);
std::vector<uint> recvIndicesForCommAfterFtoCPositions = act();
EXPECT_THAT(recvIndicesForCommAfterFtoCPositions.size(), testing::Eq(1));
@@ -384,12 +382,12 @@ TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCY, oneRec
TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCY, threeRecvIndicesY)
{
- CommunicatorDouble &comm = CommunicatorDouble::getInstance();
+ IndexExchangeDouble communicator;
std::vector<uint> expected = { 10, 20, 30 };
std::vector<uint> receivedIndicesByComm(5, 0);
std::copy(expected.begin(), expected.end(), receivedIndicesByComm.begin());
- comm.setReceivedIndices(receivedIndicesByComm);
- createTestSubject(comm);
+ communicator.setReceivedIndices(receivedIndicesByComm);
+ createTestSubject(communicator);
std::vector<uint> recvIndicesForCommAfterFtoCPositions = act();
EXPECT_THAT(recvIndicesForCommAfterFtoCPositions.size(), testing::Eq(3));
@@ -398,12 +396,12 @@ TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCY, threeR
TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCY, sixRecvIndicesY)
{
- CommunicatorDouble &comm = CommunicatorDouble::getInstance();
+ IndexExchangeDouble communicator;
std::vector<uint> expected = { 10, 20, 30, 40, 50, 60 };
std::vector<uint> receivedIndicesByComm(expected.size(), 0);
std::copy(expected.begin(), expected.end(), receivedIndicesByComm.begin());
- comm.setReceivedIndices(receivedIndicesByComm);
- createTestSubject(comm);
+ communicator.setReceivedIndices(receivedIndicesByComm);
+ createTestSubject(communicator);
std::vector<uint> recvIndicesForCommAfterFtoCPositions = act();
EXPECT_THAT(recvIndicesForCommAfterFtoCPositions.size(), testing::Eq(6));
@@ -414,9 +412,9 @@ class IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCZ : public
{
public:
- void createTestSubject(vf::gpu::Communicator &communicator)
+ void createTestSubject(vf::gpu::IndexExchange &indexExchange)
{
- sut = std::make_unique<IndexRearrangementForStreams>(para, builder, communicator);
+ sut = std::make_unique<IndexRearrangementForStreams>(para, builder, indexExchange);
}
protected:
@@ -457,9 +455,9 @@ private:
TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCZ, emptyRecvInZ)
{
- CommunicatorDouble &comm = CommunicatorDouble::getInstance();
- comm.setReceivedIndices(std::vector<uint>());
- createTestSubject(comm);
+ IndexExchangeDouble communicator;
+ communicator.setReceivedIndices(std::vector<uint>());
+ createTestSubject(communicator);
std::vector<uint> recvIndicesForCommAfterFtoCPositions = act();
EXPECT_THAT(recvIndicesForCommAfterFtoCPositions.size(), testing::Eq(0));
@@ -467,9 +465,9 @@ TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCZ, emptyR
TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCZ, zeroRecvIndexZ)
{
- CommunicatorDouble &comm = CommunicatorDouble::getInstance();
- comm.setReceivedIndices({0});
- createTestSubject(comm);
+ IndexExchangeDouble communicator;
+ communicator.setReceivedIndices({ 0 });
+ createTestSubject(communicator);
std::vector<uint> recvIndicesForCommAfterFtoCPositions = act();
EXPECT_THAT(recvIndicesForCommAfterFtoCPositions.size(), testing::Eq(0));
@@ -477,12 +475,12 @@ TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCZ, zeroRe
TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCZ, oneRecvIndexZ)
{
- CommunicatorDouble &comm = CommunicatorDouble::getInstance();
+ IndexExchangeDouble communicator;
std::vector<uint> expected = { 10 };
std::vector<uint> receivedIndicesBZComm(4, 0);
std::copy(expected.begin(), expected.end(), receivedIndicesBZComm.begin());
- comm.setReceivedIndices(receivedIndicesBZComm);
- createTestSubject(comm);
+ communicator.setReceivedIndices(receivedIndicesBZComm);
+ createTestSubject(communicator);
std::vector<uint> recvIndicesForCommAfterFtoCPositions = act();
EXPECT_THAT(recvIndicesForCommAfterFtoCPositions.size(), testing::Eq(1));
@@ -491,12 +489,12 @@ TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCZ, oneRec
TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCZ, threeRecvIndicesZ)
{
- CommunicatorDouble &comm = CommunicatorDouble::getInstance();
+ IndexExchangeDouble communicator;
std::vector<uint> expected = { 10, 20, 30 };
std::vector<uint> receivedIndicesBZComm(5, 0);
std::copy(expected.begin(), expected.end(), receivedIndicesBZComm.begin());
- comm.setReceivedIndices(receivedIndicesBZComm);
- createTestSubject(comm);
+ communicator.setReceivedIndices(receivedIndicesBZComm);
+ createTestSubject(communicator);
std::vector<uint> recvIndicesForCommAfterFtoCPositions = act();
EXPECT_THAT(recvIndicesForCommAfterFtoCPositions.size(), testing::Eq(3));
@@ -505,12 +503,12 @@ TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCZ, threeR
TEST_F(IndexRearrangementForStreamsTest_exchangeIndicesForCommAfterFtoCZ, sixRecvIndicesZ)
{
- CommunicatorDouble &comm = CommunicatorDouble::getInstance();
+ IndexExchangeDouble communicator;
std::vector<uint> expected = { 10, 20, 30, 40, 50, 60 };
std::vector<uint> receivedIndicesBZComm(expected.size(), 0);
std::copy(expected.begin(), expected.end(), receivedIndicesBZComm.begin());
- comm.setReceivedIndices(receivedIndicesBZComm);
- createTestSubject(comm);
+ communicator.setReceivedIndices(receivedIndicesBZComm);
+ createTestSubject(communicator);
std::vector<uint> recvIndicesForCommAfterFtoCPositions = act();
EXPECT_THAT(recvIndicesForCommAfterFtoCPositions.size(), testing::Eq(6));
--
GitLab