From 6c69b061b0eec7ddbb3bd064afdbc77b6091e734 Mon Sep 17 00:00:00 2001
From: Anna Wellmann <a.wellmann@tu-braunschweig.de>
Date: Wed, 6 Oct 2021 10:28:39 +0200
Subject: [PATCH] Add flag to turn on reduced communication after f to c
---
src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.cpp | 4 ++++
.../GridReaderGenerator/GridGenerator.cpp | 6 +++---
src/gpu/VirtualFluids_GPU/Parameter/Parameter.h | 4 +++-
3 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.cpp b/src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.cpp
index 0c870d1cd..324e0dbb4 100644
--- a/src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.cpp
+++ b/src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.cpp
@@ -83,6 +83,10 @@ void updateGrid27(Parameter *para, vf::gpu::Communicator *comm, CudaMemoryManage
para->getParD(level)->intFCBulk.ICellFCF,
para->getParD(level)->intFCBulk.kFC, -1);
+ if (para->useReducedCommunicationAfterFtoC) {
+ prepareExchangeMultiGPU(para, level, -1); // TODO
+ exchangeMultiGPU(para, comm, cudaManager, level, -1); // TODO
+ }
prepareExchangeMultiGPU(para, level, -1);
exchangeMultiGPU(para, comm, cudaManager, level, -1);
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
index ed1db24c5..328954fb8 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
@@ -336,7 +336,7 @@ void GridGenerator::initalValuesDomainDecompostion()
builder->getSendIndices(para->getParH(level)->sendProcessNeighborX[j].index, direction, level);
builder->getReceiveIndices(para->getParH(level)->recvProcessNeighborX[j].index, direction,
level);
- if (level != builder->getNumberOfGridLevels() - 1)
+ if (level != builder->getNumberOfGridLevels() - 1 && para->useReducedCommunicationAfterFtoC)
initCommunicationArraysForCommAfterFinetoCoarseX(level, j, direction);
////////////////////////////////////////////////////////////////////////////////////////
cudaMemoryManager->cudaCopyProcessNeighborXIndex(level, j);
@@ -405,7 +405,7 @@ void GridGenerator::initalValuesDomainDecompostion()
builder->getSendIndices(para->getParH(level)->sendProcessNeighborY[j].index, direction, level);
builder->getReceiveIndices(para->getParH(level)->recvProcessNeighborY[j].index, direction,
level);
- if (level != builder->getNumberOfGridLevels() - 1)
+ if (level != builder->getNumberOfGridLevels() - 1 && para->useReducedCommunicationAfterFtoC)
initCommunicationArraysForCommAfterFinetoCoarseY(level, j, direction);
////////////////////////////////////////////////////////////////////////////////////////
cudaMemoryManager->cudaCopyProcessNeighborYIndex(level, j);
@@ -474,7 +474,7 @@ void GridGenerator::initalValuesDomainDecompostion()
builder->getSendIndices(para->getParH(level)->sendProcessNeighborZ[j].index, direction, level);
builder->getReceiveIndices(para->getParH(level)->recvProcessNeighborZ[j].index, direction,
level);
- if (level != builder->getNumberOfGridLevels() - 1)
+ if (level != builder->getNumberOfGridLevels() - 1 && para->useReducedCommunicationAfterFtoC)
initCommunicationArraysForCommAfterFinetoCoarseZ(level, j, direction);
////////////////////////////////////////////////////////////////////////////////////////
cudaMemoryManager->cudaCopyProcessNeighborZIndex(level, j);
diff --git a/src/gpu/VirtualFluids_GPU/Parameter/Parameter.h b/src/gpu/VirtualFluids_GPU/Parameter/Parameter.h
index 0592ba945..191e54931 100644
--- a/src/gpu/VirtualFluids_GPU/Parameter/Parameter.h
+++ b/src/gpu/VirtualFluids_GPU/Parameter/Parameter.h
@@ -866,7 +866,7 @@ private:
std::function<void(real, real, real, real &, real &, real &, real &)> initialCondition;
// cuda streams
- bool useStreams { false };
+ bool useStreams{ false };
std::unique_ptr<CudaStreamManager> cudaStreamManager;
public:
@@ -886,6 +886,8 @@ public:
bool findIndexInSendNodesXZ(int level, int index);
void findCornerNodesYZ(int level);
bool findIndexInSendNodesYZ(int level, int index);
+
+ bool useReducedCommunicationAfterFtoC{ true };
};
#endif
--
GitLab