From 9542c07b69c9c6f05218851df45dc3d555942298 Mon Sep 17 00:00:00 2001
From: Anna Wellmann <a.wellmann@tu-braunschweig.de>
Date: Mon, 11 Oct 2021 20:38:33 +0200
Subject: [PATCH] Hide Communication after Fine to Coarse with f-to-c-kernel
---
apps/gpu/LBM/MusselOyster/MusselOyster.cpp | 4 ++--
.../Calculation/UpdateGrid27.cpp | 17 +++++++++++------
2 files changed, 13 insertions(+), 8 deletions(-)
diff --git a/apps/gpu/LBM/MusselOyster/MusselOyster.cpp b/apps/gpu/LBM/MusselOyster/MusselOyster.cpp
index 0cd44375f..4c18a7c84 100644
--- a/apps/gpu/LBM/MusselOyster/MusselOyster.cpp
+++ b/apps/gpu/LBM/MusselOyster/MusselOyster.cpp
@@ -140,8 +140,8 @@ void multipleLevel(const std::string& configPath)
*logging::out << logging::Logger::INFO_HIGH << "useStreams = " << useStreams << "\n";
- para->setTOut(1000);
- para->setTEnd(10000);
+ para->setTOut(10);
+ para->setTEnd(10);
para->setCalcDragLift(false);
para->setUseWale(false);
diff --git a/src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.cpp b/src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.cpp
index a8529096c..eff5f8039 100644
--- a/src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.cpp
+++ b/src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.cpp
@@ -77,16 +77,21 @@ void updateGrid27(Parameter *para, vf::gpu::Communicator *comm, CudaMemoryManage
fineToCoarseWithStream(para, level,
para->getParD(level)->intFCBorder.ICellFCC,
para->getParD(level)->intFCBorder.ICellFCF,
- para->getParD(level)->intFCBorder.kFC, -1);
+ para->getParD(level)->intFCBorder.kFC, borderStreamIndex);
+
+ // prepare exchange and trigger bulk kernel when finished
+ prepareExchangeMultiGPUAfterFtoC(para, level, borderStreamIndex);
+ if (para->getUseStreams())
+ para->getStreamManager()->triggerStartBulkKernel(borderStreamIndex);
+
+ // launch bulk kernel
+ para->getStreamManager()->waitOnStartBulkKernelEvent(bulkStreamIndex);
fineToCoarseWithStream(para, level,
para->getParD(level)->intFCBulk.ICellFCC,
para->getParD(level)->intFCBulk.ICellFCF,
- para->getParD(level)->intFCBulk.kFC, -1);
- //fineToCoarse(para, level);
-
- prepareExchangeMultiGPUAfterFtoC(para, level, -1);
- exchangeMultiGPUAfterFtoC(para, comm, cudaManager, level, -1);
+ para->getParD(level)->intFCBulk.kFC, bulkStreamIndex);
+ exchangeMultiGPUAfterFtoC(para, comm, cudaManager, level, borderStreamIndex);
coarseToFine(para, level);
} else {
fineToCoarse(para, level);
--
GitLab