From 435100db5a85ad80f0ffe0e30097b334c6f7e629 Mon Sep 17 00:00:00 2001
From: Anna Wellmann <a.wellmann@tu-bs.de>
Date: Wed, 17 Nov 2021 13:46:44 +0100
Subject: [PATCH] Fix mussel setup on 4 GPUs
---
apps/gpu/LBM/MusselOyster/MusselOyster.cpp | 8 ++++----
apps/gpu/LBM/MusselOyster/configMusselOyster.txt | 5 +++++
2 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/apps/gpu/LBM/MusselOyster/MusselOyster.cpp b/apps/gpu/LBM/MusselOyster/MusselOyster.cpp
index ca8fc87f3..5fe1c727e 100644
--- a/apps/gpu/LBM/MusselOyster/MusselOyster.cpp
+++ b/apps/gpu/LBM/MusselOyster/MusselOyster.cpp
@@ -119,7 +119,7 @@ void multipleLevel(const std::string& configPath)
bool useGridGenerator = true;
bool useStreams = true;
bool useLevels = true;
- para->useReducedCommunicationAfterFtoC = false;
+ para->useReducedCommunicationAfterFtoC = true;
if (para->getNumprocs() == 1) {
useStreams = false;
@@ -223,7 +223,7 @@ void multipleLevel(const std::string& configPath)
const uint generatePart = vf::gpu::Communicator::getInstanz()->getPID();
real overlap = (real)8.0 * dxGrid;
- gridBuilder->setNumberOfLayers(6, 8);
+ gridBuilder->setNumberOfLayers(10, 8);
if (comm->getNummberOfProcess() == 2) {
const real ySplit = bbyp - 10.0;
@@ -276,7 +276,7 @@ void multipleLevel(const std::string& configPath)
} else if (comm->getNummberOfProcess() == 4) {
const real ySplit = bbyp - 10.0;
- const real xSplit = 170.0;
+ const real xSplit = 90.0;
if (generatePart == 0) {
gridBuilder->addCoarseGrid(xGridMin, yGridMin, zGridMin, xSplit + overlap, ySplit + overlap,
@@ -378,7 +378,7 @@ void multipleLevel(const std::string& configPath)
gridBuilder->addCoarseGrid(xGridMin, yGridMin, zGridMin, xGridMax, yGridMax, zGridMax, dxGrid);
if (useLevels) {
- gridBuilder->setNumberOfLayers(6, 8);
+ gridBuilder->setNumberOfLayers(10, 8);
gridBuilder->addGrid(bivalveRef_1_STL, 1);
}
diff --git a/apps/gpu/LBM/MusselOyster/configMusselOyster.txt b/apps/gpu/LBM/MusselOyster/configMusselOyster.txt
index d5470b018..3c6ac8b5f 100644
--- a/apps/gpu/LBM/MusselOyster/configMusselOyster.txt
+++ b/apps/gpu/LBM/MusselOyster/configMusselOyster.txt
@@ -3,6 +3,11 @@
# Phoenix
# mpirun -np 2 "./VirtualFluids_dev/build/bin/MusselOyster" "./VirtualFluids_dev/apps/gpu/LBM/MusselOyster/configMusselOyster.txt"
+# Phoenix mpich
+# mpirun -np 4 nvprof -f -o MusselOyster.%q{PMI_RANK}.nvprof "./VirtualFluids_dev/build/bin/MusselOyster" "./VirtualFluids_dev/apps/gpu/LBM/SphereScaling/configPhoenix4GPU.txt"
+# Phoenix openmpi
+# mpirun -np 4 nvprof -f -o MusselOyster.%q{OMPI_COMM_WORLD_RANK}.nvprof "./VirtualFluids_dev/build/bin/MusselOyster" "./VirtualFluids_dev/apps/gpu/LBM/SphereScaling/configPhoenix4GPU.txt"
+
##################################################
#GPU Mapping
##################################################
--
GitLab