From eab6d8c29e38a2c0ce1c8745798452faf0baef0e Mon Sep 17 00:00:00 2001
From: Anna Wellmann <a.wellmann@tu-bs.de>
Date: Fri, 3 Dec 2021 17:55:41 +0100
Subject: [PATCH] Configs for Sphere on 8 GPU and use IntelMPi instead of MPICH
---
apps/gpu/LBM/SphereScaling/SphereScaling.cpp | 9 +---
.../LBM/SphereScaling/configPhoenix2GPU.txt | 4 +-
.../configPhoenix4GPU_1LevStrongStream.txt | 42 +++++++++++++++++
.../configPhoenix8GPU_1LevStrongStream.txt | 4 +-
.../configPhoenix8GPU_1LevWeakOS.txt | 42 +++++++++++++++++
.../configPhoenix8GPU_1LevWeakStream.txt | 42 +++++++++++++++++
.../gpu/LBM/SphereScaling1/SphereScaling1.cpp | 47 ++++++++-----------
.../configPhoenix8GPU_2LevStrongOS.txt | 42 +++++++++++++++++
.../configPhoenix8GPU_2LevStrongStream.txt | 42 +++++++++++++++++
.../configPhoenix8GPU_2LevWeakStream.txt | 6 +--
10 files changed, 238 insertions(+), 42 deletions(-)
create mode 100644 apps/gpu/LBM/SphereScaling/configPhoenix4GPU_1LevStrongStream.txt
create mode 100644 apps/gpu/LBM/SphereScaling/configPhoenix8GPU_1LevWeakOS.txt
create mode 100644 apps/gpu/LBM/SphereScaling/configPhoenix8GPU_1LevWeakStream.txt
create mode 100644 apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevStrongOS.txt
create mode 100644 apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevStrongStream.txt
diff --git a/apps/gpu/LBM/SphereScaling/SphereScaling.cpp b/apps/gpu/LBM/SphereScaling/SphereScaling.cpp
index 2217511b3..a7571e1a8 100644
--- a/apps/gpu/LBM/SphereScaling/SphereScaling.cpp
+++ b/apps/gpu/LBM/SphereScaling/SphereScaling.cpp
@@ -120,7 +120,7 @@ void multipleLevel(const std::string& configPath)
bool useGridGenerator = true;
bool useLevels = false;
- std::string scalingType = "strong"; // "strong" // "weak"
+ std::string scalingType = "weak"; // "strong" // "weak"
// bool useStreams = true;
// para->useReducedCommunicationAfterFtoC = true;
bool useStreams = para->getUseStreams();
@@ -252,13 +252,6 @@ void multipleLevel(const std::string& configPath)
if (useLevels) {
gridBuilder->addGeometry(new Sphere(0.0, 0.0, sideLengthCube, dSphere));
}else{
- // Sphere* sphere1 = new Sphere(0.5 * sideLengthCube, 0.5 * sideLengthCube, 0.5 * sideLengthCube, dSphere);
- // Sphere* sphere2 = new Sphere(0.5 * sideLengthCube, 0.5 * sideLengthCube, 1.5 * sideLengthCube, dSphere);
- // auto conglo = Conglomerate::makeShared();
- // conglo->add(sphere1);
- // conglo->add(sphere2);
- // gridBuilder->addGeometry(conglo.get());
-
TriangularMesh *sphereSTL = TriangularMesh::make(stlPath + "Spheres_2GPU.stl");
gridBuilder->addGeometry(sphereSTL);
}
diff --git a/apps/gpu/LBM/SphereScaling/configPhoenix2GPU.txt b/apps/gpu/LBM/SphereScaling/configPhoenix2GPU.txt
index b33d3d621..9e88253b0 100644
--- a/apps/gpu/LBM/SphereScaling/configPhoenix2GPU.txt
+++ b/apps/gpu/LBM/SphereScaling/configPhoenix2GPU.txt
@@ -36,6 +36,6 @@ GridPath=/work/y0078217/Grids/GridSphereScaling/SphereScaling2GPU/
##################################################
#simulation parameter
##################################################
-TimeEnd=100000
-TimeOut=100000
+TimeEnd=1000
+TimeOut=1000
#TimeStartOut=0
\ No newline at end of file
diff --git a/apps/gpu/LBM/SphereScaling/configPhoenix4GPU_1LevStrongStream.txt b/apps/gpu/LBM/SphereScaling/configPhoenix4GPU_1LevStrongStream.txt
new file mode 100644
index 000000000..bee817168
--- /dev/null
+++ b/apps/gpu/LBM/SphereScaling/configPhoenix4GPU_1LevStrongStream.txt
@@ -0,0 +1,42 @@
+##################################################
+#GPU Mapping
+##################################################
+Devices="0 1 2 3"
+NumberOfDevices=4
+
+##################################################
+#informations for Writing
+##################################################
+Path=/work/y0078217/Results/SphereScalingResults/4GPU/1LevStrongStream/
+#Path="F:/Work/Computations/out/SphereScaling/"
+#Prefix="SphereScaling"
+#WriteGrid=true
+##################################################
+#informations for reading
+##################################################
+GridPath=/work/y0078217/Grids/GridSphereScaling/SphereScaling4GPU/
+#GridPath="C:"
+
+##################################################
+#number of grid levels
+##################################################
+#NOGL=1
+
+##################################################
+#LBM Version
+##################################################
+#D3Qxx=27
+#MainKernelName=CumulantK17CompChim
+
+##################################################
+#simulation parameter
+##################################################
+TimeEnd=100
+TimeOut=100
+#TimeStartOut=0
+
+##################################################
+# CUDA Streams and optimized communication (only used for multiple GPUs)
+##################################################
+useStreams = true
+useReducedCommunicationInInterpolation = true
\ No newline at end of file
diff --git a/apps/gpu/LBM/SphereScaling/configPhoenix8GPU_1LevStrongStream.txt b/apps/gpu/LBM/SphereScaling/configPhoenix8GPU_1LevStrongStream.txt
index 2715162f6..85dc351c9 100644
--- a/apps/gpu/LBM/SphereScaling/configPhoenix8GPU_1LevStrongStream.txt
+++ b/apps/gpu/LBM/SphereScaling/configPhoenix8GPU_1LevStrongStream.txt
@@ -31,8 +31,8 @@ GridPath=/work/y0078217/Grids/GridSphereScaling/SphereScaling8GPU/
##################################################
#simulation parameter
##################################################
-TimeEnd=100000
-TimeOut=100000
+TimeEnd=100
+TimeOut=100
#TimeStartOut=0
##################################################
diff --git a/apps/gpu/LBM/SphereScaling/configPhoenix8GPU_1LevWeakOS.txt b/apps/gpu/LBM/SphereScaling/configPhoenix8GPU_1LevWeakOS.txt
new file mode 100644
index 000000000..ae848a288
--- /dev/null
+++ b/apps/gpu/LBM/SphereScaling/configPhoenix8GPU_1LevWeakOS.txt
@@ -0,0 +1,42 @@
+##################################################
+#GPU Mapping
+##################################################
+Devices="0 1 2 3"
+NumberOfDevices=4
+
+##################################################
+#informations for Writing
+##################################################
+Path=/work/y0078217/Results/SphereScalingResults/8GPU/1LevWeakOS/
+#Path="F:/Work/Computations/out/SphereScaling/"
+#Prefix="SphereScaling"
+#WriteGrid=true
+##################################################
+#informations for reading
+##################################################
+GridPath=/work/y0078217/Grids/GridSphereScaling/SphereScaling8GPU/
+#GridPath="C:"
+
+##################################################
+#number of grid levels
+##################################################
+#NOGL=1
+
+##################################################
+#LBM Version
+##################################################
+#D3Qxx=27
+#MainKernelName=CumulantK17CompChim
+
+##################################################
+#simulation parameter
+##################################################
+TimeEnd=100000
+TimeOut=100000
+#TimeStartOut=0
+
+##################################################
+# CUDA Streams and optimized communication (only used for multiple GPUs)
+##################################################
+useStreams = false
+useReducedCommunicationInInterpolation = false
\ No newline at end of file
diff --git a/apps/gpu/LBM/SphereScaling/configPhoenix8GPU_1LevWeakStream.txt b/apps/gpu/LBM/SphereScaling/configPhoenix8GPU_1LevWeakStream.txt
new file mode 100644
index 000000000..eeea01295
--- /dev/null
+++ b/apps/gpu/LBM/SphereScaling/configPhoenix8GPU_1LevWeakStream.txt
@@ -0,0 +1,42 @@
+##################################################
+#GPU Mapping
+##################################################
+Devices="0 1 2 3"
+NumberOfDevices=4
+
+##################################################
+#informations for Writing
+##################################################
+Path=/work/y0078217/Results/SphereScalingResults/8GPU/1LevWeakStream/
+#Path="F:/Work/Computations/out/SphereScaling/"
+#Prefix="SphereScaling"
+#WriteGrid=true
+##################################################
+#informations for reading
+##################################################
+GridPath=/work/y0078217/Grids/GridSphereScaling/SphereScaling8GPU/
+#GridPath="C:"
+
+##################################################
+#number of grid levels
+##################################################
+#NOGL=1
+
+##################################################
+#LBM Version
+##################################################
+#D3Qxx=27
+#MainKernelName=CumulantK17CompChim
+
+##################################################
+#simulation parameter
+##################################################
+TimeEnd=100000
+TimeOut=100000
+#TimeStartOut=0
+
+##################################################
+# CUDA Streams and optimized communication (only used for multiple GPUs)
+##################################################
+useStreams = true
+useReducedCommunicationInInterpolation = true
\ No newline at end of file
diff --git a/apps/gpu/LBM/SphereScaling1/SphereScaling1.cpp b/apps/gpu/LBM/SphereScaling1/SphereScaling1.cpp
index e97e5bb9b..1edaceff8 100644
--- a/apps/gpu/LBM/SphereScaling1/SphereScaling1.cpp
+++ b/apps/gpu/LBM/SphereScaling1/SphereScaling1.cpp
@@ -120,7 +120,7 @@ void multipleLevel(const std::string& configPath)
bool useGridGenerator = true;
bool useLevels = true;
- std::string scalingType = "weak"; // "strong" // "weak"
+ std::string scalingType = "strong"; // "strong" // "weak"
// bool useStreams = true;
// para->useReducedCommunicationAfterFtoC = true;
bool useStreams = para->getUseStreams();
@@ -223,19 +223,19 @@ void multipleLevel(const std::string& configPath)
gridBuilder->setNumberOfLayers(10, 8);
if (comm->getNummberOfProcess() == 2) {
- real zSplit = 0.5 * sideLengthCube;
+ real xSplit = 0.5 * sideLengthCube;
if (scalingType == "weak"){
- zSplit = zGridMax;
- zGridMax = zGridMax + sideLengthCube;
+ xSplit = xGridMax;
+ xGridMax = xGridMax + sideLengthCube;
}
if (generatePart == 0) {
- gridBuilder->addCoarseGrid(xGridMin, yGridMin, zGridMin, xGridMax, yGridMax, zSplit + overlap,
+ gridBuilder->addCoarseGrid(xGridMin, yGridMin, zGridMin, xSplit + overlap, yGridMax, zGridMax,
dxGrid);
}
if (generatePart == 1) {
- gridBuilder->addCoarseGrid(xGridMin, yGridMin, zSplit - overlap, xGridMax, yGridMax, zGridMax,
+ gridBuilder->addCoarseGrid(xSplit-overlap, yGridMin, zGridMin, xGridMax, yGridMax, zGridMax,
dxGrid);
}
@@ -243,22 +243,15 @@ void multipleLevel(const std::string& configPath)
if (scalingType == "strong"){
gridBuilder->addGrid(new Sphere(0.5 * sideLengthCube, 0.5 * sideLengthCube, 0.5 * sideLengthCube, dSphereLev1), 1);
} else if (scalingType == "weak"){
- gridBuilder->addGrid(new Cuboid( -0.5*dCubeLev1, -0.5*dCubeLev1, sideLengthCube-0.5*dCubeLev1,
- 0.5*dCubeLev1, 0.5*dCubeLev1, sideLengthCube+0.5*dCubeLev1),1);
+ gridBuilder->addGrid(new Cuboid( sideLengthCube-0.5*dCubeLev1, -0.5*dCubeLev1, -0.5*dCubeLev1,
+ sideLengthCube+0.5*dCubeLev1, 0.5*dCubeLev1, 0.5*dCubeLev1),1);
}
}
if (scalingType == "weak"){
if (useLevels) {
- gridBuilder->addGeometry(new Sphere(0.0, 0.0, sideLengthCube, dSphere));
+ gridBuilder->addGeometry(new Sphere(sideLengthCube, 0.0, 0.0, dSphere));
}else{
- // Sphere* sphere1 = new Sphere(0.5 * sideLengthCube, 0.5 * sideLengthCube, 0.5 * sideLengthCube, dSphere);
- // Sphere* sphere2 = new Sphere(0.5 * sideLengthCube, 0.5 * sideLengthCube, 1.5 * sideLengthCube, dSphere);
- // auto conglo = Conglomerate::makeShared();
- // conglo->add(sphere1);
- // conglo->add(sphere2);
- // gridBuilder->addGeometry(conglo.get());
-
TriangularMesh *sphereSTL = TriangularMesh::make(stlPath + "Spheres_2GPU.stl");
gridBuilder->addGeometry(sphereSTL);
}
@@ -268,33 +261,33 @@ void multipleLevel(const std::string& configPath)
if (generatePart == 0)
gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xGridMin, xGridMax, yGridMin, yGridMax, zGridMin, zSplit));
+ std::make_shared<BoundingBox>(xGridMin, xSplit, yGridMin, yGridMax, zGridMin, zGridMax));
if (generatePart == 1)
gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xGridMin, xGridMax, yGridMin, yGridMax, zSplit, zGridMax));
+ std::make_shared<BoundingBox>(xSplit, xGridMax, yGridMin, yGridMax, zGridMin, zGridMax));
gridBuilder->buildGrids(LBM, true); // buildGrids() has to be called before setting the BCs!!!!
if (generatePart == 0) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::PZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PZ, 1);
+ gridBuilder->findCommunicationIndices(CommunicationDirections::PX, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::PX, 1);
}
if (generatePart == 1) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::MZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MZ, 0);
+ gridBuilder->findCommunicationIndices(CommunicationDirections::MX, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::MX, 0);
}
gridBuilder->setPeriodicBoundaryCondition(false, false, false);
//////////////////////////////////////////////////////////////////////////
- gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
gridBuilder->setVelocityBoundaryCondition(SideType::MY, vxLB, 0.0, 0.0);
gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
+ gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
+ gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
if (generatePart == 0)
- gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
- if (generatePart == 1)
- gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
- gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0); // set pressure BC after velocity BCs
+ gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
+ if (generatePart == 1)
+ gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0); // set pressure BC after velocity BCs
// gridBuilder->setVelocityBoundaryCondition(SideType::GEOMETRY, 0.0, 0.0, 0.0);
//////////////////////////////////////////////////////////////////////////
diff --git a/apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevStrongOS.txt b/apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevStrongOS.txt
new file mode 100644
index 000000000..b650042a8
--- /dev/null
+++ b/apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevStrongOS.txt
@@ -0,0 +1,42 @@
+##################################################
+#GPU Mapping
+##################################################
+Devices="0 1 2 3"
+NumberOfDevices=4
+
+##################################################
+#informations for Writing
+##################################################
+Path=/work/y0078217/Results/SphereScalingResults/8GPU/2LevStrongOS/
+#Path="F:/Work/Computations/out/SphereScaling/"
+#Prefix="SphereScaling"
+#WriteGrid=true
+##################################################
+#informations for reading
+##################################################
+GridPath=/work/y0078217/Grids/GridSphereScaling/SphereScaling8GPU/
+#GridPath="C:"
+
+##################################################
+#number of grid levels
+##################################################
+#NOGL=1
+
+##################################################
+#LBM Version
+##################################################
+#D3Qxx=27
+#MainKernelName=CumulantK17CompChim
+
+##################################################
+#simulation parameter
+##################################################
+TimeEnd=100000
+TimeOut=100000
+#TimeStartOut=0
+
+##################################################
+# CUDA Streams and optimized communication (only used for multiple GPUs)
+##################################################
+useStreams = false
+useReducedCommunicationInInterpolation = false
\ No newline at end of file
diff --git a/apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevStrongStream.txt b/apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevStrongStream.txt
new file mode 100644
index 000000000..7f988e325
--- /dev/null
+++ b/apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevStrongStream.txt
@@ -0,0 +1,42 @@
+##################################################
+#GPU Mapping
+##################################################
+Devices="0 1 2 3"
+NumberOfDevices=4
+
+##################################################
+#informations for Writing
+##################################################
+Path=/work/y0078217/Results/SphereScalingResults/8GPU/2LevStrongStream/
+#Path="F:/Work/Computations/out/SphereScaling/"
+#Prefix="SphereScaling"
+#WriteGrid=true
+##################################################
+#informations for reading
+##################################################
+GridPath=/work/y0078217/Grids/GridSphereScaling/SphereScaling8GPU/
+#GridPath="C:"
+
+##################################################
+#number of grid levels
+##################################################
+#NOGL=1
+
+##################################################
+#LBM Version
+##################################################
+#D3Qxx=27
+#MainKernelName=CumulantK17CompChim
+
+##################################################
+#simulation parameter
+##################################################
+TimeEnd=100000
+TimeOut=100000
+#TimeStartOut=0
+
+##################################################
+# CUDA Streams and optimized communication (only used for multiple GPUs)
+##################################################
+useStreams = true
+useReducedCommunicationInInterpolation = true
\ No newline at end of file
diff --git a/apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevWeakStream.txt b/apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevWeakStream.txt
index 714298741..ed85622c9 100644
--- a/apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevWeakStream.txt
+++ b/apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevWeakStream.txt
@@ -1,13 +1,13 @@
##################################################
#GPU Mapping
##################################################
-Devices="0 1 2 3 4 5 6 7"
-NumberOfDevices=8
+Devices="0 1 2 3"
+NumberOfDevices=4
##################################################
#informations for Writing
##################################################
-Path=/work/y0078217/Results/SphereScalingResults/8GPU/1LevWeakStream/
+Path=/work/y0078217/Results/SphereScalingResults/8GPU/2LevWeakStream/
#Path="F:/Work/Computations/out/SphereScaling/"
#Prefix="SphereScaling"
#WriteGrid=true
--
GitLab