From e19e04211d5a7496ae1714c65a7c00e4003baefc Mon Sep 17 00:00:00 2001
From: Anna Wellmann <a.wellmann@tu-bs.de>
Date: Mon, 22 Nov 2021 12:54:04 +0100
Subject: [PATCH] Add apps for weak scaling and for 8 GPUs
---
apps/gpu/LBM/SphereScaling/SphereScaling.cpp | 225 ++++++++++++++++++-
1 file changed, 218 insertions(+), 7 deletions(-)
diff --git a/apps/gpu/LBM/SphereScaling/SphereScaling.cpp b/apps/gpu/LBM/SphereScaling/SphereScaling.cpp
index 5d9ff3ac3..31c0df608 100644
--- a/apps/gpu/LBM/SphereScaling/SphereScaling.cpp
+++ b/apps/gpu/LBM/SphereScaling/SphereScaling.cpp
@@ -118,6 +118,7 @@ void multipleLevel(const std::string& configPath)
bool useStreams = true;
bool useLevels = true;
para->useReducedCommunicationAfterFtoC = true;
+ std::string scalingType = "strong"; // "strong" // "weak"
if (para->getNumprocs() == 1) {
useStreams = false;
@@ -145,7 +146,7 @@ void multipleLevel(const std::string& configPath)
*logging::out << logging::Logger::INFO_HIGH << "useGridGenerator = " << useGridGenerator << "\n";
*logging::out << logging::Logger::INFO_HIGH << "useStreams = " << useStreams << "\n";
*logging::out << logging::Logger::INFO_HIGH << "number of processes = " << para->getNumprocs() << "\n";
- *logging::out << logging::Logger::INFO_HIGH << " para->useReducedCommunicationAfterFtoC = " << para->useReducedCommunicationAfterFtoC << "\n";
+ *logging::out << logging::Logger::INFO_HIGH << "para->useReducedCommunicationAfterFtoC = " << para->useReducedCommunicationAfterFtoC << "\n";
// para->setTOut(10);
// para->setTEnd(10);
@@ -191,9 +192,9 @@ void multipleLevel(const std::string& configPath)
if (useGridGenerator) {
const real xGridMin = -38; //-37 13702.1 MB // -38: 14388.2 MB // -39: 15111 MB bei Level 1 mit Durchmesser 20
- const real xGridMax = -xGridMin;
+ real xGridMax = -xGridMin;
const real yGridMin = xGridMin;
- const real yGridMax = xGridMax;
+ real yGridMax = xGridMax;
const real zGridMin = xGridMin;
const real zGridMax = xGridMax;
const real dSphere = 10.0;
@@ -206,7 +207,14 @@ void multipleLevel(const std::string& configPath)
gridBuilder->setNumberOfLayers(10, 8);
if (comm->getNummberOfProcess() == 2) {
- const real ySplit = 0.0;
+ real ySplit;
+
+ if(scalingType == "strong"){
+ ySplit = 0.0;
+ } else if (scalingType == "weak"){
+ ySplit = yGridMax;
+ yGridMax = yGridMax + (yGridMax-yGridMin);
+ }
if (generatePart == 0) {
gridBuilder->addCoarseGrid(xGridMin, yGridMin, zGridMin, xGridMax, ySplit + overlap, zGridMax,
@@ -219,9 +227,15 @@ void multipleLevel(const std::string& configPath)
if (useLevels) {
gridBuilder->addGrid(new Sphere(0.0, 0.0, 0.0, dSphereLev1), 1);
+ if (scalingType == "weak"){
+ gridBuilder->addGrid(new Sphere(0.0, 0.0 + 2.0 * yGridMax, 0.0, dSphereLev1), 1);
+ }
}
gridBuilder->addGeometry(new Sphere(0.0, 0.0, 0.0, dSphere));
+ if (scalingType == "weak"){
+ gridBuilder->addGrid(new Sphere(0.0, 0.0 + 2.0 * yGridMax, 0.0, dSphere));
+ }
if (generatePart == 0)
gridBuilder->setSubDomainBox(
@@ -257,9 +271,18 @@ void multipleLevel(const std::string& configPath)
//////////////////////////////////////////////////////////////////////////
} else if (comm->getNummberOfProcess() == 4) {
-
- const real ySplit = 0.0;
- const real xSplit = 0.0;
+ real xSplit;
+ real ySplit;
+
+ if(scalingType == "strong"){
+ xSplit = 0.0;
+ ySplit = 0.0;
+ } else if (scalingType == "weak"){
+ xSplit = xGridMax;
+ xGridMax = xGridMax + (xGridMax-xGridMin);
+ ySplit = yGridMax;
+ yGridMax = yGridMax + (yGridMax-yGridMin);
+ };
if (generatePart == 0) {
gridBuilder->addCoarseGrid(xGridMin, yGridMin, zGridMin, xSplit + overlap, ySplit + overlap,
@@ -280,9 +303,19 @@ void multipleLevel(const std::string& configPath)
if (useLevels) {
gridBuilder->addGrid(new Sphere(0.0, 0.0, 0.0, dSphereLev1), 1);
+ if (scalingType == "weak"){
+ gridBuilder->addGrid(new Sphere(0.0, 0.0 + 2.0 * yGridMax, 0.0, dSphereLev1), 1);
+ gridBuilder->addGrid(new Sphere(0.0 + 2.0 * xGridMax, 0.0 + 2.0 * yGridMax, 0.0, dSphereLev1), 1);
+ gridBuilder->addGrid(new Sphere(0.0 + 2.0 * xGridMax, 0.0, 0.0, dSphereLev1), 1);
+ }
}
gridBuilder->addGeometry(new Sphere(0.0, 0.0, 0.0, dSphere));
+ if (scalingType == "weak"){
+ gridBuilder->addGeometry(new Sphere(0.0, 0.0 + 2.0 * yGridMax, 0.0, dSphere));
+ gridBuilder->addGeometry(new Sphere(0.0 + 2.0 * xGridMax, 0.0 + 2.0 * yGridMax, 0.0, dSphere));
+ gridBuilder->addGeometry(new Sphere(0.0 + 2.0 * xGridMax, 0.0, 0.0, dSphere));
+ }
if (generatePart == 0)
gridBuilder->setSubDomainBox(
@@ -347,6 +380,184 @@ void multipleLevel(const std::string& configPath)
gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
// gridBuilder->setVelocityBoundaryCondition(SideType::GEOMETRY, 0.0, 0.0, 0.0);
//////////////////////////////////////////////////////////////////////////
+ } else if (comm->getNummberOfProcess() == 8) {
+ const real xSplit = 0.0;
+ const real ySplit = 0.0;
+ const real zSplit = 0.0;
+
+ if (scalingType == "weak") {
+ dxGrid = (real)dxGrid / 2.0;
+ };
+
+ if (generatePart == 0) {
+ gridBuilder->addCoarseGrid(xGridMin, yGridMin, zGridMin, xSplit + overlap, ySplit + overlap,
+ zSplit + overlap, dxGrid);
+ }
+ if (generatePart == 1) {
+ gridBuilder->addCoarseGrid(xGridMin, ySplit - overlap, zGridMin, xSplit + overlap, yGridMax,
+ zSplit + overlap, dxGrid);
+ }
+ if (generatePart == 2) {
+ gridBuilder->addCoarseGrid(xSplit - overlap, yGridMin, zGridMin, xGridMax, ySplit + overlap,
+ zSplit + overlap, dxGrid);
+ }
+ if (generatePart == 3) {
+ gridBuilder->addCoarseGrid(xSplit - overlap, ySplit - overlap, zGridMin, xGridMax, yGridMax,
+ zSplit + overlap, dxGrid);
+ }
+ if (generatePart == 4) {
+ gridBuilder->addCoarseGrid(xGridMin, yGridMin, zSplit - overlap, xSplit + overlap, ySplit + overlap,
+ zGridMax, dxGrid);
+ }
+ if (generatePart == 5) {
+ gridBuilder->addCoarseGrid(xGridMin, ySplit - overlap, zSplit - overlap, xSplit + overlap, yGridMax,
+ zGridMax, dxGrid);
+ }
+ if (generatePart == 6) {
+ gridBuilder->addCoarseGrid(xSplit - overlap, yGridMin, zSplit - overlap, xGridMax, ySplit + overlap,
+ zGridMax, dxGrid);
+ }
+ if (generatePart == 7) {
+ gridBuilder->addCoarseGrid(xSplit - overlap, ySplit - overlap, zSplit - overlap, xGridMax, yGridMax,
+ zGridMax, dxGrid);
+
+ if (generatePart == 0)
+ gridBuilder->setSubDomainBox(
+ std::make_shared<BoundingBox>(xGridMin, xSplit, yGridMin, ySplit, zGridMin, zSplit));
+ if (generatePart == 1)
+ gridBuilder->setSubDomainBox(
+ std::make_shared<BoundingBox>(xGridMin, xSplit, ySplit, yGridMax, zGridMin, zSplit));
+ if (generatePart == 2)
+ gridBuilder->setSubDomainBox(
+ std::make_shared<BoundingBox>(xSplit, xGridMax, yGridMin, ySplit, zGridMin, zSplit));
+ if (generatePart == 3)
+ gridBuilder->setSubDomainBox(
+ std::make_shared<BoundingBox>(xSplit, xGridMax, ySplit, yGridMax, zGridMin, zSplit));
+ if (generatePart == 4)
+ gridBuilder->setSubDomainBox(
+ std::make_shared<BoundingBox>(xGridMin, xSplit, yGridMin, ySplit, zSplit, zGridMax));
+ if (generatePart == 5)
+ gridBuilder->setSubDomainBox(
+ std::make_shared<BoundingBox>(xGridMin, xSplit, ySplit, yGridMax, zSplit, zGridMax));
+ if (generatePart == 6)
+ gridBuilder->setSubDomainBox(
+ std::make_shared<BoundingBox>(xSplit, xGridMax, yGridMin, ySplit, zSplit, zGridMax));
+ if (generatePart == 7)
+ gridBuilder->setSubDomainBox(
+ std::make_shared<BoundingBox>(xSplit, xGridMax, ySplit, yGridMax, zSplit, zGridMax));
+
+ gridBuilder->buildGrids(LBM, true); // buildGrids() has to be called before setting the BCs!!!!
+ gridBuilder->setPeriodicBoundaryCondition(false, false, false);
+
+ if (generatePart == 0) {
+ gridBuilder->findCommunicationIndices(CommunicationDirections::PY, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::PY, 1);
+ gridBuilder->findCommunicationIndices(CommunicationDirections::PX, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::PX, 2);
+ gridBuilder->findCommunicationIndices(CommunicationDirections::PZ, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::PZ, 4);
+ }
+ if (generatePart == 1) {
+ gridBuilder->findCommunicationIndices(CommunicationDirections::MY, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::MY, 0);
+ gridBuilder->findCommunicationIndices(CommunicationDirections::PX, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::PX, 3);
+ gridBuilder->findCommunicationIndices(CommunicationDirections::PZ, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::PZ, 5);
+ }
+ if (generatePart == 2) {
+ gridBuilder->findCommunicationIndices(CommunicationDirections::PY, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::PY, 3);
+ gridBuilder->findCommunicationIndices(CommunicationDirections::MX, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::MX, 0);
+ gridBuilder->findCommunicationIndices(CommunicationDirections::PZ, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::PZ, 6);
+ }
+ if (generatePart == 3) {
+ gridBuilder->findCommunicationIndices(CommunicationDirections::MY, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::MY, 2);
+ gridBuilder->findCommunicationIndices(CommunicationDirections::MX, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::MX, 1);
+ gridBuilder->findCommunicationIndices(CommunicationDirections::PZ, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::PZ, 7);
+ }
+ if (generatePart == 4) {
+ gridBuilder->findCommunicationIndices(CommunicationDirections::PY, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::PY, 5);
+ gridBuilder->findCommunicationIndices(CommunicationDirections::PX, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::PX, 6);
+ gridBuilder->findCommunicationIndices(CommunicationDirections::MZ, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::MZ, 0);
+ }
+ if (generatePart == 5) {
+ gridBuilder->findCommunicationIndices(CommunicationDirections::MY, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::MY, 4);
+ gridBuilder->findCommunicationIndices(CommunicationDirections::PX, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::PX, 7);
+ gridBuilder->findCommunicationIndices(CommunicationDirections::MZ, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::MZ, 1);
+ }
+ if (generatePart == 6) {
+ gridBuilder->findCommunicationIndices(CommunicationDirections::PY, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::PY, 7);
+ gridBuilder->findCommunicationIndices(CommunicationDirections::MX, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::MX, 4);
+ gridBuilder->findCommunicationIndices(CommunicationDirections::MZ, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::MZ, 2);
+ }
+ if (generatePart == 7) {
+ gridBuilder->findCommunicationIndices(CommunicationDirections::MY, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::MY, 6);
+ gridBuilder->findCommunicationIndices(CommunicationDirections::MX, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::MX, 5);
+ gridBuilder->findCommunicationIndices(CommunicationDirections::MZ, LBM);
+ gridBuilder->setCommunicationProcess(CommunicationDirections::MZ, 3);
+ }
+
+ //////////////////////////////////////////////////////////////////////////
+ if (generatePart == 0) {
+ gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
+ gridBuilder->setVelocityBoundaryCondition(SideType::MY, vxLB, 0.0, 0.0);
+ gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
+ }
+ if (generatePart == 1) {
+ gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
+ gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
+ gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
+ }
+ if (generatePart == 2) {
+ gridBuilder->setVelocityBoundaryCondition(SideType::MY, vxLB, 0.0, 0.0);
+ gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0);
+ gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
+ }
+ if (generatePart == 3) {
+ gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
+ gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0);
+ gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
+ }
+ if (generatePart == 4) {
+ gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
+ gridBuilder->setVelocityBoundaryCondition(SideType::MY, vxLB, 0.0, 0.0);
+ gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
+ }
+ if (generatePart == 5) {
+ gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
+ gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
+ gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
+ }
+ if (generatePart == 6) {
+ gridBuilder->setVelocityBoundaryCondition(SideType::MY, vxLB, 0.0, 0.0);
+ gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0);
+ gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
+ }
+ if (generatePart == 7) {
+ gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
+ gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0);
+ gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
+ }
+ // gridBuilder->setVelocityBoundaryCondition(SideType::GEOMETRY, 0.0, 0.0, 0.0);
+ //////////////////////////////////////////////////////////////////////////
+ }
}
if (para->getKernelNeedsFluidNodeIndicesToRun())
gridBuilder->findFluidNodes(useStreams);
--
GitLab