Skip to content
Snippets Groups Projects
Commit 7b76d3c4 authored by Anna Wellmann's avatar Anna Wellmann
Browse files

Setup for Sphere Scaling on 8 GPUs

parent 27f0a30d
No related branches found
No related tags found
1 merge request!104Add Communication Hiding to GPU version
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 1002 additions and 21 deletions
apps/gpu/LBM/MusselOyster/configPhoenix1GPU.txt
+ 3
− 3
View file @ 7b76d3c4
......@@ -7,7 +7,7 @@ NumberOfDevices=1
##################################################
#informations for Writing
##################################################
Path=/work/y0078217/Results/MusselOysterResults/1GPUMussel2/
Path=/work/y0078217/Results/MusselOysterResults/1GPUMussel1/
#Path="F:/Work/Computations/out/MusselOyster/"
#Prefix="MusselOyster"
#WriteGrid=true
......@@ -31,6 +31,6 @@ GridPath=/work/y0078217/Grids/GridMusselOyster/Mussel1GPU/
##################################################
#simulation parameter
##################################################
TimeEnd=500000
TimeOut=100000
TimeEnd=400000
TimeOut=200000
#TimeStartOut=0
\ No newline at end of file
apps/gpu/LBM/SphereScaling/SphereScaling.cpp
+ 15
− 13
View file @ 7b76d3c4
......@@ -119,10 +119,11 @@ void multipleLevel(const std::string& configPath)
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
bool useGridGenerator = true;
bool useStreams = false;
bool useLevels = true;
para->useReducedCommunicationAfterFtoC = false;
std::string scalingType = "weak"; // "strong" // "weak"
bool useLevels = false;
std::string scalingType = "strong"; // "strong" // "weak"
// bool useStreams = true;
// para->useReducedCommunicationAfterFtoC = true;
bool useStreams = para->getUseStreams();
if (para->getNumprocs() == 1) {
useStreams = false;
......@@ -197,11 +198,12 @@ void multipleLevel(const std::string& configPath)
if (useGridGenerator) {
real sideLengthCube;
if (useLevels)
if (useLevels){
if (scalingType == "strong")
sideLengthCube = 70.0; // Phoenix: strong scaling with two levels = 76.0
sideLengthCube = 76.0; // Phoenix: strong scaling with two levels = 76.0
else if (scalingType == "weak")
sideLengthCube = 70.0; // Phoenix: weak scaling with two levels = 70.0
}
else
sideLengthCube = 86.0; // Phoenix: 86.0
real xGridMin = 0.0;
......@@ -286,13 +288,13 @@ void multipleLevel(const std::string& configPath)
gridBuilder->setPeriodicBoundaryCondition(false, false, false);
//////////////////////////////////////////////////////////////////////////
gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0);
gridBuilder->setVelocityBoundaryCondition(SideType::MY, vxLB, 0.0, 0.0);
gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
if (generatePart == 0)
gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
if (generatePart == 1)
gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0); // set pressure BC after velocity BCs
// gridBuilder->setVelocityBoundaryCondition(SideType::GEOMETRY, 0.0, 0.0, 0.0);
//////////////////////////////////////////////////////////////////////////
......@@ -404,7 +406,7 @@ void multipleLevel(const std::string& configPath)
gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
}
gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0);
gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0); // set pressure BC after velocity BCs
// gridBuilder->setVelocityBoundaryCondition(SideType::GEOMETRY, 0.0, 0.0, 0.0);
//////////////////////////////////////////////////////////////////////////
} else if (comm->getNummberOfProcess() == 8) {
......@@ -585,13 +587,13 @@ void multipleLevel(const std::string& configPath)
}
if (generatePart == 2) {
gridBuilder->setVelocityBoundaryCondition(SideType::MY, vxLB, 0.0, 0.0);
gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0);
gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0);
}
if (generatePart == 3) {
gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0);
gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0);
}
if (generatePart == 4) {
gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
......@@ -605,13 +607,13 @@ void multipleLevel(const std::string& configPath)
}
if (generatePart == 6) {
gridBuilder->setVelocityBoundaryCondition(SideType::MY, vxLB, 0.0, 0.0);
gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0);
gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0); // set pressure BC after velocity BCs
}
if (generatePart == 7) {
gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0);
gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0); // set pressure BC after velocity BCs
}
// gridBuilder->setVelocityBoundaryCondition(SideType::GEOMETRY, 0.0, 0.0, 0.0);
//////////////////////////////////////////////////////////////////////////
......@@ -656,10 +658,10 @@ void multipleLevel(const std::string& configPath)
//////////////////////////////////////////////////////////////////////////
gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
gridBuilder->setVelocityBoundaryCondition(SideType::MY, vxLB, 0.0, 0.0);
gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0);
gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0); // set pressure BC after velocity BCs
// gridBuilder->setVelocityBoundaryCondition(SideType::GEOMETRY, 0.0, 0.0, 0.0);
//////////////////////////////////////////////////////////////////////////
......
apps/gpu/LBM/SphereScaling/configPhoenix8GPU.txt
+ 3
− 3
View file @ 7b76d3c4
......@@ -14,7 +14,7 @@ Path=/work/y0078217/Results/SphereScalingResults/8GPU/
##################################################
#informations for reading
##################################################
GridPath=/work/y0078217/Grids/GridSphereScaling/SphereScaling4GPU/
GridPath=/work/y0078217/Grids/GridSphereScaling/SphereScaling8GPU/
#GridPath="C:"
##################################################
......@@ -38,5 +38,5 @@ TimeOut=100000
##################################################
# CUDA Streams and optimized communication (only used for multiple GPUs)
##################################################
# useStreams = true
# useReducedCommunicationInInterpolation = true
\ No newline at end of file
useStreams = true
useReducedCommunicationInInterpolation = true
\ No newline at end of file
apps/gpu/LBM/SphereScaling/configPhoenix8GPU_1LevStrongOS.txt 0 → 100644
+ 42
− 0
View file @ 7b76d3c4
##################################################
#GPU Mapping
##################################################
Devices="0 1 2 3"
NumberOfDevices=4
##################################################
#informations for Writing
##################################################
Path=/work/y0078217/Results/SphereScalingResults/8GPU/1LevStrongOS/
#Path="F:/Work/Computations/out/SphereScaling/"
#Prefix="SphereScaling"
#WriteGrid=true
##################################################
#informations for reading
##################################################
GridPath=/work/y0078217/Grids/GridSphereScaling/SphereScaling8GPU/
#GridPath="C:"
##################################################
#number of grid levels
##################################################
#NOGL=1
##################################################
#LBM Version
##################################################
#D3Qxx=27
#MainKernelName=CumulantK17CompChim
##################################################
#simulation parameter
##################################################
TimeEnd=100000
TimeOut=100000
#TimeStartOut=0
##################################################
# CUDA Streams and optimized communication (only used for multiple GPUs)
##################################################
useStreams = false
useReducedCommunicationInInterpolation = false
\ No newline at end of file
apps/gpu/LBM/SphereScaling/configPhoenix8GPU_1LevStrongStream.txt 0 → 100644
+ 42
− 0
View file @ 7b76d3c4
##################################################
#GPU Mapping
##################################################
Devices="0 1 2 3"
NumberOfDevices=4
##################################################
#informations for Writing
##################################################
Path=/work/y0078217/Results/SphereScalingResults/8GPU/1LevStrongStream/
#Path="F:/Work/Computations/out/SphereScaling/"
#Prefix="SphereScaling"
#WriteGrid=true
##################################################
#informations for reading
##################################################
GridPath=/work/y0078217/Grids/GridSphereScaling/SphereScaling8GPU/
#GridPath="C:"
##################################################
#number of grid levels
##################################################
#NOGL=1
##################################################
#LBM Version
##################################################
#D3Qxx=27
#MainKernelName=CumulantK17CompChim
##################################################
#simulation parameter
##################################################
TimeEnd=100000
TimeOut=100000
#TimeStartOut=0
##################################################
# CUDA Streams and optimized communication (only used for multiple GPUs)
##################################################
useStreams = true
useReducedCommunicationInInterpolation = true
\ No newline at end of file
apps/gpu/LBM/SphereScaling1/CMakeLists.txt 0 → 100644
+ 10
− 0
View file @ 7b76d3c4
PROJECT(SphereScaling1 LANGUAGES CUDA CXX)
vf_add_library(BUILDTYPE binary PRIVATE_LINK basics VirtualFluids_GPU GridGenerator MPI::MPI_CXX FILES SphereScaling1.cpp)
set_source_files_properties(SphereScaling1.cpp PROPERTIES LANGUAGE CUDA)
set_target_properties(SphereScaling1 PROPERTIES
CUDA_SEPARABLE_COMPILATION ON)
# VS_DEBUGGER_COMMAND "C:/Program Files/Microsoft MPI/Bin/mpiexec.exe"
# VS_DEBUGGER_COMMAND_ARGUMENTS "-n 2 \"$<TARGET_FILE:SphereScaling1>\"")
\ No newline at end of file
apps/gpu/LBM/SphereScaling1/SphereScaling1.cpp 0 → 100644
+ 754
− 0
View file @ 7b76d3c4
This diff is collapsed.
apps/gpu/LBM/SphereScaling1/config.txt 0 → 100644
+ 46
− 0
View file @ 7b76d3c4
# Tesla 03
# mpiexec -n 2 "C:/Users/Master/Documents/MasterAnna/VirtualFluids_dev/build/bin/Release/SphereScaling.exe" "C:/Users/Master/Documents/MasterAnna/VirtualFluids_dev/apps/gpu/LBM/SphereScaling/config.txt"
# Phoenix
# mpirun -np 2 "./VirtualFluids_dev/build/bin/SphereScaling" "./VirtualFluids_dev/apps/gpu/LBM/SphereScaling/config.txt"
# Phoenix mpich
# mpirun -np 2 nvprof -f -o SphereScaling.%q{PMI_RANK}.nvprof "./VirtualFluids_dev/build/bin/SphereScaling" "./VirtualFluids_dev/apps/gpu/LBM/SphereScaling/configPhoenix4GPU.txt"
# Phoenix openmpi
# mpirun -np 2 nvprof -f -o SphereScaling.%q{OMPI_COMM_WORLD_RANK}.nvprof "./VirtualFluids_dev/build/bin/SphereScaling" "./VirtualFluids_dev/apps/gpu/LBM/SphereScaling/configPhoenix4GPU.txt"
##################################################
#GPU Mapping
##################################################
#Devices="0 1 2 3"
#NumberOfDevices=2
##################################################
#informations for Writing
##################################################
#Path="E:/temp/SphereScalingResults/"
Path=/work/y0078217/Results/SphereScalingResults/
#Prefix="SphereScaling"
#WriteGrid=true
##################################################
#informations for reading
##################################################
GridPath=/work/y0078217/Grids/GridSphereScaling/
#GridPath=E:/temp/GridSphereScaling/
##################################################
#number of grid levels
##################################################
#NOGL=1
##################################################
#LBM Version
##################################################
#D3Qxx=27
#MainKernelName=CumulantK17CompChim
##################################################
#simulation parameter
##################################################
#TimeEnd=10
#TimeOut=10
#TimeStartOut=0
\ No newline at end of file
apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevWeakOS.txt 0 → 100644
+ 42
− 0
View file @ 7b76d3c4
##################################################
#GPU Mapping
##################################################
Devices="0 1 2 3"
NumberOfDevices=4
##################################################
#informations for Writing
##################################################
Path=/work/y0078217/Results/SphereScalingResults/8GPU/2LevWeakOS/
#Path="F:/Work/Computations/out/SphereScaling/"
#Prefix="SphereScaling"
#WriteGrid=true
##################################################
#informations for reading
##################################################
GridPath=/work/y0078217/Grids/GridSphereScaling/SphereScaling8GPU/
#GridPath="C:"
##################################################
#number of grid levels
##################################################
#NOGL=1
##################################################
#LBM Version
##################################################
#D3Qxx=27
#MainKernelName=CumulantK17CompChim
##################################################
#simulation parameter
##################################################
TimeEnd=100000
TimeOut=100000
#TimeStartOut=0
##################################################
# CUDA Streams and optimized communication (only used for multiple GPUs)
##################################################
useStreams = false
useReducedCommunicationInInterpolation = false
\ No newline at end of file
apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevWeakStream.txt 0 → 100644
+ 42
− 0
View file @ 7b76d3c4
##################################################
#GPU Mapping
##################################################
Devices="0 1 2 3 4 5 6 7"
NumberOfDevices=8
##################################################
#informations for Writing
##################################################
Path=/work/y0078217/Results/SphereScalingResults/8GPU/1LevWeakStream/
#Path="F:/Work/Computations/out/SphereScaling/"
#Prefix="SphereScaling"
#WriteGrid=true
##################################################
#informations for reading
##################################################
GridPath=/work/y0078217/Grids/GridSphereScaling/SphereScaling8GPU/
#GridPath="C:"
##################################################
#number of grid levels
##################################################
#NOGL=1
##################################################
#LBM Version
##################################################
#D3Qxx=27
#MainKernelName=CumulantK17CompChim
##################################################
#simulation parameter
##################################################
TimeEnd=100000
TimeOut=100000
#TimeStartOut=0
##################################################
# CUDA Streams and optimized communication (only used for multiple GPUs)
##################################################
useStreams = true
useReducedCommunicationInInterpolation = true
\ No newline at end of file
gpu.cmake
+ 3
− 2
View file @ 7b76d3c4
......@@ -45,9 +45,10 @@ IF (BUILD_VF_GPU)
#add_subdirectory(apps/gpu/LBM/TGV_3D)
#add_subdirectory(apps/gpu/LBM/TGV_3D_MultiGPU)
add_subdirectory(apps/gpu/LBM/SphereScaling)
add_subdirectory(apps/gpu/LBM/SphereScaling1)
add_subdirectory(apps/gpu/LBM/MusselOyster)
add_subdirectory(apps/gpu/LBM/MusselOyster2x)
add_subdirectory(apps/gpu/LBM/MusselOyster3z)
#add_subdirectory(apps/gpu/LBM/MusselOyster2x)
#add_subdirectory(apps/gpu/LBM/MusselOyster3z)
ELSE()
MESSAGE( STATUS "exclude Virtual Fluids GPU." )
ENDIF()
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment