diff --git a/apps/gpu/LBM/ActuatorLine/ActuatorLine.cpp b/apps/gpu/LBM/ActuatorLine/ActuatorLine.cpp index 94755358f680503e19e0e204946ae51016d39802..368e80a726c1f6e75bd5300ad3de60fd840ba119 100644 --- a/apps/gpu/LBM/ActuatorLine/ActuatorLine.cpp +++ b/apps/gpu/LBM/ActuatorLine/ActuatorLine.cpp @@ -71,7 +71,7 @@ ////////////////////////////////////////////////////////////////////////// #include "VirtualFluids_GPU/LBM/Simulation.h" -#include "VirtualFluids_GPU/Communication/Communicator.h" +#include "VirtualFluids_GPU/Communication/MpiCommunicator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridProvider.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderFiles/GridReader.h" @@ -107,7 +107,7 @@ std::string simulationName("ActuatorLine"); void multipleLevel(const std::string& configPath) { - vf::gpu::Communicator& communicator = vf::gpu::Communicator::getInstance(); + vf::gpu::Communicator& communicator = vf::gpu::MpiCommunicator::getInstance(); auto gridFactory = GridFactory::make(); auto gridBuilder = MultipleGridBuilder::makeShared(gridFactory); diff --git a/apps/gpu/LBM/Basel/main.cpp b/apps/gpu/LBM/Basel/main.cpp index 4999d3418b269ae4340ca550a5f2c50fc6e45231..16161dc6a83846f2730c5920dfd64363b6e77b30 100644 --- a/apps/gpu/LBM/Basel/main.cpp +++ b/apps/gpu/LBM/Basel/main.cpp @@ -19,7 +19,7 @@ #include "Input/ConfigFileReader/ConfigFileReader.h" #include "VirtualFluids_GPU/LBM/Simulation.h" -#include "VirtualFluids_GPU/Communication/Communicator.h" +#include "VirtualFluids_GPU/Communication/MpiCommunicator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridProvider.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderFiles/GridReader.h" diff --git a/apps/gpu/LBM/BaselMultiGPU/main.cpp b/apps/gpu/LBM/BaselMultiGPU/main.cpp index bfd64d42b428907e08c3a3b7fdb99319e0f05382..346ab7ca36ccd147d1f8427974ce2510b5d637fb 100644 --- a/apps/gpu/LBM/BaselMultiGPU/main.cpp +++ b/apps/gpu/LBM/BaselMultiGPU/main.cpp @@ -19,7 +19,7 @@ #include "Input/ConfigFileReader/ConfigFileReader.h" #include "VirtualFluids_GPU/LBM/Simulation.h" -#include "VirtualFluids_GPU/Communication/Communicator.h" +#include "VirtualFluids_GPU/Communication/MpiCommunicator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridProvider.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderFiles/GridReader.h" diff --git a/apps/gpu/LBM/BaselNU/main.cpp b/apps/gpu/LBM/BaselNU/main.cpp index 9600785063018ce140606ef66b1ea466dc4bdb40..97c4c48afb5caffb89bad89e3f3fe99670c889bc 100644 --- a/apps/gpu/LBM/BaselNU/main.cpp +++ b/apps/gpu/LBM/BaselNU/main.cpp @@ -19,7 +19,7 @@ #include "Input/ConfigFileReader/ConfigFileReader.h" #include "VirtualFluids_GPU/LBM/Simulation.h" -#include "VirtualFluids_GPU/Communication/Communicator.h" +#include "VirtualFluids_GPU/Communication/MpiCommunicator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridProvider.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderFiles/GridReader.h" diff --git a/apps/gpu/LBM/BoundaryLayer/BoundaryLayer.cpp b/apps/gpu/LBM/BoundaryLayer/BoundaryLayer.cpp index 298c5d9c344a1873a2612a518f72f33f7d6b6f64..1c5f8ac0318f00a6bb819b016ff11925b2ff2da5 100644 --- a/apps/gpu/LBM/BoundaryLayer/BoundaryLayer.cpp +++ b/apps/gpu/LBM/BoundaryLayer/BoundaryLayer.cpp @@ -73,7 +73,7 @@ ////////////////////////////////////////////////////////////////////////// #include "VirtualFluids_GPU/LBM/Simulation.h" -#include "VirtualFluids_GPU/Communication/Communicator.h" +#include "VirtualFluids_GPU/Communication/MpiCommunicator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridProvider.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderFiles/GridReader.h" @@ -108,7 +108,7 @@ void multipleLevel(const std::string& configPath) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - vf::gpu::Communicator& communicator = vf::gpu::Communicator::getInstance(); + vf::gpu::Communicator& communicator = vf::gpu::MpiCommunicator::getInstance(); vf::basics::ConfigurationFile config; config.load(configPath); @@ -119,7 +119,7 @@ void multipleLevel(const std::string& configPath) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// const int nProcs = communicator.getNumberOfProcess(); - const uint procID = vf::gpu::Communicator::getInstance().getPID(); + const uint procID = vf::gpu::MpiCommunicator::getInstance().getPID(); std::vector<uint> devices(10); std::iota(devices.begin(), devices.end(), 0); para->setDevices(devices); diff --git a/apps/gpu/LBM/ChannelFlow/ChannelFlow.cpp b/apps/gpu/LBM/ChannelFlow/ChannelFlow.cpp index f553c255b882596ea4614e8e2fa33403e3d9a0f8..1482adf275da6dabae7bdde7647ac374a0d66bb1 100644 --- a/apps/gpu/LBM/ChannelFlow/ChannelFlow.cpp +++ b/apps/gpu/LBM/ChannelFlow/ChannelFlow.cpp @@ -65,7 +65,7 @@ ////////////////////////////////////////////////////////////////////////// #include "VirtualFluids_GPU/BoundaryConditions/BoundaryConditionFactory.h" -#include "VirtualFluids_GPU/Communication/Communicator.h" +#include "VirtualFluids_GPU/Communication/MpiCommunicator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridProvider.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h" #include "VirtualFluids_GPU/GPU/CudaMemoryManager.h" @@ -95,7 +95,7 @@ int main(int argc, char *argv[]) // setup simulation parameters (without config file) ////////////////////////////////////////////////////////////////////////// - vf::gpu::Communicator &communicator = vf::gpu::Communicator::getInstance(); + vf::gpu::Communicator &communicator = vf::gpu::MpiCommunicator::getInstance(); const int numberOfProcesses = communicator.getNumberOfProcess(); SPtr<Parameter> para = std::make_shared<Parameter>(numberOfProcesses, communicator.getPID()); std::vector<uint> devices(10); @@ -109,7 +109,7 @@ int main(int argc, char *argv[]) ////////////////////////////////////////////////////////////////////////// vf::logging::Logger::changeLogPath("output/vflog_process" + - std::to_string(vf::gpu::Communicator::getInstance().getPID()) + ".txt"); + std::to_string(vf::gpu::MpiCommunicator::getInstance().getPID()) + ".txt"); vf::logging::Logger::initializeLogger(); ////////////////////////////////////////////////////////////////////////// @@ -159,7 +159,7 @@ int main(int argc, char *argv[]) para->setOutputPrefix("ChannelFlow"); para->setMainKernel(vf::CollisionKernel::Compressible::CumulantK17); - const uint generatePart = vf::gpu::Communicator::getInstance().getPID(); + const uint generatePart = vf::gpu::MpiCommunicator::getInstance().getPID(); real overlap = (real)8.0 * dx; if (numberOfProcesses > 1) { diff --git a/apps/gpu/LBM/DrivenCavity/DrivenCavity.cpp b/apps/gpu/LBM/DrivenCavity/DrivenCavity.cpp index a802de12c032766f9bf14d2a43b4d16078e230f6..245d1c052c909cef7c1ab948ca55d104419fa8cd 100644 --- a/apps/gpu/LBM/DrivenCavity/DrivenCavity.cpp +++ b/apps/gpu/LBM/DrivenCavity/DrivenCavity.cpp @@ -58,7 +58,7 @@ #include "VirtualFluids_GPU/Factories/BoundaryConditionFactory.h" #include "VirtualFluids_GPU/Factories/GridScalingFactory.h" -#include "VirtualFluids_GPU/Communication/Communicator.h" +#include "VirtualFluids_GPU/Communication/MpiCommunicator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridProvider.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h" #include "VirtualFluids_GPU/GPU/CudaMemoryManager.h" @@ -164,7 +164,7 @@ int main() // set copy mesh to simulation ////////////////////////////////////////////////////////////////////////// - vf::gpu::Communicator &communicator = vf::gpu::Communicator::getInstance(); + vf::gpu::Communicator &communicator = vf::gpu::MpiCommunicator::getInstance(); auto cudaMemoryManager = std::make_shared<CudaMemoryManager>(para); SPtr<GridProvider> gridGenerator = diff --git a/apps/gpu/LBM/DrivenCavityMultiGPU/DrivenCavityMultiGPU.cpp b/apps/gpu/LBM/DrivenCavityMultiGPU/DrivenCavityMultiGPU.cpp index acab426b4868cc736710c883776c5626ec6b5753..d7f6dace74a07ba8477085b94f2146cf4500d7d1 100755 --- a/apps/gpu/LBM/DrivenCavityMultiGPU/DrivenCavityMultiGPU.cpp +++ b/apps/gpu/LBM/DrivenCavityMultiGPU/DrivenCavityMultiGPU.cpp @@ -36,7 +36,7 @@ ////////////////////////////////////////////////////////////////////////// -#include "VirtualFluids_GPU/Communication/Communicator.h" +#include "VirtualFluids_GPU/Communication/MpiCommunicator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridProvider.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderFiles/GridReader.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h" @@ -62,7 +62,7 @@ void runVirtualFluids(const vf::basics::ConfigurationFile& config) { - vf::gpu::Communicator& communicator = vf::gpu::Communicator::getInstance(); + vf::gpu::Communicator& communicator = vf::gpu::MpiCommunicator::getInstance(); auto gridFactory = GridFactory::make(); gridFactory->setTriangularMeshDiscretizationMethod(TriangularMeshDiscretizationMethod::POINT_IN_OBJECT); @@ -145,7 +145,7 @@ void runVirtualFluids(const vf::basics::ConfigurationFile& config) if (para->getNumprocs() > 1) { - const uint generatePart = vf::gpu::Communicator::getInstance().getPID(); + const uint generatePart = vf::gpu::MpiCommunicator::getInstance().getPID(); real overlap = (real)8.0 * dxGrid; gridBuilder->setNumberOfLayers(10, 8); diff --git a/apps/gpu/LBM/DrivenCavityUniform/DrivenCavity.cpp b/apps/gpu/LBM/DrivenCavityUniform/DrivenCavity.cpp index dfa1256c80e6aeb0e209638ba0c7425ff437e2d5..05c164ea82790c4a26c5c65538d944f46d1c969c 100644 --- a/apps/gpu/LBM/DrivenCavityUniform/DrivenCavity.cpp +++ b/apps/gpu/LBM/DrivenCavityUniform/DrivenCavity.cpp @@ -60,7 +60,7 @@ #include "VirtualFluids_GPU/Factories/BoundaryConditionFactory.h" #include "VirtualFluids_GPU/Factories/GridScalingFactory.h" -#include "VirtualFluids_GPU/Communication/Communicator.h" +#include "VirtualFluids_GPU/Communication/MpiCommunicator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridProvider.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h" #include "VirtualFluids_GPU/GPU/CudaMemoryManager.h" @@ -167,7 +167,7 @@ int main() // set copy mesh to simulation ////////////////////////////////////////////////////////////////////////// - vf::gpu::Communicator &communicator = vf::gpu::Communicator::getInstance(); + vf::gpu::Communicator &communicator = vf::gpu::MpiCommunicator::getInstance(); auto cudaMemoryManager = std::make_shared<CudaMemoryManager>(para); SPtr<GridProvider> gridGenerator = diff --git a/apps/gpu/LBM/MusselOyster/MusselOyster.cpp b/apps/gpu/LBM/MusselOyster/MusselOyster.cpp index b583633b50542795fe4b27aca42c08cca1a5331c..1cff9cf2675fbe9d42f4765c6c9e1bcea4bd7065 100644 --- a/apps/gpu/LBM/MusselOyster/MusselOyster.cpp +++ b/apps/gpu/LBM/MusselOyster/MusselOyster.cpp @@ -37,7 +37,7 @@ ////////////////////////////////////////////////////////////////////////// -#include "VirtualFluids_GPU/Communication/Communicator.h" +#include "VirtualFluids_GPU/Communication/MpiCommunicator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridProvider.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderFiles/GridReader.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h" @@ -86,7 +86,7 @@ const std::string simulationName("MusselOyster"); void runVirtualFluids(const vf::basics::ConfigurationFile& config) { - vf::gpu::Communicator &communicator = vf::gpu::Communicator::getInstance(); + vf::gpu::Communicator &communicator = vf::gpu::MpiCommunicator::getInstance(); auto gridFactory = GridFactory::make(); gridFactory->setTriangularMeshDiscretizationMethod(TriangularMeshDiscretizationMethod::POINT_IN_OBJECT); @@ -191,7 +191,7 @@ void runVirtualFluids(const vf::basics::ConfigurationFile& config) bivalveRef_1_STL = std::make_shared<TriangularMesh>(stlPath + bivalveType + "_Level1.stl"); if (para->getNumprocs() > 1) { - const uint generatePart = vf::gpu::Communicator::getInstance().getPID(); + const uint generatePart = vf::gpu::MpiCommunicator::getInstance().getPID(); real overlap = (real)8.0 * dxGrid; gridBuilder->setNumberOfLayers(10, 8); diff --git a/apps/gpu/LBM/SphereGPU/Sphere.cpp b/apps/gpu/LBM/SphereGPU/Sphere.cpp index a20383b8e7eab9ce61fb8d8c21de95d6033f5c5f..f70188bbd81a22088affbbc45d43dafdd418fe46 100644 --- a/apps/gpu/LBM/SphereGPU/Sphere.cpp +++ b/apps/gpu/LBM/SphereGPU/Sphere.cpp @@ -62,7 +62,7 @@ #include "VirtualFluids_GPU/DataStructureInitializer/GridProvider.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h" #include "VirtualFluids_GPU/GPU/CudaMemoryManager.h" -#include "VirtualFluids_GPU/Communication/Communicator.h" +#include "VirtualFluids_GPU/Communication/MpiCommunicator.h" #include "VirtualFluids_GPU/LBM/Simulation.h" #include "VirtualFluids_GPU/Output/FileWriter.h" #include "VirtualFluids_GPU/Parameter/Parameter.h" @@ -96,7 +96,7 @@ int main(int argc, char *argv[]) // setup simulation parameters (with or without config file) ////////////////////////// - vf::gpu::Communicator& communicator = vf::gpu::Communicator::getInstance();; + vf::gpu::Communicator& communicator = vf::gpu::MpiCommunicator::getInstance();; SPtr<Parameter> para; BoundaryConditionFactory bcFactory = BoundaryConditionFactory(); GridScalingFactory scalingFactory = GridScalingFactory(); diff --git a/apps/gpu/LBM/SphereScaling/SphereScaling.cpp b/apps/gpu/LBM/SphereScaling/SphereScaling.cpp index da80302e9e9b5b6f43c7eb3eea0ae8be08f22b93..e4291f81a288528397246f2fb56f95aa3c04ac0c 100755 --- a/apps/gpu/LBM/SphereScaling/SphereScaling.cpp +++ b/apps/gpu/LBM/SphereScaling/SphereScaling.cpp @@ -39,7 +39,7 @@ ////////////////////////////////////////////////////////////////////////// -#include "VirtualFluids_GPU/Communication/Communicator.h" +#include "VirtualFluids_GPU/Communication/MpiCommunicator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridProvider.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderFiles/GridReader.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h" @@ -65,7 +65,7 @@ void runVirtualFluids(const vf::basics::ConfigurationFile& config) { - vf::gpu::Communicator& communicator = vf::gpu::Communicator::getInstance(); + vf::gpu::Communicator& communicator = vf::gpu::MpiCommunicator::getInstance(); auto gridFactory = GridFactory::make(); gridFactory->setTriangularMeshDiscretizationMethod(TriangularMeshDiscretizationMethod::POINT_IN_OBJECT); @@ -170,7 +170,7 @@ void runVirtualFluids(const vf::basics::ConfigurationFile& config) const real dCubeLev1 = 72.0; // Phoenix: 72.0 if (para->getNumprocs() > 1) { - const uint generatePart = vf::gpu::Communicator::getInstance().getPID(); + const uint generatePart = vf::gpu::MpiCommunicator::getInstance().getPID(); real overlap = (real)8.0 * dxGrid; gridBuilder->setNumberOfLayers(10, 8); diff --git a/apps/gpu/LBM/TGV_3D/TGV_3D.cpp b/apps/gpu/LBM/TGV_3D/TGV_3D.cpp index 050efc6d0f0f2b80ca2da2df26cdb71b1e52f3ad..c813bc319c085506420638dac3ef623b110dc771 100644 --- a/apps/gpu/LBM/TGV_3D/TGV_3D.cpp +++ b/apps/gpu/LBM/TGV_3D/TGV_3D.cpp @@ -67,7 +67,7 @@ ////////////////////////////////////////////////////////////////////////// -#include "VirtualFluids_GPU/Communication/Communicator.h" +#include "VirtualFluids_GPU/Communication/MpiCommunicator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridProvider.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderFiles/GridReader.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h" @@ -132,7 +132,7 @@ std::string simulationName("TGV_3D"); void multipleLevel(const std::string& configPath) { - vf::gpu::Communicator& communicator = vf::gpu::Communicator::getInstance(); + vf::gpu::Communicator& communicator = vf::gpu::MpiCommunicator::getInstance(); //UbLog::reportingLevel() = UbLog::logLevelFromString("DEBUG5"); diff --git a/apps/gpu/LBM/TGV_3D_GridRef/TGV_3D_GridRef.cpp b/apps/gpu/LBM/TGV_3D_GridRef/TGV_3D_GridRef.cpp index 0f945e32d9fbcb2e18d4888a4fa0a2c6e03c21b4..876496783e61867a45843b0aa871bbc794eca73f 100644 --- a/apps/gpu/LBM/TGV_3D_GridRef/TGV_3D_GridRef.cpp +++ b/apps/gpu/LBM/TGV_3D_GridRef/TGV_3D_GridRef.cpp @@ -67,7 +67,7 @@ ////////////////////////////////////////////////////////////////////////// -#include "VirtualFluids_GPU/Communication/Communicator.h" +#include "VirtualFluids_GPU/Communication/MpiCommunicator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridProvider.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderFiles/GridReader.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h" @@ -131,7 +131,7 @@ std::string simulationName("TGV_3D_Gridref_noSqPress"); void multipleLevel(const std::string& configPath) { - vf::gpu::Communicator& communicator = vf::gpu::Communicator::getInstance(); + vf::gpu::Communicator& communicator = vf::gpu::MpiCommunicator::getInstance(); auto gridFactory = GridFactory::make(); //gridFactory->setTriangularMeshDiscretizationMethod(TriangularMeshDiscretizationMethod::RAYCASTING); diff --git a/apps/gpu/LBM/TGV_3D_MultiGPU/TGV_3D_MultiGPU.cpp b/apps/gpu/LBM/TGV_3D_MultiGPU/TGV_3D_MultiGPU.cpp index 0c7b9b9606201b32a8376ea637858eb14ec817bb..e643c4f72019c681bc7ae4b2711d46e3399dad71 100644 --- a/apps/gpu/LBM/TGV_3D_MultiGPU/TGV_3D_MultiGPU.cpp +++ b/apps/gpu/LBM/TGV_3D_MultiGPU/TGV_3D_MultiGPU.cpp @@ -54,7 +54,7 @@ #include "basics/config/ConfigurationFile.h" #include "VirtualFluids_GPU/LBM/Simulation.h" -#include "VirtualFluids_GPU/Communication/Communicator.h" +#include "VirtualFluids_GPU/Communication/MpiCommunicator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridProvider.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderFiles/GridReader.h" @@ -156,7 +156,7 @@ void multipleLevel(const std::string& configPath) rankY = ( mpirank % ( sideLengthX * sideLengthY ) ) / sideLengthX; rankZ = mpirank / ( sideLengthY * sideLengthX ); - vf::gpu::Communicator& communicator = vf::gpu::Communicator::getInstance(); + vf::gpu::Communicator& communicator = vf::gpu::MpiCommunicator::getInstance(); //UbLog::reportingLevel() = UbLog::logLevelFromString("DEBUG5"); diff --git a/apps/gpu/LBM/WTG_RUB/WTG_RUB.cpp b/apps/gpu/LBM/WTG_RUB/WTG_RUB.cpp index d925eef7e7f452e19b63284f9ad3a6af25740e79..00fe00a24eb454c9aa0973d6ebe20f22c6a52369 100644 --- a/apps/gpu/LBM/WTG_RUB/WTG_RUB.cpp +++ b/apps/gpu/LBM/WTG_RUB/WTG_RUB.cpp @@ -68,7 +68,7 @@ ////////////////////////////////////////////////////////////////////////// #include "VirtualFluids_GPU/LBM/Simulation.h" -#include "VirtualFluids_GPU/Communication/Communicator.h" +#include "VirtualFluids_GPU/Communication/MpiCommunicator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridProvider.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderFiles/GridReader.h" @@ -129,7 +129,7 @@ std::string chooseVariation(); void multipleLevel(const std::string& configPath) { - vf::gpu::Communicator& communicator = vf::gpu::Communicator::getInstance(); + vf::gpu::Communicator& communicator = vf::gpu::MpiCommunicator::getInstance(); auto gridFactory = GridFactory::make(); gridFactory->setTriangularMeshDiscretizationMethod(TriangularMeshDiscretizationMethod::POINT_IN_OBJECT); diff --git a/apps/gpu/LBM/gridGeneratorTest/gridGenerator.cpp b/apps/gpu/LBM/gridGeneratorTest/gridGenerator.cpp index 93ff9de0f0a156d9d9aec0ec1b6615ae728e7811..90cc4c0ab33745c65688db4bf170b25a9099da51 100644 --- a/apps/gpu/LBM/gridGeneratorTest/gridGenerator.cpp +++ b/apps/gpu/LBM/gridGeneratorTest/gridGenerator.cpp @@ -18,7 +18,7 @@ #include "basics/config/ConfigurationFile.h" #include "VirtualFluids_GPU/LBM/Simulation.h" -#include "VirtualFluids_GPU/Communication/Communicator.h" +#include "VirtualFluids_GPU/Communication/MpiCommunicator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridProvider.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderFiles/GridReader.h" @@ -65,7 +65,7 @@ void multipleLevel(const std::string& configPath) auto gridBuilder = MultipleGridBuilder::makeShared(gridFactory); - vf::gpu::Communicator& communicator = vf::gpu::Communicator::getInstance(); + vf::gpu::Communicator& communicator = vf::gpu::MpiCommunicator::getInstance(); vf::basics::ConfigurationFile config; config.load(configPath); SPtr<Parameter> para = std::make_shared<Parameter>(communicator.getNumberOfProcess(), communicator.getPID(), &config); diff --git a/apps/gpu/LBM/lbmTest/main.cpp b/apps/gpu/LBM/lbmTest/main.cpp index abe8e7488a55c773896ef6cb362db2bb8247fa4e..90640a3813a25e8249c7668a6e6c7779b0f01e4d 100644 --- a/apps/gpu/LBM/lbmTest/main.cpp +++ b/apps/gpu/LBM/lbmTest/main.cpp @@ -20,7 +20,7 @@ #include "StringUtilities/StringUtil.h" #include "VirtualFluids_GPU/LBM/Simulation.h" -#include "VirtualFluids_GPU/Communication/Communicator.h" +#include "VirtualFluids_GPU/Communication/MpiCommunicator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridProvider.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderFiles/GridReader.h" diff --git a/apps/gpu/LBM/metisTest/main.cpp b/apps/gpu/LBM/metisTest/main.cpp index b9879c16c32afd0ec27c5841a0b2dad7e4191055..9edfb9e853f1c4fff6872b15e93d4de94bd35095 100644 --- a/apps/gpu/LBM/metisTest/main.cpp +++ b/apps/gpu/LBM/metisTest/main.cpp @@ -19,7 +19,7 @@ #include "StringUtilities/StringUtil.h" #include "VirtualFluids_GPU/LBM/Simulation.h" -#include "VirtualFluids_GPU/Communication/Communicator.h" +#include "VirtualFluids_GPU/Communication/MpiCommunicator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridProvider.h" #include "VirtualFluids_GPU/DataStructureInitializer/GridReaderFiles/GridReader.h" diff --git a/apps/gpu/tests/NumericalTests/Utilities/VirtualFluidSimulationFactory/VirtualFluidSimulationFactory.cpp b/apps/gpu/tests/NumericalTests/Utilities/VirtualFluidSimulationFactory/VirtualFluidSimulationFactory.cpp index 23bcb582dd863acd813519eed4ec1402105c3618..3564f65025a38d37c577b6ea6a882e6f48fd1e65 100644 --- a/apps/gpu/tests/NumericalTests/Utilities/VirtualFluidSimulationFactory/VirtualFluidSimulationFactory.cpp +++ b/apps/gpu/tests/NumericalTests/Utilities/VirtualFluidSimulationFactory/VirtualFluidSimulationFactory.cpp @@ -9,7 +9,7 @@ #include "VirtualFluids_GPU/Parameter/Parameter.h" #include "VirtualFluids_GPU/Factories/BoundaryConditionFactory.h" -#include "VirtualFluids_GPU/Communication/Communicator.h" +#include "VirtualFluids_GPU/Communication/MpiCommunicator.h" #include "VirtualFluids_GPU/LBM/Simulation.h" std::shared_ptr<Parameter> vf::gpu::tests::makeParameter(std::shared_ptr<SimulationParameter> simPara) @@ -120,7 +120,7 @@ const std::function<void()> vf::gpu::tests::makeVirtualFluidSimulation(std::shar auto grid = makeGridReader(condition, para, cudaManager); BoundaryConditionFactory bc_factory; auto simulation = - std::make_shared<Simulation>(para, cudaManager, vf::gpu::Communicator::getInstance(), *grid.get(), &bc_factory); + std::make_shared<Simulation>(para, cudaManager, vf::gpu::MpiCommunicator::getInstance(), *grid.get(), &bc_factory); simulation->setDataWriter(dataWriter); return [simulation]() { simulation->run(); }; diff --git a/pythonbindings/src/gpu/submodules/communicator.cpp b/pythonbindings/src/gpu/submodules/communicator.cpp index 0230caf197c04c2f2cd411288e9ea24ee314c4a8..3b0550e4347b52df2cbcda6706435718e8bce354 100644 --- a/pythonbindings/src/gpu/submodules/communicator.cpp +++ b/pythonbindings/src/gpu/submodules/communicator.cpp @@ -31,7 +31,7 @@ //! \author Henry Korb //======================================================================================= #include <pybind11/pybind11.h> -#include <gpu/VirtualFluids_GPU/Communication/Communicator.h> +#include <gpu/VirtualFluids_GPU/Communication/MpiCommunicator.h> namespace communicator { @@ -39,9 +39,9 @@ namespace communicator void makeModule(py::module_ &parentModule) { - py::class_<vf::gpu::Communicator, std::unique_ptr<vf::gpu::Communicator, py::nodelete>>(parentModule, "Communicator") - .def_static("get_instance", &vf::gpu::Communicator::getInstance, py::return_value_policy::reference) - .def("get_number_of_process", &vf::gpu::Communicator::getNumberOfProcess) - .def("get_pid", &vf::gpu::Communicator::getPID); + py::class_<vf::gpu::MpiCommunicator, std::unique_ptr<vf::gpu::MpiCommunicator, py::nodelete>>(parentModule, "Communicator") + .def_static("get_instance", &vf::gpu::MpiCommunicator::getInstance, py::return_value_policy::reference) + .def("get_number_of_process", &vf::gpu::MpiCommunicator::getNumberOfProcess) + .def("get_pid", &vf::gpu::MpiCommunicator::getPID); } } \ No newline at end of file diff --git a/src/gpu/VirtualFluids_GPU/CMakeLists.txt b/src/gpu/VirtualFluids_GPU/CMakeLists.txt index ffab484b28870fcc278e6aabbb4db618d30d0e99..d58210ba6cf31403e670989ac87cc9c556170261 100644 --- a/src/gpu/VirtualFluids_GPU/CMakeLists.txt +++ b/src/gpu/VirtualFluids_GPU/CMakeLists.txt @@ -24,7 +24,6 @@ if(BUILD_VF_UNIT_TESTS) set_source_files_properties(DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreamsTest.cpp PROPERTIES LANGUAGE CUDA) set_source_files_properties(Communication/ExchangeData27Test.cpp PROPERTIES LANGUAGE CUDA) set_source_files_properties(BoundaryConditions/BoundaryConditionFactoryTest.cpp PROPERTIES LANGUAGE CUDA) - set_source_files_properties(Parameter/ParameterTest.cpp PROPERTIES LANGUAGE CUDA) target_include_directories(VirtualFluids_GPUTests PRIVATE "${VF_THIRD_DIR}/cuda_samples/") target_include_directories(VirtualFluids_GPUTests PRIVATE "${VF_ROOT_DIR}/src/gpu/GridGenerator/") endif() diff --git a/src/gpu/VirtualFluids_GPU/Communication/Communicator.h b/src/gpu/VirtualFluids_GPU/Communication/Communicator.h index 716eb6c7c2c091cb79ce502504d634d53ea40937..c52d5af9cacb4d5ae4e46090a263f67d4e63f12d 100644 --- a/src/gpu/VirtualFluids_GPU/Communication/Communicator.h +++ b/src/gpu/VirtualFluids_GPU/Communication/Communicator.h @@ -2,21 +2,10 @@ #define COMMUNICATOR_GPU_H #include <vector> - -#include <mpi.h> +#include <basics/DataTypes.h> #include "VirtualFluids_GPU_export.h" - #include "CommunicationRoutine.h" -#include <basics/DataTypes.h> - -////////////////////////////////// -#ifdef VF_DOUBLE_ACCURACY -#define MPI_Type_GPU MPI_DOUBLE -#else -#define MPI_Type_GPU MPI_FLOAT -#endif -////////////////////////////////// namespace vf::gpu { @@ -24,59 +13,25 @@ namespace vf::gpu class VIRTUALFLUIDS_GPU_EXPORT Communicator : public CommunicationRoutine { public: - static Communicator &getInstance(); - Communicator(const Communicator &) = delete; - Communicator &operator=(const Communicator &) = delete; - ~Communicator() override; - - void exchngBottomToTop(float *sbuf, float *rbuf, int count); - void exchngTopToBottom(float *sbuf, float *rbuf, int count); - void waitAll(); - void distributeGeometry(unsigned int *dataRoot, unsigned int *dataNode, int dataSizePerNode); - int getPID() const override; - int getNumberOfProcess() const; - int getNeighbourTop(); - int getNeighbourBottom(); - void exchngData(float *sbuf_t, float *rbuf_t, float *sbuf_b, float *rbuf_b, int count); - void exchngDataNB(float *sbuf_t, int count_st, float *rbuf_t, int count_rt, float *sbuf_b, int count_sb, - float *rbuf_b, int count_rb); + virtual void waitAll() = 0; + virtual int getPID() const override = 0; + virtual int getNumberOfProcess() const = 0; + virtual void exchngData(float *sbuf_t, float *rbuf_t, float *sbuf_b, float *rbuf_b, int count) = 0; ////////////////////////////////////////////////////////////////////////// - void exchngDataGPU(real *sbuf, int count_s, real *rbuf, int count_r, int nb_rank); - void sendRecvGPU(real *sbuf, int count_s, real *rbuf, int count_r, int nb_rank); - void nbRecvDataGPU(real *rbuf, int count_r, int nb_rank); - void nbSendDataGPU(real *sbuf, int count_s, int nb_rank); - void waitallGPU(); - void sendDataGPU(real *sbuf, int count_s, int nb_rank); - void waitGPU(int id); - void resetRequest(); - void barrierGPU(); - void barrier(); + virtual void exchngDataGPU(real *sbuf, int count_s, real *rbuf, int count_r, int nb_rank) = 0; + virtual void nbRecvDataGPU(real *rbuf, int count_r, int nb_rank) = 0; + virtual void nbSendDataGPU(real *sbuf, int count_s, int nb_rank) = 0; + virtual void waitallGPU() = 0; + virtual void sendDataGPU(real *sbuf, int count_s, int nb_rank) = 0; + virtual void waitGPU(int id) = 0; + virtual void resetRequest() = 0; ////////////////////////////////////////////////////////////////////////// - void exchngDataGeo(int *sbuf_t, int *rbuf_t, int *sbuf_b, int *rbuf_b, int count); - MPI_Comm getCommunicator(); - void startTimer(); - void stopTimer(); - double getTime(); - int mapCudaDevice(const int &rank, const int &size, const std::vector<unsigned int> &devices, const int &maxdev); - std::vector<double> gatherNUPS(double processNups); - double sumNups(double processNups); + virtual int mapCudaDevice(const int &rank, const int &size, const std::vector<unsigned int> &devices, const int &maxdev) = 0; + virtual double reduceSum(double quantityPerProcess) = 0; ////////////////////////////////////////////////////////////////////////// - void receive_send(uint *buffer_receive, int size_buffer_recv, int neighbor_rank_recv, uint *buffer_send, - int size_buffer_send, int neighbor_rank_send) const override; + virtual void receive_send(uint *buffer_receive, int size_buffer_recv, int neighbor_rank_recv, uint *buffer_send, + int size_buffer_send, int neighbor_rank_send) const override = 0; -private: - int numprocs, PID; - int nbrbottom, nbrtop; - MPI_Comm comm1d, commGPU; - MPI_Status status[4]; - MPI_Request request[4]; - ////////////////////////////////////////////////////////////////////////// - std::vector<MPI_Request> requestGPU; - int rcount; - ////////////////////////////////////////////////////////////////////////// - double starttime; - double endtime; - Communicator(); }; } // namespace vf::gpu diff --git a/src/gpu/VirtualFluids_GPU/Communication/Communicator.cpp b/src/gpu/VirtualFluids_GPU/Communication/MpiCommunicator.cpp similarity index 70% rename from src/gpu/VirtualFluids_GPU/Communication/Communicator.cpp rename to src/gpu/VirtualFluids_GPU/Communication/MpiCommunicator.cpp index 89f3595cf6ed4919548d27d47ae987f89053e1d5..8af5931ce92b6fa4904ab3aea7c901773f61a6b3 100644 --- a/src/gpu/VirtualFluids_GPU/Communication/Communicator.cpp +++ b/src/gpu/VirtualFluids_GPU/Communication/MpiCommunicator.cpp @@ -1,4 +1,4 @@ -#include "Communicator.h" +#include "MpiCommunicator.h" #include <mpi.h> #include <vector> @@ -16,13 +16,13 @@ namespace vf::gpu { -Communicator::Communicator() +MpiCommunicator::MpiCommunicator() { int mpiInitialized = 0; // false MPI_Initialized(&mpiInitialized); if (!mpiInitialized) { MPI_Init(NULL, NULL); - VF_LOG_TRACE("vf::gpu::Communicator(): MPI_Init"); + VF_LOG_TRACE("vf::gpu::MpiCommunicator(): MPI_Init"); } MPI_Comm_rank(MPI_COMM_WORLD, &PID); @@ -40,41 +40,41 @@ Communicator::Communicator() MPI_Cart_shift(comm1d, 0, 1, &nbrbottom, &nbrtop); } -Communicator::~Communicator() +MpiCommunicator::~MpiCommunicator() { // proof if MPI is finalized int _mpiFinalized = 0; // false MPI_Finalized(&_mpiFinalized); if (!_mpiFinalized) { MPI_Finalize(); - VF_LOG_TRACE("vf::gpu::~Communicator(): MPI_Finalize"); + VF_LOG_TRACE("vf::gpu::~MpiCommunicator(): MPI_Finalize"); } } // C++11 thread safe singelton implementation: // https://stackoverflow.com/questions/1661529/is-meyers-implementation-of-the-singleton-pattern-thread-safe -Communicator& Communicator::getInstance() +MpiCommunicator& MpiCommunicator::getInstance() { - static Communicator comm; + static MpiCommunicator comm; return comm; } -void Communicator::exchngBottomToTop(float *sbuf, float *rbuf, int count) +void MpiCommunicator::exchngBottomToTop(float *sbuf, float *rbuf, int count) { MPI_Sendrecv(sbuf, count, MPI_FLOAT, nbrtop, 0, rbuf, count, MPI_FLOAT, nbrbottom, 0, comm1d, status); } -void Communicator::exchngTopToBottom(float *sbuf, float *rbuf, int count) +void MpiCommunicator::exchngTopToBottom(float *sbuf, float *rbuf, int count) { MPI_Sendrecv(sbuf, count, MPI_FLOAT, nbrbottom, 0, rbuf, count, MPI_FLOAT, nbrtop, 0, comm1d, status); } -void Communicator::waitAll() { MPI_Waitall(4, request, status); } -void Communicator::exchngData(float *sbuf_t, float *rbuf_t, float *sbuf_b, float *rbuf_b, int count) +void MpiCommunicator::waitAll() { MPI_Waitall(4, request, status); } +void MpiCommunicator::exchngData(float *sbuf_t, float *rbuf_t, float *sbuf_b, float *rbuf_b, int count) { MPI_Sendrecv(sbuf_t, count, MPI_FLOAT, nbrtop, 0, rbuf_t, count, MPI_FLOAT, nbrbottom, 0, comm1d, status); MPI_Sendrecv(sbuf_b, count, MPI_FLOAT, nbrbottom, 0, rbuf_b, count, MPI_FLOAT, nbrtop, 0, comm1d, status); } -void Communicator::exchngDataNB(float *sbuf_t, int count_st, float *rbuf_t, int count_rt, float *sbuf_b, int count_sb, +void MpiCommunicator::exchngDataNB(float *sbuf_t, int count_st, float *rbuf_t, int count_rt, float *sbuf_b, int count_sb, float *rbuf_b, int count_rb) { MPI_Irecv(rbuf_t, count_rt, MPI_FLOAT, nbrbottom, 0, comm1d, &request[0]); @@ -85,7 +85,7 @@ void Communicator::exchngDataNB(float *sbuf_t, int count_st, float *rbuf_t, int } ////////////////////////////////////////////////////////////////////////// // Crap by Martin Sch. -void Communicator::exchngDataGPU(real *sbuf, int count_s, real *rbuf, int count_r, int nb_rank) +void MpiCommunicator::exchngDataGPU(real *sbuf, int count_s, real *rbuf, int count_r, int nb_rank) { MPI_Status MSstatus; MPI_Send(sbuf, count_s, MPI_Type_GPU, nb_rank, 0, commGPU); @@ -94,13 +94,13 @@ void Communicator::exchngDataGPU(real *sbuf, int count_s, real *rbuf, int count_ // MPI_Sendrecv(sbuf, count_s, MPI_Type_GPU, nb_rank, 0, rbuf, count_r, MPI_Type_GPU, nb_rank, 0, comm1d, // MPI_STATUSES_IGNORE); } -void Communicator::sendRecvGPU(real *sbuf, int count_s, real *rbuf, int count_r, int nb_rank) +void MpiCommunicator::sendRecvGPU(real *sbuf, int count_s, real *rbuf, int count_r, int nb_rank) { // test only - please don't use MPI_Sendrecv(sbuf, count_s, MPI_Type_GPU, nb_rank, 0, rbuf, count_r, MPI_Type_GPU, nb_rank, 0, commGPU, MPI_STATUSES_IGNORE); } -void Communicator::nbRecvDataGPU(real *rbuf, int count_r, int nb_rank) +void MpiCommunicator::nbRecvDataGPU(real *rbuf, int count_r, int nb_rank) { // printf("\n Start Recv Rank: %d, neighbor Rank: %d, request = %d \n", PID, nb_rank, (int)requestGPU.size()); // fflush(stdout); @@ -112,7 +112,7 @@ void Communicator::nbRecvDataGPU(real *rbuf, int count_r, int nb_rank) // printf("\n End Recv - Rank: %d , neighbor Rank: %d \n", PID, nb_rank); // fflush(stdout); } -void Communicator::nbSendDataGPU(real *sbuf, int count_s, int nb_rank) +void MpiCommunicator::nbSendDataGPU(real *sbuf, int count_s, int nb_rank) { // printf("\n Start Send Rank: %d, neighbor Rank: %d, request = %d \n", PID, nb_rank, (int)requestGPU.size()); // fflush(stdout); @@ -124,7 +124,7 @@ void Communicator::nbSendDataGPU(real *sbuf, int count_s, int nb_rank) // printf("\n End Send - Rank: %d , neighbor Rank: %d \n", PID, nb_rank); // fflush(stdout); } -void Communicator::waitallGPU() +void MpiCommunicator::waitallGPU() { // printf("\n Start Waitall Rank: %d, request = %d \n", PID, (int)requestGPU.size()); // fflush(stdout); @@ -136,19 +136,19 @@ void Communicator::waitallGPU() // printf("\n End Waitall \n"); // fflush(stdout); } -void Communicator::sendDataGPU(real *sbuf, int count_s, int nb_rank) +void MpiCommunicator::sendDataGPU(real *sbuf, int count_s, int nb_rank) { MPI_Send(sbuf, count_s, MPI_Type_GPU, nb_rank, 0, commGPU); } -void Communicator::waitGPU(int id) { MPI_Wait(&requestGPU[id], MPI_STATUSES_IGNORE); } -void Communicator::resetRequest() +void MpiCommunicator::waitGPU(int id) { MPI_Wait(&requestGPU[id], MPI_STATUSES_IGNORE); } +void MpiCommunicator::resetRequest() { if (requestGPU.size() > 0) { requestGPU.resize(0); rcount = 0; } } -void Communicator::barrierGPU() +void MpiCommunicator::barrierGPU() { // printf("\n Start Waitall Rank: %d, request = %d \n", PID, (int)requestGPU.size()); // fflush(stdout); @@ -158,10 +158,10 @@ void Communicator::barrierGPU() // printf("\n End Waitall \n"); // fflush(stdout); } -void Communicator::barrier() { MPI_Barrier(commGPU); } +void MpiCommunicator::barrier() { MPI_Barrier(commGPU); } ////////////////////////////////////////////////////////////////////////// -void Communicator::exchngDataGeo(int *sbuf_t, int *rbuf_t, int *sbuf_b, int *rbuf_b, int count) +void MpiCommunicator::exchngDataGeo(int *sbuf_t, int *rbuf_t, int *sbuf_b, int *rbuf_b, int count) { MPI_Irecv(rbuf_t, count, MPI_INT, nbrbottom, 0, comm1d, &request[0]); MPI_Irecv(rbuf_b, count, MPI_INT, nbrtop, 0, comm1d, &request[1]); @@ -169,19 +169,16 @@ void Communicator::exchngDataGeo(int *sbuf_t, int *rbuf_t, int *sbuf_b, int *rbu MPI_Isend(sbuf_b, count, MPI_INT, nbrbottom, 0, comm1d, &request[3]); MPI_Waitall(4, request, status); } -int Communicator::getPID() const { return PID; } -int Communicator::getNumberOfProcess() const { return numprocs; } -int Communicator::getNeighbourTop() { return nbrtop; } -int Communicator::getNeighbourBottom() { return nbrbottom; } -MPI_Comm Communicator::getCommunicator() { return comm1d; } -void Communicator::startTimer() { starttime = MPI_Wtime(); } -void Communicator::stopTimer() { endtime = MPI_Wtime(); } -double Communicator::getTime() { return endtime - starttime; } -void Communicator::distributeGeometry(unsigned int *dataRoot, unsigned int *dataNode, int dataSizePerNode) +int MpiCommunicator::getPID() const { return PID; } +int MpiCommunicator::getNumberOfProcess() const { return numprocs; } +int MpiCommunicator::getNeighbourTop() { return nbrtop; } +int MpiCommunicator::getNeighbourBottom() { return nbrbottom; } +MPI_Comm MpiCommunicator::getMpiCommunicator() { return comm1d; } +void MpiCommunicator::distributeGeometry(unsigned int *dataRoot, unsigned int *dataNode, int dataSizePerNode) { MPI_Scatter(dataRoot, dataSizePerNode, MPI_UNSIGNED, dataNode, dataSizePerNode, MPI_UNSIGNED, 0, MPI_COMM_WORLD); } -int Communicator::mapCudaDevice(const int &rank, const int &size, const std::vector<unsigned int> &devices, +int MpiCommunicator::mapCudaDevice(const int &rank, const int &size, const std::vector<unsigned int> &devices, const int &maxdev) { int device = -1; @@ -219,21 +216,9 @@ int Communicator::mapCudaDevice(const int &rank, const int &size, const std::vec return device; } -std::vector<double> Communicator::gatherNUPS(double processNups) +double MpiCommunicator::reduceSum(double quantityPerProcess) { - double *buffer_send = &processNups; - double *buffer_recv = (double *)malloc(sizeof(double) * this->numprocs); - - MPI_Gather(buffer_send, 1, MPI_DOUBLE, buffer_recv, 1, MPI_DOUBLE, 0, commGPU); - - if (this->PID == 0) - return std::vector<double>(buffer_recv, buffer_recv + this->numprocs); - return std::vector<double>(); -} - -double Communicator::sumNups(double processNups) -{ - double *buffer_send = &processNups; + double *buffer_send = &quantityPerProcess; double *buffer_recv = (double *)malloc(sizeof(double)); MPI_Reduce(buffer_send, buffer_recv, 1, MPI_DOUBLE, MPI_SUM, 0, commGPU); @@ -241,7 +226,7 @@ double Communicator::sumNups(double processNups) return *buffer_recv; } -void Communicator::receive_send(uint *buffer_receive, int size_buffer_recv, int neighbor_rank_recv, uint *buffer_send, +void MpiCommunicator::receive_send(uint *buffer_receive, int size_buffer_recv, int neighbor_rank_recv, uint *buffer_send, int size_buffer_send, int neighbor_rank_send) const { MPI_Request recv_request; diff --git a/src/gpu/VirtualFluids_GPU/Communication/MpiCommunicator.h b/src/gpu/VirtualFluids_GPU/Communication/MpiCommunicator.h new file mode 100644 index 0000000000000000000000000000000000000000..c6a71c0bf2e292133db90f5a1e2110cb1c484c31 --- /dev/null +++ b/src/gpu/VirtualFluids_GPU/Communication/MpiCommunicator.h @@ -0,0 +1,78 @@ +#ifndef MPIMpiCommunicator_GPU_H +#define MPIMpiCommunicator_GPU_H + +#include <vector> + +#include <mpi.h> + +#include "VirtualFluids_GPU_export.h" + +#include "Communicator.h" +#include <basics/DataTypes.h> + +////////////////////////////////// +#ifdef VF_DOUBLE_ACCURACY +#define MPI_Type_GPU MPI_DOUBLE +#else +#define MPI_Type_GPU MPI_FLOAT +#endif +////////////////////////////////// + +namespace vf::gpu +{ + +class VIRTUALFLUIDS_GPU_EXPORT MpiCommunicator : public Communicator +{ +public: + static MpiCommunicator &getInstance(); + MpiCommunicator(const MpiCommunicator &) = delete; + MpiCommunicator &operator=(const MpiCommunicator &) = delete; + ~MpiCommunicator() override; + + void exchngBottomToTop(float *sbuf, float *rbuf, int count); + void exchngTopToBottom(float *sbuf, float *rbuf, int count); + void waitAll() override; + void distributeGeometry(unsigned int *dataRoot, unsigned int *dataNode, int dataSizePerNode); + int getPID() const override; + int getNumberOfProcess() const override; + int getNeighbourTop(); + int getNeighbourBottom(); + void exchngData(float *sbuf_t, float *rbuf_t, float *sbuf_b, float *rbuf_b, int count) override; + void exchngDataNB(float *sbuf_t, int count_st, float *rbuf_t, int count_rt, float *sbuf_b, int count_sb, + float *rbuf_b, int count_rb); + ////////////////////////////////////////////////////////////////////////// + void exchngDataGPU(real *sbuf, int count_s, real *rbuf, int count_r, int nb_rank) override; + void sendRecvGPU(real *sbuf, int count_s, real *rbuf, int count_r, int nb_rank); + void nbRecvDataGPU(real *rbuf, int count_r, int nb_rank) override; + void nbSendDataGPU(real *sbuf, int count_s, int nb_rank) override; + void waitallGPU() override; + void sendDataGPU(real *sbuf, int count_s, int nb_rank) override; + void waitGPU(int id) override; + void resetRequest() override; + void barrierGPU(); + void barrier(); + ////////////////////////////////////////////////////////////////////////// + void exchngDataGeo(int *sbuf_t, int *rbuf_t, int *sbuf_b, int *rbuf_b, int count); + MPI_Comm getMpiCommunicator(); + int mapCudaDevice(const int &rank, const int &size, const std::vector<unsigned int> &devices, const int &maxdev) override; + double reduceSum(double quantityPerProcess) override; + ////////////////////////////////////////////////////////////////////////// + void receive_send(uint *buffer_receive, int size_buffer_recv, int neighbor_rank_recv, uint *buffer_send, + int size_buffer_send, int neighbor_rank_send) const override; + +private: + int numprocs, PID; + int nbrbottom, nbrtop; + MPI_Comm comm1d, commGPU; + MPI_Status status[4]; + MPI_Request request[4]; + ////////////////////////////////////////////////////////////////////////// + std::vector<MPI_Request> requestGPU; + int rcount; + ////////////////////////////////////////////////////////////////////////// + MpiCommunicator(); +}; + +} // namespace vf::gpu + +#endif diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGeneratorTest.cpp b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGeneratorTest.cpp index 23d858f5bb5d8abcfda34a9ccfb5b3ff91ff313c..1d3fb8220c9085b07708809e05814c960d52e61d 100644 --- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGeneratorTest.cpp +++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGeneratorTest.cpp @@ -1,7 +1,7 @@ #include "GridGenerator.h" #include <gmock/gmock.h> -#include "Communication/Communicator.h" +#include "Communication/MpiCommunicator.h" #include "DataTypes.h" #include "GPU/CudaMemoryManager.h" #include "IndexRearrangementForStreams.h" @@ -113,7 +113,7 @@ private: para->setNumprocs(2); builder = std::make_shared<LevelGridBuilderStub>(nullptr); - vf::gpu::Communicator &communicator = vf::gpu::Communicator::getInstance(); + vf::gpu::Communicator &communicator = vf::gpu::MpiCommunicator::getInstance(); gridGenerator = std::make_shared<GridGenerator>(builder, para, std::make_shared<CudaMemoryManagerDouble>(para), communicator); diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreamsTest.cpp b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreamsTest.cpp index 0d4419755b391c28f5e985a9f08d67c2ce493d3f..081986add72b21314d0095c7da9df047e860f0d2 100644 --- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreamsTest.cpp +++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/IndexRearrangementForStreamsTest.cpp @@ -13,7 +13,7 @@ #include "gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h" #include "gpu/GridGenerator/grid/GridImp.h" #include "gpu/GridGenerator/utilities/communication.h" -#include "gpu/VirtualFluids_GPU/Communication/Communicator.cpp" +#include "gpu/VirtualFluids_GPU/Communication/MpiCommunicator.cpp" namespace indexRearrangementTests { @@ -148,7 +148,7 @@ private: para->initProcessNeighborsAfterFtoCX(sendIndices.level); testSubject = std::make_unique<IndexRearrangementForStreams>( - IndexRearrangementForStreams(para, builder, vf::gpu::Communicator::getInstance())); + IndexRearrangementForStreams(para, builder, vf::gpu::MpiCommunicator::getInstance())); }; }; @@ -608,7 +608,7 @@ private: para = testingVF::createParameterForLevel(ri.level); testSubject = std::make_unique<IndexRearrangementForStreams>( - IndexRearrangementForStreams(para, builder, vf::gpu::Communicator::getInstance())); + IndexRearrangementForStreams(para, builder, vf::gpu::MpiCommunicator::getInstance())); }; }; diff --git a/src/gpu/VirtualFluids_GPU/Output/EdgeNodeDebugWriter.hpp b/src/gpu/VirtualFluids_GPU/Output/EdgeNodeDebugWriter.hpp index fb0423de958e16727eca8e5a40af2c3e32faf1ae..ee5333dfc130ac7dfdf7ab8c4de812a2916777fa 100644 --- a/src/gpu/VirtualFluids_GPU/Output/EdgeNodeDebugWriter.hpp +++ b/src/gpu/VirtualFluids_GPU/Output/EdgeNodeDebugWriter.hpp @@ -13,7 +13,7 @@ #include <basics/writer/WbWriterVtkXmlBinary.h> #include <cmath> -#include "VirtualFluids_GPU/Communication/Communicator.h" +#include "VirtualFluids_GPU/Communication/MpiCommunicator.h" namespace EdgeNodeDebugWriter { @@ -54,7 +54,7 @@ void writeEdgeNodesXZ_Send(SPtr<Parameter> para) nodeCount++; } std::string filenameVec = para->getFName() + "_writeEdgeNodesXZ_Send_PID_" + - std::to_string(vf::gpu::Communicator::getInstance().getPID()) + "_" + + std::to_string(vf::gpu::MpiCommunicator::getInstance().getPID()) + "_" + StringUtil::toString<int>(level); WbWriterVtkXmlBinary::getInstance()->writeNodesWithNodeData(filenameVec, nodesVec, datanames, nodedata); @@ -90,7 +90,7 @@ void writeEdgeNodesXZ_Recv(SPtr<Parameter> para) nodeCount++; } std::string filenameVec = para->getFName() + "_writeEdgeNodesXZ_Recv_PID_" + - std::to_string(vf::gpu::Communicator::getInstance().getPID()) + "_" + + std::to_string(vf::gpu::MpiCommunicator::getInstance().getPID()) + "_" + StringUtil::toString<int>(level); WbWriterVtkXmlBinary::getInstance()->writeNodesWithNodeData(filenameVec, nodesVec, datanames, nodedata); diff --git a/src/gpu/VirtualFluids_GPU/Output/InterfaceDebugWriter.hpp b/src/gpu/VirtualFluids_GPU/Output/InterfaceDebugWriter.hpp index 705d86992d9eac39b66cf5033f7c32b5cb4fb602..4af9a50a123588e25f1ca9faa5c18581601f69d2 100644 --- a/src/gpu/VirtualFluids_GPU/Output/InterfaceDebugWriter.hpp +++ b/src/gpu/VirtualFluids_GPU/Output/InterfaceDebugWriter.hpp @@ -11,7 +11,7 @@ #include <basics/writer/WbWriterVtkXmlBinary.h> #include <cmath> -#include "VirtualFluids_GPU/Communication/Communicator.h" +#include "VirtualFluids_GPU/Communication/MpiCommunicator.h" namespace InterfaceDebugWriter { @@ -650,7 +650,7 @@ void writeInterfaceFCC_Send(Parameter *para) nodeCount++; } std::string filenameVec = para->getFName() + "_writeInterfaceFCC_Send_PID_" + - std::to_string(vf::gpu::Communicator::getInstance().getPID()) + "_" + + std::to_string(vf::gpu::MpiCommunicator::getInstance().getPID()) + "_" + StringUtil::toString<int>(level); WbWriterVtkXmlBinary::getInstance()->writeNodesWithNodeData(filenameVec, nodesVec, datanames, nodedata); @@ -703,7 +703,7 @@ void writeInterfaceCFC_Recv(Parameter *para) nodeCount++; } std::string filenameVec = para->getFName() + "_writeInterfaceCFC_Recv_PID_" + - std::to_string(vf::gpu::Communicator::getInstance().getPID()) + "_" + + std::to_string(vf::gpu::MpiCommunicator::getInstance().getPID()) + "_" + StringUtil::toString<int>(level); WbWriterVtkXmlBinary::getInstance()->writeNodesWithNodeData(filenameVec, nodesVec, datanames, nodedata); @@ -808,7 +808,7 @@ void writeSendNodesStream(Parameter *para) } } std::string filenameVec = para->getFName() + "_writeSendNodesStreams_PID_" + - std::to_string(vf::gpu::Communicator::getInstance().getPID()) + "_" + + std::to_string(vf::gpu::MpiCommunicator::getInstance().getPID()) + "_" + StringUtil::toString<int>(level); WbWriterVtkXmlBinary::getInstance()->writeNodesWithNodeData(filenameVec, nodesVec, datanames, nodedata); @@ -894,7 +894,7 @@ void writeRecvNodesStream(Parameter *para) // Recv are nodes ghost nodes and therefore they can't be coarse cells for the interpolation from coarse to fine std::string filenameVec = para->getFName() + "_writeRecvNodesStreams_PID_" + - std::to_string(vf::gpu::Communicator::getInstance().getPID()) + "_" + + std::to_string(vf::gpu::MpiCommunicator::getInstance().getPID()) + "_" + StringUtil::toString<int>(level); WbWriterVtkXmlBinary::getInstance()->writeNodesWithNodeData(filenameVec, nodesVec, datanames, nodedata); diff --git a/src/gpu/VirtualFluids_GPU/Output/Timer.cpp b/src/gpu/VirtualFluids_GPU/Output/Timer.cpp index 5a5e010944a776038416386267c3bf6477d47e9f..f6efff58440bd786d57a3ccb44d2271c29761323 100644 --- a/src/gpu/VirtualFluids_GPU/Output/Timer.cpp +++ b/src/gpu/VirtualFluids_GPU/Output/Timer.cpp @@ -52,7 +52,7 @@ void Timer::outputPerformance(uint t, Parameter* para, vf::gpu::Communicator& co // When using multiple GPUs, sum the nups of all processes if (communicator.getNumberOfProcess() > 1) { - double nupsSum = communicator.sumNups(fnups); + double nupsSum = communicator.reduceSum(fnups); if (communicator.getPID() == 0) VF_LOG_INFO("Sum of all {} processes: Nups in Mio: {:.1f}", communicator.getNumberOfProcess(), nupsSum); } diff --git a/src/gpu/VirtualFluids_GPU/Parameter/ParameterTest.cpp b/src/gpu/VirtualFluids_GPU/Parameter/ParameterTest.cpp index bdd22ab27669363ef7b52ac1b16854e5fa25a3c8..6073a4d6dd2aa7ea145bee7f6e0225f90d3ec171 100644 --- a/src/gpu/VirtualFluids_GPU/Parameter/ParameterTest.cpp +++ b/src/gpu/VirtualFluids_GPU/Parameter/ParameterTest.cpp @@ -250,6 +250,41 @@ TEST(ParameterTest, whenCreatingParameterClassWithGridRefinement_afterCallingIni EXPECT_THAT(para->getParH(1), testing::Ne(nullptr)); } +class MockCommunicator : public vf::gpu::Communicator +{ +public: + void waitAll() override {}; + int getPID() const override + { + return 0; + }; + int getNumberOfProcess() const override + { + return 1; + }; + void exchngData(float *sbuf_t, float *rbuf_t, float *sbuf_b, float *rbuf_b, int count) override {}; + ////////////////////////////////////////////////////////////////////////// + void exchngDataGPU(real *sbuf, int count_s, real *rbuf, int count_r, int nb_rank) override {}; + void nbRecvDataGPU(real *rbuf, int count_r, int nb_rank) override {}; + void nbSendDataGPU(real *sbuf, int count_s, int nb_rank) override {}; + void waitallGPU() override {}; + void sendDataGPU(real *sbuf, int count_s, int nb_rank) override {}; + void waitGPU(int id) override {}; + void resetRequest() override {}; + ////////////////////////////////////////////////////////////////////////// + int mapCudaDevice(const int &rank, const int &size, const std::vector<unsigned int> &devices, const int &maxdev) override + { + return 0; + }; + double reduceSum(double quantityPerProcess) override + { + return 0; + }; + ////////////////////////////////////////////////////////////////////////// + void receive_send(uint *buffer_receive, int size_buffer_recv, int neighbor_rank_recv, uint *buffer_send, int size_buffer_send, int neighbor_rank_send) const override {}; + +}; + TEST(ParameterTest, whenCreatingParameterClassWithGridRefinement_afterCallingSimulationConstructor_shouldNotThrow) { spdlog::set_level(spdlog::level::warn); // avoids logger spam in output @@ -258,7 +293,7 @@ TEST(ParameterTest, whenCreatingParameterClassWithGridRefinement_afterCallingSim para->setMaxLevel(2); SPtr<CudaMemoryManager> cudaMemoryManager = std::make_shared<CudaMemoryManager>(para); - vf::gpu::Communicator &communicator = vf::gpu::Communicator::getInstance(); + MockCommunicator communicator = MockCommunicator(); auto gridFactory = GridFactory::make(); auto gridBuilder = MultipleGridBuilder::makeShared(gridFactory); SPtr<GridProvider> gridGenerator =