From 8c732a94ca90c0e2f75e9154ce1c7161075694ca Mon Sep 17 00:00:00 2001
From: Anna Wellmann <a.wellmann@tu-braunschweig.de>
Date: Thu, 16 Dec 2021 14:08:19 +0100
Subject: [PATCH] Remove duplicate apps
---
apps/gpu/LBM/MusselOyster2x/CMakeLists.txt | 10 -
.../gpu/LBM/MusselOyster2x/MusselOyster2x.cpp | 416 ----------
.../LBM/MusselOyster2x/MusselOyster2xMPI.bat | 4 -
.../LBM/MusselOyster2x/configMusselOyster.txt | 36 -
apps/gpu/LBM/MusselOyster3z/CMakeLists.txt | 10 -
.../gpu/LBM/MusselOyster3z/MusselOyster3z.cpp | 419 ----------
.../LBM/MusselOyster3z/MusselOyster3zMPI.bat | 4 -
.../LBM/MusselOyster3z/configMusselOyster.txt | 36 -
.../gpu/LBM/MusselOysterOyster/CMakeLists.txt | 10 -
.../MusselOysterOyster/MusselOysterOyster.cpp | 728 -----------------
.../MusselOysterOyster/configPhoenix1GPU.txt | 36 -
.../LBM/MusselOysterOyster/configPhoenix8.txt | 42 -
apps/gpu/LBM/SphereScaling1/CMakeLists.txt | 10 -
.../gpu/LBM/SphereScaling1/SphereScaling1.cpp | 742 ------------------
apps/gpu/LBM/SphereScaling1/config.txt | 46 --
.../configPhoenix4GPU_2LevStrongStream.txt | 42 -
.../configPhoenix4GPU_2LevWeakStream.txt | 42 -
.../configPhoenix8GPU_2LevStrongOS.txt | 42 -
.../configPhoenix8GPU_2LevStrongStream.txt | 42 -
.../configPhoenix8GPU_2LevWeakOS.txt | 42 -
.../configPhoenix8GPU_2LevWeakStream.txt | 42 -
gpu.cmake | 4 -
22 files changed, 2805 deletions(-)
delete mode 100644 apps/gpu/LBM/MusselOyster2x/CMakeLists.txt
delete mode 100644 apps/gpu/LBM/MusselOyster2x/MusselOyster2x.cpp
delete mode 100644 apps/gpu/LBM/MusselOyster2x/MusselOyster2xMPI.bat
delete mode 100644 apps/gpu/LBM/MusselOyster2x/configMusselOyster.txt
delete mode 100644 apps/gpu/LBM/MusselOyster3z/CMakeLists.txt
delete mode 100644 apps/gpu/LBM/MusselOyster3z/MusselOyster3z.cpp
delete mode 100644 apps/gpu/LBM/MusselOyster3z/MusselOyster3zMPI.bat
delete mode 100644 apps/gpu/LBM/MusselOyster3z/configMusselOyster.txt
delete mode 100644 apps/gpu/LBM/MusselOysterOyster/CMakeLists.txt
delete mode 100644 apps/gpu/LBM/MusselOysterOyster/MusselOysterOyster.cpp
delete mode 100644 apps/gpu/LBM/MusselOysterOyster/configPhoenix1GPU.txt
delete mode 100644 apps/gpu/LBM/MusselOysterOyster/configPhoenix8.txt
delete mode 100644 apps/gpu/LBM/SphereScaling1/CMakeLists.txt
delete mode 100644 apps/gpu/LBM/SphereScaling1/SphereScaling1.cpp
delete mode 100644 apps/gpu/LBM/SphereScaling1/config.txt
delete mode 100644 apps/gpu/LBM/SphereScaling1/configPhoenix4GPU_2LevStrongStream.txt
delete mode 100644 apps/gpu/LBM/SphereScaling1/configPhoenix4GPU_2LevWeakStream.txt
delete mode 100644 apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevStrongOS.txt
delete mode 100644 apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevStrongStream.txt
delete mode 100644 apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevWeakOS.txt
delete mode 100644 apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevWeakStream.txt
diff --git a/apps/gpu/LBM/MusselOyster2x/CMakeLists.txt b/apps/gpu/LBM/MusselOyster2x/CMakeLists.txt
deleted file mode 100644
index 881b64c44..000000000
--- a/apps/gpu/LBM/MusselOyster2x/CMakeLists.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-PROJECT(MusselOyster LANGUAGES CUDA CXX)
-
-vf_add_library(BUILDTYPE binary PRIVATE_LINK basics VirtualFluids_GPU GridGenerator MPI::MPI_CXX FILES MusselOyster2x.cpp)
-
-set_source_files_properties(MusselOyster2x.cpp PROPERTIES LANGUAGE CUDA)
-
-set_target_properties(MusselOyster2x PROPERTIES
- CUDA_SEPARABLE_COMPILATION ON)
-# VS_DEBUGGER_COMMAND "C:/Program Files/Microsoft MPI/Bin/mpiexec.exe"
-# VS_DEBUGGER_COMMAND_ARGUMENTS "-n 2 \"$<TARGET_FILE:MusselOyster2x>\"")
\ No newline at end of file
diff --git a/apps/gpu/LBM/MusselOyster2x/MusselOyster2x.cpp b/apps/gpu/LBM/MusselOyster2x/MusselOyster2x.cpp
deleted file mode 100644
index c6e309f5e..000000000
--- a/apps/gpu/LBM/MusselOyster2x/MusselOyster2x.cpp
+++ /dev/null
@@ -1,416 +0,0 @@
-
-#define _USE_MATH_DEFINES
-#include <math.h>
-#include <string>
-#include <sstream>
-#include <iostream>
-#include <stdexcept>
-#include <fstream>
-#include <exception>
-#include <memory>
-
-#include "mpi.h"
-
-//////////////////////////////////////////////////////////////////////////
-
-#include "basics/Core/DataTypes.h"
-#include "basics/PointerDefinitions.h"
-#include "basics/Core/VectorTypes.h"
-
-#include "basics/Core/LbmOrGks.h"
-#include "basics/Core/StringUtilities/StringUtil.h"
-#include "basics/config/ConfigurationFile.h"
-#include "basics/Core/Logger/Logger.h"
-
-//////////////////////////////////////////////////////////////////////////
-
-#include "GridGenerator/grid/GridBuilder/LevelGridBuilder.h"
-#include "GridGenerator/grid/GridBuilder/MultipleGridBuilder.h"
-#include "GridGenerator/grid/BoundaryConditions/Side.h"
-#include "GridGenerator/grid/GridFactory.h"
-
-#include "geometries/Sphere/Sphere.h"
-#include "geometries/TriangularMesh/TriangularMesh.h"
-
-#include "GridGenerator/io/SimulationFileWriter/SimulationFileWriter.h"
-#include "GridGenerator/io/GridVTKWriter/GridVTKWriter.h"
-#include "GridGenerator/io/STLReaderWriter/STLReader.h"
-#include "GridGenerator/io/STLReaderWriter/STLWriter.h"
-
-//////////////////////////////////////////////////////////////////////////
-
-#include "VirtualFluids_GPU/LBM/Simulation.h"
-#include "VirtualFluids_GPU/Communication/Communicator.h"
-#include "VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h"
-#include "VirtualFluids_GPU/DataStructureInitializer/GridProvider.h"
-#include "VirtualFluids_GPU/DataStructureInitializer/GridReaderFiles/GridReader.h"
-#include "VirtualFluids_GPU/Parameter/Parameter.h"
-#include "VirtualFluids_GPU/Output/FileWriter.h"
-
-#include "VirtualFluids_GPU/Kernel/Utilities/KernelFactory/KernelFactoryImp.h"
-#include "VirtualFluids_GPU/PreProcessor/PreProcessorFactory/PreProcessorFactoryImp.h"
-
-#include "VirtualFluids_GPU/GPU/CudaMemoryManager.h"
-
-//////////////////////////////////////////////////////////////////////////
-
-#include "utilities/communication.h"
-
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-//
-// U s e r s e t t i n g s
-//
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-std::string outPath("E:/temp/MusselOysterResults");
-std::string gridPathParent = "E:/temp/GridMussel/";
-std::string stlPath("C:/Users/Master/Documents/MasterAnna/STL/");
-std::string simulationName("MusselOyster");
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-void multipleLevel(const std::string& configPath)
-{
- logging::Logger::addStream(&std::cout);
- logging::Logger::setDebugLevel(logging::Logger::Level::INFO_LOW);
- logging::Logger::timeStamp(logging::Logger::ENABLE);
- logging::Logger::enablePrintedRankNumbers(logging::Logger::ENABLE);
-
- auto gridFactory = GridFactory::make();
- gridFactory->setGridStrategy(Device::CPU);
- gridFactory->setTriangularMeshDiscretizationMethod(TriangularMeshDiscretizationMethod::POINT_IN_OBJECT);
-
- auto gridBuilder = MultipleGridBuilder::makeShared(gridFactory);
-
- vf::gpu::Communicator* comm = vf::gpu::Communicator::getInstanz();
- vf::basics::ConfigurationFile config;
- std::cout << configPath << std::endl;
- config.load(configPath);
- SPtr<Parameter> para = std::make_shared<Parameter>(config, comm->getNummberOfProcess(), comm->getPID());
-
-
-
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
- bool useGridGenerator = true;
- bool useMultiGPU = false;
- bool useStreams = false;
- bool useLevels = true;
- para->useReducedCommunicationAfterFtoC = true;
- para->setCalcTurbulenceIntensity(true);
-
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- std::string bivalveType = "MUSSEL"; // "MUSSEL" "OYSTER"
- std::string gridPath(gridPathParent + bivalveType); // only for GridGenerator, for GridReader the gridPath needs to be set in the config file
-
- real dxGrid = (real)1.0;
- real vxLB = (real)0.051; // LB units
- real Re = (real)300.0;
- real viscosityLB = (vxLB * dxGrid) / Re;
-
- para->setVelocity(vxLB);
- para->setViscosity(viscosityLB);
- para->setVelocityRatio((real) 58.82352941);
- para->setViscosityRatio((real) 0.058823529);
- para->setDensityRatio((real) 998.0);
-
- *logging::out << logging::Logger::INFO_HIGH << "velocity LB [dx/dt] = " << vxLB << " \n";
- *logging::out << logging::Logger::INFO_HIGH << "viscosity LB [dx^2/dt] = " << viscosityLB << "\n";
- *logging::out << logging::Logger::INFO_HIGH << "velocity real [m/s] = " << vxLB * para->getVelocityRatio()<< " \n";
- *logging::out << logging::Logger::INFO_HIGH << "viscosity real [m^2/s] = " << viscosityLB * para->getViscosityRatio() << "\n";
- *logging::out << logging::Logger::INFO_HIGH << "dxGrid = " << dxGrid << "\n";
- *logging::out << logging::Logger::INFO_HIGH << "useGridGenerator = " << useGridGenerator << "\n";
- *logging::out << logging::Logger::INFO_HIGH << "useMultiGPU = " << useMultiGPU << "\n";
- *logging::out << logging::Logger::INFO_HIGH << "useStreams = " << useStreams << "\n";
-
-
- //para->setTOut(1000);
- //para->setTEnd(10000);
-
- para->setCalcDragLift(false);
- para->setUseWale(false);
-
- if (para->getOutputPath().size() == 0) {
- para->setOutputPath(outPath);
- }
- para->setOutputPrefix(simulationName);
- para->setFName(para->getOutputPath() + "/" + para->getOutputPrefix());
- para->setPrintFiles(true);
-
- if (useLevels)
- para->setMaxLevel(2);
- else
- para->setMaxLevel(1);
-
-
- if (useStreams)
- para->setUseStreams();
- //para->setMainKernel("CumulantK17CompChim");
- para->setMainKernel("CumulantK17CompChimStream");
- *logging::out << logging::Logger::INFO_HIGH << "Kernel: " << para->getMainKernel() << "\n";
-
- //if (useMultiGPU) {
- // para->setDevices(std::vector<uint>{ (uint)0, (uint)1 });
- // para->setMaxDev(2);
- //} else
- // para->setDevices(std::vector<uint>{ (uint)0 });
-
-
-
- //////////////////////////////////////////////////////////////////////////
-
-
- if (useGridGenerator) {
- // bounding box mussel:
- const real bbxm = 0.0;
- const real bbxp = 76.0;
- const real bbym = 0.0;
- const real bbyp = 35.0;
- const real bbzm = 0.0;
- const real bbzp = 18.3;
- // bounding box oyster:
- // const real bbxm = 0.0;
- // const real bbxp = 115.0;
- // const real bbym = 0.0;
- // const real bbyp = 27.0;
- // const real bbzm = 0.0;
- // const real bbzp = 63.0;
-
- const real xGridMin = bbxm - 40.0;
- const real xGridMax = bbxp + 250.0;
- const real yGridMin = bbym + 1.0;
- const real yGridMax = bbyp + 60.0;
- const real zGridMin = bbzm - 30.0;
- const real zGridMax = bbzp + 30.0;
-
- TriangularMesh *bivalveSTL = TriangularMesh::make(stlPath + bivalveType + ".stl");
- TriangularMesh *bivalveRef_1_STL = nullptr;
- if (useLevels)
- bivalveRef_1_STL = TriangularMesh::make(stlPath + bivalveType + "_Level1.stl");
-
-
- if (useMultiGPU) {
- const uint generatePart = vf::gpu::Communicator::getInstanz()->getPID();
-
- real overlap = (real)8.0 * dxGrid;
- const real xSplit = round((bbxp + bbxm)*0.5);
-
- if (generatePart == 0) {
- gridBuilder->addCoarseGrid( xGridMin, yGridMin, zGridMin,
- xSplit+overlap, yGridMax, zGridMax, dxGrid);
- }
- if (generatePart == 1) {
- gridBuilder->addCoarseGrid(xSplit - overlap, yGridMin, zGridMin,
- xGridMax, yGridMax, zGridMax, dxGrid);
- }
-
- if (useLevels) {
- gridBuilder->setNumberOfLayers(6, 8);
- gridBuilder->addGrid(bivalveRef_1_STL, 1);
- }
-
- gridBuilder->addGeometry(bivalveSTL);
-
- if (generatePart == 0)
- gridBuilder->setSubDomainBox(std::make_shared<BoundingBox>(xGridMin, xSplit,
- yGridMin, yGridMax,
- zGridMin, zGridMax));
- if (generatePart == 1)
- gridBuilder->setSubDomainBox(std::make_shared<BoundingBox>(xSplit, xGridMax,
- yGridMin, yGridMax,
- zGridMin, zGridMax));
-
- gridBuilder->setPeriodicBoundaryCondition(false, false, true);
-
- gridBuilder->buildGrids(LBM, true); // buildGrids() has to be called before setting the BCs!!!!
-
- if (generatePart == 0) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::PX, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PX, 1);
- }
-
- if (generatePart == 1) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::MX, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MX, 0);
- }
-
- //////////////////////////////////////////////////////////////////////////
- if (generatePart == 0)
- gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
- if (generatePart == 1)
- gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MY, 0.0, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::GEOMETRY, 0.0, 0.0, 0.0);
- //////////////////////////////////////////////////////////////////////////
- if (para->getKernelNeedsFluidNodeIndicesToRun())
- gridBuilder->findFluidNodes(useStreams);
-
- //gridBuilder->writeGridsToVtk(outPath + "/" + bivalveType + "/grid/part" + std::to_string(generatePart) + "_");
- //gridBuilder->writeGridsToVtk(outPath + "/" + bivalveType + "/" + std::to_string(generatePart) + "/grid/");
- //gridBuilder->writeArrows(outPath + "/" + bivalveType + "/" + std::to_string(generatePart) + " /arrow");
-
- SimulationFileWriter::write(gridPath + "/" + std::to_string(generatePart) + "/", gridBuilder, FILEFORMAT::BINARY);
-
- } else {
-
- gridBuilder->addCoarseGrid(xGridMin, yGridMin, zGridMin, xGridMax, yGridMax, zGridMax, dxGrid);
-
- if (useLevels) {
- gridBuilder->setNumberOfLayers(6, 8);
- gridBuilder->addGrid(bivalveRef_1_STL, 1);
- }
-
- gridBuilder->addGeometry(bivalveSTL);
-
- gridBuilder->setPeriodicBoundaryCondition(false, false, true);
-
- gridBuilder->buildGrids(LBM, true); // buildGrids() has to be called before setting the BCs!!!!
-
- //////////////////////////////////////////////////////////////////////////
- gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MY, 0.0, 0.0, 0.0);
- gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
-
- gridBuilder->setVelocityBoundaryCondition(SideType::GEOMETRY, 0.0, 0.0, 0.0);
- //////////////////////////////////////////////////////////////////////////
- if (para->getKernelNeedsFluidNodeIndicesToRun())
- gridBuilder->findFluidNodes(useStreams);
-
- // gridBuilder->writeGridsToVtk("E:/temp/MusselOyster/" + bivalveType + "/grid/");
- // gridBuilder->writeArrows ("E:/temp/MusselOyster/" + bivalveType + "/arrow");
-
- SimulationFileWriter::write(gridPath, gridBuilder, FILEFORMAT::BINARY);
- }
-
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
- // const real velocityLB = velocity * dt / dx; // LB units
-
- // const real vx = velocityLB / (real)sqrt(2.0); // LB units
- // const real vy = velocityLB / (real)sqrt(2.0); // LB units
-
- // const real viscosityLB = nx * velocityLB / Re; // LB units
-
- //*logging::out << logging::Logger::INFO_HIGH << "velocity [dx/dt] = " << velocityLB << " \n";
- //*logging::out << logging::Logger::INFO_HIGH << "viscosity [dx^2/dt] = " << viscosityLB << "\n";
-
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-
- // para->setVelocity(velocityLB);
- // para->setViscosity(viscosityLB);
-
- // para->setVelocityRatio(velocity/ velocityLB);
-
- // para->setMainKernel("CumulantK17CompChim");
-
- // para->setInitialCondition([&](real coordX, real coordY, real coordZ, real &rho, real &vx, real &vy, real &vz)
- // {
- // rho = (real)0.0;
- // vx = (real)0.0; //(6 * velocityLB * coordZ * (L - coordZ) / (L * L));
- // vy = (real)0.0;
- // vz = (real)0.0;
- // });
-
-
-
- //return;
- }
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
- SPtr<CudaMemoryManager> cudaMemoryManager = CudaMemoryManager::make(para);
-
- SPtr<GridProvider> gridGenerator;
- if (useGridGenerator)
- gridGenerator = GridProvider::makeGridGenerator(gridBuilder, para, cudaMemoryManager);
- else {
- gridGenerator = GridProvider::makeGridReader(FILEFORMAT::BINARY, para, cudaMemoryManager);
- }
-
- Simulation sim;
- SPtr<FileWriter> fileWriter = SPtr<FileWriter>(new FileWriter());
- SPtr<KernelFactoryImp> kernelFactory = KernelFactoryImp::getInstance();
- SPtr<PreProcessorFactoryImp> preProcessorFactory = PreProcessorFactoryImp::getInstance();
- sim.setFactories(kernelFactory, preProcessorFactory);
- sim.init(para, gridGenerator, fileWriter, cudaMemoryManager);
- sim.run();
- sim.free();
-
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-}
-
-int main( int argc, char* argv[])
-{
- MPI_Init(&argc, &argv);
- std::string str, str2, configFile;
- if ( argv != NULL )
- {
- //str = static_cast<std::string>(argv[0]);
-
- try {
- //////////////////////////////////////////////////////////////////////////
-
- std::string targetPath;
-
- targetPath = __FILE__;
-
- if (argc == 2) {
- configFile = argv[1];
- std::cout << "Using configFile command line argument: " << configFile << std::endl;
- }
-
-#ifdef _WIN32
- targetPath = targetPath.substr(0, targetPath.find_last_of('\\') + 1);
-#else
- targetPath = targetPath.substr(0, targetPath.find_last_of('/') + 1);
-#endif
-
- std::cout << targetPath << std::endl;
-
- if (configFile.size() == 0) {
- configFile = targetPath + "configMusselOyster.txt";
- }
-
- multipleLevel(configFile);
-
- //////////////////////////////////////////////////////////////////////////
- }
- catch (const std::bad_alloc& e)
- {
- *logging::out << logging::Logger::LOGGER_ERROR << "Bad Alloc:" << e.what() << "\n";
- }
- catch (const std::exception& e)
- {
- *logging::out << logging::Logger::LOGGER_ERROR << e.what() << "\n";
- }
- catch (...)
- {
- *logging::out << logging::Logger::LOGGER_ERROR << "Unknown exception!\n";
- }
- }
-
- MPI_Finalize();
- return 0;
-}
diff --git a/apps/gpu/LBM/MusselOyster2x/MusselOyster2xMPI.bat b/apps/gpu/LBM/MusselOyster2x/MusselOyster2xMPI.bat
deleted file mode 100644
index f868c5ea7..000000000
--- a/apps/gpu/LBM/MusselOyster2x/MusselOyster2xMPI.bat
+++ /dev/null
@@ -1,4 +0,0 @@
-:: don't close cmd on error
-if not defined in_subprocess (cmd /k set in_subprocess=y ^& %0 %*) & exit )
-:: @ECHO OFF
-mpiexec -n 2 C:\Users\Master\Documents\MasterAnna\VirtualFluids_dev\build\bin\Release\MusselOyster2.exe
\ No newline at end of file
diff --git a/apps/gpu/LBM/MusselOyster2x/configMusselOyster.txt b/apps/gpu/LBM/MusselOyster2x/configMusselOyster.txt
deleted file mode 100644
index 1cc651641..000000000
--- a/apps/gpu/LBM/MusselOyster2x/configMusselOyster.txt
+++ /dev/null
@@ -1,36 +0,0 @@
-##################################################
-#GPU Mapping
-##################################################
-#Devices="0 1 2 3"
-#NumberOfDevices=4
-
-##################################################
-#informations for Writing
-##################################################
-Path=E:/temp/MusselOysterResults
-#Path="F:/Work/Computations/out/MusselOyster/"
-#Prefix="MusselOyster"
-#WriteGrid=true
-##################################################
-#informations for reading
-##################################################
-GridPath=E:/temp/GridMussel/MUSSEL
-#GridPath="C:"
-
-##################################################
-#number of grid levels
-##################################################
-#NOGL=1
-
-##################################################
-#LBM Version
-##################################################
-#D3Qxx=27
-#MainKernelName=CumulantK17CompChim
-
-##################################################
-#simulation parameter
-##################################################
-#TimeEnd=100000
-#TimeOut=1000
-#TimeStartOut=0
\ No newline at end of file
diff --git a/apps/gpu/LBM/MusselOyster3z/CMakeLists.txt b/apps/gpu/LBM/MusselOyster3z/CMakeLists.txt
deleted file mode 100644
index 329241bcf..000000000
--- a/apps/gpu/LBM/MusselOyster3z/CMakeLists.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-PROJECT(MusselOyster LANGUAGES CUDA CXX)
-
-vf_add_library(BUILDTYPE binary PRIVATE_LINK basics VirtualFluids_GPU GridGenerator MPI::MPI_CXX FILES MusselOyster3z.cpp)
-
-set_source_files_properties(MusselOyster3z.cpp PROPERTIES LANGUAGE CUDA)
-
-set_target_properties(MusselOyster3z PROPERTIES
- CUDA_SEPARABLE_COMPILATION ON
- VS_DEBUGGER_COMMAND "C:/Program Files/Microsoft MPI/Bin/mpiexec.exe"
- VS_DEBUGGER_COMMAND_ARGUMENTS "-n 2 \"$<TARGET_FILE:MusselOyster3z>\"")
\ No newline at end of file
diff --git a/apps/gpu/LBM/MusselOyster3z/MusselOyster3z.cpp b/apps/gpu/LBM/MusselOyster3z/MusselOyster3z.cpp
deleted file mode 100644
index 224896fb6..000000000
--- a/apps/gpu/LBM/MusselOyster3z/MusselOyster3z.cpp
+++ /dev/null
@@ -1,419 +0,0 @@
-
-#define _USE_MATH_DEFINES
-#include <math.h>
-#include <string>
-#include <sstream>
-#include <iostream>
-#include <stdexcept>
-#include <fstream>
-#include <exception>
-#include <memory>
-
-#include "mpi.h"
-
-//////////////////////////////////////////////////////////////////////////
-
-#include "basics/Core/DataTypes.h"
-#include "basics/PointerDefinitions.h"
-#include "basics/Core/VectorTypes.h"
-
-#include "basics/Core/LbmOrGks.h"
-#include "basics/Core/StringUtilities/StringUtil.h"
-#include "basics/config/ConfigurationFile.h"
-#include "basics/Core/Logger/Logger.h"
-
-//////////////////////////////////////////////////////////////////////////
-
-#include "GridGenerator/grid/GridBuilder/LevelGridBuilder.h"
-#include "GridGenerator/grid/GridBuilder/MultipleGridBuilder.h"
-#include "GridGenerator/grid/BoundaryConditions/Side.h"
-#include "GridGenerator/grid/GridFactory.h"
-
-#include "geometries/Sphere/Sphere.h"
-#include "geometries/TriangularMesh/TriangularMesh.h"
-
-#include "GridGenerator/io/SimulationFileWriter/SimulationFileWriter.h"
-#include "GridGenerator/io/GridVTKWriter/GridVTKWriter.h"
-#include "GridGenerator/io/STLReaderWriter/STLReader.h"
-#include "GridGenerator/io/STLReaderWriter/STLWriter.h"
-
-//////////////////////////////////////////////////////////////////////////
-
-#include "VirtualFluids_GPU/LBM/Simulation.h"
-#include "VirtualFluids_GPU/Communication/Communicator.h"
-#include "VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h"
-#include "VirtualFluids_GPU/DataStructureInitializer/GridProvider.h"
-#include "VirtualFluids_GPU/DataStructureInitializer/GridReaderFiles/GridReader.h"
-#include "VirtualFluids_GPU/Parameter/Parameter.h"
-#include "VirtualFluids_GPU/Output/FileWriter.h"
-
-#include "VirtualFluids_GPU/Kernel/Utilities/KernelFactory/KernelFactoryImp.h"
-#include "VirtualFluids_GPU/PreProcessor/PreProcessorFactory/PreProcessorFactoryImp.h"
-
-#include "VirtualFluids_GPU/GPU/CudaMemoryManager.h"
-
-//////////////////////////////////////////////////////////////////////////
-
-#include "utilities/communication.h"
-
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-//
-// U s e r s e t t i n g s
-//
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-std::string outPath("E:/temp/MusselOysterResults");
-std::string gridPathParent = "E:/temp/GridMussel/";
-std::string stlPath("C:/Users/Master/Documents/MasterAnna/STL/");
-std::string simulationName("MusselOyster");
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-void multipleLevel(const std::string& configPath)
-{
- logging::Logger::addStream(&std::cout);
- logging::Logger::setDebugLevel(logging::Logger::Level::INFO_LOW);
- logging::Logger::timeStamp(logging::Logger::ENABLE);
- logging::Logger::enablePrintedRankNumbers(logging::Logger::ENABLE);
-
- auto gridFactory = GridFactory::make();
- gridFactory->setGridStrategy(Device::CPU);
- gridFactory->setTriangularMeshDiscretizationMethod(TriangularMeshDiscretizationMethod::POINT_IN_OBJECT);
-
- auto gridBuilder = MultipleGridBuilder::makeShared(gridFactory);
-
- vf::gpu::Communicator* comm = vf::gpu::Communicator::getInstanz();
- vf::basics::ConfigurationFile config;
- std::cout << configPath << std::endl;
- config.load(configPath);
- SPtr<Parameter> para = std::make_shared<Parameter>(config, comm->getNummberOfProcess(), comm->getPID());
-
-
-
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
- bool useGridGenerator = true;
- bool useMultiGPU = true;
- bool useStreams = true;
- bool useLevels = true;
- para->useReducedCommunicationAfterFtoC = true;
- para->setCalcTurbulenceIntensity(true);
-
-
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- std::string bivalveType = "MUSSEL"; // "MUSSEL" "OYSTER"
- std::string gridPath(gridPathParent + bivalveType); // only for GridGenerator, for GridReader the gridPath needs to be set in the config file
-
- real dxGrid = (real)1.0;
- real vxLB = (real)0.051; // LB units
- real Re = (real)300.0;
- real viscosityLB = (vxLB * dxGrid) / Re;
-
- para->setVelocity(vxLB);
- para->setViscosity(viscosityLB);
- para->setVelocityRatio((real) 58.82352941);
- para->setViscosityRatio((real) 0.058823529);
- para->setDensityRatio((real) 998.0);
-
- *logging::out << logging::Logger::INFO_HIGH << "velocity LB [dx/dt] = " << vxLB << " \n";
- *logging::out << logging::Logger::INFO_HIGH << "viscosity LB [dx^2/dt] = " << viscosityLB << "\n";
- *logging::out << logging::Logger::INFO_HIGH << "velocity real [m/s] = " << vxLB * para->getVelocityRatio()<< " \n";
- *logging::out << logging::Logger::INFO_HIGH << "viscosity real [m^2/s] = " << viscosityLB * para->getViscosityRatio() << "\n";
- *logging::out << logging::Logger::INFO_HIGH << "dxGrid = " << dxGrid << "\n";
- *logging::out << logging::Logger::INFO_HIGH << "useGridGenerator = " << useGridGenerator << "\n";
- *logging::out << logging::Logger::INFO_HIGH << "useMultiGPU = " << useMultiGPU << "\n";
- *logging::out << logging::Logger::INFO_HIGH << "useStreams = " << useStreams << "\n";
-
- //para->setTOut(1000);
- //para->setTEnd(10000);
-
- para->setCalcDragLift(false);
- para->setUseWale(false);
-
- if (para->getOutputPath().size() == 0) {
- para->setOutputPath(outPath);
- }
- para->setOutputPrefix(simulationName);
- para->setFName(para->getOutputPath() + "/" + para->getOutputPrefix());
- para->setPrintFiles(true);
-
- if (useLevels)
- para->setMaxLevel(2);
- else
- para->setMaxLevel(1);
-
-
- if (useStreams)
- para->setUseStreams();
- //para->setMainKernel("CumulantK17CompChim");
- para->setMainKernel("CumulantK17CompChimStream");
- *logging::out << logging::Logger::INFO_HIGH << "Kernel: " << para->getMainKernel() << "\n";
-
- //if (useMultiGPU) {
- // para->setDevices(std::vector<uint>{ (uint)0, (uint)1 });
- // para->setMaxDev(2);
- //} else
- // para->setDevices(std::vector<uint>{ (uint)0 });
-
-
-
- //////////////////////////////////////////////////////////////////////////
-
-
- if (useGridGenerator) {
- // bounding box mussel:
- const real bbxm = 0.0;
- const real bbxp = 76.0;
- const real bbym = 0.0;
- const real bbyp = 35.0;
- const real bbzm = 0.0;
- const real bbzp = 18.3;
- // bounding box oyster:
- // const real bbxm = 0.0;
- // const real bbxp = 115.0;
- // const real bbym = 0.0;
- // const real bbyp = 27.0;
- // const real bbzm = 0.0;
- // const real bbzp = 63.0;
-
- const real xGridMin = bbxm - 40.0;
- const real xGridMax = bbxp + 250.0;
- const real yGridMin = bbym + 1.0;
- const real yGridMax = bbyp + 60.0;
- const real zGridMin = bbzm - 30.0;
- const real zGridMax = bbzp + 30.0;
-
- TriangularMesh *bivalveSTL = TriangularMesh::make(stlPath + bivalveType + ".stl");
- TriangularMesh *bivalveRef_1_STL = nullptr;
- if (useLevels)
- bivalveRef_1_STL = TriangularMesh::make(stlPath + bivalveType + "_Level1.stl");
-
-
- if (useMultiGPU) {
- const uint generatePart = vf::gpu::Communicator::getInstanz()->getPID();
-
- real overlap = (real)10.0 * dxGrid;
- const real zSplit = round(((double)bbzp + bbzm) * 0.5);
-
- if (generatePart == 0) {
- gridBuilder->addCoarseGrid( xGridMin, yGridMin, zGridMin,
- xGridMax, yGridMax, zSplit+overlap, dxGrid);
- }
- if (generatePart == 1) {
- gridBuilder->addCoarseGrid(xGridMin, yGridMin, zSplit-overlap,
- xGridMax, yGridMax, zGridMax, dxGrid);
- }
-
- if (useLevels) {
- gridBuilder->setNumberOfLayers(8, 10);
- gridBuilder->addGrid(bivalveRef_1_STL, 1);
- }
-
- gridBuilder->addGeometry(bivalveSTL);
-
- if (generatePart == 0)
- gridBuilder->setSubDomainBox(std::make_shared<BoundingBox>(xGridMin, xGridMax,
- yGridMin, yGridMax,
- zGridMin, zSplit));
- if (generatePart == 1)
- gridBuilder->setSubDomainBox(std::make_shared<BoundingBox>(xGridMin, xGridMax,
- yGridMin, yGridMax,
- zSplit, zGridMax));
-
- gridBuilder->setPeriodicBoundaryCondition(false, false, false);
-
- gridBuilder->buildGrids(LBM, true); // buildGrids() has to be called before setting the BCs!!!!
-
- if (generatePart == 0) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::PZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PZ, 1);
- }
-
- if (generatePart == 1) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::MZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MZ, 0);
- }
-
- //////////////////////////////////////////////////////////////////////////
- if (generatePart == 0)
- gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
- if (generatePart == 1)
- gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
- gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MY, 0.0, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::GEOMETRY, 0.0, 0.0, 0.0);
- //////////////////////////////////////////////////////////////////////////
- if (para->getKernelNeedsFluidNodeIndicesToRun())
- gridBuilder->findFluidNodes(useStreams);
-
- //gridBuilder->writeGridsToVtk(outPath + "/" + bivalveType + "/grid/part" + std::to_string(generatePart) + "_");
- //gridBuilder->writeGridsToVtk(outPath + "/" + bivalveType + "/" + std::to_string(generatePart) + "/grid/");
- //gridBuilder->writeArrows(outPath + "/" + bivalveType + "/" + std::to_string(generatePart) + " /arrow");
-
- SimulationFileWriter::write(gridPath + "/" + std::to_string(generatePart) + "/", gridBuilder, FILEFORMAT::BINARY);
-
- } else {
-
- gridBuilder->addCoarseGrid(xGridMin, yGridMin, zGridMin, xGridMax, yGridMax, zGridMax, dxGrid);
-
- if (useLevels) {
- gridBuilder->setNumberOfLayers(6, 8);
- gridBuilder->addGrid(bivalveRef_1_STL, 1);
- }
-
- gridBuilder->addGeometry(bivalveSTL);
-
- gridBuilder->setPeriodicBoundaryCondition(false, false, true);
-
- gridBuilder->buildGrids(LBM, true); // buildGrids() has to be called before setting the BCs!!!!
-
- //////////////////////////////////////////////////////////////////////////
- gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MY, 0.0, 0.0, 0.0);
- gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
-
- gridBuilder->setVelocityBoundaryCondition(SideType::GEOMETRY, 0.0, 0.0, 0.0);
- //////////////////////////////////////////////////////////////////////////
- if (para->getKernelNeedsFluidNodeIndicesToRun())
- gridBuilder->findFluidNodes(useStreams);
-
- // gridBuilder->writeGridsToVtk("E:/temp/MusselOyster/" + bivalveType + "/grid/");
- // gridBuilder->writeArrows ("E:/temp/MusselOyster/" + bivalveType + "/arrow");
-
- SimulationFileWriter::write(gridPath, gridBuilder, FILEFORMAT::BINARY);
- }
-
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
- // const real velocityLB = velocity * dt / dx; // LB units
-
- // const real vx = velocityLB / (real)sqrt(2.0); // LB units
- // const real vy = velocityLB / (real)sqrt(2.0); // LB units
-
- // const real viscosityLB = nx * velocityLB / Re; // LB units
-
- //*logging::out << logging::Logger::INFO_HIGH << "velocity [dx/dt] = " << velocityLB << " \n";
- //*logging::out << logging::Logger::INFO_HIGH << "viscosity [dx^2/dt] = " << viscosityLB << "\n";
-
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-
- // para->setVelocity(velocityLB);
- // para->setViscosity(viscosityLB);
-
- // para->setVelocityRatio(velocity/ velocityLB);
-
- // para->setMainKernel("CumulantK17CompChim");
-
- // para->setInitialCondition([&](real coordX, real coordY, real coordZ, real &rho, real &vx, real &vy, real &vz)
- // {
- // rho = (real)0.0;
- // vx = (real)0.0; //(6 * velocityLB * coordZ * (L - coordZ) / (L * L));
- // vy = (real)0.0;
- // vz = (real)0.0;
- // });
-
-
-
- //return;
- }
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
- SPtr<CudaMemoryManager> cudaMemoryManager = CudaMemoryManager::make(para);
-
- SPtr<GridProvider> gridGenerator;
- if (useGridGenerator)
- gridGenerator = GridProvider::makeGridGenerator(gridBuilder, para, cudaMemoryManager);
- else {
- gridGenerator = GridProvider::makeGridReader(FILEFORMAT::BINARY, para, cudaMemoryManager);
- }
-
- Simulation sim;
- SPtr<FileWriter> fileWriter = SPtr<FileWriter>(new FileWriter());
- SPtr<KernelFactoryImp> kernelFactory = KernelFactoryImp::getInstance();
- SPtr<PreProcessorFactoryImp> preProcessorFactory = PreProcessorFactoryImp::getInstance();
- sim.setFactories(kernelFactory, preProcessorFactory);
- sim.init(para, gridGenerator, fileWriter, cudaMemoryManager);
- sim.run();
- sim.free();
-
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-}
-
-int main( int argc, char* argv[])
-{
- MPI_Init(&argc, &argv);
- std::string str, str2, configFile;
- if ( argv != NULL )
- {
- //str = static_cast<std::string>(argv[0]);
-
- try
- {
- //////////////////////////////////////////////////////////////////////////
-
- std::string targetPath;
-
- targetPath = __FILE__;
-
- if (argc == 2) {
- configFile = argv[1];
- std::cout << "Using configFile command line argument: " << configFile << std::endl;
- }
-
-#ifdef _WIN32
- targetPath = targetPath.substr(0, targetPath.find_last_of('\\') + 1);
-#else
- targetPath = targetPath.substr(0, targetPath.find_last_of('/') + 1);
-#endif
-
- std::cout << targetPath << std::endl;
-
- if (configFile.size() == 0) {
- configFile = targetPath + "configMusselOyster.txt";
- }
-
- multipleLevel(configFile);
-
- //////////////////////////////////////////////////////////////////////////
- }
- catch (const std::bad_alloc& e)
- {
- *logging::out << logging::Logger::LOGGER_ERROR << "Bad Alloc:" << e.what() << "\n";
- }
- catch (const std::exception& e)
- {
- *logging::out << logging::Logger::LOGGER_ERROR << e.what() << "\n";
- }
- catch (...)
- {
- *logging::out << logging::Logger::LOGGER_ERROR << "Unknown exception!\n";
- }
- }
-
- MPI_Finalize();
- return 0;
-}
diff --git a/apps/gpu/LBM/MusselOyster3z/MusselOyster3zMPI.bat b/apps/gpu/LBM/MusselOyster3z/MusselOyster3zMPI.bat
deleted file mode 100644
index f868c5ea7..000000000
--- a/apps/gpu/LBM/MusselOyster3z/MusselOyster3zMPI.bat
+++ /dev/null
@@ -1,4 +0,0 @@
-:: don't close cmd on error
-if not defined in_subprocess (cmd /k set in_subprocess=y ^& %0 %*) & exit )
-:: @ECHO OFF
-mpiexec -n 2 C:\Users\Master\Documents\MasterAnna\VirtualFluids_dev\build\bin\Release\MusselOyster2.exe
\ No newline at end of file
diff --git a/apps/gpu/LBM/MusselOyster3z/configMusselOyster.txt b/apps/gpu/LBM/MusselOyster3z/configMusselOyster.txt
deleted file mode 100644
index 1cc651641..000000000
--- a/apps/gpu/LBM/MusselOyster3z/configMusselOyster.txt
+++ /dev/null
@@ -1,36 +0,0 @@
-##################################################
-#GPU Mapping
-##################################################
-#Devices="0 1 2 3"
-#NumberOfDevices=4
-
-##################################################
-#informations for Writing
-##################################################
-Path=E:/temp/MusselOysterResults
-#Path="F:/Work/Computations/out/MusselOyster/"
-#Prefix="MusselOyster"
-#WriteGrid=true
-##################################################
-#informations for reading
-##################################################
-GridPath=E:/temp/GridMussel/MUSSEL
-#GridPath="C:"
-
-##################################################
-#number of grid levels
-##################################################
-#NOGL=1
-
-##################################################
-#LBM Version
-##################################################
-#D3Qxx=27
-#MainKernelName=CumulantK17CompChim
-
-##################################################
-#simulation parameter
-##################################################
-#TimeEnd=100000
-#TimeOut=1000
-#TimeStartOut=0
\ No newline at end of file
diff --git a/apps/gpu/LBM/MusselOysterOyster/CMakeLists.txt b/apps/gpu/LBM/MusselOysterOyster/CMakeLists.txt
deleted file mode 100644
index 6966f6e48..000000000
--- a/apps/gpu/LBM/MusselOysterOyster/CMakeLists.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-PROJECT(MusselOysterOyster LANGUAGES CUDA CXX)
-
-vf_add_library(BUILDTYPE binary PRIVATE_LINK basics VirtualFluids_GPU GridGenerator MPI::MPI_CXX FILES MusselOysterOyster.cpp)
-
-set_source_files_properties(MusselOysterOyster.cpp PROPERTIES LANGUAGE CUDA)
-
-set_target_properties(MusselOysterOyster PROPERTIES
- CUDA_SEPARABLE_COMPILATION ON
- VS_DEBUGGER_COMMAND "C:/Program Files/Microsoft MPI/Bin/mpiexec.exe"
- VS_DEBUGGER_COMMAND_ARGUMENTS "-n 2 \"$<TARGET_FILE:MusselOysterOyster>\"")
\ No newline at end of file
diff --git a/apps/gpu/LBM/MusselOysterOyster/MusselOysterOyster.cpp b/apps/gpu/LBM/MusselOysterOyster/MusselOysterOyster.cpp
deleted file mode 100644
index 477bae807..000000000
--- a/apps/gpu/LBM/MusselOysterOyster/MusselOysterOyster.cpp
+++ /dev/null
@@ -1,728 +0,0 @@
-
-#define _USE_MATH_DEFINES
-#include <math.h>
-#include <string>
-#include <sstream>
-#include <iostream>
-#include <stdexcept>
-#include <fstream>
-#include <exception>
-#include <memory>
-
-#include "mpi.h"
-
-//////////////////////////////////////////////////////////////////////////
-
-#include "basics/Core/DataTypes.h"
-#include "basics/PointerDefinitions.h"
-#include "basics/Core/VectorTypes.h"
-
-#include "basics/Core/LbmOrGks.h"
-#include "basics/Core/StringUtilities/StringUtil.h"
-#include "basics/config/ConfigurationFile.h"
-#include "basics/Core/Logger/Logger.h"
-
-//////////////////////////////////////////////////////////////////////////
-
-#include "GridGenerator/grid/GridBuilder/LevelGridBuilder.h"
-#include "GridGenerator/grid/GridBuilder/MultipleGridBuilder.h"
-#include "GridGenerator/grid/BoundaryConditions/Side.h"
-#include "GridGenerator/grid/GridFactory.h"
-
-#include "geometries/Sphere/Sphere.h"
-#include "geometries/TriangularMesh/TriangularMesh.h"
-
-#include "GridGenerator/io/SimulationFileWriter/SimulationFileWriter.h"
-#include "GridGenerator/io/GridVTKWriter/GridVTKWriter.h"
-#include "GridGenerator/io/STLReaderWriter/STLReader.h"
-#include "GridGenerator/io/STLReaderWriter/STLWriter.h"
-
-//////////////////////////////////////////////////////////////////////////
-
-#include "VirtualFluids_GPU/LBM/Simulation.h"
-#include "VirtualFluids_GPU/Communication/Communicator.h"
-#include "VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h"
-#include "VirtualFluids_GPU/DataStructureInitializer/GridProvider.h"
-#include "VirtualFluids_GPU/DataStructureInitializer/GridReaderFiles/GridReader.h"
-#include "VirtualFluids_GPU/Parameter/Parameter.h"
-#include "VirtualFluids_GPU/Output/FileWriter.h"
-
-#include "VirtualFluids_GPU/Kernel/Utilities/KernelFactory/KernelFactoryImp.h"
-#include "VirtualFluids_GPU/PreProcessor/PreProcessorFactory/PreProcessorFactoryImp.h"
-
-#include "VirtualFluids_GPU/GPU/CudaMemoryManager.h"
-
-//////////////////////////////////////////////////////////////////////////
-
-#include "utilities/communication.h"
-
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-//
-// U s e r s e t t i n g s
-//
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-// Tesla 03
-// std::string outPath("E:/temp/MusselOysterResults/");
-// std::string gridPathParent = "E:/temp/GridMussel/";
-// std::string stlPath("C:/Users/Master/Documents/MasterAnna/STL/");
-// std::string simulationName("MusselOyster");
-
-// Phoenix
-std::string outPath("/work/y0078217/Results/MusselOysterResults/");
-std::string gridPathParent = "/work/y0078217/Grids/GridMusselOyster/";
-std::string stlPath("/home/y0078217/STL/MusselOyster/");
-std::string simulationName("MusselOyster");
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-void multipleLevel(const std::string& configPath)
-{
- logging::Logger::addStream(&std::cout);
- logging::Logger::setDebugLevel(logging::Logger::Level::INFO_LOW);
- logging::Logger::timeStamp(logging::Logger::ENABLE);
- logging::Logger::enablePrintedRankNumbers(logging::Logger::ENABLE);
-
- auto gridFactory = GridFactory::make();
- gridFactory->setGridStrategy(Device::CPU);
- gridFactory->setTriangularMeshDiscretizationMethod(TriangularMeshDiscretizationMethod::POINT_IN_OBJECT);
-
- auto gridBuilder = MultipleGridBuilder::makeShared(gridFactory);
-
- vf::gpu::Communicator* comm = vf::gpu::Communicator::getInstanz();
- vf::basics::ConfigurationFile config;
- std::cout << configPath << std::endl;
- config.load(configPath);
- SPtr<Parameter> para = std::make_shared<Parameter>(config, comm->getNummberOfProcess(), comm->getPID());
-
-
-
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
- bool useGridGenerator = true;
- bool useStreams = true;
- bool useLevels = true;
- para->useReducedCommunicationAfterFtoC = true;
- para->setCalcTurbulenceIntensity(true);
-
- if (para->getNumprocs() == 1) {
- useStreams = false;
- para->useReducedCommunicationAfterFtoC = false;
- }
-
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- std::string bivalveType = "OYSTER"; // "MUSSEL" "OYSTER"
- std::string gridPath(gridPathParent + bivalveType); // only for GridGenerator, for GridReader the gridPath needs to be set in the config file
-
- // real dxGrid = (real)2.0; // 2.0
- real dxGrid = (real)1.0; // 1.0
- if (para->getNumprocs() == 8)
- dxGrid = 0.5;
- real vxLB = (real)0.051; // LB units
- real Re = (real)300.0;
-
- real heightBivalve;
- if (bivalveType == "MUSSEL")
- heightBivalve = (real)35.0;
- else if (bivalveType == "OYSTER")
- heightBivalve = (real)72.0;
- else
- std::cerr << "Error: unknown bivalveType" << std::endl;
- real length = 1.0 / dxGrid; // heightBivalve / dxGrid
- real viscosityLB = (vxLB * length) / Re;
-
- para->setVelocity(vxLB);
- para->setViscosity(viscosityLB);
- para->setVelocityRatio((real) 58.82352941);
- para->setViscosityRatio((real) 0.058823529);
- para->setDensityRatio((real) 998.0);
-
- *logging::out << logging::Logger::INFO_HIGH << "bivalveType = " << bivalveType << " \n";
- *logging::out << logging::Logger::INFO_HIGH << "velocity LB [dx/dt] = " << vxLB << " \n";
- *logging::out << logging::Logger::INFO_HIGH << "viscosity LB [dx^2/dt] = " << viscosityLB << "\n";
- *logging::out << logging::Logger::INFO_HIGH << "velocity real [m/s] = " << vxLB * para->getVelocityRatio()<< " \n";
- *logging::out << logging::Logger::INFO_HIGH << "viscosity real [m^2/s] = " << viscosityLB * para->getViscosityRatio() << "\n";
- *logging::out << logging::Logger::INFO_HIGH << "dxGrid = " << dxGrid << "\n";
- *logging::out << logging::Logger::INFO_HIGH << "useGridGenerator = " << useGridGenerator << "\n";
- *logging::out << logging::Logger::INFO_HIGH << "useStreams = " << useStreams << "\n";
- *logging::out << logging::Logger::INFO_HIGH << "number of processes = " << para->getNumprocs() << "\n";
-
-
- // para->setTOut(1000);
- // para->setTEnd(10000);
-
- para->setCalcDragLift(false);
- para->setUseWale(false);
-
- if (para->getOutputPath().size() == 0) {
- para->setOutputPath(outPath);
- }
- para->setOutputPrefix(simulationName);
- para->setFName(para->getOutputPath() + para->getOutputPrefix());
- para->setPrintFiles(true);
- std::cout << "Write result files to " << para->getFName() << std::endl;
-
-
- if (useLevels)
- para->setMaxLevel(2);
- else
- para->setMaxLevel(1);
-
-
- if (useStreams)
- para->setUseStreams();
- // para->setMainKernel("CumulantK17CompChim");
- para->setMainKernel("CumulantK17CompChimStream");
- *logging::out << logging::Logger::INFO_HIGH << "Kernel: " << para->getMainKernel() << "\n";
-
- // if (para->getNumprocs() > 1) {
- // para->setDevices(std::vector<uint>{ (uint)0, (uint)1 });
- // para->setMaxDev(2);
- // } else
- // para->setDevices(std::vector<uint>{ (uint)0 });
-
-
-
- //////////////////////////////////////////////////////////////////////////
-
-
- if (useGridGenerator) {
- real bbzm;
- real bbzp;
- if (bivalveType == "MUSSEL")
- bbzp = 9.0;
- if (bivalveType == "OYSTER")
- bbzp = 13.0;
- bbzm = -bbzp;
- // bounding box mussel:
- // const real bbxm = 0.0;
- // const real bbxp = 76.0;
- // const real bbym = 0.0;
- // const real bbyp = 35.0;
- // const real bbzm = -9.15;
- // const real bbzp = 9.15;
- // bounding box oyster:
- // const real bbxm = 0.0;
- // const real bbxp = 102.0;
- // const real bbym = 0.0;
- // const real bbyp = 72.0;
- // const real bbzm = -13.0;
- // const real bbzp = 13.0;
-
- const real xGridMin = -100.0; // -100.0;
- const real xGridMax = 470.0; // alt 540.0 // neu 440 // mit groesserem Level 1 470
- const real yGridMin = 1.0; // 1.0;
- const real yGridMax = 350.0; // alt 440.0; // neu 350
- const real zGridMin = -85; // -85;
- const real zGridMax = 85.0; // 85;
-
- TriangularMesh *bivalveSTL = TriangularMesh::make(stlPath + bivalveType + ".stl");
- TriangularMesh *bivalveRef_1_STL = nullptr;
- if (useLevels)
- bivalveRef_1_STL = TriangularMesh::make(stlPath + bivalveType + "_Level1.stl");
-
-
- if (para->getNumprocs() > 1) {
- const uint generatePart = vf::gpu::Communicator::getInstanz()->getPID();
-
- real overlap = (real)8.0 * dxGrid;
- gridBuilder->setNumberOfLayers(10, 8);
-
- if (comm->getNummberOfProcess() == 2) {
- const real zSplit = 0.0; //round(((double)bbzp + bbzm) * 0.5);
-
- if (generatePart == 0) {
- gridBuilder->addCoarseGrid( xGridMin, yGridMin, zGridMin,
- xGridMax, yGridMax, zSplit+overlap, dxGrid);
- }
- if (generatePart == 1) {
- gridBuilder->addCoarseGrid( xGridMin, yGridMin, zSplit-overlap,
- xGridMax, yGridMax, zGridMax, dxGrid);
- }
-
-
- if (useLevels) {
- gridBuilder->addGrid(bivalveRef_1_STL, 1);
- }
-
- gridBuilder->addGeometry(bivalveSTL);
-
- if (generatePart == 0){
- gridBuilder->setSubDomainBox(std::make_shared<BoundingBox>(xGridMin, xGridMax,
- yGridMin, yGridMax,
- zGridMin, zSplit));
- }
- if (generatePart == 1){
- gridBuilder->setSubDomainBox(std::make_shared<BoundingBox>(xGridMin, xGridMax,
- yGridMin, yGridMax,
- zSplit, zGridMax));
- }
-
- gridBuilder->buildGrids(LBM, true); // buildGrids() has to be called before setting the BCs!!!!
-
- if (generatePart == 0) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::PZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PZ, 1);
- }
-
- if (generatePart == 1) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::MZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MZ, 0);
- }
-
- gridBuilder->setPeriodicBoundaryCondition(false, false, false);
- //////////////////////////////////////////////////////////////////////////
- if (generatePart == 0)
- gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
- if (generatePart == 1)
- gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MY, 0.0, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::GEOMETRY, 0.0, 0.0, 0.0);
- gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0); // set pressure BC after velocity BCs
- //////////////////////////////////////////////////////////////////////////
- } else if (comm->getNummberOfProcess() == 4) {
-
- const real xSplit = 100.0;
- const real zSplit = 0.0;
-
- if (generatePart == 0) {
- gridBuilder->addCoarseGrid(xGridMin, yGridMin, zGridMin, xSplit + overlap, yGridMax,
- zSplit + overlap, dxGrid);
- }
- if (generatePart == 1) {
- gridBuilder->addCoarseGrid(xSplit - overlap, yGridMin, zGridMin, xGridMax, yGridMax,
- zSplit+overlap, dxGrid);
- }
- if (generatePart == 2) {
- gridBuilder->addCoarseGrid(xGridMin, yGridMin, zSplit-overlap, xSplit + overlap, yGridMax,
- zGridMax, dxGrid);
- }
- if (generatePart == 3) {
- gridBuilder->addCoarseGrid(xSplit - overlap, yGridMin, zSplit-overlap, xGridMax, yGridMax,
- zGridMax, dxGrid);
- }
-
- if (useLevels) {
- gridBuilder->addGrid(bivalveRef_1_STL, 1);
- }
-
- gridBuilder->addGeometry(bivalveSTL);
-
- if (generatePart == 0)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xGridMin, xSplit, yGridMin, yGridMax, zGridMin, zSplit));
- if (generatePart == 1)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xSplit, xGridMax, yGridMin, yGridMax, zGridMin, zSplit));
- if (generatePart == 2)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xGridMin, xSplit, yGridMin, yGridMax, zSplit, zGridMax));
- if (generatePart == 3)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xSplit, xGridMax, yGridMin, yGridMax, zSplit, zGridMax));
-
- gridBuilder->buildGrids(LBM, true); // buildGrids() has to be called before setting the BCs!!!!
-
- if (generatePart == 0) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::PX, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PX, 1);
- gridBuilder->findCommunicationIndices(CommunicationDirections::PZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PZ, 2);
- }
- if (generatePart == 1) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::MX, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MX, 0);
- gridBuilder->findCommunicationIndices(CommunicationDirections::PZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PZ, 3);
- }
- if (generatePart == 2) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::PX, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PX, 3);
- gridBuilder->findCommunicationIndices(CommunicationDirections::MZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MZ, 0);
- }
- if (generatePart == 3) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::MX, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MX, 2);
- gridBuilder->findCommunicationIndices(CommunicationDirections::MZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MZ, 1);
- }
-
- gridBuilder->setPeriodicBoundaryCondition(false, false, false);
- //////////////////////////////////////////////////////////////////////////
- if (generatePart == 0) {
- gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
- }
- if (generatePart == 2) {
- gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
- }
- gridBuilder->setVelocityBoundaryCondition(SideType::GEOMETRY, 0.0, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MY, 0.0, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
- if (generatePart == 3) {
- gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
- gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0); // set pressure BC after velocity BCs
- }
- if (generatePart == 1) {
- gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
- gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0); // set pressure BC after velocity BCs
- }
- //////////////////////////////////////////////////////////////////////////
- } else if (comm->getNummberOfProcess() == 8) {
- real xSplit = 140.0; // 100.0 // mit groesserem Level 1 140.0
- real ySplit = 32.0; // 32.0
- real zSplit = 0.0;
-
- if (generatePart == 0) {
- gridBuilder->addCoarseGrid(xGridMin, yGridMin, zGridMin, xSplit + overlap, ySplit + overlap,
- zSplit + overlap, dxGrid);
- }
- if (generatePart == 1) {
- gridBuilder->addCoarseGrid(xGridMin, ySplit - overlap, zGridMin, xSplit + overlap, yGridMax,
- zSplit + overlap, dxGrid);
- }
- if (generatePart == 2) {
- gridBuilder->addCoarseGrid(xSplit - overlap, yGridMin, zGridMin, xGridMax, ySplit + overlap,
- zSplit + overlap, dxGrid);
- }
- if (generatePart == 3) {
- gridBuilder->addCoarseGrid(xSplit - overlap, ySplit - overlap, zGridMin, xGridMax, yGridMax,
- zSplit + overlap, dxGrid);
- }
- if (generatePart == 4) {
- gridBuilder->addCoarseGrid(xGridMin, yGridMin, zSplit - overlap, xSplit + overlap, ySplit + overlap,
- zGridMax, dxGrid);
- }
- if (generatePart == 5) {
- gridBuilder->addCoarseGrid(xGridMin, ySplit - overlap, zSplit - overlap, xSplit + overlap, yGridMax,
- zGridMax, dxGrid);
- }
- if (generatePart == 6) {
- gridBuilder->addCoarseGrid(xSplit - overlap, yGridMin, zSplit - overlap, xGridMax, ySplit + overlap,
- zGridMax, dxGrid);
- }
- if (generatePart == 7) {
- gridBuilder->addCoarseGrid(xSplit - overlap, ySplit - overlap, zSplit - overlap, xGridMax, yGridMax,
- zGridMax, dxGrid);
- }
-
- if (useLevels) {
- gridBuilder->addGrid(bivalveRef_1_STL, 1);
- }
-
- gridBuilder->addGeometry(bivalveSTL);
-
- if (generatePart == 0)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xGridMin, xSplit, yGridMin, ySplit, zGridMin, zSplit));
- if (generatePart == 1)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xGridMin, xSplit, ySplit, yGridMax, zGridMin, zSplit));
- if (generatePart == 2)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xSplit, xGridMax, yGridMin, ySplit, zGridMin, zSplit));
- if (generatePart == 3)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xSplit, xGridMax, ySplit, yGridMax, zGridMin, zSplit));
- if (generatePart == 4)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xGridMin, xSplit, yGridMin, ySplit, zSplit, zGridMax));
- if (generatePart == 5)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xGridMin, xSplit, ySplit, yGridMax, zSplit, zGridMax));
- if (generatePart == 6)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xSplit, xGridMax, yGridMin, ySplit, zSplit, zGridMax));
- if (generatePart == 7)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xSplit, xGridMax, ySplit, yGridMax, zSplit, zGridMax));
-
- gridBuilder->buildGrids(LBM, true); // buildGrids() has to be called before setting the BCs!!!!
- gridBuilder->setPeriodicBoundaryCondition(false, false, false);
-
- if (generatePart == 0) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::PY, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PY, 1);
- gridBuilder->findCommunicationIndices(CommunicationDirections::PX, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PX, 2);
- gridBuilder->findCommunicationIndices(CommunicationDirections::PZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PZ, 4);
- }
- if (generatePart == 1) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::MY, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MY, 0);
- gridBuilder->findCommunicationIndices(CommunicationDirections::PX, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PX, 3);
- gridBuilder->findCommunicationIndices(CommunicationDirections::PZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PZ, 5);
- }
- if (generatePart == 2) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::PY, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PY, 3);
- gridBuilder->findCommunicationIndices(CommunicationDirections::MX, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MX, 0);
- gridBuilder->findCommunicationIndices(CommunicationDirections::PZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PZ, 6);
- }
- if (generatePart == 3) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::MY, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MY, 2);
- gridBuilder->findCommunicationIndices(CommunicationDirections::MX, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MX, 1);
- gridBuilder->findCommunicationIndices(CommunicationDirections::PZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PZ, 7);
- }
- if (generatePart == 4) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::PY, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PY, 5);
- gridBuilder->findCommunicationIndices(CommunicationDirections::PX, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PX, 6);
- gridBuilder->findCommunicationIndices(CommunicationDirections::MZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MZ, 0);
- }
- if (generatePart == 5) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::MY, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MY, 4);
- gridBuilder->findCommunicationIndices(CommunicationDirections::PX, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PX, 7);
- gridBuilder->findCommunicationIndices(CommunicationDirections::MZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MZ, 1);
- }
- if (generatePart == 6) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::PY, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PY, 7);
- gridBuilder->findCommunicationIndices(CommunicationDirections::MX, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MX, 4);
- gridBuilder->findCommunicationIndices(CommunicationDirections::MZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MZ, 2);
- }
- if (generatePart == 7) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::MY, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MY, 6);
- gridBuilder->findCommunicationIndices(CommunicationDirections::MX, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MX, 5);
- gridBuilder->findCommunicationIndices(CommunicationDirections::MZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MZ, 3);
- }
-
- //////////////////////////////////////////////////////////////////////////
- if (generatePart == 0) {
- gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MY, 0.0, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
- }
- if (generatePart == 1) {
- gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
- }
- if (generatePart == 2) {
- gridBuilder->setVelocityBoundaryCondition(SideType::MY, 0.0, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
- gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0);
- }
- if (generatePart == 3) {
- gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
- gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0);
- }
- if (generatePart == 4) {
- gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MY, 0.0, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
- }
- if (generatePart == 5) {
- gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
- }
- if (generatePart == 6) {
- gridBuilder->setVelocityBoundaryCondition(SideType::MY, 0.0, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
- gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0); // set pressure BC after velocity BCs
- }
- if (generatePart == 7) {
- gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
- gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0); // set pressure BC after velocity BCs
- }
- // gridBuilder->setVelocityBoundaryCondition(SideType::GEOMETRY, 0.0, 0.0, 0.0);
- //////////////////////////////////////////////////////////////////////////
- }
- if (para->getKernelNeedsFluidNodeIndicesToRun())
- gridBuilder->findFluidNodes(useStreams);
-
- // gridBuilder->writeGridsToVtk(outPath + bivalveType + "/grid/part" + std::to_string(generatePart) + "_");
- // gridBuilder->writeGridsToVtk(outPath + bivalveType + "/" + std::to_string(generatePart) + "/grid/");
- // gridBuilder->writeArrows(outPath + bivalveType + "/" + std::to_string(generatePart) + " /arrow");
-
- SimulationFileWriter::write(gridPath + std::to_string(generatePart) + "/", gridBuilder,
- FILEFORMAT::BINARY);
- } else {
-
- gridBuilder->addCoarseGrid(xGridMin, yGridMin, zGridMin, xGridMax, yGridMax, zGridMax, dxGrid);
-
- if (useLevels) {
- gridBuilder->setNumberOfLayers(10, 8);
- gridBuilder->addGrid(bivalveRef_1_STL, 1);
- }
-
- gridBuilder->addGeometry(bivalveSTL);
-
- gridBuilder->buildGrids(LBM, true); // buildGrids() has to be called before setting the BCs!!!!
-
- gridBuilder->setPeriodicBoundaryCondition(false, false, false);
- //////////////////////////////////////////////////////////////////////////
- gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MY, 0.0, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::GEOMETRY, 0.0, 0.0, 0.0);
- gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0); // set pressure BC after velocity BCs
-
- //////////////////////////////////////////////////////////////////////////
- if (para->getKernelNeedsFluidNodeIndicesToRun())
- gridBuilder->findFluidNodes(useStreams);
-
- // gridBuilder->writeGridsToVtk(outPath + bivalveType + "/grid/");
- // gridBuilder->writeArrows ((outPath + bivalveType + "/arrow");
-
- SimulationFileWriter::write(gridPath, gridBuilder, FILEFORMAT::BINARY);
- }
-
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
- // const real velocityLB = velocity * dt / dx; // LB units
-
- // const real vx = velocityLB / (real)sqrt(2.0); // LB units
- // const real vy = velocityLB / (real)sqrt(2.0); // LB units
-
- // const real viscosityLB = nx * velocityLB / Re; // LB units
-
- //*logging::out << logging::Logger::INFO_HIGH << "velocity [dx/dt] = " << velocityLB << " \n";
- //*logging::out << logging::Logger::INFO_HIGH << "viscosity [dx^2/dt] = " << viscosityLB << "\n";
-
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-
- // para->setVelocity(velocityLB);
- // para->setViscosity(viscosityLB);
-
- // para->setVelocityRatio(velocity/ velocityLB);
-
- // para->setMainKernel("CumulantK17CompChim");
-
- // para->setInitialCondition([&](real coordX, real coordY, real coordZ, real &rho, real &vx, real &vy, real &vz)
- // {
- // rho = (real)0.0;
- // vx = (real)0.0; //(6 * velocityLB * coordZ * (L - coordZ) / (L * L));
- // vy = (real)0.0;
- // vz = (real)0.0;
- // });
-
-
-
- //return;
- }
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
- SPtr<CudaMemoryManager> cudaMemoryManager = CudaMemoryManager::make(para);
-
- SPtr<GridProvider> gridGenerator;
- if (useGridGenerator)
- gridGenerator = GridProvider::makeGridGenerator(gridBuilder, para, cudaMemoryManager);
- else {
- gridGenerator = GridProvider::makeGridReader(FILEFORMAT::BINARY, para, cudaMemoryManager);
- }
-
- Simulation sim;
- SPtr<FileWriter> fileWriter = SPtr<FileWriter>(new FileWriter());
- SPtr<KernelFactoryImp> kernelFactory = KernelFactoryImp::getInstance();
- SPtr<PreProcessorFactoryImp> preProcessorFactory = PreProcessorFactoryImp::getInstance();
- sim.setFactories(kernelFactory, preProcessorFactory);
- sim.init(para, gridGenerator, fileWriter, cudaMemoryManager);
- sim.run();
- sim.free();
-
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-}
-
-int main( int argc, char* argv[])
-{
- MPI_Init(&argc, &argv);
- std::string str, str2, configFile;
-
- if ( argv != NULL )
- {
- //str = static_cast<std::string>(argv[0]);
-
- try {
- //////////////////////////////////////////////////////////////////////////
-
- std::string targetPath;
-
- targetPath = __FILE__;
-
- if (argc == 2) {
- configFile = argv[1];
- std::cout << "Using configFile command line argument: " << configFile << std::endl;
- }
-
-#ifdef _WIN32
- targetPath = targetPath.substr(0, targetPath.find_last_of('\\') + 1);
-#else
- targetPath = targetPath.substr(0, targetPath.find_last_of('/') + 1);
-#endif
-
- std::cout << targetPath << std::endl;
-
- if (configFile.size() == 0) {
- configFile = targetPath + "configMusselOyster.txt";
- }
-
- multipleLevel(configFile);
-
- //////////////////////////////////////////////////////////////////////////
- }
- catch (const std::bad_alloc& e)
- {
- *logging::out << logging::Logger::LOGGER_ERROR << "Bad Alloc:" << e.what() << "\n";
- }
- catch (const std::exception& e)
- {
- *logging::out << logging::Logger::LOGGER_ERROR << e.what() << "\n";
- }
- catch (...)
- {
- *logging::out << logging::Logger::LOGGER_ERROR << "Unknown exception!\n";
- }
- }
-
- MPI_Finalize();
- return 0;
-}
diff --git a/apps/gpu/LBM/MusselOysterOyster/configPhoenix1GPU.txt b/apps/gpu/LBM/MusselOysterOyster/configPhoenix1GPU.txt
deleted file mode 100644
index 09d92d28c..000000000
--- a/apps/gpu/LBM/MusselOysterOyster/configPhoenix1GPU.txt
+++ /dev/null
@@ -1,36 +0,0 @@
-##################################################
-#GPU Mapping
-##################################################
-Devices="0"
-NumberOfDevices=1
-
-##################################################
-#informations for Writing
-##################################################
-Path=/work/y0078217/Results/MusselOysterResults/1GPUOyster1/
-#Path="F:/Work/Computations/out/MusselOyster/"
-#Prefix="MusselOyster"
-#WriteGrid=true
-##################################################
-#informations for reading
-##################################################
-GridPath=/work/y0078217/Grids/GridMusselOyster/Oyster1GPU/
-#GridPath="C:"
-
-##################################################
-#number of grid levels
-##################################################
-#NOGL=1
-
-##################################################
-#LBM Version
-##################################################
-#D3Qxx=27
-#MainKernelName=CumulantK17CompChim
-
-##################################################
-#simulation parameter
-##################################################
-TimeEnd=400000 #400000 / 200000
-TimeOut=200000 #200000 / 100000
-#TimeStartOut=0
\ No newline at end of file
diff --git a/apps/gpu/LBM/MusselOysterOyster/configPhoenix8.txt b/apps/gpu/LBM/MusselOysterOyster/configPhoenix8.txt
deleted file mode 100644
index 7dfef6c2a..000000000
--- a/apps/gpu/LBM/MusselOysterOyster/configPhoenix8.txt
+++ /dev/null
@@ -1,42 +0,0 @@
-##################################################
-#GPU Mapping
-##################################################
-Devices="0 1 2 3"
-NumberOfDevices=4
-
-##################################################
-#informations for Writing
-##################################################
-Path=/work/y0078217/Results/MusselOysterResults/8GPUOyster05/
-#Path="F:/Work/Computations/out/SphereScaling/"
-#Prefix="SphereScaling"
-#WriteGrid=true
-##################################################
-#informations for reading
-##################################################
-GridPath=/work/y0078217/Grids/GridMusselOyster/Oyster8GPU/
-#GridPath="C:"
-
-##################################################
-#number of grid levels
-##################################################
-#NOGL=1
-
-##################################################
-#LBM Version
-##################################################
-#D3Qxx=27
-#MainKernelName=CumulantK17CompChim
-
-##################################################
-#simulation parameter
-##################################################
-TimeEnd=800000
-TimeOut=400000
-#TimeStartOut=0
-
-##################################################
-# CUDA Streams and optimized communication (only used for multiple GPUs)
-##################################################
-useStreams = true
-useReducedCommunicationInInterpolation = true
\ No newline at end of file
diff --git a/apps/gpu/LBM/SphereScaling1/CMakeLists.txt b/apps/gpu/LBM/SphereScaling1/CMakeLists.txt
deleted file mode 100644
index a759675bd..000000000
--- a/apps/gpu/LBM/SphereScaling1/CMakeLists.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-PROJECT(SphereScaling1 LANGUAGES CUDA CXX)
-
-vf_add_library(BUILDTYPE binary PRIVATE_LINK basics VirtualFluids_GPU GridGenerator MPI::MPI_CXX FILES SphereScaling1.cpp)
-
-set_source_files_properties(SphereScaling1.cpp PROPERTIES LANGUAGE CUDA)
-
-set_target_properties(SphereScaling1 PROPERTIES
- CUDA_SEPARABLE_COMPILATION ON)
- # VS_DEBUGGER_COMMAND "C:/Program Files/Microsoft MPI/Bin/mpiexec.exe"
- # VS_DEBUGGER_COMMAND_ARGUMENTS "-n 2 \"$<TARGET_FILE:SphereScaling1>\"")
\ No newline at end of file
diff --git a/apps/gpu/LBM/SphereScaling1/SphereScaling1.cpp b/apps/gpu/LBM/SphereScaling1/SphereScaling1.cpp
deleted file mode 100644
index 14d151402..000000000
--- a/apps/gpu/LBM/SphereScaling1/SphereScaling1.cpp
+++ /dev/null
@@ -1,742 +0,0 @@
-
-#define _USE_MATH_DEFINES
-#include <math.h>
-#include <string>
-#include <sstream>
-#include <iostream>
-#include <stdexcept>
-#include <fstream>
-#include <exception>
-#include <memory>
-
-#include "mpi.h"
-
-//////////////////////////////////////////////////////////////////////////
-
-#include "basics/Core/DataTypes.h"
-#include "basics/PointerDefinitions.h"
-#include "basics/Core/VectorTypes.h"
-
-#include "basics/Core/LbmOrGks.h"
-#include "basics/Core/StringUtilities/StringUtil.h"
-#include "basics/config/ConfigurationFile.h"
-#include "basics/Core/Logger/Logger.h"
-
-//////////////////////////////////////////////////////////////////////////
-
-#include "GridGenerator/grid/GridBuilder/LevelGridBuilder.h"
-#include "GridGenerator/grid/GridBuilder/MultipleGridBuilder.h"
-#include "GridGenerator/grid/BoundaryConditions/Side.h"
-#include "GridGenerator/grid/GridFactory.h"
-
-#include "geometries/Sphere/Sphere.h"
-#include "geometries/Cuboid/Cuboid.h"
-#include "geometries/Conglomerate/Conglomerate.h"
-#include "geometries/TriangularMesh/TriangularMesh.h"
-
-#include "GridGenerator/io/SimulationFileWriter/SimulationFileWriter.h"
-#include "GridGenerator/io/GridVTKWriter/GridVTKWriter.h"
-#include "GridGenerator/io/STLReaderWriter/STLReader.h"
-#include "GridGenerator/io/STLReaderWriter/STLWriter.h"
-
-//////////////////////////////////////////////////////////////////////////
-
-#include "VirtualFluids_GPU/LBM/Simulation.h"
-#include "VirtualFluids_GPU/Communication/Communicator.h"
-#include "VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h"
-#include "VirtualFluids_GPU/DataStructureInitializer/GridProvider.h"
-#include "VirtualFluids_GPU/DataStructureInitializer/GridReaderFiles/GridReader.h"
-#include "VirtualFluids_GPU/Parameter/Parameter.h"
-#include "VirtualFluids_GPU/Output/FileWriter.h"
-
-#include "VirtualFluids_GPU/Kernel/Utilities/KernelFactory/KernelFactoryImp.h"
-#include "VirtualFluids_GPU/PreProcessor/PreProcessorFactory/PreProcessorFactoryImp.h"
-
-#include "VirtualFluids_GPU/GPU/CudaMemoryManager.h"
-
-//////////////////////////////////////////////////////////////////////////
-
-#include "utilities/communication.h"
-
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-//
-// U s e r s e t t i n g s
-//
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-// Tesla 03
-// std::string outPath("E:/temp/SphereScalingResults/");
-// std::string gridPathParent = "E:/temp/GridSphereScaling/";
-// std::string simulationName("SphereScaling");
-// std::string stlPath("C:/Users/Master/Documents/MasterAnna/STL/Sphere/");
-
-// Phoenix
-std::string outPath("/work/y0078217/Results/SphereScalingResults/");
-std::string gridPathParent = "/work/y0078217/Grids/GridSphereScaling/";
-std::string simulationName("SphereScaling");
-std::string stlPath("/home/y0078217/STL/Sphere/");
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-void multipleLevel(const std::string& configPath)
-{
- logging::Logger::addStream(&std::cout);
- logging::Logger::setDebugLevel(logging::Logger::Level::INFO_LOW);
- logging::Logger::timeStamp(logging::Logger::ENABLE);
- logging::Logger::enablePrintedRankNumbers(logging::Logger::ENABLE);
-
- auto gridFactory = GridFactory::make();
- gridFactory->setGridStrategy(Device::CPU);
- gridFactory->setTriangularMeshDiscretizationMethod(TriangularMeshDiscretizationMethod::POINT_IN_OBJECT);
-
- auto gridBuilder = MultipleGridBuilder::makeShared(gridFactory);
-
- vf::gpu::Communicator* comm = vf::gpu::Communicator::getInstanz();
- vf::basics::ConfigurationFile config;
- std::cout << configPath << std::endl;
- config.load(configPath);
- SPtr<Parameter> para = std::make_shared<Parameter>(config, comm->getNummberOfProcess(), comm->getPID());
-
-
-
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
- bool useGridGenerator = true;
- bool useLevels = true;
- std::string scalingType = "weak"; // "strong" // "weak"
- // bool useStreams = true;
- // para->useReducedCommunicationAfterFtoC = true;
- bool useStreams = para->getUseStreams();
-
- if (para->getNumprocs() == 1) {
- useStreams = false;
- para->useReducedCommunicationAfterFtoC = false;
- }
- if (scalingType != "weak" && scalingType != "strong")
- std::cerr << "unknown scaling type" << std::endl;
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- std::string gridPath(gridPathParent); // only for GridGenerator, for GridReader the gridPath needs to be set in the config file
-
- real dxGrid = (real)0.2;
- real vxLB = (real)0.0005; // LB units
- real viscosityLB = 0.001; //(vxLB * dxGrid) / Re;
-
- para->setVelocity(vxLB);
- para->setViscosity(viscosityLB);
- para->setVelocityRatio((real) 58.82352941);
- para->setViscosityRatio((real) 0.058823529);
- para->setDensityRatio((real) 998.0);
-
- *logging::out << logging::Logger::INFO_HIGH << "velocity LB [dx/dt] = " << vxLB << " \n";
- *logging::out << logging::Logger::INFO_HIGH << "viscosity LB [dx^2/dt] = " << viscosityLB << "\n";
- *logging::out << logging::Logger::INFO_HIGH << "velocity real [m/s] = " << vxLB * para->getVelocityRatio()<< " \n";
- *logging::out << logging::Logger::INFO_HIGH << "viscosity real [m^2/s] = " << viscosityLB * para->getViscosityRatio() << "\n";
- *logging::out << logging::Logger::INFO_HIGH << "dxGrid = " << dxGrid << "\n";
- *logging::out << logging::Logger::INFO_HIGH << "useGridGenerator = " << useGridGenerator << "\n";
- *logging::out << logging::Logger::INFO_HIGH << "useStreams = " << useStreams << "\n";
- *logging::out << logging::Logger::INFO_HIGH << "number of processes = " << para->getNumprocs() << "\n";
- *logging::out << logging::Logger::INFO_HIGH << "para->useReducedCommunicationAfterFtoC = " << para->useReducedCommunicationAfterFtoC << "\n";
- *logging::out << logging::Logger::INFO_HIGH << "scalingType = " << scalingType << "\n";
-
- // para->setTOut(10);
- // para->setTEnd(10);
-
- para->setCalcDragLift(false);
- para->setUseWale(false);
-
- if (para->getOutputPath().size() == 0) {
- para->setOutputPath(outPath);
- }
- para->setOutputPrefix(simulationName);
- para->setFName(para->getOutputPath() + para->getOutputPrefix());
- para->setPrintFiles(true);
- std::cout << "Write result files to " << para->getFName() << std::endl;
-
- if (useLevels)
- para->setMaxLevel(2);
- else
- para->setMaxLevel(1);
-
-
- if (useStreams)
- para->setUseStreams();
- //para->setMainKernel("CumulantK17CompChim");
- para->setMainKernel("CumulantK17CompChimStream");
- *logging::out << logging::Logger::INFO_HIGH << "Kernel: " << para->getMainKernel() << "\n";
-
- // if (para->getNumprocs() == 4) {
- // para->setDevices(std::vector<uint>{ 0u, 1u, 2u, 3u });
- // para->setMaxDev(4);
- // } else if (para->getNumprocs() == 2) {
- // para->setDevices(std::vector<uint>{ 2u, 3u });
- // para->setMaxDev(2);
- // } else
- // para->setDevices(std::vector<uint>{ 0u });
- // para->setMaxDev(1);
-
-
-
- //////////////////////////////////////////////////////////////////////////
-
-
- if (useGridGenerator) {
- real sideLengthCube;
- if (useLevels){
- if (scalingType == "strong")
- sideLengthCube = 76.0; // Phoenix: strong scaling with two levels = 76.0
- else if (scalingType == "weak")
- sideLengthCube = 70.0; // Phoenix: weak scaling with two levels = 70.0
- }
- else
- sideLengthCube = 86.0; // Phoenix: 86.0
- real xGridMin = 0.0;
- real xGridMax = sideLengthCube;
- real yGridMin = 0.0;
- real yGridMax = sideLengthCube;
- real zGridMin = 0.0;
- real zGridMax = sideLengthCube;
- const real dSphere = 10.0;
- const real dSphereLev1 = 22.0; // Phoenix: 22.0
- const real dCubeLev1 = 72.0; // Phoenix: 72.0
-
- if (para->getNumprocs() > 1) {
- const uint generatePart = vf::gpu::Communicator::getInstanz()->getPID();
-
- real overlap = (real)8.0 * dxGrid;
- gridBuilder->setNumberOfLayers(10, 8);
-
- if (comm->getNummberOfProcess() == 2) {
- real xSplit = 0.5 * sideLengthCube;
-
- if (scalingType == "weak"){
- xSplit = xGridMax;
- xGridMax = xGridMax + sideLengthCube;
- }
-
- if (generatePart == 0) {
- gridBuilder->addCoarseGrid(xGridMin, yGridMin, zGridMin, xSplit + overlap, yGridMax, zGridMax,
- dxGrid);
- }
- if (generatePart == 1) {
- gridBuilder->addCoarseGrid(xSplit-overlap, yGridMin, zGridMin, xGridMax, yGridMax, zGridMax,
- dxGrid);
- }
-
- if (useLevels) {
- if (scalingType == "strong"){
- gridBuilder->addGrid(new Sphere(0.5 * sideLengthCube, 0.5 * sideLengthCube, 0.5 * sideLengthCube, dSphereLev1), 1);
- } else if (scalingType == "weak"){
- gridBuilder->addGrid(new Cuboid( sideLengthCube-0.5*dCubeLev1, -0.5*dCubeLev1, -0.5*dCubeLev1,
- sideLengthCube+0.5*dCubeLev1, 0.5*dCubeLev1, 0.5*dCubeLev1),1);
- }
- }
-
- if (scalingType == "weak"){
- if (useLevels) {
- gridBuilder->addGeometry(new Sphere(sideLengthCube, 0.0, 0.0, dSphere));
- }else{
- TriangularMesh *sphereSTL = TriangularMesh::make(stlPath + "Spheres_2GPU.stl");
- gridBuilder->addGeometry(sphereSTL);
- }
- } else if (scalingType == "strong") {
- gridBuilder->addGeometry(new Sphere(0.5 * sideLengthCube, 0.5 * sideLengthCube, 0.5 * sideLengthCube, dSphere));
- }
-
- if (generatePart == 0)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xGridMin, xSplit, yGridMin, yGridMax, zGridMin, zGridMax));
- if (generatePart == 1)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xSplit, xGridMax, yGridMin, yGridMax, zGridMin, zGridMax));
-
- gridBuilder->buildGrids(LBM, true); // buildGrids() has to be called before setting the BCs!!!!
-
- if (generatePart == 0) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::PX, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PX, 1);
- }
-
- if (generatePart == 1) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::MX, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MX, 0);
- }
-
- gridBuilder->setPeriodicBoundaryCondition(false, false, false);
- //////////////////////////////////////////////////////////////////////////
- gridBuilder->setVelocityBoundaryCondition(SideType::MY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
- if (generatePart == 0)
- gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
- if (generatePart == 1)
- gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0); // set pressure BC after velocity BCs
- // gridBuilder->setVelocityBoundaryCondition(SideType::GEOMETRY, 0.0, 0.0, 0.0);
- //////////////////////////////////////////////////////////////////////////
-
- } else if (comm->getNummberOfProcess() == 4) {
- real ySplit= 0.5 * sideLengthCube;
- real zSplit= 0.5 * sideLengthCube;
-
- if (scalingType == "weak") {
- ySplit = yGridMax;
- yGridMax = yGridMax + (yGridMax-yGridMin);
- zSplit = zGridMax;
- zGridMax = zGridMax + (zGridMax-zGridMin);
- }
-
- if (generatePart == 0) {
- gridBuilder->addCoarseGrid(xGridMin, yGridMin, zGridMin, xGridMax , ySplit + overlap,
- zSplit + overlap, dxGrid);
- }
- if (generatePart == 1) {
- gridBuilder->addCoarseGrid(xGridMin, ySplit - overlap, zGridMin, xGridMax, yGridMax,
- zSplit + overlap, dxGrid);
- }
- if (generatePart == 2) {
- gridBuilder->addCoarseGrid(xGridMin, yGridMin, zSplit - overlap, xGridMax, ySplit + overlap,
- zGridMax, dxGrid);
- }
- if (generatePart == 3) {
- gridBuilder->addCoarseGrid(xGridMin, ySplit - overlap, zSplit - overlap, xGridMax, yGridMax,
- zGridMax, dxGrid);
- }
-
- if (useLevels) {
- if (scalingType == "strong"){
- gridBuilder->addGrid(new Sphere(0.5 * sideLengthCube, 0.5 * sideLengthCube, 0.5 * sideLengthCube, dSphereLev1), 1);
- } else if (scalingType == "weak"){
- gridBuilder->addGrid(new Cuboid( -0.5*dCubeLev1, sideLengthCube-0.5*dCubeLev1, sideLengthCube-0.5*dCubeLev1,
- 0.5*dCubeLev1, sideLengthCube+0.5*dCubeLev1, sideLengthCube+0.5*dCubeLev1),1);
- }
- }
-
- if (scalingType == "weak"){
- if (useLevels) {
- gridBuilder->addGeometry(new Sphere(0.0, sideLengthCube, sideLengthCube, dSphere));
- }else{
- TriangularMesh *sphereSTL = TriangularMesh::make(stlPath + "Spheres_4GPU.stl");
- gridBuilder->addGeometry(sphereSTL);
- }
- } else if (scalingType == "strong") {
- gridBuilder->addGeometry(new Sphere(0.5 * sideLengthCube, 0.5 * sideLengthCube, 0.5 * sideLengthCube, dSphere));
- }
-
- if (generatePart == 0)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xGridMin, xGridMax, yGridMin, ySplit, zGridMin, zSplit));
- if (generatePart == 1)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xGridMin, xGridMax, ySplit, yGridMax, zGridMin, zSplit));
- if (generatePart == 2)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xGridMin, xGridMax, yGridMin, ySplit, zSplit, zGridMax));
- if (generatePart == 3)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xGridMin, xGridMax, ySplit, yGridMax, zSplit, zGridMax));
-
-
- gridBuilder->buildGrids(LBM, true); // buildGrids() has to be called before setting the BCs!!!!
- gridBuilder->setPeriodicBoundaryCondition(false, false, false);
-
- if (generatePart == 0) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::PY, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PY, 1);
- gridBuilder->findCommunicationIndices(CommunicationDirections::PZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PZ, 2);
- }
- if (generatePart == 1) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::MY, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MY, 0);
- gridBuilder->findCommunicationIndices(CommunicationDirections::PZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PZ, 3);
- }
- if (generatePart == 2) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::PY, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PY, 3);
- gridBuilder->findCommunicationIndices(CommunicationDirections::MZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MZ, 0);
- }
- if (generatePart == 3) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::MY, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MY, 2);
- gridBuilder->findCommunicationIndices(CommunicationDirections::MZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MZ, 1);
- }
-
- //////////////////////////////////////////////////////////////////////////
- if (generatePart == 0) {
- gridBuilder->setVelocityBoundaryCondition(SideType::MY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
- }
- if (generatePart == 1) {
- gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
- }
- if (generatePart == 2) {
- gridBuilder->setVelocityBoundaryCondition(SideType::MY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
- }
- if (generatePart == 3) {
- gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
- }
- gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
- gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0); // set pressure BC after velocity BCs
- // gridBuilder->setVelocityBoundaryCondition(SideType::GEOMETRY, 0.0, 0.0, 0.0);
- //////////////////////////////////////////////////////////////////////////
- } else if (comm->getNummberOfProcess() == 8) {
- real xSplit = 0.5 * sideLengthCube;
- real ySplit = 0.5 * sideLengthCube;
- real zSplit = 0.5 * sideLengthCube;
-
- if (scalingType == "weak") {
- xSplit = xGridMax;
- xGridMax = xGridMax + (xGridMax-xGridMin);
- ySplit = yGridMax;
- yGridMax = yGridMax + (yGridMax-yGridMin);
- zSplit = zGridMax;
- zGridMax = zGridMax + (zGridMax-zGridMin);
- }
-
- if (generatePart == 0) {
- gridBuilder->addCoarseGrid(xGridMin, yGridMin, zGridMin, xSplit + overlap, ySplit + overlap,
- zSplit + overlap, dxGrid);
- }
- if (generatePart == 1) {
- gridBuilder->addCoarseGrid(xGridMin, ySplit - overlap, zGridMin, xSplit + overlap, yGridMax,
- zSplit + overlap, dxGrid);
- }
- if (generatePart == 2) {
- gridBuilder->addCoarseGrid(xSplit - overlap, yGridMin, zGridMin, xGridMax, ySplit + overlap,
- zSplit + overlap, dxGrid);
- }
- if (generatePart == 3) {
- gridBuilder->addCoarseGrid(xSplit - overlap, ySplit - overlap, zGridMin, xGridMax, yGridMax,
- zSplit + overlap, dxGrid);
- }
- if (generatePart == 4) {
- gridBuilder->addCoarseGrid(xGridMin, yGridMin, zSplit - overlap, xSplit + overlap, ySplit + overlap,
- zGridMax, dxGrid);
- }
- if (generatePart == 5) {
- gridBuilder->addCoarseGrid(xGridMin, ySplit - overlap, zSplit - overlap, xSplit + overlap, yGridMax,
- zGridMax, dxGrid);
- }
- if (generatePart == 6) {
- gridBuilder->addCoarseGrid(xSplit - overlap, yGridMin, zSplit - overlap, xGridMax, ySplit + overlap,
- zGridMax, dxGrid);
- }
- if (generatePart == 7) {
- gridBuilder->addCoarseGrid(xSplit - overlap, ySplit - overlap, zSplit - overlap, xGridMax, yGridMax,
- zGridMax, dxGrid);
- }
-
- if (useLevels) {
- if (scalingType == "strong"){
- gridBuilder->addGrid(new Sphere(0.5 * sideLengthCube, 0.5 * sideLengthCube, 0.5 * sideLengthCube, dSphereLev1), 1);
- } else if (scalingType == "weak"){
- gridBuilder->addGrid(new Cuboid( sideLengthCube-0.5*dCubeLev1, sideLengthCube-0.5*dCubeLev1, sideLengthCube-0.5*dCubeLev1,
- sideLengthCube+0.5*dCubeLev1, sideLengthCube+0.5*dCubeLev1, sideLengthCube+0.5*dCubeLev1),1);
- }
- }
-
- if (scalingType == "weak"){
- if (useLevels) {
- gridBuilder->addGeometry(new Sphere(sideLengthCube, sideLengthCube, sideLengthCube, dSphere));
- }else{
- TriangularMesh *sphereSTL = TriangularMesh::make(stlPath + "Spheres_8GPU.stl");
- gridBuilder->addGeometry(sphereSTL);
- }
- } else if (scalingType == "strong") {
- gridBuilder->addGeometry(new Sphere(0.5 * sideLengthCube, 0.5 * sideLengthCube, 0.5 * sideLengthCube, dSphere));
- }
-
- if (generatePart == 0)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xGridMin, xSplit, yGridMin, ySplit, zGridMin, zSplit));
- if (generatePart == 1)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xGridMin, xSplit, ySplit, yGridMax, zGridMin, zSplit));
- if (generatePart == 2)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xSplit, xGridMax, yGridMin, ySplit, zGridMin, zSplit));
- if (generatePart == 3)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xSplit, xGridMax, ySplit, yGridMax, zGridMin, zSplit));
- if (generatePart == 4)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xGridMin, xSplit, yGridMin, ySplit, zSplit, zGridMax));
- if (generatePart == 5)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xGridMin, xSplit, ySplit, yGridMax, zSplit, zGridMax));
- if (generatePart == 6)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xSplit, xGridMax, yGridMin, ySplit, zSplit, zGridMax));
- if (generatePart == 7)
- gridBuilder->setSubDomainBox(
- std::make_shared<BoundingBox>(xSplit, xGridMax, ySplit, yGridMax, zSplit, zGridMax));
-
- gridBuilder->buildGrids(LBM, true); // buildGrids() has to be called before setting the BCs!!!!
- gridBuilder->setPeriodicBoundaryCondition(false, false, false);
-
- if (generatePart == 0) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::PY, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PY, 1);
- gridBuilder->findCommunicationIndices(CommunicationDirections::PX, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PX, 2);
- gridBuilder->findCommunicationIndices(CommunicationDirections::PZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PZ, 4);
- }
- if (generatePart == 1) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::MY, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MY, 0);
- gridBuilder->findCommunicationIndices(CommunicationDirections::PX, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PX, 3);
- gridBuilder->findCommunicationIndices(CommunicationDirections::PZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PZ, 5);
- }
- if (generatePart == 2) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::PY, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PY, 3);
- gridBuilder->findCommunicationIndices(CommunicationDirections::MX, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MX, 0);
- gridBuilder->findCommunicationIndices(CommunicationDirections::PZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PZ, 6);
- }
- if (generatePart == 3) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::MY, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MY, 2);
- gridBuilder->findCommunicationIndices(CommunicationDirections::MX, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MX, 1);
- gridBuilder->findCommunicationIndices(CommunicationDirections::PZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PZ, 7);
- }
- if (generatePart == 4) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::PY, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PY, 5);
- gridBuilder->findCommunicationIndices(CommunicationDirections::PX, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PX, 6);
- gridBuilder->findCommunicationIndices(CommunicationDirections::MZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MZ, 0);
- }
- if (generatePart == 5) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::MY, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MY, 4);
- gridBuilder->findCommunicationIndices(CommunicationDirections::PX, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PX, 7);
- gridBuilder->findCommunicationIndices(CommunicationDirections::MZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MZ, 1);
- }
- if (generatePart == 6) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::PY, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::PY, 7);
- gridBuilder->findCommunicationIndices(CommunicationDirections::MX, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MX, 4);
- gridBuilder->findCommunicationIndices(CommunicationDirections::MZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MZ, 2);
- }
- if (generatePart == 7) {
- gridBuilder->findCommunicationIndices(CommunicationDirections::MY, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MY, 6);
- gridBuilder->findCommunicationIndices(CommunicationDirections::MX, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MX, 5);
- gridBuilder->findCommunicationIndices(CommunicationDirections::MZ, LBM);
- gridBuilder->setCommunicationProcess(CommunicationDirections::MZ, 3);
- }
-
- //////////////////////////////////////////////////////////////////////////
- if (generatePart == 0) {
- gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
- }
- if (generatePart == 1) {
- gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
- }
- if (generatePart == 2) {
- gridBuilder->setVelocityBoundaryCondition(SideType::MY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
- gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0);
- }
- if (generatePart == 3) {
- gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
- gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0);
- }
- if (generatePart == 4) {
- gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
- }
- if (generatePart == 5) {
- gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
- }
- if (generatePart == 6) {
- gridBuilder->setVelocityBoundaryCondition(SideType::MY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
- gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0); // set pressure BC after velocity BCs
- }
- if (generatePart == 7) {
- gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
- gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0); // set pressure BC after velocity BCs
- }
- // gridBuilder->setVelocityBoundaryCondition(SideType::GEOMETRY, 0.0, 0.0, 0.0);
- //////////////////////////////////////////////////////////////////////////
- }
- if (para->getKernelNeedsFluidNodeIndicesToRun())
- gridBuilder->findFluidNodes(useStreams);
-
- // gridBuilder->writeGridsToVtk(outPath + "grid/part" +
- // std::to_string(generatePart) + "_"); gridBuilder->writeGridsToVtk(outPath +
- // std::to_string(generatePart) + "/grid/"); gridBuilder->writeArrows(outPath +
- // std::to_string(generatePart) + " /arrow");
-
- SimulationFileWriter::write(gridPath + std::to_string(generatePart) + "/", gridBuilder,
- FILEFORMAT::BINARY);
- } else {
-
- gridBuilder->addCoarseGrid(xGridMin, yGridMin, zGridMin, xGridMax, yGridMax, zGridMax, dxGrid);
-
- if (useLevels) {
- gridBuilder->setNumberOfLayers(10, 8);
- if(scalingType == "strong"){
- gridBuilder->addGrid(new Sphere(0.5 * sideLengthCube, 0.5 * sideLengthCube, 0.5 * sideLengthCube, dSphereLev1), 1);
- } else if (scalingType == "weak")
- gridBuilder->addGrid(new Cuboid( sideLengthCube-0.5*dCubeLev1, sideLengthCube-0.5*dCubeLev1, sideLengthCube-0.5*dCubeLev1,
- sideLengthCube+0.5*dCubeLev1, sideLengthCube+0.5*dCubeLev1, sideLengthCube+0.5*dCubeLev1),1);
- }
-
- if (scalingType == "weak"){
- if(useLevels){
- gridBuilder->addGeometry(new Sphere(sideLengthCube, sideLengthCube, sideLengthCube, dSphere));
- }else{
- TriangularMesh *sphereSTL = TriangularMesh::make(stlPath + "Spheres_1GPU.stl");
- gridBuilder->addGeometry(sphereSTL);
- }
- } else {
- gridBuilder->addGeometry(new Sphere(0.5 * sideLengthCube, 0.5 * sideLengthCube, 0.5 * sideLengthCube, dSphere));
- }
-
- gridBuilder->buildGrids(LBM, true); // buildGrids() has to be called before setting the BCs!!!!
-
- gridBuilder->setPeriodicBoundaryCondition(false, false, false);
- //////////////////////////////////////////////////////////////////////////
- gridBuilder->setVelocityBoundaryCondition(SideType::PY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MY, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MX, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vxLB, 0.0, 0.0);
- gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, 0.0, 0.0);
- gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0); // set pressure BC after velocity BCs
-
- // gridBuilder->setVelocityBoundaryCondition(SideType::GEOMETRY, 0.0, 0.0, 0.0);
- //////////////////////////////////////////////////////////////////////////
- if (para->getKernelNeedsFluidNodeIndicesToRun())
- gridBuilder->findFluidNodes(useStreams);
-
- // gridBuilder->writeGridsToVtk("E:/temp/MusselOyster/" + "/grid/");
- // gridBuilder->writeArrows ("E:/temp/MusselOyster/" + "/arrow");
-
- SimulationFileWriter::write(gridPath, gridBuilder, FILEFORMAT::BINARY);
- }
- }
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
- SPtr<CudaMemoryManager> cudaMemoryManager = CudaMemoryManager::make(para);
-
- SPtr<GridProvider> gridGenerator;
- if (useGridGenerator)
- gridGenerator = GridProvider::makeGridGenerator(gridBuilder, para, cudaMemoryManager);
- else {
- gridGenerator = GridProvider::makeGridReader(FILEFORMAT::BINARY, para, cudaMemoryManager);
- }
-
- Simulation sim;
- SPtr<FileWriter> fileWriter = SPtr<FileWriter>(new FileWriter());
- SPtr<KernelFactoryImp> kernelFactory = KernelFactoryImp::getInstance();
- SPtr<PreProcessorFactoryImp> preProcessorFactory = PreProcessorFactoryImp::getInstance();
- sim.setFactories(kernelFactory, preProcessorFactory);
- sim.init(para, gridGenerator, fileWriter, cudaMemoryManager);
- sim.run();
- sim.free();
-
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-}
-
-int main( int argc, char* argv[])
-{
- MPI_Init(&argc, &argv);
- std::string str, str2, configFile;
-
- if ( argv != NULL )
- {
-
- try
- {
- //////////////////////////////////////////////////////////////////////////
-
- std::string targetPath;
-
- targetPath = __FILE__;
-
- if (argc == 2) {
- configFile = argv[1];
- std::cout << "Using configFile command line argument: " << configFile << std::endl;
- }
-
-#ifdef _WIN32
- targetPath = targetPath.substr(0, targetPath.find_last_of('\\') + 1);
-#else
- targetPath = targetPath.substr(0, targetPath.find_last_of('/') + 1);
-#endif
-
- std::cout << targetPath << std::endl;
-
- if (configFile.size()==0) {
- configFile = targetPath + "config.txt";
- }
-
- multipleLevel(configFile);
-
- //////////////////////////////////////////////////////////////////////////
- }
- catch (const std::bad_alloc& e)
- {
- *logging::out << logging::Logger::LOGGER_ERROR << "Bad Alloc:" << e.what() << "\n";
- }
- catch (const std::exception& e)
- {
- *logging::out << logging::Logger::LOGGER_ERROR << e.what() << "\n";
- }
- catch (...)
- {
- *logging::out << logging::Logger::LOGGER_ERROR << "Unknown exception!\n";
- }
- }
-
- MPI_Finalize();
- return 0;
-}
diff --git a/apps/gpu/LBM/SphereScaling1/config.txt b/apps/gpu/LBM/SphereScaling1/config.txt
deleted file mode 100644
index 44c5fedb2..000000000
--- a/apps/gpu/LBM/SphereScaling1/config.txt
+++ /dev/null
@@ -1,46 +0,0 @@
-# Tesla 03
-# mpiexec -n 2 "C:/Users/Master/Documents/MasterAnna/VirtualFluids_dev/build/bin/Release/SphereScaling.exe" "C:/Users/Master/Documents/MasterAnna/VirtualFluids_dev/apps/gpu/LBM/SphereScaling/config.txt"
-# Phoenix
-# mpirun -np 2 "./VirtualFluids_dev/build/bin/SphereScaling" "./VirtualFluids_dev/apps/gpu/LBM/SphereScaling/config.txt"
-
-# Phoenix mpich
-# mpirun -np 2 nvprof -f -o SphereScaling.%q{PMI_RANK}.nvprof "./VirtualFluids_dev/build/bin/SphereScaling" "./VirtualFluids_dev/apps/gpu/LBM/SphereScaling/configPhoenix4GPU.txt"
-# Phoenix openmpi
-# mpirun -np 2 nvprof -f -o SphereScaling.%q{OMPI_COMM_WORLD_RANK}.nvprof "./VirtualFluids_dev/build/bin/SphereScaling" "./VirtualFluids_dev/apps/gpu/LBM/SphereScaling/configPhoenix4GPU.txt"
-
-##################################################
-#GPU Mapping
-##################################################
-#Devices="0 1 2 3"
-#NumberOfDevices=2
-
-##################################################
-#informations for Writing
-##################################################
-#Path="E:/temp/SphereScalingResults/"
-Path=/work/y0078217/Results/SphereScalingResults/
-#Prefix="SphereScaling"
-#WriteGrid=true
-##################################################
-#informations for reading
-##################################################
-GridPath=/work/y0078217/Grids/GridSphereScaling/
-#GridPath=E:/temp/GridSphereScaling/
-
-##################################################
-#number of grid levels
-##################################################
-#NOGL=1
-
-##################################################
-#LBM Version
-##################################################
-#D3Qxx=27
-#MainKernelName=CumulantK17CompChim
-
-##################################################
-#simulation parameter
-##################################################
-#TimeEnd=10
-#TimeOut=10
-#TimeStartOut=0
\ No newline at end of file
diff --git a/apps/gpu/LBM/SphereScaling1/configPhoenix4GPU_2LevStrongStream.txt b/apps/gpu/LBM/SphereScaling1/configPhoenix4GPU_2LevStrongStream.txt
deleted file mode 100644
index 234ca79bf..000000000
--- a/apps/gpu/LBM/SphereScaling1/configPhoenix4GPU_2LevStrongStream.txt
+++ /dev/null
@@ -1,42 +0,0 @@
-##################################################
-#GPU Mapping
-##################################################
-Devices="0 1 2 3"
-NumberOfDevices=4
-
-##################################################
-#informations for Writing
-##################################################
-Path=/work/y0078217/Results/SphereScalingResults/4GPU/2LevStrongStream/
-#Path="F:/Work/Computations/out/SphereScaling/"
-#Prefix="SphereScaling"
-#WriteGrid=true
-##################################################
-#informations for reading
-##################################################
-GridPath=/work/y0078217/Grids/GridSphereScaling/SphereScaling4GPU/
-#GridPath="C:"
-
-##################################################
-#number of grid levels
-##################################################
-#NOGL=1
-
-##################################################
-#LBM Version
-##################################################
-#D3Qxx=27
-#MainKernelName=CumulantK17CompChim
-
-##################################################
-#simulation parameter
-##################################################
-TimeEnd=100000
-TimeOut=100000
-#TimeStartOut=0
-
-##################################################
-# CUDA Streams and optimized communication (only used for multiple GPUs)
-##################################################
-useStreams = true
-useReducedCommunicationInInterpolation = true
\ No newline at end of file
diff --git a/apps/gpu/LBM/SphereScaling1/configPhoenix4GPU_2LevWeakStream.txt b/apps/gpu/LBM/SphereScaling1/configPhoenix4GPU_2LevWeakStream.txt
deleted file mode 100644
index efbc2f125..000000000
--- a/apps/gpu/LBM/SphereScaling1/configPhoenix4GPU_2LevWeakStream.txt
+++ /dev/null
@@ -1,42 +0,0 @@
-##################################################
-#GPU Mapping
-##################################################
-Devices="0 1 2 3"
-NumberOfDevices=4
-
-##################################################
-#informations for Writing
-##################################################
-Path=/work/y0078217/Results/SphereScalingResults/4GPU/2LevWeakStream/
-#Path="F:/Work/Computations/out/SphereScaling/"
-#Prefix="SphereScaling"
-#WriteGrid=true
-##################################################
-#informations for reading
-##################################################
-GridPath=/work/y0078217/Grids/GridSphereScaling/SphereScaling4GPU/
-#GridPath="C:"
-
-##################################################
-#number of grid levels
-##################################################
-#NOGL=1
-
-##################################################
-#LBM Version
-##################################################
-#D3Qxx=27
-#MainKernelName=CumulantK17CompChim
-
-##################################################
-#simulation parameter
-##################################################
-TimeEnd=1000
-TimeOut=1000
-#TimeStartOut=0
-
-##################################################
-# CUDA Streams and optimized communication (only used for multiple GPUs)
-##################################################
-useStreams = true
-useReducedCommunicationInInterpolation = true
\ No newline at end of file
diff --git a/apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevStrongOS.txt b/apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevStrongOS.txt
deleted file mode 100644
index b650042a8..000000000
--- a/apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevStrongOS.txt
+++ /dev/null
@@ -1,42 +0,0 @@
-##################################################
-#GPU Mapping
-##################################################
-Devices="0 1 2 3"
-NumberOfDevices=4
-
-##################################################
-#informations for Writing
-##################################################
-Path=/work/y0078217/Results/SphereScalingResults/8GPU/2LevStrongOS/
-#Path="F:/Work/Computations/out/SphereScaling/"
-#Prefix="SphereScaling"
-#WriteGrid=true
-##################################################
-#informations for reading
-##################################################
-GridPath=/work/y0078217/Grids/GridSphereScaling/SphereScaling8GPU/
-#GridPath="C:"
-
-##################################################
-#number of grid levels
-##################################################
-#NOGL=1
-
-##################################################
-#LBM Version
-##################################################
-#D3Qxx=27
-#MainKernelName=CumulantK17CompChim
-
-##################################################
-#simulation parameter
-##################################################
-TimeEnd=100000
-TimeOut=100000
-#TimeStartOut=0
-
-##################################################
-# CUDA Streams and optimized communication (only used for multiple GPUs)
-##################################################
-useStreams = false
-useReducedCommunicationInInterpolation = false
\ No newline at end of file
diff --git a/apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevStrongStream.txt b/apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevStrongStream.txt
deleted file mode 100644
index 6f3849abb..000000000
--- a/apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevStrongStream.txt
+++ /dev/null
@@ -1,42 +0,0 @@
-##################################################
-#GPU Mapping
-##################################################
-Devices="0 1 2 3"
-NumberOfDevices=4
-
-##################################################
-#informations for Writing
-##################################################
-Path=/work/y0078217/Results/SphereScalingResults/8GPU/2LevStrongStream/
-#Path="F:/Work/Computations/out/SphereScaling/"
-#Prefix="SphereScaling"
-#WriteGrid=true
-##################################################
-#informations for reading
-##################################################
-GridPath=/work/y0078217/Grids/GridSphereScaling/SphereScaling8GPU/
-#GridPath="C:"
-
-##################################################
-#number of grid levels
-##################################################
-#NOGL=1
-
-##################################################
-#LBM Version
-##################################################
-#D3Qxx=27
-#MainKernelName=CumulantK17CompChim
-
-##################################################
-#simulation parameter
-##################################################
-TimeEnd=100000
-TimeOut=100000
-#TimeStartOut=0
-
-##################################################
-# CUDA Streams and optimized communication (only used for multiple GPUs)
-##################################################
-useStreams = true
-useReducedCommunicationInInterpolation = true
\ No newline at end of file
diff --git a/apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevWeakOS.txt b/apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevWeakOS.txt
deleted file mode 100644
index 63b1e745a..000000000
--- a/apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevWeakOS.txt
+++ /dev/null
@@ -1,42 +0,0 @@
-##################################################
-#GPU Mapping
-##################################################
-Devices="0 1 2 3"
-NumberOfDevices=4
-
-##################################################
-#informations for Writing
-##################################################
-Path=/work/y0078217/Results/SphereScalingResults/8GPU/2LevWeakOS/
-#Path="F:/Work/Computations/out/SphereScaling/"
-#Prefix="SphereScaling"
-#WriteGrid=true
-##################################################
-#informations for reading
-##################################################
-GridPath=/work/y0078217/Grids/GridSphereScaling/SphereScaling8GPU/
-#GridPath="C:"
-
-##################################################
-#number of grid levels
-##################################################
-#NOGL=1
-
-##################################################
-#LBM Version
-##################################################
-#D3Qxx=27
-#MainKernelName=CumulantK17CompChim
-
-##################################################
-#simulation parameter
-##################################################
-TimeEnd=100000
-TimeOut=100000
-#TimeStartOut=0
-
-##################################################
-# CUDA Streams and optimized communication (only used for multiple GPUs)
-##################################################
-useStreams = false
-useReducedCommunicationInInterpolation = false
\ No newline at end of file
diff --git a/apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevWeakStream.txt b/apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevWeakStream.txt
deleted file mode 100644
index 9dbdd9203..000000000
--- a/apps/gpu/LBM/SphereScaling1/configPhoenix8GPU_2LevWeakStream.txt
+++ /dev/null
@@ -1,42 +0,0 @@
-##################################################
-#GPU Mapping
-##################################################
-Devices="0 1 2 3"
-NumberOfDevices=4
-
-##################################################
-#informations for Writing
-##################################################
-Path=/work/y0078217/Results/SphereScalingResults/8GPU/2LevWeakStream/
-#Path="F:/Work/Computations/out/SphereScaling/"
-#Prefix="SphereScaling"
-#WriteGrid=true
-##################################################
-#informations for reading
-##################################################
-GridPath=/work/y0078217/Grids/GridSphereScaling/SphereScaling8GPU/
-#GridPath="C:"
-
-##################################################
-#number of grid levels
-##################################################
-#NOGL=1
-
-##################################################
-#LBM Version
-##################################################
-#D3Qxx=27
-#MainKernelName=CumulantK17CompChim
-
-##################################################
-#simulation parameter
-##################################################
-TimeEnd=100000
-TimeOut=100000
-#TimeStartOut=0
-
-##################################################
-# CUDA Streams and optimized communication (only used for multiple GPUs)
-##################################################
-useStreams = true
-useReducedCommunicationInInterpolation = true
\ No newline at end of file
diff --git a/gpu.cmake b/gpu.cmake
index 370e16397..bd27b92e3 100644
--- a/gpu.cmake
+++ b/gpu.cmake
@@ -45,11 +45,7 @@ IF (BUILD_VF_GPU)
#add_subdirectory(apps/gpu/LBM/TGV_3D)
#add_subdirectory(apps/gpu/LBM/TGV_3D_MultiGPU)
add_subdirectory(apps/gpu/LBM/SphereScaling)
- add_subdirectory(apps/gpu/LBM/SphereScaling1)
add_subdirectory(apps/gpu/LBM/MusselOyster)
- add_subdirectory(apps/gpu/LBM/MusselOysterOyster)
- #add_subdirectory(apps/gpu/LBM/MusselOyster2x)
- #add_subdirectory(apps/gpu/LBM/MusselOyster3z)
ELSE()
MESSAGE( STATUS "exclude Virtual Fluids GPU." )
ENDIF()
--
GitLab