diff --git a/.gitignore b/.gitignore index 63bd843b606911d303ce1476fc002c009b8ebc9a..1755c0de2430c33f98e442d3c7fac4eb9597c51f 100644 --- a/.gitignore +++ b/.gitignore @@ -39,4 +39,7 @@ stl/ .DS_Store # Settings -.gitconfig \ No newline at end of file +.gitconfig + +# User Settings +CMakeUserPresets.json \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ad5efff0767898377e58c37a86773fe9b3d20aa7..84feed6dc40bce2b59ad483fb17973703d9ce387 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -49,7 +49,7 @@ stages: - cd $CI_PROJECT_DIR/$BUILD_FOLDER - rm -r -f ./* - cmake .. -LAH - --preset=all_make + --preset=make_all -DBUILD_WARNINGS_AS_ERRORS=ON -DCMAKE_CUDA_ARCHITECTURES=60 - make -j4 @@ -99,7 +99,7 @@ msvc_17: - cd $CI_PROJECT_DIR - md -force $env:BUILD_FOLDER - cd $env:BUILD_FOLDER - - cmake .. --preset=all_msvc -DCMAKE_CUDA_ARCHITECTURES=61 -DBUILD_WARNINGS_AS_ERRORS=ON + - cmake .. --preset=msvc_all -DCMAKE_CUDA_ARCHITECTURES=61 -DBUILD_WARNINGS_AS_ERRORS=ON - MSBuild.exe VirtualFluids.sln /property:Configuration=$env:BUILD_CONFIGURATION /verbosity:minimal /maxcpucount:4 artifacts: @@ -281,7 +281,7 @@ gpu_numerical_tests: - cd $CI_PROJECT_DIR/build - rm -r -f ./* - cmake .. - --preset=gpu_numerical_tests_make + --preset=make_numerical_tests_gpu -DCMAKE_CUDA_ARCHITECTURES=60 -DPATH_NUMERICAL_TESTS=/tmp/test_data/numerical_tests_gpu - make -j4 @@ -352,7 +352,7 @@ clang_build_analyzer_clang_10: - mkdir -p $CI_PROJECT_DIR/build - cd $CI_PROJECT_DIR/build - cmake .. - --preset=all_make + --preset=make_all -DCMAKE_CUDA_ARCHITECTURES=60 -DCMAKE_CXX_FLAGS=-ftime-trace - ClangBuildAnalyzer --start . @@ -384,7 +384,7 @@ include_what_you_use_clang_10: - mkdir -p $CI_PROJECT_DIR/build - cd $CI_PROJECT_DIR/build - cmake .. - --preset=all_make + --preset=make_all -DCMAKE_CUDA_ARCHITECTURES=60 -DBUILD_VF_INCLUDE_WHAT_YOU_USE=ON - make @@ -461,7 +461,7 @@ gcov_gcc_9: - mkdir -p $CI_PROJECT_DIR/build - cd $CI_PROJECT_DIR/build - cmake .. - --preset=all_make + --preset=make_all -DCMAKE_CUDA_ARCHITECTURES=60 -DBUILD_VF_COVERAGE=ON - make -j4 diff --git a/CMake/FileUtilities.cmake b/CMake/FileUtilities.cmake index 151000a681795923d4e31ed8c5f06dfd1e7af7fd..13057ef832b5aa2d7ce303fe55e95a91284f5f56 100644 --- a/CMake/FileUtilities.cmake +++ b/CMake/FileUtilities.cmake @@ -5,7 +5,7 @@ ## After function call the files are stored in: MY_SRCS ################################################################################# -macro(includeAllFiles targetName file_path) +macro(includeAllFiles folderName targetName file_path) if(NOT DEFINED collectTestFiles) set(collectTestFiles ON) endif() @@ -14,11 +14,11 @@ macro(includeAllFiles targetName file_path) set(collectProductionFiles ON) endif() - includeFiles(${targetName} "${file_path}") + includeFiles(${folderName} ${targetName} "${file_path}") endmacro(includeAllFiles) -macro(includeProductionFiles targetName file_path) +macro(includeProductionFiles folderName targetName file_path) if(NOT DEFINED collectTestFiles) set(collectTestFiles OFF) endif() @@ -27,12 +27,12 @@ macro(includeProductionFiles targetName file_path) set(collectProductionFiles ON) endif() - includeFiles(${targetName} "${file_path}") + includeFiles(${folderName} ${targetName} "${file_path}") endmacro(includeProductionFiles) -macro(includeTestFiles targetName file_paths) +macro(includeTestFiles folderName file_paths) if(NOT DEFINED collectTestFiles) set(collectTestFiles ON) endif() @@ -41,13 +41,13 @@ macro(includeTestFiles targetName file_paths) set(collectProductionFiles OFF) endif() - includeFiles(${targetName} "${file_paths}") + includeFiles(${folderName} ${folderName} "${file_paths}") endmacro(includeTestFiles) -macro(includeFiles targetName file_paths) +macro(includeFiles folderName targetName file_paths) foreach(file ${file_paths}) @@ -57,7 +57,7 @@ macro(includeFiles targetName file_paths) collectFilesFrom(${file}) if (package_dir) - setSourceGroupForFilesIn(${file} ${package_dir} ${targetName}) + setSourceGroupForFilesIn(${file} ${package_dir} ${targetName} ${folderName}) endif() endforeach() @@ -90,9 +90,9 @@ endmacro() -macro(setSourceGroupForFilesIn file package_dir targetName) +macro(setSourceGroupForFilesIn file package_dir targetName folderName) #input: target_name PACKAGE_SRCS - buildSourceGroup(${targetName} ${package_dir}) + buildSourceGroup(${folderName} ${package_dir}) if(isAllTestSuite) source_group(${targetName}\\${SOURCE_GROUP} FILES ${file}) @@ -105,20 +105,20 @@ endmacro(setSourceGroupForFilesIn) -macro(buildSourceGroup targetName path) -#input: targetName (e.g. lib name, exe name) +macro(buildSourceGroup folderName path) +#input: folderName (e.g. name of folder after src/) unset(SOURCE_GROUP) string(REPLACE "/" ";" folderListFromPath ${path}) - set(findTargetName 0) + set(findFolderName 0) foreach(folder ${folderListFromPath}) - if(findTargetName) + if(findFolderName) set(SOURCE_GROUP ${SOURCE_GROUP}\\${folder}) endif() - if(${folder} STREQUAL ${targetName}) - SET(findTargetName 1) + if(${folder} STREQUAL ${folderName}) + SET(findFolderName 1) endif() endforeach() diff --git a/CMake/VirtualFluidsMacros.cmake b/CMake/VirtualFluidsMacros.cmake index 63503f5f14221bb8cec7670dbdda6aa92497d327..4fd163b2cc1b53fe461ef482d906f4cb1255a76c 100644 --- a/CMake/VirtualFluidsMacros.cmake +++ b/CMake/VirtualFluidsMacros.cmake @@ -105,14 +105,15 @@ function(vf_add_library) set( options ) set( oneValueArgs NAME BUILDTYPE) - set( multiValueArgs PUBLIC_LINK PRIVATE_LINK FILES FOLDER EXCLUDE) + set( multiValueArgs PUBLIC_LINK PRIVATE_LINK FILES FOLDER EXCLUDE MODULEFOLDER) cmake_parse_arguments( ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) - if(DEFINED ARG_NAME) + if(DEFINED ARG_NAME) set(library_name ${ARG_NAME}) else() vf_get_library_name (library_name) endif() + vf_get_library_name (folder_name) # folder_name is not equal to library_name when ARG_NAME was set if(NOT DEFINED ARG_BUILDTYPE) if(BUILD_SHARED_LIBS) @@ -122,12 +123,16 @@ function(vf_add_library) endif() endif() + if(DEFINED ARG_MODULEFOLDER) + set(folder_name ${ARG_MODULEFOLDER}) + endif() + status("Configuring the target: ${library_name} (type=${ARG_BUILDTYPE})...") collectFiles(sourceFiles "${ARG_FILES}" "${ARG_FOLDER}" "${ARG_EXCLUDE}") - includeProductionFiles (${library_name} "${sourceFiles}") + includeProductionFiles (${folder_name} ${library_name} "${sourceFiles}") ################################################################# ### ADD TARGET ### @@ -325,4 +330,4 @@ function(vf_load_user_apps) foreach(app IN LISTS USER_APPS) add_subdirectory(${app}) endforeach() -endfunction() \ No newline at end of file +endfunction() diff --git a/CMakePresets.json b/CMakePresets.json index c794d45cc3818eca53f9559e77ad2a03c9510e1e..37951b31c46dd5a39c29c84b9a6fe572fbdba732 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -1,15 +1,15 @@ { - "version": 2, + "version": 3, "cmakeMinimumRequired": { "major": 3, - "minor": 20, + "minor": 21, "patch": 0 }, "configurePresets": [ { "name": "default", - "hidden": true, - "binaryDir": "${sourceDir}/build/" + "binaryDir": "build", + "hidden": true }, { "name": "msvc", @@ -20,7 +20,12 @@ { "name": "make", "hidden": true, - "generator": "Unix Makefiles" + "generator": "Unix Makefiles", + "condition": { + "type": "notEquals", + "lhs": "${hostSystemName}", + "rhs": "Windows" + } }, { "name": "unit_tests", @@ -49,34 +54,6 @@ "BUILD_VF_DOUBLE_ACCURACY": "OFF" } }, - { - "name": "debug", - "hidden": true, - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug" - } - }, - { - "name": "release", - "hidden": true, - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Release" - } - }, - { - "name": "min_size_rel", - "hidden": true, - "cacheVariables": { - "CMAKE_BUILD_TYPE": "MinSizeRel" - } - }, - { - "name": "rel_with_deb_info", - "hidden": true, - "cacheVariables": { - "CMAKE_BUILD_TYPE": "RelWithDebInfo" - } - }, { "name": "gpu_numerical_tests", "inherits": [ @@ -91,7 +68,7 @@ } }, { - "name": "all_make", + "name": "make_all", "inherits": [ "cpu", "gpu", @@ -101,162 +78,66 @@ "displayName": "all make configuration" }, { - "name": "all_msvc", + "name": "make_cpu", "inherits": [ "cpu", - "gpu", "unit_tests", - "msvc" - ], - "displayName": "all msvc configuration" - }, - { - "name": "gpu_numerical_tests_make", - "inherits": [ - "gpu_numerical_tests", "make" ], - "displayName": "gpu numerical tests make configuration" + "displayName": "cpu make configuration" }, { - "name": "gpu_numerical_tests_msvc", - "inherits": [ - "msvc", - "gpu_numerical_tests" - ], - "displayName": "gpu numerical tests msvc configuration" - }, - { - "name": "debug_make_gpu", - "displayName": "Debug GPU Make", - "inherits": [ - "gpu", - "make", - "debug" - ] - }, - { - "name": "release_make_gpu", - "displayName": "Release GPU Make", + "name": "make_gpu", "inherits": [ "gpu", - "make", - "release" - ] - }, - { - "name": "min_size_rel_make_gpu", - "displayName": "MinSizeRel GPU Make", - "inherits": [ - "gpu", - "make", - "min_size_rel" - ] + "unit_tests", + "make" + ], + "displayName": "gpu make configuration" }, { - "name": "rel_with_deb_info_make_gpu", - "displayName": "RelWithDebInfo GPU Make", + "name": "msvc_all", "inherits": [ + "cpu", "gpu", - "make", - "rel_with_deb_info" - ] + "unit_tests", + "msvc" + ], + "displayName": "all msvc configuration" }, { - "name": "debug_msvc_gpu", - "displayName": "Debug GPU MSVC", + "name": "msvc_cpu", "inherits": [ - "gpu", - "msvc", - "debug" - ] + "cpu", + "unit_tests", + "msvc" + ], + "displayName": "cpu msvc configuration" }, { - "name": "release_msvc_gpu", - "displayName": "Release GPU MSVC", + "name": "msvc_gpu", "inherits": [ "gpu", - "msvc", - "release" - ] + "unit_tests", + "msvc" + ], + "displayName": "gpu msvc configuration" }, { - "name": "min_size_rel_msvc_gpu", - "displayName": "MinSizeRel GPU MSVC", + "name": "make_numerical_tests_gpu", "inherits": [ - "gpu", - "msvc", - "min_size_rel" - ] + "gpu_numerical_tests", + "make" + ], + "displayName": "gpu numerical tests make configuration" }, { - "name": "rel_with_deb_info_msvc_gpu", - "displayName": "RelWithDebInfo GPU MSVC", + "name": "msvc_numerical_tests_gpu", "inherits": [ - "gpu", "msvc", - "rel_with_deb_info" - ] - } - ], - "buildPresets": [ - { - "name": "Default", - "hidden": true, - "configurePreset": "default", - "jobs": 4 - }, - { - "name": "GPU", - "hidden": true, - "configurePreset": "gpu", - "targets": [ - "ActuatorLine", - "DrivenCavity", - "BoundaryLayer" + "gpu_numerical_tests" ], - "inherits": [ - "Default" - ] - }, - { - "name": "Release", - "hidden": true, - "configurePreset": "release" - }, - { - "name": "Debug_Make_GPU", - "displayName": "Debug", - "description": "Compile GPU version with debug information", - "configurePreset": "debug_make_gpu", - "inherits": [ - "GPU" - ] - }, - { - "name": "MinSizeRel_Make_GPU", - "displayName": "MinSizeRel", - "configurePreset": "min_size_rel_make_gpu", - "inherits": [ - "GPU" - ] - }, - { - "name": "RelWithDebInfo_GPU", - "displayName": "RelWithDebInfo", - "configurePreset": "rel_with_deb_info_make_gpu", - "inherits": [ - "GPU" - ] - }, - { - "name": "Release_GPU", - "description": "Build release version of GPU", - "displayName": "Release GPU", - "configurePreset": "release_make_gpu", - "inherits": [ - "GPU" - ] + "displayName": "gpu numerical tests msvc configuration" } ] } \ No newline at end of file diff --git a/apps/gpu/LBM/DrivenCavityUniform/CMakeLists.txt b/apps/gpu/LBM/DrivenCavityUniform/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..8384e1bc6fcfa3fd2514434b620b266e96b3626a --- /dev/null +++ b/apps/gpu/LBM/DrivenCavityUniform/CMakeLists.txt @@ -0,0 +1,9 @@ +PROJECT(DrivenCavity LANGUAGES CUDA CXX) + +#LIST(APPEND CS_COMPILER_FLAGS_CXX "-DOMPI_SKIP_MPICXX" ) + +vf_add_library(BUILDTYPE binary PRIVATE_LINK basics VirtualFluids_GPU GridGenerator MPI::MPI_CXX FILES DrivenCavity.cpp) + +set_source_files_properties(DrivenCavity.cpp PROPERTIES LANGUAGE CUDA) + +set_target_properties(DrivenCavity PROPERTIES CUDA_SEPARABLE_COMPILATION ON) diff --git a/apps/gpu/LBM/DrivenCavityUniform/DrivenCavity.cpp b/apps/gpu/LBM/DrivenCavityUniform/DrivenCavity.cpp new file mode 100644 index 0000000000000000000000000000000000000000..958ef4714118aac34b8cfb0bec3aab97b108b01d --- /dev/null +++ b/apps/gpu/LBM/DrivenCavityUniform/DrivenCavity.cpp @@ -0,0 +1,231 @@ +//======================================================================================= +// ____ ____ __ ______ __________ __ __ __ __ +// \ \ | | | | | _ \ |___ ___| | | | | / \ | | +// \ \ | | | | | |_) | | | | | | | / \ | | +// \ \ | | | | | _ / | | | | | | / /\ \ | | +// \ \ | | | | | | \ \ | | | \__/ | / ____ \ | |____ +// \ \ | | |__| |__| \__\ |__| \________/ /__/ \__\ |_______| +// \ \ | | ________________________________________________________________ +// \ \ | | | ______________________________________________________________| +// \ \| | | | __ __ __ __ ______ _______ +// \ | | |_____ | | | | | | | | | _ \ / _____) +// \ | | _____| | | | | | | | | | | \ \ \_______ +// \ | | | | |_____ | \_/ | | | | |_/ / _____ | +// \ _____| |__| |________| \_______/ |__| |______/ (_______/ +// +// This file is part of VirtualFluids. VirtualFluids is free software: you can +// redistribute it and/or modify it under the terms of the GNU General Public +// License as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// VirtualFluids is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// for more details. +// +// You should have received a copy of the GNU General Public License along +// with VirtualFluids (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>. +// +//! \file LidDrivenCavity.cpp +//! \ingroup Applications +//! \author Martin Schoenherr, Stephan Lenz +//======================================================================================= +#define _USE_MATH_DEFINES +#include <exception> +#include <fstream> +#include <iostream> +#include <memory> +#include <sstream> +#include <stdexcept> +#include <string> + +////////////////////////////////////////////////////////////////////////// + +#include "Core/DataTypes.h" +#include "Core/LbmOrGks.h" +#include "Core/Logger/Logger.h" +#include "Core/VectorTypes.h" +#include "PointerDefinitions.h" + +#include <logger/Logger.h> + +////////////////////////////////////////////////////////////////////////// + +#include "GridGenerator/grid/BoundaryConditions/Side.h" +#include "GridGenerator/grid/GridBuilder/LevelGridBuilder.h" +#include "GridGenerator/grid/GridBuilder/MultipleGridBuilder.h" +#include "GridGenerator/grid/GridFactory.h" +#include "GridGenerator/geometries/Cuboid/Cuboid.h" + +////////////////////////////////////////////////////////////////////////// + +#include "VirtualFluids_GPU/Factories/BoundaryConditionFactory.h" +#include "VirtualFluids_GPU/Factories/GridScalingFactory.h" +#include "VirtualFluids_GPU/Communication/Communicator.h" +#include "VirtualFluids_GPU/DataStructureInitializer/GridProvider.h" +#include "VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.h" +#include "VirtualFluids_GPU/GPU/CudaMemoryManager.h" +#include "VirtualFluids_GPU/LBM/Simulation.h" +#include "VirtualFluids_GPU/Output/FileWriter.h" +#include "VirtualFluids_GPU/Parameter/Parameter.h" +#include "VirtualFluids_GPU/Factories/GridScalingFactory.h" + +////////////////////////////////////////////////////////////////////////// + +int main() +{ + try { + vf::logging::Logger::initalizeLogger(); + ////////////////////////////////////////////////////////////////////////// + // Simulation parameters + ////////////////////////////////////////////////////////////////////////// + std::string path("./output/DrivenCavity_uniform"); + std::string simulationName("LidDrivenCavity"); + + const real L = 1.0; + const real Re = 1000.0; + const real velocity = 1.0; + const real dt = (real)0.5e-3; + const uint nx = 64; + + const uint timeStepOut = 1000; + const uint timeStepEnd = 10000; + + ////////////////////////////////////////////////////////////////////////// + // setup logger + ////////////////////////////////////////////////////////////////////////// + + logging::Logger::addStream(&std::cout); + logging::Logger::setDebugLevel(logging::Logger::Level::INFO_LOW); + logging::Logger::timeStamp(logging::Logger::ENABLE); + logging::Logger::enablePrintedRankNumbers(logging::Logger::ENABLE); + + ////////////////////////////////////////////////////////////////////////// + // setup gridGenerator + ////////////////////////////////////////////////////////////////////////// + + auto gridFactory = GridFactory::make(); + gridFactory->setTriangularMeshDiscretizationMethod(TriangularMeshDiscretizationMethod::POINT_IN_OBJECT); + auto gridBuilder = MultipleGridBuilder::makeShared(gridFactory); + + ////////////////////////////////////////////////////////////////////////// + // create grid + ////////////////////////////////////////////////////////////////////////// + + real dx = L / real(nx); + + gridBuilder->addCoarseGrid(-0.5 * L, -0.5 * L, -0.5 * L, 0.5 * L, 0.5 * L, 0.5 * L, dx); + + // gridBuilder->addGrid(new Cuboid(-0.25, -0.25, -0.25, 0.25, 0.25, 0.25), 1); // add fine grid + GridScalingFactory scalingFactory = GridScalingFactory(); + scalingFactory.setScalingFactory(GridScalingFactory::GridScaling::ScaleCompressible); + + gridBuilder->setPeriodicBoundaryCondition(false, false, false); + + gridBuilder->buildGrids(LbmOrGks::LBM, false); + + ////////////////////////////////////////////////////////////////////////// + // compute parameters in lattice units + ////////////////////////////////////////////////////////////////////////// + + const real velocityLB = velocity * dt / dx; // LB units + + const real vxLB = velocityLB / sqrt(2.0); // LB units + const real vyLB = velocityLB / sqrt(2.0); // LB units + + const real viscosityLB = nx * velocityLB / Re; // LB units + + ////////////////////////////////////////////////////////////////////////// + // set parameters + ////////////////////////////////////////////////////////////////////////// + SPtr<Parameter> para = std::make_shared<Parameter>(); + + para->setOutputPath(path); + para->setOutputPrefix(simulationName); + + para->setPrintFiles(true); + + para->setVelocityLB(velocityLB); + para->setViscosityLB(viscosityLB); + + para->setVelocityRatio(velocity / velocityLB); + para->setDensityRatio(1.0); + + para->setTimestepOut(timeStepOut); + para->setTimestepEnd(timeStepEnd); + + para->setMainKernel("CumulantK17"); + + ////////////////////////////////////////////////////////////////////////// + // set boundary conditions + ////////////////////////////////////////////////////////////////////////// + + gridBuilder->setNoSlipBoundaryCondition(SideType::PX); + gridBuilder->setNoSlipBoundaryCondition(SideType::MX); + gridBuilder->setNoSlipBoundaryCondition(SideType::PY); + gridBuilder->setNoSlipBoundaryCondition(SideType::MY); + gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vxLB, vyLB, 0.0); + gridBuilder->setNoSlipBoundaryCondition(SideType::MZ); + + BoundaryConditionFactory bcFactory; + + bcFactory.setNoSlipBoundaryCondition(BoundaryConditionFactory::NoSlipBC::NoSlipBounceBack); + bcFactory.setVelocityBoundaryCondition(BoundaryConditionFactory::VelocityBC::VelocitySimpleBounceBackCompressible); + + ////////////////////////////////////////////////////////////////////////// + // set copy mesh to simulation + ////////////////////////////////////////////////////////////////////////// + + vf::gpu::Communicator &communicator = vf::gpu::Communicator::getInstance(); + + auto cudaMemoryManager = std::make_shared<CudaMemoryManager>(para); + SPtr<GridProvider> gridGenerator = + GridProvider::makeGridGenerator(gridBuilder, para, cudaMemoryManager, communicator); + + + ////////////////////////////////////////////////////////////////////////// + // run simulation + ////////////////////////////////////////////////////////////////////////// + + VF_LOG_INFO("Start Running DrivenCavity Showcase..."); + printf("\n"); + VF_LOG_INFO("world parameter:"); + VF_LOG_INFO("--------------"); + VF_LOG_INFO("dt [s] = {}", dt); + VF_LOG_INFO("world_length [m] = {}", L); + VF_LOG_INFO("world_velocity [m/s] = {}", velocity); + VF_LOG_INFO("dx [m] = {}", dx); + printf("\n"); + VF_LOG_INFO("LB parameter:"); + VF_LOG_INFO("--------------"); + VF_LOG_INFO("Re = {}", Re); + VF_LOG_INFO("lb_velocity [dx/dt] = {}", velocityLB); + VF_LOG_INFO("lb_viscosity [dx^2/dt] = {}", viscosityLB); + VF_LOG_INFO("lb_vx [dx/dt] (lb_velocity/sqrt(2)) = {}", vxLB); + VF_LOG_INFO("lb_vy [dx/dt] (lb_velocity/sqrt(2)) = {}", vyLB); + printf("\n"); + VF_LOG_INFO("simulation parameter:"); + VF_LOG_INFO("--------------"); + VF_LOG_INFO("nx = {}", nx); + VF_LOG_INFO("ny = {}", nx); + VF_LOG_INFO("nz = {}", nx); + VF_LOG_INFO("number of nodes = {}", nx * nx * nx); + VF_LOG_INFO("n timesteps = {}", timeStepOut); + VF_LOG_INFO("write_nth_timestep = {}", timeStepEnd); + VF_LOG_INFO("output_path = {}", path); + + Simulation sim(para, cudaMemoryManager, communicator, *gridGenerator, &bcFactory, &scalingFactory); + sim.run(); + + } catch (const spdlog::spdlog_ex &ex) { + std::cout << "Log initialization failed: " << ex.what() << std::endl; + } catch (const std::bad_alloc &e) { + VF_LOG_CRITICAL("Bad Alloc: {}", e.what()); + } catch (const std::exception &e) { + VF_LOG_CRITICAL("exception: {}", e.what()); + } catch (...) { + VF_LOG_CRITICAL("Unknown exception!"); + } + + return 0; +} diff --git a/apps/gpu/LBM/DrivenCavityUniform/configDrivenCavity.txt b/apps/gpu/LBM/DrivenCavityUniform/configDrivenCavity.txt new file mode 100644 index 0000000000000000000000000000000000000000..458346a67c7f001580494af1dc9262034613be68 --- /dev/null +++ b/apps/gpu/LBM/DrivenCavityUniform/configDrivenCavity.txt @@ -0,0 +1,34 @@ +################################################## +#GPU Mapping +################################################## +#Devices="0 1 2 3" +#NumberOfDevices=4 + +################################################## +#informations for Writing +################################################## +#Path = "output/" +#Prefix="DrivenCavity" +#WriteGrid=true +################################################## +#informations for reading +################################################## +#GridPath="grid/" + +################################################## +#number of grid levels +################################################## +#NOGL=1 + +################################################## +#LBM Version +################################################## +#D3Qxx=27 +#MainKernelName=CumulantAA2016CompSP27 + +################################################## +#simulation parameter +################################################## +#TimeEnd=100000 +#TimeOut=1000 +#TimeStartOut=0 \ No newline at end of file diff --git a/pythonbindings/CMakeLists.txt b/pythonbindings/CMakeLists.txt index 945dcd08b3bf13fb2e7c569aea811a16fd71f9f0..815a4b59cf6c3e4e5ac4a7a72a5bd4e374d64c96 100644 --- a/pythonbindings/CMakeLists.txt +++ b/pythonbindings/CMakeLists.txt @@ -20,7 +20,7 @@ IF(BUILD_VF_GPU) target_include_directories(python_bindings PRIVATE ${VF_THIRD_DIR}/cuda_samples/) target_compile_definitions(python_bindings PRIVATE VF_GPU_PYTHONBINDINGS) - target_link_libraries(python_bindings PRIVATE GridGenerator VirtualFluids_GPU lbmCuda) + target_link_libraries(python_bindings PRIVATE GridGenerator VirtualFluids_GPU lbm) ENDIF() IF(BUILD_VF_CPU) diff --git a/regression-tests/driven_cavity_test.sh b/regression-tests/driven_cavity_test.sh index 27e2dea9e5c8d4a60f7476c06c6086b0a87c6422..a00e550f0d00181d6e6ab00fc97a619ad8301c38 100755 --- a/regression-tests/driven_cavity_test.sh +++ b/regression-tests/driven_cavity_test.sh @@ -8,7 +8,7 @@ # in this case adding -DUSER_APPS="apps/gpu/LBM/DrivenCavity to the cmake command is not necessary, because the DrivenCavity is added to VirtualFluids by default. mkdir -p build cmake -B build --preset=release_make_gpu -DCMAKE_CUDA_ARCHITECTURES=75 #-DUSER_APPS="apps/gpu/LBM/DrivenCavity" -cd build && make -j 8 && cd .. +cmake --build build --parallel 8 # execute VirtualFluids ./build/bin/DrivenCavity @@ -21,4 +21,4 @@ PATH_TO_DIR=output/DrivenCavity PATH_TO_REFERENCE_DIR=regression-tests/reference_data/regression_tests/gpu/DrivenCavity_2Levels # execute fieldcompare (A more comprehensive manual can be found here https://gitlab.com/dglaeser/fieldcompare) -fieldcompare dir $PATH_TO_DIR --reference $PATH_TO_REFERENCE_DIR --include-files "*.vtu" +fieldcompare dir $PATH_TO_DIR $PATH_TO_REFERENCE_DIR --include-files "*.vtu" diff --git a/regression-tests/driven_cavity_uniform_test.sh b/regression-tests/driven_cavity_uniform_test.sh new file mode 100755 index 0000000000000000000000000000000000000000..9337eb7f9e607a88e884728b599e863f5b746a63 --- /dev/null +++ b/regression-tests/driven_cavity_uniform_test.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +################################# +# Driven Cavity Regression Test +################################# + +# build VirtualFluids accordingly to our specific test scenario. +# in this case adding -DUSER_APPS="apps/gpu/LBM/DrivenCavity to the cmake command is not necessary, because the DrivenCavity is added to VirtualFluids by default. +mkdir -p build +cmake -B build --preset=release_make_gpu -DCMAKE_CUDA_ARCHITECTURES=75 -DUSER_APPS="apps/gpu/LBM/DrivenCavityUniform" +cmake --build build --parallel 8 + +# execute VirtualFluids +./build/bin/DrivenCavity + +# set the path to the produced data +PATH_TO_DIR=output/DrivenCavity_uniform + +# set the path to the reference data. +# `regression-tests/reference_data` is fix `regression_tests/gpu/DrivenCavity_uniform_2022_12_16` must match the structure in https://github.com/irmb/test_data: +PATH_TO_REFERENCE_DIR=regression-tests/reference_data/regression_tests/gpu/DrivenCavity_uniform_2022_12_16 + +# execute fieldcompare (A more comprehensive manual can be found here https://gitlab.com/dglaeser/fieldcompare) +fieldcompare dir $PATH_TO_DIR $PATH_TO_REFERENCE_DIR --include-files "*.vtu" diff --git a/regression-tests/regression-tests.sh b/regression-tests/regression-tests.sh index 5b7d227907594b727103be91d2382c05a07b9c6f..9f5dc8cf758b380709fcc9ad8020d1335f760f64 100755 --- a/regression-tests/regression-tests.sh +++ b/regression-tests/regression-tests.sh @@ -13,11 +13,10 @@ git clone https://github.com/irmb/test_data regression-tests/reference_data # by cloning our meshio patch and fieldcompare into a venv python3 -m venv .venv source .venv/bin/activate -pip install rich -pip install git+https://github.com/soerenPeters/meshio@update-pyproject-version -pip install git+https://gitlab.com/dglaeser/fieldcompare +pip install fieldcompare # 3. Running the specific tests +./regression-tests/driven_cavity_uniform_test.sh ./regression-tests/driven_cavity_test.sh diff --git a/src/gpu/GksGpu/CMakeLists.txt b/src/gpu/GksGpu/CMakeLists.txt index 5dbc533cc5f45c006c29a12242350f0433518bbf..6db6cbac1ff60c76986c3c22cc8017300d4f71ea 100644 --- a/src/gpu/GksGpu/CMakeLists.txt +++ b/src/gpu/GksGpu/CMakeLists.txt @@ -1,6 +1,6 @@ project(GksGpu LANGUAGES CUDA CXX) -vf_add_library(PRIVATE_LINK basics lbmCuda GksMeshAdapter OpenMP::OpenMP_CXX MPI::MPI_CXX) +vf_add_library(PRIVATE_LINK basics lbm GksMeshAdapter OpenMP::OpenMP_CXX MPI::MPI_CXX) target_include_directories(GksGpu PRIVATE "${VF_THIRD_DIR}/cuda_samples/") diff --git a/src/gpu/GksMeshAdapter/CMakeLists.txt b/src/gpu/GksMeshAdapter/CMakeLists.txt index b9a2d12df4d0bee9396a706c6636b5f4056b2d3a..8ac5e69513eca94710797db1f971b2461336b769 100644 --- a/src/gpu/GksMeshAdapter/CMakeLists.txt +++ b/src/gpu/GksMeshAdapter/CMakeLists.txt @@ -1,3 +1,3 @@ project(GksMeshAdapter LANGUAGES CUDA CXX) -vf_add_library(PRIVATE_LINK basics GridGenerator lbmCuda) +vf_add_library(PRIVATE_LINK basics GridGenerator lbm) diff --git a/src/gpu/VirtualFluids_GPU/CMakeLists.txt b/src/gpu/VirtualFluids_GPU/CMakeLists.txt index 759528e5346ba8d9899cb90eb64503b20a44c4fc..ed647cb406bca23ef90667b7d17171c7b3f46283 100644 --- a/src/gpu/VirtualFluids_GPU/CMakeLists.txt +++ b/src/gpu/VirtualFluids_GPU/CMakeLists.txt @@ -8,7 +8,7 @@ if(MSVC) set(additional_libraries ws2_32 Traffic) # ws_32 throws an error on Phoenix endif() -vf_add_library(PUBLIC_LINK basics lbmCuda PRIVATE_LINK ${additional_libraries} GridGenerator MPI::MPI_CXX vf_cuda) +vf_add_library(PUBLIC_LINK basics lbm PRIVATE_LINK ${additional_libraries} GridGenerator MPI::MPI_CXX vf_cuda) #SET(TPN_WIN32 "/EHsc") #https://stackoverflow.com/questions/6832666/lnk2019-when-including-asio-headers-solution-generated-with-cmake diff --git a/src/gpu/VirtualFluids_GPU/Kernel/Kernels/BasicKernels/FluidFlow/Compressible/BGKUnified/BGKUnified.cu b/src/gpu/VirtualFluids_GPU/Kernel/Kernels/BasicKernels/FluidFlow/Compressible/BGKUnified/BGKUnified.cu index 2431809170561c838e1e9905466ba1abfdbbe27b..1107d343801f8ac3626b03a93ca92415217732ac 100644 --- a/src/gpu/VirtualFluids_GPU/Kernel/Kernels/BasicKernels/FluidFlow/Compressible/BGKUnified/BGKUnified.cu +++ b/src/gpu/VirtualFluids_GPU/Kernel/Kernels/BasicKernels/FluidFlow/Compressible/BGKUnified/BGKUnified.cu @@ -6,6 +6,7 @@ #include "../RunLBMKernel.cuh" #include <lbm/BGK.h> +#include <lbm/KernelParameter.h> namespace vf diff --git a/src/gpu/VirtualFluids_GPU/Output/NeighborDebugWriter.hpp b/src/gpu/VirtualFluids_GPU/Output/NeighborDebugWriter.hpp index d325a0c9777954705d1b8eb5f848ac5207633641..57139d25ae4d046e1dd1be1f3ef5e179daf0872e 100644 --- a/src/gpu/VirtualFluids_GPU/Output/NeighborDebugWriter.hpp +++ b/src/gpu/VirtualFluids_GPU/Output/NeighborDebugWriter.hpp @@ -5,44 +5,48 @@ #include "Logger.h" #include "Parameter/Parameter.h" #include "basics/utilities/UbSystem.h" -#include "grid/NodeValues.h" +#include "gpu/GridGenerator/grid/NodeValues.h" #include "lbm/constants/D3Q27.h" #include <basics/writer/WbWriterVtkXmlBinary.h> -#include "Utilities/FindNeighbors.h" -#include "VirtualFluids_GPU/Communication/Communicator.h" #include "Core/StringUtilities/StringUtil.h" +#include "Utilities/FindNeighbors.h" +#include "gpu/VirtualFluids_GPU/Communication/Communicator.h" namespace NeighborDebugWriter { -inline void writeNeighborLinkLines(Parameter *para, const int level, const unsigned long long numberOfNodes, const int direction, - const std::string &name) +inline void writeNeighborLinkLines(LBMSimulationParameter *parH, int direction, const std::string &name, + WbWriter *writer) { VF_LOG_INFO("Write node links in direction {}.", direction); - std::vector<UbTupleFloat3> nodes(numberOfNodes * 2); - std::vector<UbTupleInt2> cells(numberOfNodes); + + const unsigned long long numberOfNodes = parH->numberOfNodes; + std::vector<UbTupleFloat3> nodes; + nodes.reserve(numberOfNodes); + std::vector<UbTupleInt2> cells; + cells.reserve(numberOfNodes/2); for (size_t position = 0; position < numberOfNodes; position++) { - if (para->getParH(level)->typeOfGridNode[position] != GEO_FLUID) + if (parH->typeOfGridNode[position] != GEO_FLUID) continue; - const double x1 = para->getParH(level)->coordinateX[position]; - const double x2 = para->getParH(level)->coordinateY[position]; - const double x3 = para->getParH(level)->coordinateZ[position]; + const double x1 = parH->coordinateX[position]; + const double x2 = parH->coordinateY[position]; + const double x3 = parH->coordinateZ[position]; - const uint positionNeighbor = getNeighborIndex(para->getParH(level).get(), (uint)position, direction); + const uint positionNeighbor = getNeighborIndex(parH, (uint)position, direction); - const double x1Neighbor = para->getParH(level)->coordinateX[positionNeighbor]; - const double x2Neighbor = para->getParH(level)->coordinateY[positionNeighbor]; - const double x3Neighbor = para->getParH(level)->coordinateZ[positionNeighbor]; + const double x1Neighbor = parH->coordinateX[positionNeighbor]; + const double x2Neighbor = parH->coordinateY[positionNeighbor]; + const double x3Neighbor = parH->coordinateZ[positionNeighbor]; nodes.emplace_back(float(x1), float(x2), float(x3)); nodes.emplace_back(float(x1Neighbor), float(x2Neighbor), float(x3Neighbor)); cells.emplace_back((int)nodes.size() - 2, (int)nodes.size() - 1); } - WbWriterVtkXmlBinary::getInstance()->writeLines(name, nodes, cells); + writer->writeLines(name, nodes, cells); } inline void writeNeighborLinkLinesDebug(Parameter *para) @@ -51,7 +55,7 @@ inline void writeNeighborLinkLinesDebug(Parameter *para) for (size_t direction = vf::lbm::dir::STARTDIR; direction <= vf::lbm::dir::ENDDIR; direction++) { const std::string fileName = para->getFName() + "_" + StringUtil::toString<int>(level) + "_Link_" + std::to_string(direction) + "_Debug.vtk"; - writeNeighborLinkLines(para, level, para->getParH(level)->numberOfNodes, (int)direction, fileName); + writeNeighborLinkLines(para->getParH(level).get(), (int)direction, fileName, WbWriterVtkXmlBinary::getInstance()); } } } diff --git a/src/gpu/VirtualFluids_GPU/Output/NeighborDebugWriterTest.cpp b/src/gpu/VirtualFluids_GPU/Output/NeighborDebugWriterTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a19ed3d723f28998f5d27cd15ebf4bab8ba061c4 --- /dev/null +++ b/src/gpu/VirtualFluids_GPU/Output/NeighborDebugWriterTest.cpp @@ -0,0 +1,79 @@ +#include <gmock/gmock.h> +#include "NeighborDebugWriter.hpp" +#include "gpu/VirtualFluids_GPU/Utilities/testUtilitiesGPU.h" + +class WbWriterSpy : public WbWriter +{ +public: + std::string writeLines(const std::string & /*filename*/, std::vector<UbTupleFloat3> &nodes, + std::vector<UbTupleInt2> &lines) override + { + this->nodes = nodes; + this->lines = lines; + return ""; + } + std::vector<UbTupleFloat3> nodes; + std::vector<UbTupleInt2> lines; + + std::string getFileExtension() override { return ""; } +}; + +class NeighborDebugWriterTest : public testing::Test +{ +protected: + void SetUp() override + { + typeOfGridNode = std::vector<uint>(numberOfNodes, GEO_FLUID); + neighbors = std::vector<uint>(numberOfNodes, 2); + coordinates = std::vector<real>(numberOfNodes, 1.0); + coordinates[2] = 3.0; + + parH->numberOfNodes = numberOfNodes; + parH->coordinateX = coordinates.data(); + parH->coordinateY = coordinates.data(); + parH->coordinateZ = coordinates.data(); + parH->neighborX = neighbors.data(); + parH->typeOfGridNode = typeOfGridNode.data(); + } + + const int level = 0; + const unsigned long long numberOfNodes = 3; + const uint direction = vf::lbm::dir::DIR_P00; // x + std::unique_ptr<LBMSimulationParameter> parH = std::make_unique<LBMSimulationParameter>(); + WbWriterSpy writerSpy; + std::vector<uint> typeOfGridNode; + std::vector<uint> neighbors; + std::vector<real> coordinates; +}; + +TEST_F(NeighborDebugWriterTest, writeNeighborLinkLines_onlyFLuidNodes_writesAllNodes) +{ + UbTupleFloat3 oneCoord(1.0, 1.0, 1.0); + UbTupleFloat3 threeCoord(3.0, 3.0, 3.0); + std::vector<UbTupleFloat3> expectedNodes = { oneCoord, threeCoord, oneCoord, threeCoord, threeCoord, threeCoord }; + std::vector<UbTupleInt2> expectedLines = { UbTupleInt2(0, 1), UbTupleInt2(2, 3), UbTupleInt2(4, 5) }; + + NeighborDebugWriter::writeNeighborLinkLines(parH.get(), direction, "name", &writerSpy); + + EXPECT_THAT(writerSpy.nodes.size(), testing::Eq(numberOfNodes * 2)); + EXPECT_THAT(writerSpy.lines.size(), testing::Eq(numberOfNodes)); + EXPECT_THAT(writerSpy.nodes, testing::Eq(expectedNodes)); + EXPECT_THAT(writerSpy.lines, testing::Eq(expectedLines)); +} + +TEST_F(NeighborDebugWriterTest, writeNeighborLinkLines_fluidAndSolidNodes_writesOnlyFluidNodes) +{ + typeOfGridNode[2] = GEO_SOLID; + + UbTupleFloat3 oneCoord(1.0, 1.0, 1.0); + UbTupleFloat3 threeCoord(3.0, 3.0, 3.0); + std::vector<UbTupleFloat3> expectedNodes = { oneCoord, threeCoord, oneCoord, threeCoord}; + std::vector<UbTupleInt2> expectedLines = { UbTupleInt2(0, 1), UbTupleInt2(2, 3)}; + + NeighborDebugWriter::writeNeighborLinkLines(parH.get(), direction, "name", &writerSpy); + + EXPECT_THAT(writerSpy.nodes.size(), testing::Eq((numberOfNodes-1) * 2)); + EXPECT_THAT(writerSpy.lines.size(), testing::Eq(numberOfNodes-1)); + EXPECT_THAT(writerSpy.nodes, testing::Eq(expectedNodes)); + EXPECT_THAT(writerSpy.lines, testing::Eq(expectedLines)); +} diff --git a/src/lbm/CMakeLists.txt b/src/lbm/CMakeLists.txt index afa90bdd3f95bb71cf7f1eda6407f9b38766072a..7a9a96ace1c7377b7ad0c67937464d1f2c00cce6 100644 --- a/src/lbm/CMakeLists.txt +++ b/src/lbm/CMakeLists.txt @@ -1,12 +1,12 @@ -if(BUILD_VF_CPU) - project(lbm LANGUAGES CXX) - vf_add_library(NAME lbm PUBLIC_LINK basics) - target_link_libraries(lbm PRIVATE project_warnings) - - vf_add_tests() -endif() +vf_add_library(PUBLIC_LINK basics) if(BUILD_VF_GPU OR BUILD_VF_GKS) - add_subdirectory(cuda) + set_target_properties(lbm PROPERTIES CUDA_SEPARABLE_COMPILATION ON POSITION_INDEPENDENT_CODE ON) + + set_source_files_properties(KernelParameter.cpp PROPERTIES LANGUAGE CUDA) + set_source_files_properties(CumulantChimera.cpp PROPERTIES LANGUAGE CUDA) + set_source_files_properties(BGK.cpp PROPERTIES LANGUAGE CUDA) endif() + +vf_add_tests() \ No newline at end of file diff --git a/src/lbm/KernelParameter.cpp b/src/lbm/KernelParameter.cpp index e039214d218ef19f35e8adf927f36d3a6f1aa355..7bf5a369d0e5d4e673d79dcb30bc22fc2c330e68 100644 --- a/src/lbm/KernelParameter.cpp +++ b/src/lbm/KernelParameter.cpp @@ -5,11 +5,8 @@ #include "MacroscopicQuantities.h" -namespace vf +namespace vf::lbm { -namespace lbm -{ - inline __host__ __device__ real Distribution27::getDensity_() const @@ -17,8 +14,6 @@ inline __host__ __device__ real Distribution27::getDensity_() const return getDensity(f); } - - __host__ __device__ real abs_internal(real value) { #ifdef __CUDA_ARCH__ @@ -30,4 +25,3 @@ __host__ __device__ real abs_internal(real value) } -} diff --git a/src/lbm/KernelParameter.h b/src/lbm/KernelParameter.h index 95226628110637f3794c8a1f7e6f6c1f6dda937b..18c4f2a4b20b84d9d519993f3ddb54cf612d4306 100644 --- a/src/lbm/KernelParameter.h +++ b/src/lbm/KernelParameter.h @@ -11,9 +11,7 @@ #include <basics/Core/DataTypes.h> -namespace vf -{ -namespace lbm +namespace vf::lbm { struct Distribution27 @@ -35,9 +33,6 @@ struct KernelParameter }; - - -} } #endif diff --git a/src/lbm/cuda/CMakeLists.txt b/src/lbm/cuda/CMakeLists.txt deleted file mode 100644 index 4142b7c3b1c46275c3257e3dfd657cc6b30c841d..0000000000000000000000000000000000000000 --- a/src/lbm/cuda/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -project(lbmCuda LANGUAGES CUDA CXX) - - -vf_add_library(NAME lbmCuda BUILDTYPE static PUBLIC_LINK basics FOLDER ../../lbm) - - -set_target_properties(lbmCuda PROPERTIES CUDA_SEPARABLE_COMPILATION ON POSITION_INDEPENDENT_CODE ON) - - -set_source_files_properties(../KernelParameter.cpp PROPERTIES LANGUAGE CUDA) -set_source_files_properties(../CumulantChimera.cpp PROPERTIES LANGUAGE CUDA) -set_source_files_properties(../BGK.cpp PROPERTIES LANGUAGE CUDA)