Skip to content
Snippets Groups Projects
Commit 930c20d6 authored by LEGOLAS\lenz's avatar LEGOLAS\lenz
Browse files

fixes bug in boundary conditions, with multiple GPUs

parent 5528a268
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
src/GksGpu/BoundaryConditions/BoundaryCondition.cpp
+ 2
− 0
View file @ 930c20d6
......@@ -54,6 +54,8 @@ void BoundaryCondition::findBoundaryCells(GksMeshAdapter & adapter, bool allowGh
if( cell.type != STOPPER_OUT_OF_GRID && cell.type != STOPPER_OUT_OF_GRID_BOUNDARY && cell.type != STOPPER_SOLID ) continue;
if( cell.isRecvCell ) continue;
// look in all directions
uint maximalSearchDirection = 27;
......
src/GksGpu/DataBase/DataBase.cpp
+ 1
− 1
View file @ 930c20d6
......@@ -9,7 +9,7 @@
#include "DataBaseAllocator.h"
#include "DataBaseStruct.h"
#include "core/Logger/Logger.h"
#include "Core/Logger/Logger.h"
#include "GksMeshAdapter/GksMeshAdapter.h"
#include "Communication/Communicator.h"
......
src/GridGenerator/grid/GridImp.cu
+ 2
− 1
View file @ 930c20d6
......@@ -1415,11 +1415,12 @@ void GridImp::findCommunicationIndices(int direction, SPtr<BoundingBox> subDomai
if( this->getFieldEntry(index) == INVALID_OUT_OF_GRID ||
this->getFieldEntry(index) == INVALID_SOLID ||
this->getFieldEntry(index) == INVALID_COARSE_UNDER_FINE ||
this->getFieldEntry(index) == STOPPER_SOLID ||
//this->getFieldEntry(index) == STOPPER_SOLID ||
this->getFieldEntry(index) == STOPPER_OUT_OF_GRID ||
this->getFieldEntry(index) == STOPPER_COARSE_UNDER_FINE ) continue;
if( lbmOrGks == LBM && this->getFieldEntry(index) == STOPPER_OUT_OF_GRID_BOUNDARY ) continue;
if( lbmOrGks == LBM && this->getFieldEntry(index) == STOPPER_SOLID ) continue;
if( direction == CommunicationDirections::MX ) findCommunicationIndex( index, x, subDomainBox->minX, direction);
if( direction == CommunicationDirections::PX ) findCommunicationIndex( index, x, subDomainBox->maxX, direction);
......
targets/apps/GKS/MultiGPU_nD/MultiGPU_nD.cpp
+ 8
− 8
View file @ 930c20d6
......@@ -306,8 +306,8 @@ void performanceTest( std::string path, std::string simulationName, uint decompo
{
real U = 0.1;
real ULocal = /*0.1*/ + U * sin( 2.0 * M_PI * cellCenter.x ) * cos( 2.0 * M_PI * cellCenter.y ) * cos( 2.0 * M_PI * cellCenter.z );
real VLocal = /*0.1*/ - U * cos( 2.0 * M_PI * cellCenter.x ) * sin( 2.0 * M_PI * cellCenter.y ) * cos( 2.0 * M_PI * cellCenter.z );
real ULocal = 0.1 + U * sin( 2.0 * M_PI * cellCenter.x ) * cos( 2.0 * M_PI * cellCenter.y ) * cos( 2.0 * M_PI * cellCenter.z );
real VLocal = 0.1 - U * cos( 2.0 * M_PI * cellCenter.x ) * sin( 2.0 * M_PI * cellCenter.y ) * cos( 2.0 * M_PI * cellCenter.z );
real WLocal = 0.1;
real rho = 1.0;
......@@ -337,11 +337,11 @@ void performanceTest( std::string path, std::string simulationName, uint decompo
Initializer::initializeDataUpdate(dataBase);
//dataBase->copyDataDeviceToHost();
dataBase->copyDataDeviceToHost();
//if( rank == 0 ) writeVtkXMLParallelSummaryFile( dataBase, parameters, path + simulationName + "_0", mpiWorldSize );
if( rank == 0 ) writeVtkXMLParallelSummaryFile( dataBase, parameters, path + simulationName + "_0", mpiWorldSize );
//writeVtkXML( dataBase, parameters, 0, path + simulationName + "_0" + "_rank_" + std::to_string(rank) );
writeVtkXML( dataBase, parameters, 0, path + simulationName + "_0" + "_rank_" + std::to_string(rank) );
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
......@@ -365,11 +365,11 @@ void performanceTest( std::string path, std::string simulationName, uint decompo
//////////////////////////////////////////////////////////////////////////
//dataBase->copyDataDeviceToHost();
dataBase->copyDataDeviceToHost();
//if( rank == 0 ) writeVtkXMLParallelSummaryFile( dataBase, parameters, path + simulationName + "_final", mpiWorldSize );
if( rank == 0 ) writeVtkXMLParallelSummaryFile( dataBase, parameters, path + simulationName + "_final", mpiWorldSize );
//writeVtkXML( dataBase, parameters, 0, path + simulationName + "_final_rank_" + std::to_string(rank) );
writeVtkXML( dataBase, parameters, 0, path + simulationName + "_final_rank_" + std::to_string(rank) );
//////////////////////////////////////////////////////////////////////////
......
targets/apps/GKS/RoomFireExtended/RoomFireExtended.cpp
+ 68
− 41
View file @ 930c20d6
......@@ -3,6 +3,7 @@
#define _USE_MATH_DEFINES
#include <math.h>
#include <string>
#include <sstream>
#include <iostream>
#include <exception>
#include <fstream>
......@@ -188,6 +189,8 @@ void thermalCavity( std::string path, std::string simulationName, uint windowInd
gridBuilder->addCoarseGrid(-2.1, -1.6, -0.1,
2.1, 6.0, 5.0, dx);
//gridBuilder->addCoarseGrid(-1.1, -1.2, 1.1,
//1.2, 1.2, 2.2, dx);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
......@@ -228,41 +231,7 @@ void thermalCavity( std::string path, std::string simulationName, uint windowInd
Cuboid boxCoarse ( -2.0, -3.0, -0.5,
3.0, 3.0, 3.5 );
//gridBuilder->addGrid( &boxCoarse, 1 );
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Cuboid roomRef( -2.1, -1.8, -1.0,
2.1, 1.7, 10.0 );
Cuboid windowRef( -1.1, 1.6, 0.9,
1.1, 2.0, 3.0 );
Conglomerate refRegion1;
refRegion1.add( &roomRef );
refRegion1.add( &windowRef );
gridBuilder->setNumberOfLayers(0,22);
//gridBuilder->addGrid( &refRegion1, 2 );
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Cuboid boxRef ( -0.6 * LBurner, -0.6 * LBurner, -1.0,
0.6 * LBurner, 0.6 * LBurner, 10.0 );
Cuboid beamRef( -10.0, -0.25, 2.4, 10.0, 0.25, 10.0 );
Conglomerate refRegion2;
refRegion2.add( &boxRef );
refRegion2.add( &beamRef );
gridBuilder->setNumberOfLayers(0,22);
//gridBuilder->addGrid( &refRegion2, 3 );
uint maxLevel = gridBuilder->getNumberOfGridLevels() - 1;
gridBuilder->addGrid( &boxCoarse, 1 );
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
......@@ -303,10 +272,46 @@ void thermalCavity( std::string path, std::string simulationName, uint windowInd
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Cuboid roomRef( -2.1, -1.8, -1.0,
2.1, 1.7, 10.0 );
Cuboid windowRef( -1.1, 1.6, 0.9,
1.1, 2.0, 3.0 );
Conglomerate refRegion1;
refRegion1.add( &roomRef );
refRegion1.add( &windowRef );
gridBuilder->setNumberOfLayers(0,22);
//gridBuilder->addGrid( &refRegion1, 2 );
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Cuboid boxRef ( -0.6 * LBurner, -0.6 * LBurner, -1.0,
0.6 * LBurner, 0.6 * LBurner, 10.0 );
Cuboid beamRef( -10.0, -0.25, 2.4, 10.0, 0.25, 10.0 );
Conglomerate refRegion2;
refRegion2.add( &boxRef );
refRegion2.add( &beamRef );
gridBuilder->setNumberOfLayers(0,22);
//gridBuilder->addGrid( &refRegion2, 3 );
uint maxLevel = gridBuilder->getNumberOfGridLevels() - 1;
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
gridBuilder->setPeriodicBoundaryCondition(false, false, false);
gridBuilder->buildGrids(GKS, false);
MPI_Barrier(MPI_COMM_WORLD);
//gridBuilder->writeGridsToVtk(path + "Grid_rank_" + std::to_string( rank ) + "_lev_");
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
......@@ -358,7 +363,7 @@ void thermalCavity( std::string path, std::string simulationName, uint windowInd
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
CudaUtility::setCudaDevice( rank % CudaUtility::getCudaDeviceCount() );
//CudaUtility::setCudaDevice( rank % CudaUtility::getCudaDeviceCount() );
auto dataBase = std::make_shared<DataBase>( "GPU" );
......@@ -407,8 +412,6 @@ void thermalCavity( std::string path, std::string simulationName, uint windowInd
dataBase->boundaryConditions.push_back( bcWall );
//dataBase->boundaryConditions.push_back( bcTop );
dataBase->boundaryConditions.push_back( bcOpen );
dataBase->boundaryConditions.push_back( bcPressure );
......@@ -457,7 +460,11 @@ void thermalCavity( std::string path, std::string simulationName, uint windowInd
{
Initializer::interpret(dataBase, [&](Vec3 cellCenter) -> ConservedVariables {
return toConservedVariables(prim, parameters.K);
PrimitiveVariables primLocal = prim;
//if( cellCenter.x > 0 ) primLocal.rho = 1.21;
return toConservedVariables(primLocal, parameters.K);
});
if (rank == 0) writeVtkXMLParallelSummaryFile(dataBase, parameters, path + simulationName + "_0", mpiWorldSize);
......@@ -502,6 +509,8 @@ void thermalCavity( std::string path, std::string simulationName, uint windowInd
*logging::out << logging::Logger::INFO_HIGH << "================================================================================\n";
*logging::out << logging::Logger::INFO_HIGH << "================================================================================\n";
MPI_Barrier(MPI_COMM_WORLD);
cupsAnalyzer.start();
for( uint iter = startIter + 1; iter <= 100000000; iter++ )
......@@ -524,7 +533,7 @@ void thermalCavity( std::string path, std::string simulationName, uint windowInd
//////////////////////////////////////////////////////////////////////////
pointTimeSeriesCollector->run(iter, parameters);
//pointTimeSeriesCollector->run(iter, parameters);
int crashCellIndex = dataBase->getCrashCellIndex();
if( crashCellIndex >= 0 )
......@@ -536,7 +545,7 @@ void thermalCavity( std::string path, std::string simulationName, uint windowInd
break;
}
if( iter % 100 == 0 )
if( iter % 1000 == 0 )
{
dataBase->copyDataDeviceToHost();
......@@ -599,6 +608,22 @@ int main( int argc, char* argv[])
logging::Logger::setDebugLevel(logging::Logger::Level::INFO_LOW);
logging::Logger::timeStamp(logging::Logger::ENABLE);
//////////////////////////////////////////////////////////////////////////
// Important: for Cuda-Aware MPI the device must be set before MPI_Init()
int deviceCount = CudaUtility::getCudaDeviceCount();
if(deviceCount == 0)
{
std::stringstream msg;
msg << "No devices devices found!" << std::endl;
*logging::out << logging::Logger::WARNING << msg.str(); msg.str("");
}
CudaUtility::setCudaDevice( rank % deviceCount );
//////////////////////////////////////////////////////////////////////////
if( sizeof(real) == 4 )
*logging::out << logging::Logger::INFO_HIGH << "Using Single Precision\n";
else
......@@ -633,6 +658,8 @@ int main( int argc, char* argv[])
logFile.close();
MPI_Finalize();
return 0;
}
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment