Newer
Older
#include "LBM/LB.h"
#include "GridGenerator/grid/GridBuilder/GridBuilder.h"
#include "GPU/CudaMemoryManager.h"
#include "IndexRearrangementForStreams.h"
#include <sstream>
#include <iostream>
#include <algorithm>
#include "Output/QDebugWriter.hpp"
#include "GridGenerator/VelocitySetter/VelocitySetter.h"
#include "utilities/communication.h"
#include "Communication/Communicator.h"
using namespace vf::lbm::dir;
GridGenerator::GridGenerator(std::shared_ptr<GridBuilder> builder, std::shared_ptr<Parameter> para, std::shared_ptr<CudaMemoryManager> cudaMemoryManager, vf::gpu::Communicator& communicator):
mpiProcessID(communicator.getPID()), builder(builder)
this->cudaMemoryManager = cudaMemoryManager;
Anna Wellmann
committed
this->indexRearrangement = std::make_unique<IndexRearrangementForStreams>(para, builder, communicator);
GridGenerator::~GridGenerator() = default;
void GridGenerator::initalGridInformations()
{
if (para->getKernelNeedsFluidNodeIndicesToRun())
builder->findFluidNodes(para->getUseStreams());
std::vector<int> gridX, gridY, gridZ;
std::vector<int> distX, distY, distZ;
const int numberOfGridLevels = builder->getNumberOfGridLevels();
builder->getGridInformations(gridX, gridY, gridZ, distX, distY, distZ);
para->setMaxLevel(numberOfGridLevels);
para->setGridX(gridX);
para->setGridY(gridY);
para->setGridZ(gridZ);
para->setDistX(distX);
para->setDistY(distY);
para->setDistZ(distZ);
}
const uint numberOfLevels = builder->getNumberOfGridLevels();
std::cout << "Number of Level: " << numberOfLevels << std::endl;
int numberOfNodesGlobal = 0;
std::cout << "Number of Nodes: " << std::endl;
for (uint level = 0; level < numberOfLevels; level++)
{
const int numberOfNodesPerLevel = builder->getNumberOfNodes(level) + 1;
numberOfNodesGlobal += numberOfNodesPerLevel;
std::cout << "Level " << level << " = " << numberOfNodesPerLevel << " Nodes" << std::endl;
setNumberOfNodes(numberOfNodesPerLevel, level);
cudaMemoryManager->cudaAllocCoord(level);
cudaMemoryManager->cudaAllocSP(level);
//cudaMemoryManager->cudaAllocF3SP(level);
cudaMemoryManager->cudaAllocNeighborWSB(level);
cudaMemoryManager->cudaAllocTurbulentViscosity(level);
if(para->getIsBodyForce())
cudaMemoryManager->cudaAllocBodyForce(level);
builder->getNodeValues(
para->getParH(level)->coordinateX,
para->getParH(level)->coordinateY,
para->getParH(level)->coordinateZ,
para->getParH(level)->neighborX,
para->getParH(level)->neighborY,
para->getParH(level)->neighborZ,
para->getParH(level)->neighborInverse,
para->getParH(level)->typeOfGridNode,
level);
setInitalNodeValues(numberOfNodesPerLevel, level);
cudaMemoryManager->cudaCopyNeighborWSB(level);
cudaMemoryManager->cudaCopySP(level);
cudaMemoryManager->cudaCopyCoord(level);
if(para->getIsBodyForce())
cudaMemoryManager->cudaCopyBodyForce(level);
//std::cout << verifyNeighborIndices(level);
}
std::cout << "Number of Nodes: " << numberOfNodesGlobal << std::endl;
std::cout << "-----finish Coord, Neighbor, Geo------" << std::endl;
void GridGenerator::allocArrays_taggedFluidNodes() {
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
for (uint level = 0; level < builder->getNumberOfGridLevels(); level++)
{
for ( int tag = 0; tag < int(CollisionTemplate::LAST); tag++ )
{ //TODO: Need to add CollisionTemplate to GridBuilder to allow as argument and get rid of indivual get funtions for fluid node indices
switch(static_cast<CollisionTemplate>(tag))
{
case CollisionTemplate::Default:
this->setNumberOfTaggedFluidNodes(builder->getNumberOfFluidNodes(level), CollisionTemplate::Default, level);
cudaMemoryManager->cudaAllocTaggedFluidNodeIndices(CollisionTemplate::Default, level);
builder->getFluidNodeIndices(para->getParH(level)->taggedFluidNodeIndices[CollisionTemplate::Default], level);
cudaMemoryManager->cudaCopyTaggedFluidNodeIndices(CollisionTemplate::Default, level);
break;
case CollisionTemplate::Border:
this->setNumberOfTaggedFluidNodes(builder->getNumberOfFluidNodesBorder(level), CollisionTemplate::Border, level);
cudaMemoryManager->cudaAllocTaggedFluidNodeIndices(CollisionTemplate::Border, level);
builder->getFluidNodeIndicesBorder(para->getParH(level)->taggedFluidNodeIndices[CollisionTemplate::Border], level);
cudaMemoryManager->cudaCopyTaggedFluidNodeIndices(CollisionTemplate::Border, level);
break;
case CollisionTemplate::WriteMacroVars:
this->setNumberOfTaggedFluidNodes(builder->getNumberOfFluidNodesMacroVars(level), CollisionTemplate::WriteMacroVars, level);
cudaMemoryManager->cudaAllocTaggedFluidNodeIndices(CollisionTemplate::WriteMacroVars, level);
builder->getFluidNodeIndicesMacroVars(para->getParH(level)->taggedFluidNodeIndices[CollisionTemplate::WriteMacroVars], level);
cudaMemoryManager->cudaCopyTaggedFluidNodeIndices(CollisionTemplate::WriteMacroVars, level);
break;
case CollisionTemplate::ApplyBodyForce:
this->setNumberOfTaggedFluidNodes(builder->getNumberOfFluidNodesApplyBodyForce(level), CollisionTemplate::ApplyBodyForce, level);
cudaMemoryManager->cudaAllocTaggedFluidNodeIndices(CollisionTemplate::ApplyBodyForce, level);
builder->getFluidNodeIndicesApplyBodyForce(para->getParH(level)->taggedFluidNodeIndices[CollisionTemplate::ApplyBodyForce], level);
cudaMemoryManager->cudaCopyTaggedFluidNodeIndices(CollisionTemplate::ApplyBodyForce, level);
break;
case CollisionTemplate::AllFeatures:
this->setNumberOfTaggedFluidNodes(builder->getNumberOfFluidNodesAllFeatures(level), CollisionTemplate::AllFeatures, level);
cudaMemoryManager->cudaAllocTaggedFluidNodeIndices(CollisionTemplate::AllFeatures, level);
builder->getFluidNodeIndicesAllFeatures(para->getParH(level)->taggedFluidNodeIndices[CollisionTemplate::AllFeatures], level);
cudaMemoryManager->cudaCopyTaggedFluidNodeIndices(CollisionTemplate::AllFeatures, level);
break;
default:
break;
}
}
VF_LOG_INFO("Number of tagged nodes on level {}:", level);
VF_LOG_INFO("Default: {}, Border: {}, WriteMacroVars: {}, ApplyBodyForce: {}, AllFeatures: {}",
para->getParH(level)->numberOfTaggedFluidNodes[CollisionTemplate::Default],
para->getParH(level)->numberOfTaggedFluidNodes[CollisionTemplate::Border],
para->getParH(level)->numberOfTaggedFluidNodes[CollisionTemplate::WriteMacroVars],
para->getParH(level)->numberOfTaggedFluidNodes[CollisionTemplate::ApplyBodyForce],
para->getParH(level)->numberOfTaggedFluidNodes[CollisionTemplate::AllFeatures] );
void GridGenerator::tagFluidNodeIndices(std::vector<uint> taggedFluidNodeIndices, CollisionTemplate tag, uint level) {
switch(tag)
{
case CollisionTemplate::WriteMacroVars:
builder->addFluidNodeIndicesMacroVars( taggedFluidNodeIndices, level );
break;
case CollisionTemplate::ApplyBodyForce:
builder->addFluidNodeIndicesApplyBodyForce( taggedFluidNodeIndices, level );
break;
case CollisionTemplate::AllFeatures:
builder->addFluidNodeIndicesAllFeatures( taggedFluidNodeIndices, level );
break;
case CollisionTemplate::Default:
case CollisionTemplate::Border:
throw std::runtime_error("Cannot tag fluid nodes as Default or Border!");
default:
throw std::runtime_error("Tagging fluid nodes with invald tag!");
break;
}
}
void GridGenerator::sortFluidNodeTags() {
VF_LOG_INFO("Start sorting tagged fluid nodes...");
for (uint level = 0; level < builder->getNumberOfGridLevels(); level++)
{
builder->sortFluidNodeIndicesAllFeatures(level); //has to be called first!
builder->sortFluidNodeIndicesMacroVars(level);
builder->sortFluidNodeIndicesApplyBodyForce(level);
std::cout << "------read BoundaryValues------" << std::endl;
int blocks = 0;
for (uint level = 0; level < builder->getNumberOfGridLevels(); level++) {
const auto numberOfPressureValues = int(builder->getPressureSize(level));
*logging::out << logging::Logger::INFO_INTERMEDIATE << "size pressure level " << level << " : " << numberOfPressureValues << "\n";
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->pressureBC.numberOfBCnodes = 0;
para->getParD(level)->outflowPressureCorrectionFactor = para->getOutflowPressureCorrectionFactor();
if (numberOfPressureValues > 1)
{
blocks = (numberOfPressureValues / para->getParH(level)->numberofthreads) + 1;
para->getParH(level)->pressureBC.numberOfBCnodes = blocks * para->getParH(level)->numberofthreads;
cudaMemoryManager->cudaAllocPress(level);
builder->getPressureValues(para->getParH(level)->pressureBC.RhoBC, para->getParH(level)->pressureBC.k, para->getParH(level)->pressureBC.kN, level);
cudaMemoryManager->cudaCopyPress(level);
para->getParD(level)->pressureBC.numberOfBCnodes = para->getParH(level)->pressureBC.numberOfBCnodes;
para->getParH(level)->numberOfPressureBCnodesRead = para->getParH(level)->pressureBC.numberOfBCnodes * para->getD3Qxx();
para->getParD(level)->numberOfPressureBCnodesRead = para->getParH(level)->pressureBC.numberOfBCnodes * para->getD3Qxx();
Henrik Asmuth
committed
for (uint level = 0; level < builder->getNumberOfGridLevels(); level++) {
const auto numberOfSlipValues = int(builder->getSlipSize(level));
*logging::out << logging::Logger::INFO_INTERMEDIATE << "size slip level " << level << " : " << numberOfSlipValues << "\n";
Henrik Asmuth
committed
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->slipBC.numberOfBCnodes = 0;
Henrik Asmuth
committed
if (numberOfSlipValues > 1)
{
blocks = (numberOfSlipValues / para->getParH(level)->numberofthreads) + 1;
para->getParH(level)->slipBC.numberOfBCnodes = blocks * para->getParH(level)->numberofthreads;
Henrik Asmuth
committed
cudaMemoryManager->cudaAllocSlipBC(level);
builder->getSlipValues(para->getParH(level)->slipBC.normalX, para->getParH(level)->slipBC.normalY, para->getParH(level)->slipBC.normalZ, para->getParH(level)->slipBC.k, level);
Henrik Asmuth
committed
cudaMemoryManager->cudaCopySlipBC(level);
}
para->getParD(level)->slipBC.numberOfBCnodes = para->getParH(level)->slipBC.numberOfBCnodes;
para->getParH(level)->numberOfSlipBCnodesRead = para->getParH(level)->slipBC.numberOfBCnodes * para->getD3Qxx();
para->getParD(level)->numberOfSlipBCnodesRead = para->getParH(level)->slipBC.numberOfBCnodes * para->getD3Qxx();
Henrik Asmuth
committed
}
Henrik Asmuth
committed
for (uint level = 0; level < builder->getNumberOfGridLevels(); level++) {
const auto numberOfStressValues = int(builder->getStressSize(level));
*logging::out << logging::Logger::INFO_INTERMEDIATE << "size stress level " << level << " : " << numberOfStressValues << "\n";
Henrik Asmuth
committed
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->stressBC.numberOfBCnodes = 0;
Henrik Asmuth
committed
if (numberOfStressValues > 1)
{
blocks = (numberOfStressValues / para->getParH(level)->numberofthreads) + 1;
para->getParH(level)->stressBC.numberOfBCnodes = blocks * para->getParH(level)->numberofthreads;
Henrik Asmuth
committed
cudaMemoryManager->cudaAllocStressBC(level);
Henrik Asmuth
committed
cudaMemoryManager->cudaAllocWallModel(level, para->getHasWallModelMonitor());
builder->getStressValues( para->getParH(level)->stressBC.normalX, para->getParH(level)->stressBC.normalY, para->getParH(level)->stressBC.normalZ,
para->getParH(level)->stressBC.Vx, para->getParH(level)->stressBC.Vy, para->getParH(level)->stressBC.Vz,
para->getParH(level)->stressBC.Vx1, para->getParH(level)->stressBC.Vy1, para->getParH(level)->stressBC.Vz1,
para->getParH(level)->stressBC.k, para->getParH(level)->stressBC.kN,
Henrik Asmuth
committed
para->getParH(level)->wallModel.samplingOffset, para->getParH(level)->wallModel.z0,
level);
Henrik Asmuth
committed
cudaMemoryManager->cudaCopyStressBC(level);
Henrik Asmuth
committed
cudaMemoryManager->cudaCopyWallModel(level, para->getHasWallModelMonitor());
Henrik Asmuth
committed
}
para->getParD(level)->stressBC.numberOfBCnodes = para->getParH(level)->stressBC.numberOfBCnodes;
para->getParH(level)->numberOfStressBCnodesRead = para->getParH(level)->stressBC.numberOfBCnodes * para->getD3Qxx();
para->getParD(level)->numberOfStressBCnodesRead = para->getParH(level)->stressBC.numberOfBCnodes * para->getD3Qxx();
Henrik Asmuth
committed
}
for (uint level = 0; level < builder->getNumberOfGridLevels(); level++) {
const auto numberOfVelocityValues = int(builder->getVelocitySize(level));
*logging::out << logging::Logger::INFO_INTERMEDIATE << "size velocity level " << level << " : " << numberOfVelocityValues << "\n";
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->velocityBC.numberOfBCnodes = 0;
if (numberOfVelocityValues > 1)
{
blocks = (numberOfVelocityValues / para->getParH(level)->numberofthreads) + 1;
para->getParH(level)->velocityBC.numberOfBCnodes = blocks * para->getParH(level)->numberofthreads;
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
cudaMemoryManager->cudaAllocVeloBC(level);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
builder->getVelocityValues(para->getParH(level)->velocityBC.Vx, para->getParH(level)->velocityBC.Vy, para->getParH(level)->velocityBC.Vz, para->getParH(level)->velocityBC.k, level);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
cudaMemoryManager->cudaCopyVeloBC(level);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// advection - diffusion stuff
if (para->getDiffOn()==true){
//////////////////////////////////////////////////////////////////////////
para->getParH(level)->TempVel.kTemp = para->getParH(level)->velocityBC.numberOfBCnodes;
//cout << "Groesse kTemp = " << para->getParH(i)->TempPress.kTemp << endl;
std::cout << "getTemperatureInit = " << para->getTemperatureInit() << std::endl;
std::cout << "getTemperatureBC = " << para->getTemperatureBC() << std::endl;
//////////////////////////////////////////////////////////////////////////
cudaMemoryManager->cudaAllocTempVeloBC(level);
//cout << "nach alloc " << endl;
//////////////////////////////////////////////////////////////////////////
for (uint m = 0; m < para->getParH(level)->velocityBC.numberOfBCnodes; m++)
{
para->getParH(level)->TempVel.temp[m] = para->getTemperatureInit();
para->getParH(level)->TempVel.tempPulse[m] = para->getTemperatureBC();
para->getParH(level)->TempVel.velo[m] = para->getVelocity();
para->getParH(level)->TempVel.k[m] = para->getParH(level)->velocityBC.k[m];
}
//////////////////////////////////////////////////////////////////////////
//cout << "vor copy " << endl;
cudaMemoryManager->cudaCopyTempVeloBCHD(level);
//cout << "nach copy " << endl;
//////////////////////////////////////////////////////////////////////////
}
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
}
para->getParD(level)->velocityBC.numberOfBCnodes = para->getParH(level)->velocityBC.numberOfBCnodes;
para->getParH(level)->numberOfVeloBCnodesRead = para->getParH(level)->velocityBC.numberOfBCnodes * para->getD3Qxx();
para->getParD(level)->numberOfVeloBCnodesRead = para->getParH(level)->velocityBC.numberOfBCnodes * para->getD3Qxx();
for (uint level = 0; level < builder->getNumberOfGridLevels(); level++) {
const auto numberOfPrecursorValues = int(builder->getPrecursorSize(level));
std::cout << "size precursor level " << level << " : " << numberOfPrecursorValues << std::endl;
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
int blocks = (numberOfPrecursorValues / para->getParH(level)->numberofthreads) + 1;
para->getParH(level)->precursorBC.sizeQ = blocks * para->getParH(level)->numberofthreads;
para->getParD(level)->precursorBC.sizeQ = para->getParH(level)->precursorBC.sizeQ;
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->precursorBC.numberOfBCnodes = numberOfPrecursorValues;
para->getParD(level)->precursorBC.numberOfBCnodes = numberOfPrecursorValues;
para->getParH(level)->numberOfPrecursorBCnodesRead = numberOfPrecursorValues * para->getD3Qxx();
para->getParD(level)->numberOfPrecursorBCnodesRead = numberOfPrecursorValues * para->getD3Qxx();
if (numberOfPrecursorValues > 1)
{
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
cudaMemoryManager->cudaAllocPrecursorBC(level);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
builder->getPrecursorValues(
para->getParH(level)->precursorBC.planeNeighborNT, para->getParH(level)->precursorBC.planeNeighborNB,
para->getParH(level)->precursorBC.planeNeighborST, para->getParH(level)->precursorBC.planeNeighborSB,
para->getParH(level)->precursorBC.weightsNT, para->getParH(level)->precursorBC.weightsNB,
para->getParH(level)->precursorBC.weightsST, para->getParH(level)->precursorBC.weightsSB,
para->getParH(level)->precursorBC.k, para->getParH(level)->velocityReader, para->getParH(level)->precursorBC.numberOfPrecursorNodes,
para->getParH(level)->precursorBC.numberOfQuantities, para->getParH(level)->precursorBC.nTRead,
para->getParH(level)->precursorBC.velocityX, para->getParH(level)->precursorBC.velocityY, para->getParH(level)->precursorBC.velocityZ,
level);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
para->getParD(level)->precursorBC.numberOfPrecursorNodes = para->getParH(level)->precursorBC.numberOfPrecursorNodes;
para->getParD(level)->precursorBC.numberOfQuantities = para->getParH(level)->precursorBC.numberOfQuantities;
para->getParD(level)->precursorBC.nTRead = para->getParH(level)->precursorBC.nTRead;
para->getParD(level)->precursorBC.velocityX = para->getParH(level)->precursorBC.velocityX;
para->getParD(level)->precursorBC.velocityY = para->getParH(level)->precursorBC.velocityY;
para->getParD(level)->precursorBC.velocityZ = para->getParH(level)->precursorBC.velocityZ;
for(auto reader : para->getParH(level)->velocityReader)
{
if(reader->getNumberOfQuantities() != para->getParD(level)->precursorBC.numberOfQuantities) throw std::runtime_error("Number of quantities in reader and number of quantities needed for precursor don't match!");
// read first timestep of precursor into next and copy to next on device
for(auto reader : para->getParH(level)->velocityReader)
{
reader->getNextData(para->getParH(level)->precursorBC.next, para->getParH(level)->precursorBC.numberOfPrecursorNodes, 0);
cudaMemoryManager->cudaCopyPrecursorData(level);
real* tmp = para->getParD(level)->precursorBC.last;
para->getParD(level)->precursorBC.last = para->getParD(level)->precursorBC.next;
para->getParD(level)->precursorBC.next = tmp;
//read second timestep of precursor into next and copy next to device
real nextTime = para->getParD(level)->precursorBC.nTRead*pow(2,-((real)level))*para->getTimeRatio();
for(auto reader : para->getParH(level)->velocityReader)
{
reader->getNextData(para->getParH(level)->precursorBC.next, para->getParH(level)->precursorBC.numberOfPrecursorNodes, nextTime);
para->getParD(level)->precursorBC.nPrecursorReads = 1;
tmp = para->getParD(level)->precursorBC.current;
para->getParD(level)->precursorBC.current = para->getParD(level)->precursorBC.next;
para->getParD(level)->precursorBC.next = tmp;
//start usual cycle of loading, i.e. read velocities of timestep after current and copy asynchronously to device
for(auto reader : para->getParH(level)->velocityReader)
{
reader->getNextData(para->getParH(level)->precursorBC.next, para->getParH(level)->precursorBC.numberOfPrecursorNodes, 2*nextTime);
para->getParD(level)->precursorBC.nPrecursorReads = 2;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// advection - diffusion stuff
if (para->getDiffOn()==true){
throw std::runtime_error(" Advection Diffusion not implemented for Precursor!");
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if (builder->hasGeometryValues()) {
para->setUseGeometryValues(true);
for (uint level = 0; level < builder->getNumberOfGridLevels(); level++) {
int numberOfGeometryValues = builder->getGeometrySize(level);
*logging::out << logging::Logger::INFO_INTERMEDIATE << "size geometry values, Level " << level << " : " << numberOfGeometryValues << "\n";
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->geometryBC.numberOfBCnodes = 0;
if (numberOfGeometryValues > 0)
{
blocks = (numberOfGeometryValues / para->getParH(level)->numberofthreads) + 1;
para->getParH(level)->geometryBC.numberOfBCnodes = blocks * para->getParH(level)->numberofthreads;
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
cudaMemoryManager->cudaAllocGeomValuesBC(level);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
builder->getGeometryValues(para->getParH(level)->geometryBC.Vx, para->getParH(level)->geometryBC.Vy, para->getParH(level)->geometryBC.Vz, level);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
for (uint m = 0; m < para->getParH(level)->geometryBC.numberOfBCnodes; m++)
para->getParH(level)->geometryBC.Vx[m] = para->getParH(level)->geometryBC.Vx[m] / para->getVelocityRatio();
para->getParH(level)->geometryBC.Vy[m] = para->getParH(level)->geometryBC.Vy[m] / para->getVelocityRatio();
para->getParH(level)->geometryBC.Vz[m] = para->getParH(level)->geometryBC.Vz[m] / para->getVelocityRatio();
cudaMemoryManager->cudaCopyGeomValuesBC(level);
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//// advection - diffusion stuff
//if (para->getDiffOn()==true){
// //////////////////////////////////////////////////////////////////////////
// para->getParH(i)->Temp.kTemp = temp4;
// cout << "Groesse kTemp = " << para->getParH(i)->Temp.kTemp << "\n";
// //////////////////////////////////////////////////////////////////////////
// para->cudaAllocTempNoSlipBC(i);
// //////////////////////////////////////////////////////////////////////////
// for (int m = 0; m < temp4; m++)
// {
// para->getParH(i)->Temp.temp[m] = para->getTemperatureInit();
// para->getParH(i)->Temp.k[m] = para->getParH(i)->geometryBC.k[m];
// }
// //////////////////////////////////////////////////////////////////////////
// para->cudaCopyTempNoSlipBCHD(i);
// //////////////////////////////////////////////////////////////////////////
//}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
}
para->getParD(level)->geometryBC.numberOfBCnodes = para->getParH(level)->geometryBC.numberOfBCnodes;
Anna Wellmann
committed
initalValuesDomainDecompostion();
}
void GridGenerator::initalValuesDomainDecompostion()
{
if (para->getNumprocs() < 2)
return;
if ((para->getNumprocs() > 1) /*&& (procNeighborsSendX.size() == procNeighborsRecvX.size())*/) {
// direction has to be changed in case of periodic BCs and multiple sub domains
std::vector<int> fillOrder = { 0, 1, 2, 3, 4, 5 };
for (int direction = 0; direction < 6; direction++) {
if (direction % 2 > 0 && mpiProcessID % 2 > 0 && (builder->getCommunicationProcess(direction) == builder->getCommunicationProcess(direction - 1)))
{
int temp = fillOrder[direction];
fillOrder[direction] = fillOrder[direction-1];
fillOrder[direction-1] = temp;
}
}
for (int direction : fillOrder) {
if (builder->getCommunicationProcess(direction) == INVALID_INDEX)
continue;
for (uint level = 0; level < builder->getNumberOfGridLevels(); level++) {
if (direction == CommunicationDirections::MX || direction == CommunicationDirections::PX) {
Soeren Peters
committed
int j = (int)para->getParH(level)->sendProcessNeighborX.size();
para->getParH(level)->sendProcessNeighborX.emplace_back();
para->getParD(level)->sendProcessNeighborX.emplace_back();
para->getParH(level)->recvProcessNeighborX.emplace_back();
para->getParD(level)->recvProcessNeighborX.emplace_back();
if (para->getDiffOn() == true) {
para->getParH(level)->sendProcessNeighborADX.emplace_back();
para->getParD(level)->sendProcessNeighborADX.emplace_back();
para->getParH(level)->recvProcessNeighborADX.emplace_back();
para->getParD(level)->recvProcessNeighborADX.emplace_back();
}
int tempSend = builder->getNumberOfSendIndices(direction, level);
int tempRecv = builder->getNumberOfReceiveIndices(direction, level);
if (tempSend > 0) {
////////////////////////////////////////////////////////////////////////////////////////
// send
para->getParH(level)->sendProcessNeighborX.back().rankNeighbor =
builder->getCommunicationProcess(direction);
////////////////////////////////////////////////////////////////////////////////////////
*logging::out << logging::Logger::INFO_INTERMEDIATE << "size of Data for X send buffer, \t\tLevel " << level << " : " << tempSend
<< " \t(neighbor rank: " << builder->getCommunicationProcess(direction) << ")\n";
////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->sendProcessNeighborX.back().numberOfNodes = tempSend;
para->getParD(level)->sendProcessNeighborX.back().numberOfNodes = tempSend;
para->getParH(level)->sendProcessNeighborX.back().numberOfFs = para->getD3Qxx() * tempSend;
para->getParD(level)->sendProcessNeighborX.back().numberOfFs = para->getD3Qxx() * tempSend;
para->getParH(level)->sendProcessNeighborX.back().memsizeIndex =
sizeof(unsigned int) * tempSend;
para->getParD(level)->sendProcessNeighborX.back().memsizeIndex =
sizeof(unsigned int) * tempSend;
para->getParH(level)->sendProcessNeighborX.back().memsizeFs = sizeof(real) * tempSend;
para->getParD(level)->sendProcessNeighborX.back().memsizeFs = sizeof(real) * tempSend;
////////////////////////////////////////////////////////////////////////////////////////
// recv
*logging::out << logging::Logger::INFO_INTERMEDIATE << "size of Data for X receive buffer, \tLevel " << level << " : " << tempRecv
<< " \t(neighbor rank: " << builder->getCommunicationProcess(direction) << ")\n";
////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->recvProcessNeighborX.back().rankNeighbor =
builder->getCommunicationProcess(direction);
////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->recvProcessNeighborX.back().numberOfNodes = tempRecv;
para->getParD(level)->recvProcessNeighborX.back().numberOfNodes = tempRecv;
para->getParH(level)->recvProcessNeighborX.back().numberOfFs = para->getD3Qxx() * tempRecv;
para->getParD(level)->recvProcessNeighborX.back().numberOfFs = para->getD3Qxx() * tempRecv;
para->getParH(level)->recvProcessNeighborX.back().memsizeIndex =
sizeof(unsigned int) * tempRecv;
para->getParD(level)->recvProcessNeighborX.back().memsizeIndex =
sizeof(unsigned int) * tempRecv;
para->getParH(level)->recvProcessNeighborX.back().memsizeFs = sizeof(real) * tempRecv;
para->getParD(level)->recvProcessNeighborX.back().memsizeFs = sizeof(real) * tempRecv;
////////////////////////////////////////////////////////////////////////////////////////
// malloc on host and device
cudaMemoryManager->cudaAllocProcessNeighborX(level, j);
////////////////////////////////////////////////////////////////////////////////////////
// init index arrays
builder->getSendIndices(para->getParH(level)->sendProcessNeighborX[j].index, direction, level);
builder->getReceiveIndices(para->getParH(level)->recvProcessNeighborX[j].index, direction,
level);
if (level != builder->getNumberOfGridLevels() - 1 && para->useReducedCommunicationAfterFtoC)
indexRearrangement->initCommunicationArraysForCommAfterFinetoCoarseX(level, j, direction);
////////////////////////////////////////////////////////////////////////////////////////
cudaMemoryManager->cudaCopyProcessNeighborXIndex(level, j);
////////////////////////////////////////////////////////////////////////////////////////
// int sendIDX = para->getParH(level)->sendProcessNeighborX[j].index[0];
// int recvIDX = para->getParH(level)->recvProcessNeighborX[j].index[0];
if (direction == CommunicationDirections::MY || direction == CommunicationDirections::PY) {
Soeren Peters
committed
int j = (int)para->getParH(level)->sendProcessNeighborY.size();
para->getParH(level)->sendProcessNeighborY.emplace_back();
para->getParD(level)->sendProcessNeighborY.emplace_back();
para->getParH(level)->recvProcessNeighborY.emplace_back();
para->getParD(level)->recvProcessNeighborY.emplace_back();
if (para->getDiffOn() == true) {
para->getParH(level)->sendProcessNeighborADY.emplace_back();
para->getParD(level)->sendProcessNeighborADY.emplace_back();
para->getParH(level)->recvProcessNeighborADY.emplace_back();
para->getParD(level)->recvProcessNeighborADY.emplace_back();
}
int tempSend = builder->getNumberOfSendIndices(direction, level);
int tempRecv = builder->getNumberOfReceiveIndices(direction, level);
if (tempSend > 0) {
////////////////////////////////////////////////////////////////////////////////////////
// send
*logging::out << logging::Logger::INFO_INTERMEDIATE << "size of Data for Y send buffer, \t\tLevel " << level << " : " << tempSend
<< " \t(neighbor rank: " << builder->getCommunicationProcess(direction) << ")\n";
////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->sendProcessNeighborY.back().rankNeighbor =
builder->getCommunicationProcess(direction);
////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->sendProcessNeighborY.back().numberOfNodes = tempSend;
para->getParD(level)->sendProcessNeighborY.back().numberOfNodes = tempSend;
para->getParH(level)->sendProcessNeighborY.back().numberOfFs = para->getD3Qxx() * tempSend;
para->getParD(level)->sendProcessNeighborY.back().numberOfFs = para->getD3Qxx() * tempSend;
para->getParH(level)->sendProcessNeighborY.back().memsizeIndex =
sizeof(unsigned int) * tempSend;
para->getParD(level)->sendProcessNeighborY.back().memsizeIndex =
sizeof(unsigned int) * tempSend;
para->getParH(level)->sendProcessNeighborY.back().memsizeFs = sizeof(real) * tempSend;
para->getParD(level)->sendProcessNeighborY.back().memsizeFs = sizeof(real) * tempSend;
////////////////////////////////////////////////////////////////////////////////////////
// recv
*logging::out << logging::Logger::INFO_INTERMEDIATE << "size of Data for Y receive buffer, \tLevel " << level << " : " << tempRecv
<< " \t(neighbor rank: " << builder->getCommunicationProcess(direction) << ")\n";
////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->recvProcessNeighborY.back().rankNeighbor =
builder->getCommunicationProcess(direction);
////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->recvProcessNeighborY.back().numberOfNodes = tempRecv;
para->getParD(level)->recvProcessNeighborY.back().numberOfNodes = tempRecv;
para->getParH(level)->recvProcessNeighborY.back().numberOfFs = para->getD3Qxx() * tempRecv;
para->getParD(level)->recvProcessNeighborY.back().numberOfFs = para->getD3Qxx() * tempRecv;
para->getParH(level)->recvProcessNeighborY.back().memsizeIndex =
sizeof(unsigned int) * tempRecv;
para->getParD(level)->recvProcessNeighborY.back().memsizeIndex =
sizeof(unsigned int) * tempRecv;
para->getParH(level)->recvProcessNeighborY.back().memsizeFs = sizeof(real) * tempRecv;
para->getParD(level)->recvProcessNeighborY.back().memsizeFs = sizeof(real) * tempRecv;
////////////////////////////////////////////////////////////////////////////////////////
// malloc on host and device
cudaMemoryManager->cudaAllocProcessNeighborY(level, j);
////////////////////////////////////////////////////////////////////////////////////////
// init index arrays
builder->getSendIndices(para->getParH(level)->sendProcessNeighborY[j].index, direction, level);
builder->getReceiveIndices(para->getParH(level)->recvProcessNeighborY[j].index, direction,
level);
if (level != builder->getNumberOfGridLevels() - 1 && para->useReducedCommunicationAfterFtoC)
indexRearrangement->initCommunicationArraysForCommAfterFinetoCoarseY(level, j, direction);
////////////////////////////////////////////////////////////////////////////////////////
cudaMemoryManager->cudaCopyProcessNeighborYIndex(level, j);
////////////////////////////////////////////////////////////////////////////////////////
}
if (direction == CommunicationDirections::MZ || direction == CommunicationDirections::PZ) {
Soeren Peters
committed
int j = (int)para->getParH(level)->sendProcessNeighborZ.size();
para->getParH(level)->sendProcessNeighborZ.emplace_back();
para->getParD(level)->sendProcessNeighborZ.emplace_back();
para->getParH(level)->recvProcessNeighborZ.emplace_back();
para->getParD(level)->recvProcessNeighborZ.emplace_back();
if (para->getDiffOn() == true) {
para->getParH(level)->sendProcessNeighborADZ.emplace_back();
para->getParD(level)->sendProcessNeighborADZ.emplace_back();
para->getParH(level)->recvProcessNeighborADZ.emplace_back();
para->getParD(level)->recvProcessNeighborADZ.emplace_back();
}
int tempSend = builder->getNumberOfSendIndices(direction, level);
int tempRecv = builder->getNumberOfReceiveIndices(direction, level);
if (tempSend > 0) {
////////////////////////////////////////////////////////////////////////////////////////
// send
*logging::out << logging::Logger::INFO_INTERMEDIATE << "size of Data for Z send buffer, \t\tLevel " << level << " : " << tempSend
<< " \t(neighbor rank: " << builder->getCommunicationProcess(direction) << ")\n";
////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->sendProcessNeighborZ.back().rankNeighbor =
builder->getCommunicationProcess(direction);
////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->sendProcessNeighborZ.back().numberOfNodes = tempSend;
para->getParD(level)->sendProcessNeighborZ.back().numberOfNodes = tempSend;
para->getParH(level)->sendProcessNeighborZ.back().numberOfFs = para->getD3Qxx() * tempSend;
para->getParD(level)->sendProcessNeighborZ.back().numberOfFs = para->getD3Qxx() * tempSend;
para->getParH(level)->sendProcessNeighborZ.back().memsizeIndex =
sizeof(unsigned int) * tempSend;
para->getParD(level)->sendProcessNeighborZ.back().memsizeIndex =
sizeof(unsigned int) * tempSend;
para->getParH(level)->sendProcessNeighborZ.back().memsizeFs = sizeof(real) * tempSend;
para->getParD(level)->sendProcessNeighborZ.back().memsizeFs = sizeof(real) * tempSend;
////////////////////////////////////////////////////////////////////////////////////////
// recv
*logging::out << logging::Logger::INFO_INTERMEDIATE << "size of Data for Z receive buffer, \tLevel " << level << " : " << tempRecv
<< " \t(neighbor rank: " << builder->getCommunicationProcess(direction) << ")\n";
////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->recvProcessNeighborZ.back().rankNeighbor =
builder->getCommunicationProcess(direction);
////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->recvProcessNeighborZ.back().numberOfNodes = tempRecv;
para->getParD(level)->recvProcessNeighborZ.back().numberOfNodes = tempRecv;
para->getParH(level)->recvProcessNeighborZ.back().numberOfFs = para->getD3Qxx() * tempRecv;
para->getParD(level)->recvProcessNeighborZ.back().numberOfFs = para->getD3Qxx() * tempRecv;
para->getParH(level)->recvProcessNeighborZ.back().memsizeIndex =
sizeof(unsigned int) * tempRecv;
para->getParD(level)->recvProcessNeighborZ.back().memsizeIndex =
sizeof(unsigned int) * tempRecv;
para->getParH(level)->recvProcessNeighborZ.back().memsizeFs = sizeof(real) * tempRecv;
para->getParD(level)->recvProcessNeighborZ.back().memsizeFs = sizeof(real) * tempRecv;
////////////////////////////////////////////////////////////////////////////////////////
// malloc on host and device
cudaMemoryManager->cudaAllocProcessNeighborZ(level, j);
////////////////////////////////////////////////////////////////////////////////////////
// init index arrays
builder->getSendIndices(para->getParH(level)->sendProcessNeighborZ[j].index, direction, level);
builder->getReceiveIndices(para->getParH(level)->recvProcessNeighborZ[j].index, direction,
level);
if (level != builder->getNumberOfGridLevels() - 1 && para->useReducedCommunicationAfterFtoC)
indexRearrangement->initCommunicationArraysForCommAfterFinetoCoarseZ(level, j, direction);
////////////////////////////////////////////////////////////////////////////////////////
cudaMemoryManager->cudaCopyProcessNeighborZIndex(level, j);
////////////////////////////////////////////////////////////////////////////////////////
}
}
}
}
// data exchange for F3 / G6
if ((para->getNumprocs() > 1) && (para->getIsF3())) {
for (int direction = 0; direction < 6; direction++) {
if (builder->getCommunicationProcess(direction) == INVALID_INDEX)
continue;
for (uint level = 0; level < builder->getNumberOfGridLevels(); level++) {
if (direction == CommunicationDirections::MX || direction == CommunicationDirections::PX) {
Soeren Peters
committed
int j = (int)para->getParH(level)->sendProcessNeighborF3X.size();
para->getParH(level)->sendProcessNeighborF3X.emplace_back();
para->getParD(level)->sendProcessNeighborF3X.emplace_back();
para->getParH(level)->recvProcessNeighborF3X.emplace_back();
para->getParD(level)->recvProcessNeighborF3X.emplace_back();
int tempSend = builder->getNumberOfSendIndices(direction, level);
int tempRecv = builder->getNumberOfReceiveIndices(direction, level);
if (tempSend > 0) {
////////////////////////////////////////////////////////////////////////////////////////
// send
*logging::out << logging::Logger::INFO_INTERMEDIATE << "size of Data for X send buffer, \t\tLevel " << level << " : " << tempSend
<< " \t(neighbor rank: " << builder->getCommunicationProcess(direction) << ")\n";
////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->sendProcessNeighborF3X.back().rankNeighbor =
builder->getCommunicationProcess(direction);
////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->sendProcessNeighborF3X.back().numberOfNodes = tempSend;
para->getParD(level)->sendProcessNeighborF3X.back().numberOfNodes = tempSend;
para->getParH(level)->sendProcessNeighborF3X.back().numberOfGs = 6 * tempSend;
para->getParD(level)->sendProcessNeighborF3X.back().numberOfGs = 6 * tempSend;
para->getParH(level)->sendProcessNeighborF3X.back().memsizeIndex =
sizeof(unsigned int) * tempSend;
para->getParD(level)->sendProcessNeighborF3X.back().memsizeIndex =
sizeof(unsigned int) * tempSend;
para->getParH(level)->sendProcessNeighborF3X.back().memsizeGs =
sizeof(real) * para->getParH(level)->sendProcessNeighborF3X.back().numberOfGs;
para->getParD(level)->sendProcessNeighborF3X.back().memsizeGs =
sizeof(real) * para->getParH(level)->sendProcessNeighborF3X.back().numberOfGs;
////////////////////////////////////////////////////////////////////////////////////////
// recv
*logging::out << logging::Logger::INFO_INTERMEDIATE << "size of Data for X receive buffer, \tLevel " << level << " : " << tempRecv
<< " \t(neighbor rank: " << builder->getCommunicationProcess(direction) << ")\n";
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->recvProcessNeighborF3X.back().rankNeighbor =
builder->getCommunicationProcess(direction);
////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->recvProcessNeighborF3X.back().numberOfNodes = tempRecv;
para->getParD(level)->recvProcessNeighborF3X.back().numberOfNodes = tempRecv;
para->getParH(level)->recvProcessNeighborF3X.back().numberOfGs = 6 * tempRecv;
para->getParD(level)->recvProcessNeighborF3X.back().numberOfGs = 6 * tempRecv;
para->getParH(level)->recvProcessNeighborF3X.back().memsizeIndex =
sizeof(unsigned int) * tempRecv;
para->getParD(level)->recvProcessNeighborF3X.back().memsizeIndex =
sizeof(unsigned int) * tempRecv;
para->getParH(level)->recvProcessNeighborF3X.back().memsizeGs =
sizeof(real) * para->getParH(level)->recvProcessNeighborF3X.back().numberOfGs;
para->getParD(level)->recvProcessNeighborF3X.back().memsizeGs =
sizeof(real) * para->getParH(level)->recvProcessNeighborF3X.back().numberOfGs;
////////////////////////////////////////////////////////////////////////////////////////
// malloc on host and device
cudaMemoryManager->cudaAllocProcessNeighborF3X(level, j);
////////////////////////////////////////////////////////////////////////////////////////
// init index arrays
builder->getSendIndices(para->getParH(level)->sendProcessNeighborF3X[j].index, direction,
level);
builder->getReceiveIndices(para->getParH(level)->recvProcessNeighborF3X[j].index, direction,
level);
////////////////////////////////////////////////////////////////////////////////////////
cudaMemoryManager->cudaCopyProcessNeighborF3XIndex(level, j);
////////////////////////////////////////////////////////////////////////////////////////
}
}
if (direction == CommunicationDirections::MY || direction == CommunicationDirections::PY) {
Soeren Peters
committed
int j = (int)para->getParH(level)->sendProcessNeighborF3Y.size();
para->getParH(level)->sendProcessNeighborF3Y.emplace_back();
para->getParD(level)->sendProcessNeighborF3Y.emplace_back();
para->getParH(level)->recvProcessNeighborF3Y.emplace_back();
para->getParD(level)->recvProcessNeighborF3Y.emplace_back();
int tempSend = builder->getNumberOfSendIndices(direction, level);
int tempRecv = builder->getNumberOfReceiveIndices(direction, level);
if (tempSend > 0) {
////////////////////////////////////////////////////////////////////////////////////////
// send
*logging::out << logging::Logger::INFO_INTERMEDIATE << "size of Data for Y send buffer, \t\tLevel " << level << " : " << tempSend
<< " \t(neighbor rank: " << builder->getCommunicationProcess(direction) << ")\n";
////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->sendProcessNeighborF3Y.back().rankNeighbor =
builder->getCommunicationProcess(direction);
////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->sendProcessNeighborF3Y.back().numberOfNodes = tempSend;
para->getParD(level)->sendProcessNeighborF3Y.back().numberOfNodes = tempSend;
para->getParH(level)->sendProcessNeighborF3Y.back().numberOfGs = 6 * tempSend;
para->getParD(level)->sendProcessNeighborF3Y.back().numberOfGs = 6 * tempSend;
para->getParH(level)->sendProcessNeighborF3Y.back().memsizeIndex =
sizeof(unsigned int) * tempSend;
para->getParD(level)->sendProcessNeighborF3Y.back().memsizeIndex =
sizeof(unsigned int) * tempSend;
para->getParH(level)->sendProcessNeighborF3Y.back().memsizeGs =
sizeof(real) * para->getParH(level)->sendProcessNeighborF3Y.back().numberOfGs;
para->getParD(level)->sendProcessNeighborF3Y.back().memsizeGs =
sizeof(real) * para->getParH(level)->sendProcessNeighborF3Y.back().numberOfGs;
////////////////////////////////////////////////////////////////////////////////////////
// recv
*logging::out << logging::Logger::INFO_INTERMEDIATE << "size of Data for Y receive buffer, \tLevel " << level << " : " << tempRecv
<< " \t(neighbor rank: " << builder->getCommunicationProcess(direction) << ")\n";
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->recvProcessNeighborF3Y.back().rankNeighbor =
builder->getCommunicationProcess(direction);
////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->recvProcessNeighborF3Y.back().numberOfNodes = tempRecv;
para->getParD(level)->recvProcessNeighborF3Y.back().numberOfNodes = tempRecv;
para->getParH(level)->recvProcessNeighborF3Y.back().numberOfGs = 6 * tempRecv;
para->getParD(level)->recvProcessNeighborF3Y.back().numberOfGs = 6 * tempRecv;
para->getParH(level)->recvProcessNeighborF3Y.back().memsizeIndex =
sizeof(unsigned int) * tempRecv;
para->getParD(level)->recvProcessNeighborF3Y.back().memsizeIndex =
sizeof(unsigned int) * tempRecv;
para->getParH(level)->recvProcessNeighborF3Y.back().memsizeGs =
sizeof(real) * para->getParH(level)->recvProcessNeighborF3Y.back().numberOfGs;
para->getParD(level)->recvProcessNeighborF3Y.back().memsizeGs =
sizeof(real) * para->getParH(level)->recvProcessNeighborF3Y.back().numberOfGs;
////////////////////////////////////////////////////////////////////////////////////////
// malloc on host and device
cudaMemoryManager->cudaAllocProcessNeighborF3Y(level, j);
////////////////////////////////////////////////////////////////////////////////////////
// init index arrays
builder->getSendIndices(para->getParH(level)->sendProcessNeighborF3Y[j].index, direction,
level);
builder->getReceiveIndices(para->getParH(level)->recvProcessNeighborF3Y[j].index, direction,
level);
////////////////////////////////////////////////////////////////////////////////////////
cudaMemoryManager->cudaCopyProcessNeighborF3YIndex(level, j);
////////////////////////////////////////////////////////////////////////////////////////
}
}
if (direction == CommunicationDirections::MZ || direction == CommunicationDirections::PZ) {
Soeren Peters
committed
int j = (int)para->getParH(level)->sendProcessNeighborF3Z.size();
para->getParH(level)->sendProcessNeighborF3Z.emplace_back();
para->getParD(level)->sendProcessNeighborF3Z.emplace_back();
para->getParH(level)->recvProcessNeighborF3Z.emplace_back();
para->getParD(level)->recvProcessNeighborF3Z.emplace_back();
int tempSend = builder->getNumberOfSendIndices(direction, level);
int tempRecv = builder->getNumberOfReceiveIndices(direction, level);
if (tempSend > 0) {
////////////////////////////////////////////////////////////////////////////////////////
// send
*logging::out << logging::Logger::INFO_INTERMEDIATE << "size of Data for Z send buffer, \t\tLevel " << level << " : " << tempSend
<< " \t(neighbor rank: " << builder->getCommunicationProcess(direction) << ")\n";
////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->sendProcessNeighborF3Z.back().rankNeighbor =
builder->getCommunicationProcess(direction);
////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->sendProcessNeighborF3Z.back().numberOfNodes = tempSend;
para->getParD(level)->sendProcessNeighborF3Z.back().numberOfNodes = tempSend;
para->getParH(level)->sendProcessNeighborF3Z.back().numberOfGs = 6 * tempSend;
para->getParD(level)->sendProcessNeighborF3Z.back().numberOfGs = 6 * tempSend;
para->getParH(level)->sendProcessNeighborF3Z.back().memsizeIndex =
sizeof(unsigned int) * tempSend;
para->getParD(level)->sendProcessNeighborF3Z.back().memsizeIndex =
sizeof(unsigned int) * tempSend;
para->getParH(level)->sendProcessNeighborF3Z.back().memsizeGs =
sizeof(real) * para->getParH(level)->sendProcessNeighborF3Z.back().numberOfGs;
para->getParD(level)->sendProcessNeighborF3Z.back().memsizeGs =
sizeof(real) * para->getParH(level)->sendProcessNeighborF3Z.back().numberOfGs;
////////////////////////////////////////////////////////////////////////////////////////
// recv
*logging::out << logging::Logger::INFO_INTERMEDIATE << "size of Data for Z receive buffer, \tLevel " << level << " : " << tempRecv
<< " \t(neighbor rank: " << builder->getCommunicationProcess(direction) << ")\n";
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->recvProcessNeighborF3Z.back().rankNeighbor =
builder->getCommunicationProcess(direction);
////////////////////////////////////////////////////////////////////////////////////////
para->getParH(level)->recvProcessNeighborF3Z.back().numberOfNodes = tempRecv;
para->getParD(level)->recvProcessNeighborF3Z.back().numberOfNodes = tempRecv;
para->getParH(level)->recvProcessNeighborF3Z.back().numberOfGs = 6 * tempRecv;
para->getParD(level)->recvProcessNeighborF3Z.back().numberOfGs = 6 * tempRecv;
para->getParH(level)->recvProcessNeighborF3Z.back().memsizeIndex =
sizeof(unsigned int) * tempRecv;
para->getParD(level)->recvProcessNeighborF3Z.back().memsizeIndex =
sizeof(unsigned int) * tempRecv;
para->getParH(level)->recvProcessNeighborF3Z.back().memsizeGs =
sizeof(real) * para->getParH(level)->recvProcessNeighborF3Z.back().numberOfGs;
para->getParD(level)->recvProcessNeighborF3Z.back().memsizeGs =
sizeof(real) * para->getParH(level)->recvProcessNeighborF3Z.back().numberOfGs;
////////////////////////////////////////////////////////////////////////////////////////
// malloc on host and device
cudaMemoryManager->cudaAllocProcessNeighborF3Z(level, j);
////////////////////////////////////////////////////////////////////////////////////////
// init index arrays
builder->getSendIndices(para->getParH(level)->sendProcessNeighborF3Z[j].index, direction,
level);
builder->getReceiveIndices(para->getParH(level)->recvProcessNeighborF3Z[j].index, direction,
level);
////////////////////////////////////////////////////////////////////////////////////////
cudaMemoryManager->cudaCopyProcessNeighborF3ZIndex(level, j);
////////////////////////////////////////////////////////////////////////////////////////
}
}
}
}
}
}
std::cout << "------read BoundaryQs-------" << std::endl;
for (uint i = 0; i < builder->getNumberOfGridLevels(); i++) {
const auto numberOfPressureValues = (int)builder->getPressureSize(i);
if (numberOfPressureValues > 0)
{
*logging::out << logging::Logger::INFO_INTERMEDIATE << "size Pressure: " << i << " : " << numberOfPressureValues << "\n";
//cout << "Groesse Pressure: " << i << " : " << temp1 << "MyID: " << para->getMyID() << endl;
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//preprocessing
real* QQ = para->getParH(i)->pressureBC.q27[0];
unsigned int sizeQ = para->getParH(i)->pressureBC.numberOfBCnodes;
getPointersToBoundaryConditions(Q, QQ, sizeQ);
builder->getPressureQs(Q.q27, i);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// advection - diffusion stuff
//cout << "vor advec diff" << endl;
if (para->getDiffOn() == true) {
//////////////////////////////////////////////////////////////////////////
//cout << "vor setzen von kTemp" << endl;
para->getParH(i)->TempPress.kTemp = numberOfPressureValues;
para->getParD(i)->TempPress.kTemp = numberOfPressureValues;
std::cout << "Groesse TempPress.kTemp = " << para->getParH(i)->TempPress.kTemp << std::endl;
//////////////////////////////////////////////////////////////////////////
cudaMemoryManager->cudaAllocTempPressBC(i);
//cout << "nach alloc" << endl;
//////////////////////////////////////////////////////////////////////////
for (int m = 0; m < numberOfPressureValues; m++)
{
para->getParH(i)->TempPress.temp[m] = para->getTemperatureInit();
para->getParH(i)->TempPress.velo[m] = (real)0.0;
para->getParH(i)->TempPress.k[m] = para->getParH(i)->pressureBC.k[m];
}
//////////////////////////////////////////////////////////////////////////
//cout << "vor copy" << endl;
cudaMemoryManager->cudaCopyTempPressBCHD(i);
//cout << "nach copy" << endl;
//////////////////////////////////////////////////////////////////////////
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
}//ende if
}//ende oberste for schleife
Henrik Asmuth
committed
for (uint i = 0; i < builder->getNumberOfGridLevels(); i++) {
int numberOfSlipValues = (int)builder->getSlipSize(i);
if (numberOfSlipValues > 0)
{
*logging::out << logging::Logger::INFO_INTERMEDIATE << "size Slip: " << i << " : " << numberOfSlipValues << "\n";
Henrik Asmuth
committed
//cout << "Groesse Pressure: " << i << " : " << temp1 << "MyID: " << para->getMyID() << endl;
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//preprocessing
real* QQ = para->getParH(i)->slipBC.q27[0];
unsigned int sizeQ = para->getParH(i)->slipBC.numberOfBCnodes;
Henrik Asmuth
committed
QforBoundaryConditions Q;
getPointersToBoundaryConditions(Q, QQ, sizeQ);
Henrik Asmuth
committed
builder->getSlipQs(Q.q27, i);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
cudaMemoryManager->cudaCopySlipBC(i);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
}//ende if
}//ende oberste for schleife
Henrik Asmuth
committed
for (uint i = 0; i < builder->getNumberOfGridLevels(); i++) {
int numberOfStressValues = (int)builder->getStressSize(i);
if (numberOfStressValues > 0)
{
*logging::out << logging::Logger::INFO_INTERMEDIATE << "size Stress: " << i << " : " << numberOfStressValues << "\n";
Henrik Asmuth
committed
//cout << "Groesse Pressure: " << i << " : " << temp1 << "MyID: " << para->getMyID() << endl;
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//preprocessing
real* QQ = para->getParH(i)->stressBC.q27[0];
unsigned int sizeQ = para->getParH(i)->stressBC.numberOfBCnodes;
Henrik Asmuth
committed
QforBoundaryConditions Q;
getPointersToBoundaryConditions(Q, QQ, sizeQ);
Henrik Asmuth
committed
builder->getStressQs(Q.q27, i);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
cudaMemoryManager->cudaCopyStressBC(i);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
}//ende if
}//ende oberste for schleife
for (uint i = 0; i < builder->getNumberOfGridLevels(); i++) {
const auto numberOfVelocityNodes = int(builder->getVelocitySize(i));