diff --git a/src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.cpp b/src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.cpp index 75b7f65da990b8d2075995eb379e655fb813e304..af0511662cdd6cc6ecc4e58fb015da476c9120a2 100644 --- a/src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.cpp +++ b/src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.cpp @@ -48,6 +48,7 @@ void updateGrid27(Parameter *para, vf::gpu::Communicator *comm, CudaMemoryManage para->getParD(level)->numberOfFluidNodes, -1); else collision(para, pm, level, t, kernels); + prepareExchangeMultiGPU(para, level, -1); exchangeMultiGPU(para, comm, cudaManager, level, -1); } @@ -70,15 +71,27 @@ void updateGrid27(Parameter *para, vf::gpu::Communicator *comm, CudaMemoryManage preCollisionBC(para, cudaManager, level, t); ////////////////////////////////////////////////////////////////////////// - if( level != para->getFine() ) { - fineToCoarse(para, level); + if (para->getUseStreams() && para->getNumprocs() > 1) { + } else { + if (para->getKernelNeedsFluidNodeIndicesToRun()) { + fineToCoarseUsingIndex(para, level, para->getParD(level)->fluidNodeIndices, + para->getParD(level)->numberOfFluidNodes, -1); - prepareExchangeMultiGPU(para, level, -1); - exchangeMultiGPU(para, comm, cudaManager, level, -1); + prepareExchangeMultiGPU(para, level, -1); + exchangeMultiGPU(para, comm, cudaManager, level, -1); + + coarseToFine(para, level); + } else { + fineToCoarse(para, level); - coarseToFine(para, level); + prepareExchangeMultiGPU(para, level, -1); + exchangeMultiGPU(para, comm, cudaManager, level, -1); + + coarseToFine(para, level); + } + } } } @@ -1126,6 +1139,58 @@ void fineToCoarse(Parameter* para, int level) } +void fineToCoarseUsingIndex(Parameter *para, int level, uint *fluidNodeIndices, uint numberOfFluidNodes, + int streamIndex) +{ + cudaStream_t stream = (streamIndex == -1) ? CU_STREAM_LEGACY : para->getStreamManager()->getStream(streamIndex); + ScaleFC_RhoSq_comp_27_Stream( + para->getParD(level)->d0SP.f[0], para->getParD(level + 1)->d0SP.f[0], para->getParD(level)->neighborX_SP, + para->getParD(level)->neighborY_SP, para->getParD(level)->neighborZ_SP, para->getParD(level + 1)->neighborX_SP, + para->getParD(level + 1)->neighborY_SP, para->getParD(level + 1)->neighborZ_SP, + para->getParD(level)->size_Mat_SP, para->getParD(level + 1)->size_Mat_SP, para->getParD(level)->evenOrOdd, + para->getParD(level)->intFC.ICellFCC, para->getParD(level)->intFC.ICellFCF, para->getParD(level)->K_FC, + para->getParD(level)->omega, para->getParD(level + 1)->omega, para->getParD(level)->vis, + para->getParD(level)->nx, para->getParD(level)->ny, para->getParD(level + 1)->nx, para->getParD(level + 1)->ny, + para->getParD(level)->offFC, fluidNodeIndices, numberOfFluidNodes, stream); + getLastCudaError("ScaleFC27_RhoSq_comp_Stream execution failed"); + + ////////////////////////////////////////////////////////////////////////// + // A D V E C T I O N D I F F U S I O N + ////////////////////////////////////////////////////////////////////////// + + if (para->getDiffOn()) { + if (para->getDiffMod() == 7) { + // TODO + printf("fineToCoarseUsingIndex Advection Diffusion not implemented"); + //ScaleFCThSMG7(para->getParD(level)->d0SP.f[0], para->getParD(level + 1)->d0SP.f[0], + // para->getParD(level)->d7.f[0], para->getParD(level + 1)->d7.f[0], + // para->getParD(level)->neighborX_SP, para->getParD(level)->neighborY_SP, + // para->getParD(level)->neighborZ_SP, para->getParD(level + 1)->neighborX_SP, + // para->getParD(level + 1)->neighborY_SP, para->getParD(level + 1)->neighborZ_SP, + // para->getParD(level)->size_Mat_SP, para->getParD(level + 1)->size_Mat_SP, + // para->getParD(level)->evenOrOdd, para->getParD(level)->intFC.ICellFCC, + // para->getParD(level)->intFC.ICellFCF, para->getParD(level)->K_FC, para->getParD(level)->vis, + // para->getParD(level)->diffusivity, para->getParD(level)->numberofthreads, + // para->getParD(level)->offFC); + //getLastCudaError("ScaleFCTh7 execution failed"); + } else if (para->getDiffMod() == 27) { + // TODO + printf("fineToCoarseUsingIndex Advection Diffusion not implemented"); + //ScaleFCThS27(para->getParD(level)->d0SP.f[0], para->getParD(level + 1)->d0SP.f[0], + // para->getParD(level)->d27.f[0], para->getParD(level + 1)->d27.f[0], + // para->getParD(level)->neighborX_SP, para->getParD(level)->neighborY_SP, + // para->getParD(level)->neighborZ_SP, para->getParD(level + 1)->neighborX_SP, + // para->getParD(level + 1)->neighborY_SP, para->getParD(level + 1)->neighborZ_SP, + // para->getParD(level)->size_Mat_SP, para->getParD(level + 1)->size_Mat_SP, + // para->getParD(level)->evenOrOdd, para->getParD(level)->intFC.ICellFCC, + // para->getParD(level)->intFC.ICellFCF, para->getParD(level)->K_FC, para->getParD(level)->vis, + // para->getParD(level)->diffusivity, para->getParD(level)->numberofthreads, + // para->getParD(level)->offFC); + //getLastCudaError("ScaleFCTh27 execution failed"); + } + } +} + void coarseToFine(Parameter* para, int level) { //ScaleCF_comp_D3Q27F3(para->getParD(level)->d0SP.f[0], para->getParD(level+1)->d0SP.f[0], para->getParD(level+1)->g6.g[0], diff --git a/src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.h b/src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.h index 39a985435e04c3e406b5293462ee3ae60bb91486..ef3cfab2deb13f4592f958f59b0484c56f24dffe 100644 --- a/src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.h +++ b/src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.h @@ -40,7 +40,11 @@ extern "C" void calcMacroscopicQuantities(Parameter* para, int level); extern "C" void preCollisionBC(Parameter* para, CudaMemoryManager* cudaManager, int level, unsigned int t); extern "C" void fineToCoarse(Parameter* para, int level); +extern "C" void fineToCoarseUsingIndex(Parameter *para, int level, uint *fluidNodeIndices = nullptr, + uint numberOfFluidNodes = 0, int stream = -1); extern "C" void coarseToFine(Parameter* para, int level); +extern "C" void coarseToFineUsingIndex(Parameter *para, int level, uint *fluidNodeIndices = nullptr, + uint numberOfFluidNodes = 0, int stream = -1); #endif