From 33618fccef0cd476f8cd1abf844cb0c272f1cb62 Mon Sep 17 00:00:00 2001
From: Anna Wellmann <a.wellmann@tu-braunschweig.de>
Date: Mon, 6 Sep 2021 14:13:20 +0200
Subject: [PATCH] Use new f to c kernel in upgradeGrid27
---
.../Calculation/UpdateGrid27.cpp | 75 +++++++++++++++++--
.../Calculation/UpdateGrid27.h | 4 +
2 files changed, 74 insertions(+), 5 deletions(-)
diff --git a/src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.cpp b/src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.cpp
index 75b7f65da..af0511662 100644
--- a/src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.cpp
+++ b/src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.cpp
@@ -48,6 +48,7 @@ void updateGrid27(Parameter *para, vf::gpu::Communicator *comm, CudaMemoryManage
para->getParD(level)->numberOfFluidNodes, -1);
else
collision(para, pm, level, t, kernels);
+
prepareExchangeMultiGPU(para, level, -1);
exchangeMultiGPU(para, comm, cudaManager, level, -1);
}
@@ -70,15 +71,27 @@ void updateGrid27(Parameter *para, vf::gpu::Communicator *comm, CudaMemoryManage
preCollisionBC(para, cudaManager, level, t);
//////////////////////////////////////////////////////////////////////////
-
if( level != para->getFine() )
{
- fineToCoarse(para, level);
+ if (para->getUseStreams() && para->getNumprocs() > 1) {
+ } else {
+ if (para->getKernelNeedsFluidNodeIndicesToRun()) {
+ fineToCoarseUsingIndex(para, level, para->getParD(level)->fluidNodeIndices,
+ para->getParD(level)->numberOfFluidNodes, -1);
- prepareExchangeMultiGPU(para, level, -1);
- exchangeMultiGPU(para, comm, cudaManager, level, -1);
+ prepareExchangeMultiGPU(para, level, -1);
+ exchangeMultiGPU(para, comm, cudaManager, level, -1);
+
+ coarseToFine(para, level);
+ } else {
+ fineToCoarse(para, level);
- coarseToFine(para, level);
+ prepareExchangeMultiGPU(para, level, -1);
+ exchangeMultiGPU(para, comm, cudaManager, level, -1);
+
+ coarseToFine(para, level);
+ }
+ }
}
}
@@ -1126,6 +1139,58 @@ void fineToCoarse(Parameter* para, int level)
}
+void fineToCoarseUsingIndex(Parameter *para, int level, uint *fluidNodeIndices, uint numberOfFluidNodes,
+ int streamIndex)
+{
+ cudaStream_t stream = (streamIndex == -1) ? CU_STREAM_LEGACY : para->getStreamManager()->getStream(streamIndex);
+ ScaleFC_RhoSq_comp_27_Stream(
+ para->getParD(level)->d0SP.f[0], para->getParD(level + 1)->d0SP.f[0], para->getParD(level)->neighborX_SP,
+ para->getParD(level)->neighborY_SP, para->getParD(level)->neighborZ_SP, para->getParD(level + 1)->neighborX_SP,
+ para->getParD(level + 1)->neighborY_SP, para->getParD(level + 1)->neighborZ_SP,
+ para->getParD(level)->size_Mat_SP, para->getParD(level + 1)->size_Mat_SP, para->getParD(level)->evenOrOdd,
+ para->getParD(level)->intFC.ICellFCC, para->getParD(level)->intFC.ICellFCF, para->getParD(level)->K_FC,
+ para->getParD(level)->omega, para->getParD(level + 1)->omega, para->getParD(level)->vis,
+ para->getParD(level)->nx, para->getParD(level)->ny, para->getParD(level + 1)->nx, para->getParD(level + 1)->ny,
+ para->getParD(level)->offFC, fluidNodeIndices, numberOfFluidNodes, stream);
+ getLastCudaError("ScaleFC27_RhoSq_comp_Stream execution failed");
+
+ //////////////////////////////////////////////////////////////////////////
+ // A D V E C T I O N D I F F U S I O N
+ //////////////////////////////////////////////////////////////////////////
+
+ if (para->getDiffOn()) {
+ if (para->getDiffMod() == 7) {
+ // TODO
+ printf("fineToCoarseUsingIndex Advection Diffusion not implemented");
+ //ScaleFCThSMG7(para->getParD(level)->d0SP.f[0], para->getParD(level + 1)->d0SP.f[0],
+ // para->getParD(level)->d7.f[0], para->getParD(level + 1)->d7.f[0],
+ // para->getParD(level)->neighborX_SP, para->getParD(level)->neighborY_SP,
+ // para->getParD(level)->neighborZ_SP, para->getParD(level + 1)->neighborX_SP,
+ // para->getParD(level + 1)->neighborY_SP, para->getParD(level + 1)->neighborZ_SP,
+ // para->getParD(level)->size_Mat_SP, para->getParD(level + 1)->size_Mat_SP,
+ // para->getParD(level)->evenOrOdd, para->getParD(level)->intFC.ICellFCC,
+ // para->getParD(level)->intFC.ICellFCF, para->getParD(level)->K_FC, para->getParD(level)->vis,
+ // para->getParD(level)->diffusivity, para->getParD(level)->numberofthreads,
+ // para->getParD(level)->offFC);
+ //getLastCudaError("ScaleFCTh7 execution failed");
+ } else if (para->getDiffMod() == 27) {
+ // TODO
+ printf("fineToCoarseUsingIndex Advection Diffusion not implemented");
+ //ScaleFCThS27(para->getParD(level)->d0SP.f[0], para->getParD(level + 1)->d0SP.f[0],
+ // para->getParD(level)->d27.f[0], para->getParD(level + 1)->d27.f[0],
+ // para->getParD(level)->neighborX_SP, para->getParD(level)->neighborY_SP,
+ // para->getParD(level)->neighborZ_SP, para->getParD(level + 1)->neighborX_SP,
+ // para->getParD(level + 1)->neighborY_SP, para->getParD(level + 1)->neighborZ_SP,
+ // para->getParD(level)->size_Mat_SP, para->getParD(level + 1)->size_Mat_SP,
+ // para->getParD(level)->evenOrOdd, para->getParD(level)->intFC.ICellFCC,
+ // para->getParD(level)->intFC.ICellFCF, para->getParD(level)->K_FC, para->getParD(level)->vis,
+ // para->getParD(level)->diffusivity, para->getParD(level)->numberofthreads,
+ // para->getParD(level)->offFC);
+ //getLastCudaError("ScaleFCTh27 execution failed");
+ }
+ }
+}
+
void coarseToFine(Parameter* para, int level)
{
//ScaleCF_comp_D3Q27F3(para->getParD(level)->d0SP.f[0], para->getParD(level+1)->d0SP.f[0], para->getParD(level+1)->g6.g[0],
diff --git a/src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.h b/src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.h
index 39a985435..ef3cfab2d 100644
--- a/src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.h
+++ b/src/gpu/VirtualFluids_GPU/Calculation/UpdateGrid27.h
@@ -40,7 +40,11 @@ extern "C" void calcMacroscopicQuantities(Parameter* para, int level);
extern "C" void preCollisionBC(Parameter* para, CudaMemoryManager* cudaManager, int level, unsigned int t);
extern "C" void fineToCoarse(Parameter* para, int level);
+extern "C" void fineToCoarseUsingIndex(Parameter *para, int level, uint *fluidNodeIndices = nullptr,
+ uint numberOfFluidNodes = 0, int stream = -1);
extern "C" void coarseToFine(Parameter* para, int level);
+extern "C" void coarseToFineUsingIndex(Parameter *para, int level, uint *fluidNodeIndices = nullptr,
+ uint numberOfFluidNodes = 0, int stream = -1);
#endif
--
GitLab