From 978a49e418d590aa4cd74f1853ac971eb77bf298 Mon Sep 17 00:00:00 2001
From: Anna Wellmann <a.wellmann@tu-braunschweig.de>
Date: Mon, 18 Oct 2021 14:36:03 +0200
Subject: [PATCH] Split ICellCF into border and bulk
---
.../gpu/LBM/MusselOyster3z/MusselOyster3z.cpp | 1 -
src/gpu/GridGenerator/grid/Grid.h | 5 ++
.../grid/GridBuilder/GridBuilder.h | 6 +-
.../grid/GridBuilder/LevelGridBuilder.cpp | 12 ++++
.../grid/GridBuilder/LevelGridBuilder.h | 8 ++-
src/gpu/GridGenerator/grid/GridImp.cu | 59 ++++++++++++++++++-
src/gpu/GridGenerator/grid/GridImp.h | 7 ++-
.../GridReaderGenerator/GridGenerator.cpp | 21 +++++++
.../VirtualFluids_GPU/Parameter/Parameter.h | 2 +
9 files changed, 115 insertions(+), 6 deletions(-)
diff --git a/apps/gpu/LBM/MusselOyster3z/MusselOyster3z.cpp b/apps/gpu/LBM/MusselOyster3z/MusselOyster3z.cpp
index 68c32a5db..c572d969e 100644
--- a/apps/gpu/LBM/MusselOyster3z/MusselOyster3z.cpp
+++ b/apps/gpu/LBM/MusselOyster3z/MusselOyster3z.cpp
@@ -139,7 +139,6 @@ void multipleLevel(const std::string& configPath)
*logging::out << logging::Logger::INFO_HIGH << "useMultiGPU = " << useMultiGPU << "\n";
*logging::out << logging::Logger::INFO_HIGH << "useStreams = " << useStreams << "\n";
-
para->setTOut(1000);
para->setTEnd(10000);
diff --git a/src/gpu/GridGenerator/grid/Grid.h b/src/gpu/GridGenerator/grid/Grid.h
index 6b179de2d..4b10743dd 100644
--- a/src/gpu/GridGenerator/grid/Grid.h
+++ b/src/gpu/GridGenerator/grid/Grid.h
@@ -55,6 +55,11 @@ public:
uint *iCellFcfBorder, uint *&iCellFcfBulk,
uint &intFCBorderKfc, uint &intFCBulkKfc,
int level) const = 0;
+ CUDA_HOST virtual void getGridInterfaceIndicesCFBorderBulk(uint *iCellCfcBorder, uint *&iCellCfcBulk,
+ uint *iCellCffBorder, uint *&iCellCffBulk,
+ uint &intCFBorderKfc, uint &intCFBulkKfc,
+ uint *neighborX_SP, uint *neighborY_SP,
+ uint *neighborZ_SP, int level) const = 0;
CUDA_HOST virtual int *getNeighborsX() const = 0;
CUDA_HOST virtual int *getNeighborsY() const = 0;
diff --git a/src/gpu/GridGenerator/grid/GridBuilder/GridBuilder.h b/src/gpu/GridGenerator/grid/GridBuilder/GridBuilder.h
index 55dd90545..be42a6ab0 100644
--- a/src/gpu/GridGenerator/grid/GridBuilder/GridBuilder.h
+++ b/src/gpu/GridGenerator/grid/GridBuilder/GridBuilder.h
@@ -58,7 +58,11 @@ public:
virtual void getGridInterfaceIndices(uint* iCellCfc, uint* iCellCff, uint* iCellFcc, uint* iCellFcf, int level) const = 0;
virtual void getGridInterfaceIndicesFCBorderBulk(uint *iCellFccBorder, uint *&iCellFccBulk, uint *iCellFcfBorder,
uint *&iCellFcfBulk, uint &intFCBorderKfc, uint &intFCBulkKfc,
- int level) const = 0;
+ int level) const = 0;
+ virtual void getGridInterfaceIndicesCFBorderBulk(uint *iCellCfcBorder, uint *&iCellCfcBulk, uint *iCellCffBorder,
+ uint *&iCellCffBulk, uint &intCFBorderKfc, uint &intCFBulkKfc,
+ uint *neighborX_SP, uint *neighborY_SP, uint *neighborZ_SP,
+ int level) const = 0;
virtual void getOffsetFC(real* xOffCf, real* yOffCf, real* zOffCf, int level) = 0;
virtual void getOffsetCF(real* xOffFc, real* yOffFc, real* zOffFc, int level) = 0;
diff --git a/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.cpp b/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.cpp
index 04a0cf721..0bcd595c1 100644
--- a/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.cpp
+++ b/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.cpp
@@ -215,6 +215,18 @@ void LevelGridBuilder::getGridInterfaceIndicesFCBorderBulk(uint *iCellFccBorder,
intFCBorderKfc, intFCBulkKfc, level);
}
+void LevelGridBuilder::getGridInterfaceIndicesCFBorderBulk(uint *iCellCfcBorder, uint *&iCellCfcBulk,
+ uint *iCellCffBorder,
+ uint *&iCellCffBulk, uint &intCFBorderKfc, uint &intCFBulkKfc,
+ uint *neighborX_SP, uint *neighborY_SP, uint *neighborZ_SP, int level) const
+{
+ this->grids[level]->getGridInterfaceIndicesCFBorderBulk(iCellCfcBorder, iCellCfcBulk,
+ iCellCffBorder, iCellCffBulk,
+ intCFBorderKfc, intCFBulkKfc,
+ neighborX_SP, neighborY_SP, neighborZ_SP,
+ level);
+}
+
void LevelGridBuilder::getOffsetFC(real * xOffFC, real * yOffFC, real * zOffFC, int level)
{
for (uint i = 0; i < getNumberOfNodesFC(level); i++)
diff --git a/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h b/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h
index ae889715a..d5fa5f538 100644
--- a/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h
+++ b/src/gpu/GridGenerator/grid/GridBuilder/LevelGridBuilder.h
@@ -140,7 +140,13 @@ public:
GRIDGENERATOR_EXPORT void getGridInterfaceIndicesFCBorderBulk(uint *iCellFccBorder, uint *&iCellFccBulk,
uint *iCellFcfBorder, uint *&iCellFcfBulk,
uint &intFCBorderKfc, uint &intFCBulkKfc,
- int level) const override;
+ int level) const override;
+
+ GRIDGENERATOR_EXPORT void getGridInterfaceIndicesCFBorderBulk(uint *iCellCfcBorder, uint *&iCellCfcBulk,
+ uint *iCellCffBorder, uint *&iCellCffBulk,
+ uint &intCFBorderKfc, uint &intCFBulkKfc,
+ uint *neighborX_SP, uint *neighborY_SP,
+ uint *neighborZ_SP, int level) const override;
GRIDGENERATOR_EXPORT void getOffsetFC(real* xOffCf, real* yOffCf, real* zOffCf, int level) override;
GRIDGENERATOR_EXPORT void getOffsetCF(real* xOffFc, real* yOffFc, real* zOffFc, int level) override;
diff --git a/src/gpu/GridGenerator/grid/GridImp.cu b/src/gpu/GridGenerator/grid/GridImp.cu
index 766b134ef..2a2159188 100644
--- a/src/gpu/GridGenerator/grid/GridImp.cu
+++ b/src/gpu/GridGenerator/grid/GridImp.cu
@@ -1958,8 +1958,7 @@ void GridImp::getGridInterfaceIndicesFCBorderBulk(uint *iCellFccBorder, uint *&i
std::vector<uint> iCellFcfBulkVector;
for (uint i = 0; i < intFCKfcAll; i++)
- if (std::find(this->fluidNodeIndicesBorder.begin(), this->fluidNodeIndicesBorder.end(), iCellFccAll[i]) !=
- this->fluidNodeIndicesBorder.end()) {
+ if (isSparseIndexInFluidNodeIndicesBorder(iCellFccAll[i])) {
iCellFccBorderVector.push_back(iCellFccAll[i]);
iCellFcfBorderVector.push_back(iCellFcfAll[i]);
} else {
@@ -1982,6 +1981,62 @@ void GridImp::getGridInterfaceIndicesFCBorderBulk(uint *iCellFccBorder, uint *&i
}
}
+void GridImp::getGridInterfaceIndicesCFBorderBulk(uint *iCellCfcBorder, uint *&iCellCfcBulk, uint *iCellCffBorder,
+ uint *&iCellCffBulk, uint &intCFBorderKfc, uint &intCFBulkKfc,
+ uint *neighborX_SP, uint *neighborY_SP, uint *neighborZ_SP,
+ int level) const
+{
+ // reorder the array of CFC/CFF indices and return pointers and sizes of the new subarrays
+ uint *iCellCfcAll = iCellCfcBorder;
+ uint *iCellCffAll = iCellCffBorder;
+ uint intCFKfcAll = this->gridInterface->fc.numberOfEntries;
+ std::vector<uint> iCellCfcBorderVector;
+ std::vector<uint> iCellCfcBulkVector;
+ std::vector<uint> iCellCffBorderVector;
+ std::vector<uint> iCellCffBulkVector;
+
+ uint sparseIndexOfICellBSW;
+ for (uint i = 0; i < intCFKfcAll; i++) {
+ sparseIndexOfICellBSW = iCellCfcAll[i];
+
+ if (isSparseIndexInFluidNodeIndicesBorder(sparseIndexOfICellBSW)
+ || isSparseIndexInFluidNodeIndicesBorder(neighborX_SP[sparseIndexOfICellBSW])
+ || isSparseIndexInFluidNodeIndicesBorder(neighborY_SP[sparseIndexOfICellBSW])
+ || isSparseIndexInFluidNodeIndicesBorder(neighborZ_SP[sparseIndexOfICellBSW])
+ || isSparseIndexInFluidNodeIndicesBorder(neighborY_SP[neighborX_SP[sparseIndexOfICellBSW]])
+ || isSparseIndexInFluidNodeIndicesBorder(neighborZ_SP[neighborX_SP[sparseIndexOfICellBSW]])
+ || isSparseIndexInFluidNodeIndicesBorder(neighborZ_SP[neighborY_SP[sparseIndexOfICellBSW]])
+ || isSparseIndexInFluidNodeIndicesBorder(neighborZ_SP[neighborY_SP[neighborX_SP[sparseIndexOfICellBSW]]])) {
+
+ iCellCfcBorderVector.push_back(iCellCfcAll[i]);
+ iCellCffBorderVector.push_back(iCellCffAll[i]);
+ } else {
+ iCellCfcBulkVector.push_back(iCellCfcAll[i]);
+ iCellCffBulkVector.push_back(iCellCffAll[i]);
+ }
+ }
+
+ intCFBorderKfc = (uint)iCellCfcBorderVector.size();
+ intCFBulkKfc = (uint)iCellCfcBulkVector.size();
+ iCellCfcBulk = iCellCfcBorder + intCFBorderKfc;
+ iCellCffBulk = iCellCffBorder + intCFBorderKfc;
+
+ for (uint i = 0; i < (uint)iCellCfcBorderVector.size(); i++) {
+ iCellCfcBorder[i] = iCellCfcBorderVector[i];
+ iCellCffBorder[i] = iCellCffBorderVector[i];
+ }
+ for (uint i = 0; i < (uint)iCellCfcBulkVector.size(); i++) {
+ iCellCfcBulk[i] = iCellCfcBulkVector[i];
+ iCellCffBulk[i] = iCellCffBulkVector[i];
+ }
+}
+
+bool GridImp::isSparseIndexInFluidNodeIndicesBorder(uint &sparseIndex) const
+{
+ return std::find(this->fluidNodeIndicesBorder.begin(), this->fluidNodeIndicesBorder.end(), sparseIndex) !=
+ this->fluidNodeIndicesBorder.end();
+}
+
#define GEOFLUID 19
#define GEOSOLID 16
diff --git a/src/gpu/GridGenerator/grid/GridImp.h b/src/gpu/GridGenerator/grid/GridImp.h
index d9daa415c..78596952b 100644
--- a/src/gpu/GridGenerator/grid/GridImp.h
+++ b/src/gpu/GridGenerator/grid/GridImp.h
@@ -57,6 +57,7 @@ private:
HOSTDEVICE bool isOutSideOfGrid(Cell &cell) const;
HOSTDEVICE bool contains(Cell &cell, char type) const;
HOSTDEVICE void setNodeTo(Cell &cell, char type);
+ CUDA_HOST bool isSparseIndexInFluidNodeIndicesBorder(uint &sparseIndex) const;
HOSTDEVICE bool nodeInPreviousCellIs(int index, char type) const;
HOSTDEVICE bool nodeInCellIs(Cell& cell, char type) const override;
@@ -221,7 +222,11 @@ public:
CUDA_HOST static void getGridInterface(uint* gridInterfaceList, const uint* oldGridInterfaceList, uint size);
CUDA_HOST void getGridInterfaceIndicesFCBorderBulk(uint *iCellFccBorder, uint *&iCellFccBulk, uint *iCellFcfBorder,
uint *&iCellFcfBulk, uint &intFCBorderKfc, uint &intFCBulkKfc,
- int level) const override;
+ int level) const override;
+ CUDA_HOST void getGridInterfaceIndicesCFBorderBulk(uint *iCellCfcBorder, uint *&iCellCfcBulk, uint *iCellCffBorder,
+ uint *&iCellCffBulk, uint &intCFBorderKfc, uint &intCFBulkKfc,
+ uint *neighborX_SP, uint *neighborY_SP, uint *neighborZ_SP,
+ int level) const override;
int* getNeighborsX() const override;
int* getNeighborsY() const override;
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
index 4096ddab5..4c0736bef 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
@@ -1392,6 +1392,27 @@ void GridGenerator::allocArrays_OffsetScale()
para->getParD(level)->intFCBulk.ICellFCC = para->getParD(level)->intFCBorder.ICellFCC + para->getParD(level)->intFCBorder.kFC;
para->getParD(level)->intFCBorder.ICellFCF = para->getParD(level)->intFC.ICellFCF;
para->getParD(level)->intFCBulk.ICellFCF = para->getParD(level)->intFCBorder.ICellFCF + para->getParD(level)->intFCBorder.kFC;
+
+ // split coarse-to-fine indices into border and bulk
+ para->getParH(level)->intCFBorder.ICellCFC = para->getParH(level)->intCF.ICellCFC;
+ para->getParH(level)->intCFBorder.ICellCFF = para->getParH(level)->intCF.ICellCFF;
+
+ builder->getGridInterfaceIndicesCFBorderBulk(
+ para->getParH(level)->intCFBorder.ICellCFC, para->getParH(level)->intCFBulk.ICellCFC,
+ para->getParH(level)->intCFBorder.ICellCFF, para->getParH(level)->intCFBulk.ICellCFF,
+ para->getParH(level)->intCFBorder.kCF, para->getParH(level)->intCFBulk.kCF,
+ this->para->getParH(level)->neighborX_SP, this->para->getParH(level)->neighborY_SP,
+ this->para->getParH(level)->neighborZ_SP, level);
+
+ para->getParD(level)->intCFBorder.kCF = para->getParH(level)->intCFBorder.kCF;
+ para->getParD(level)->intCFBulk.kCF = para->getParH(level)->intCFBulk.kCF;
+ para->getParD(level)->intCFBorder.ICellCFC = para->getParD(level)->intCF.ICellCFC;
+ para->getParD(level)->intCFBulk.ICellCFC = para->getParD(level)->intCFBorder.ICellCFC + para->getParD(level)->intCFBorder.kCF;
+ para->getParD(level)->intCFBorder.ICellCFF = para->getParD(level)->intCF.ICellCFF;
+ para->getParD(level)->intCFBulk.ICellCFF = para->getParD(level)->intCFBorder.ICellCFF + para->getParD(level)->intCFBorder.kCF;
+
+
+
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//copy
diff --git a/src/gpu/VirtualFluids_GPU/Parameter/Parameter.h b/src/gpu/VirtualFluids_GPU/Parameter/Parameter.h
index d05da93f3..e468fe15b 100644
--- a/src/gpu/VirtualFluids_GPU/Parameter/Parameter.h
+++ b/src/gpu/VirtualFluids_GPU/Parameter/Parameter.h
@@ -183,6 +183,8 @@ struct LBMSimulationParameter
InterpolationCellFC intFCBorder;
InterpolationCellFC intFCBulk;
+ InterpolationCellCF intCFBorder;
+ InterpolationCellCF intCFBulk;
// offset//////////////////
OffsetCF offCF;
--
GitLab