From 62620ace13b1a527f715db7520c56f2571ccdc60 Mon Sep 17 00:00:00 2001
From: Anna Wellmann <a.wellmann@tu-braunschweig.de>
Date: Fri, 15 Oct 2021 12:42:25 +0200
Subject: [PATCH] Fix bug by replacing k_CF with intCF.CF in GridGenerator
Not all sendNodes for communication after fine to coarse were found.
This was only a problem when cutting the domain in the z-direction.
---
apps/gpu/LBM/MusselOyster3z/MusselOyster3z.cpp | 4 ++--
.../GridReaderGenerator/GridGenerator.cpp | 6 +++---
src/gpu/VirtualFluids_GPU/Output/FileWriter.cpp | 9 ++++++++-
src/gpu/VirtualFluids_GPU/Output/FileWriter.h | 5 +++--
4 files changed, 16 insertions(+), 8 deletions(-)
diff --git a/apps/gpu/LBM/MusselOyster3z/MusselOyster3z.cpp b/apps/gpu/LBM/MusselOyster3z/MusselOyster3z.cpp
index d02b49721..68c32a5db 100644
--- a/apps/gpu/LBM/MusselOyster3z/MusselOyster3z.cpp
+++ b/apps/gpu/LBM/MusselOyster3z/MusselOyster3z.cpp
@@ -206,7 +206,7 @@ void multipleLevel(const std::string& configPath)
if (useMultiGPU) {
const uint generatePart = vf::gpu::Communicator::getInstanz()->getPID();
- real overlap = (real)8.0 * dxGrid;
+ real overlap = (real)10.0 * dxGrid;
const real zSplit = round(((double)bbzp + bbzm) * 0.5);
if (generatePart == 0) {
@@ -219,7 +219,7 @@ void multipleLevel(const std::string& configPath)
}
if (useLevels) {
- gridBuilder->setNumberOfLayers(6, 8);
+ gridBuilder->setNumberOfLayers(8, 10);
gridBuilder->addGrid(bivalveRef_1_STL, 1);
}
diff --git a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
index 2c1636769..4096ddab5 100644
--- a/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
+++ b/src/gpu/VirtualFluids_GPU/DataStructureInitializer/GridReaderGenerator/GridGenerator.cpp
@@ -891,7 +891,7 @@ void GridGenerator::reorderSendIndicesForCommAfterFtoC(int *sendIndices, int &nu
*logging::out << logging::Logger::INFO_INTERMEDIATE
<< "reorder send indices for communication after fine to coarse: level: " << level
<< " direction: " << direction;
- if (para->getParH(level)->K_CF == 0 || para->getParH(level)->K_FC == 0)
+ if (para->getParH(level)->intCF.kCF == 0 || para->getParH(level)->intFC.kFC == 0)
*logging::out << logging::Logger::LOGGER_ERROR
<< "reorderSendIndicesForCommAfterFtoC(): iCellFCC needs to be inititalized before calling "
"this function "
@@ -907,7 +907,7 @@ void GridGenerator::reorderSendIndicesForCommAfterFtoC(int *sendIndices, int &nu
for (uint posInSendIndices = 0; posInSendIndices < numberOfSendIndices; posInSendIndices++) {
neighbors.fill(-1);
sparseIndexSend = sendIndices[posInSendIndices];
- if (isSparseIndexInICellFCC(para->getParH(level)->K_CF, sparseIndexSend, level))
+ if (isSparseIndexInICellFCC(para->getParH(level)->intFC.kFC, sparseIndexSend, level))
addUniqueIndexToCommunicationVectors(sendIndicesAfterFtoC, sparseIndexSend,
sendIndicesForCommAfterFtoCPositions, posInSendIndices);
}
@@ -961,7 +961,7 @@ void GridGenerator::aggregateNodesInICellCFC(int level, std::vector<uint> &nodes
uint *neighborY = para->getParH(level)->neighborY_SP;
uint *neighborZ = para->getParH(level)->neighborZ_SP;
- for (uint x = 0; x < para->getParH(level)->K_FC; x++) {
+ for (uint x = 0; x < para->getParH(level)->intCF.kCF; x++) {
sparseIndex = para->getParH(level)->intCF.ICellCFC[x];
nodesCFC.push_back(sparseIndex);
nodesCFC.push_back(neighborX[sparseIndex]);
diff --git a/src/gpu/VirtualFluids_GPU/Output/FileWriter.cpp b/src/gpu/VirtualFluids_GPU/Output/FileWriter.cpp
index be7540dcc..0a9e34166 100644
--- a/src/gpu/VirtualFluids_GPU/Output/FileWriter.cpp
+++ b/src/gpu/VirtualFluids_GPU/Output/FileWriter.cpp
@@ -230,9 +230,10 @@ void FileWriter::writeUnstrucuredGridLT(std::shared_ptr<Parameter> para, int lev
nodedata[5][dn1] = (double)para->getParH(level)->geoSP[pos];
//int sendNode = 0; // 0 - not a sendNode; 1 - sendNode; 2 - sendNode in communication after fine to coarse
- // testForSendNodeY(para, level, pos, sendNode); // slow and should not be done multiple times --> use for debugging only!
+ // testForSendNodeZ(para, level, pos, sendNode); // slow and should not be done multiple times --> use for debugging only!
//nodedata[6][dn1] = (double) sendNode;
// nodedata[7][dn1] = (double) pos;
+
//////////////////////////////////////////////////////////////////////////
number2 = para->getParH(level)->neighborX_SP[number1];
number3 = para->getParH(level)->neighborY_SP[number2];
@@ -283,6 +284,12 @@ void FileWriter::testForSendNodeY(std::shared_ptr<Parameter> ¶, int level, u
¶->getParH(level)->sendProcessNeighborsAfterFtoCY);
}
+void FileWriter::testForSendNodeZ(std::shared_ptr<Parameter> ¶, int level, unsigned int pos, int &sendNode)
+{
+ testForCommunicationNode(para, level, pos, sendNode, ¶->getParH(level)->sendProcessNeighborZ,
+ ¶->getParH(level)->sendProcessNeighborsAfterFtoCZ);
+}
+
void FileWriter::testForCommunicationNode(std::shared_ptr<Parameter> ¶, int level, unsigned int pos, int &sendNode,
std::vector<PN27> *sendOrRecvProcessNeighbor,
std::vector<PN27> *sendOrRecvProcessNeighborAfterFtoC)
diff --git a/src/gpu/VirtualFluids_GPU/Output/FileWriter.h b/src/gpu/VirtualFluids_GPU/Output/FileWriter.h
index 28286034a..99c82cbd7 100644
--- a/src/gpu/VirtualFluids_GPU/Output/FileWriter.h
+++ b/src/gpu/VirtualFluids_GPU/Output/FileWriter.h
@@ -27,9 +27,10 @@ private:
void VIRTUALFLUIDS_GPU_EXPORT writeUnstrucuredGridLT(std::shared_ptr<Parameter> para, int level,
std::vector<std::string> &fname);
void testForSendNodeY(std::shared_ptr<Parameter> ¶, int level, unsigned int pos, int &sendNode);
+ void testForSendNodeZ(std::shared_ptr<Parameter> ¶, int level, unsigned int pos, int &sendNode);
void testForCommunicationNode(std::shared_ptr<Parameter> ¶, int level, unsigned int pos, int &sendNode,
- std::vector<PN27> *sendOrRecvProcessNeighbor,
- std::vector<PN27> *sendOrRecvProcessNeighborAfterFtoC);
+ std::vector<PN27> *sendOrRecvProcessNeighbor,
+ std::vector<PN27> *sendOrRecvProcessNeighborAfterFtoC);
void VIRTUALFLUIDS_GPU_EXPORT writeUnstrucuredGridLTConc(std::shared_ptr<Parameter> para, int level, std::vector<std::string >& fname);
void VIRTUALFLUIDS_GPU_EXPORT writeUnstrucuredGridMedianLT(std::shared_ptr<Parameter> para, int level, std::vector<std::string >& fname);
void VIRTUALFLUIDS_GPU_EXPORT writeUnstrucuredGridMedianLTConc(std::shared_ptr<Parameter> para, int level, std::vector<std::string >& fname);
--
GitLab