From 0378d83262c7d0693a5565a3d6ca87d95a53490f Mon Sep 17 00:00:00 2001
From: Anna Wellmann <a.wellmann@tu-bs.de>
Date: Fri, 8 Apr 2022 14:13:33 +0200
Subject: [PATCH] Add writeInterfaceCFC_Recv to InterfaceDebugWriter
---
src/gpu/VirtualFluids_GPU/LBM/Simulation.cpp | 1 +
.../Output/InterfaceDebugWriter.hpp | 156 +++++++++++++-----
2 files changed, 112 insertions(+), 45 deletions(-)
diff --git a/src/gpu/VirtualFluids_GPU/LBM/Simulation.cpp b/src/gpu/VirtualFluids_GPU/LBM/Simulation.cpp
index 77f678ab0..5d56b48f4 100644
--- a/src/gpu/VirtualFluids_GPU/LBM/Simulation.cpp
+++ b/src/gpu/VirtualFluids_GPU/LBM/Simulation.cpp
@@ -392,6 +392,7 @@ void Simulation::init(SPtr<Parameter> para, SPtr<GridProvider> gridProvider, std
// writers for Version with communication hiding
if(para->getNumprocs() > 1 && para->getUseStreams()){
InterfaceDebugWriter::writeInterfaceFCC_Send(para.get());
+ InterfaceDebugWriter::writeInterfaceCFC_Recv(para.get());
InterfaceDebugWriter::writeSendNodesStream(para.get());
InterfaceDebugWriter::writeRecvNodesStream(para.get());
}
diff --git a/src/gpu/VirtualFluids_GPU/Output/InterfaceDebugWriter.hpp b/src/gpu/VirtualFluids_GPU/Output/InterfaceDebugWriter.hpp
index 6039991a7..66b59324f 100644
--- a/src/gpu/VirtualFluids_GPU/Output/InterfaceDebugWriter.hpp
+++ b/src/gpu/VirtualFluids_GPU/Output/InterfaceDebugWriter.hpp
@@ -613,55 +613,70 @@ namespace InterfaceDebugWriter
//////////////////////////////////////////////////////////////////////////
// Functions for version with streams
//////////////////////////////////////////////////////////////////////////
- void checkForSendNodeInX(int pos, int& sendDir, int& sendDirectionInCommAfterFtoC, Parameter* para, int level){
- for(uint pn=0; pn<(uint)para->getParH(level)->sendProcessNeighborX.size(); pn++)
+ void checkForSendOrRecvNode(int pos, int& commDir, int& commDirectionInCommAfterFtoC, std::vector<ProcessNeighbor27>& sendRecvProcessNeighbor,
+ std::vector<ProcessNeighbor27>& sendRecvProcessNeighborsAfterFtoC, double indicator ){
+ for(uint pn=0; pn<(uint)sendRecvProcessNeighbor.size(); pn++)
{
- for(int j=0; j<para->getParH(level)->sendProcessNeighborX[pn].numberOfNodes; j++)
+ for(int j=0; j<sendRecvProcessNeighbor[pn].numberOfNodes; j++)
{
- if (pos == para->getParH(level)->sendProcessNeighborX[pn].index[j]) {
- sendDir=2.0;
- if(j < para->getParH(level)->sendProcessNeighborsAfterFtoCX[pn].numberOfNodes)
+ if (pos == sendRecvProcessNeighbor[pn].index[j]) {
+ commDir=indicator;
+ if(j < sendRecvProcessNeighborsAfterFtoC[pn].numberOfNodes)
{
- sendDirectionInCommAfterFtoC=2.0;
+ commDirectionInCommAfterFtoC=indicator;
}
return;
}
}
}
}
-
- void checkForSendNodeInY(const int pos, int& sendDir, int& sendDirectionInCommAfterFtoC, Parameter* para, int level){
- for(uint pn=0; pn<(uint)para->getParH(level)->sendProcessNeighborY.size(); pn++)
- {
- for(int j=0; j<para->getParH(level)->sendProcessNeighborY[pn].numberOfNodes; j++)
- {
- if (pos == para->getParH(level)->sendProcessNeighborY[pn].index[j]) {
- sendDir+=4.0;
- if(j < para->getParH(level)->sendProcessNeighborsAfterFtoCY[pn].numberOfNodes)
- {
- sendDirectionInCommAfterFtoC+=4.0;
- }
- return;
- }
- }
- }
+
+ void checkForRecvNodeX(int pos, int& recvDir, int& recvDirectionInCommAfterFtoC, Parameter* para, int level)
+ {
+ checkForSendOrRecvNode(pos, recvDir, recvDirectionInCommAfterFtoC,
+ para->getParH(level)->recvProcessNeighborX,
+ para->getParH(level)->recvProcessNeighborsAfterFtoCX,
+ 2.0);
}
- void checkForSendNodeInZ(const int pos, int& sendDir, int& sendDirectionInCommAfterFtoC, Parameter* para, int level){
- for(uint pn=0; pn<(uint)para->getParH(level)->sendProcessNeighborZ.size(); pn++)
- {
- for(int j=0; j<para->getParH(level)->sendProcessNeighborZ[pn].numberOfNodes; j++)
- {
- if (pos == para->getParH(level)->sendProcessNeighborZ[pn].index[j]) {
- sendDir+=8.0;
- if(j < para->getParH(level)->sendProcessNeighborsAfterFtoCZ[pn].numberOfNodes)
- {
- sendDirectionInCommAfterFtoC+=8.0;
- }
- return;
- }
- }
- }
+ void checkForRecvNodeY(int pos, int& recvDir, int& recvDirectionInCommAfterFtoC, Parameter* para, int level)
+ {
+ checkForSendOrRecvNode(pos, recvDir, recvDirectionInCommAfterFtoC,
+ para->getParH(level)->recvProcessNeighborY,
+ para->getParH(level)->recvProcessNeighborsAfterFtoCY,
+ 4.0);
+ }
+
+ void checkForRecvNodeZ(int pos, int& recvDir, int& recvDirectionInCommAfterFtoC, Parameter* para, int level)
+ {
+ checkForSendOrRecvNode(pos, recvDir, recvDirectionInCommAfterFtoC,
+ para->getParH(level)->recvProcessNeighborZ,
+ para->getParH(level)->recvProcessNeighborsAfterFtoCZ,
+ 8.0);
+ }
+
+ void checkForSendNodeX(int pos, int& sendDir, int& sendDirectionInCommAfterFtoC, Parameter* para, int level)
+ {
+ checkForSendOrRecvNode(pos, sendDir, sendDirectionInCommAfterFtoC,
+ para->getParH(level)->sendProcessNeighborX,
+ para->getParH(level)->sendProcessNeighborsAfterFtoCX,
+ 2.0);
+ }
+
+ void checkForSendNodeY(int pos, int& sendDir, int& sendDirectionInCommAfterFtoC, Parameter* para, int level)
+ {
+ checkForSendOrRecvNode(pos, sendDir, sendDirectionInCommAfterFtoC,
+ para->getParH(level)->sendProcessNeighborY,
+ para->getParH(level)->sendProcessNeighborsAfterFtoCY,
+ 4.0);
+ }
+
+ void checkForSendNodeZ(int pos, int& sendDir, int& sendDirectionInCommAfterFtoC, Parameter* para, int level)
+ {
+ checkForSendOrRecvNode(pos, sendDir, sendDirectionInCommAfterFtoC,
+ para->getParH(level)->sendProcessNeighborZ,
+ para->getParH(level)->sendProcessNeighborsAfterFtoCZ,
+ 8.0);
}
void writeInterfaceFCC_Send(Parameter* para){
@@ -697,15 +712,15 @@ namespace InterfaceDebugWriter
nodesVec[nodeCount]=( makeUbTuple( (float)(x1),(float)(x2),(float)(x3) ) );
// nodedata section
- int sendDir = 0;
- int sendDirectionInCommAfterFtoC = 0;
+ nodedata[1][nodeCount]= u < para->getParH(level)->intFCBorder.kFC;
+ int sendDir = 0.0;
+ int sendDirectionInCommAfterFtoC = 0.0;
- checkForSendNodeInX(pos, sendDir, sendDirectionInCommAfterFtoC, para, level);
- checkForSendNodeInY(pos, sendDir, sendDirectionInCommAfterFtoC, para, level);
- checkForSendNodeInZ(pos, sendDir, sendDirectionInCommAfterFtoC, para, level);
+ checkForSendNodeX(pos, sendDir, sendDirectionInCommAfterFtoC, para, level);
+ checkForSendNodeY(pos, sendDir, sendDirectionInCommAfterFtoC, para, level);
+ checkForSendNodeZ(pos, sendDir, sendDirectionInCommAfterFtoC, para, level);
nodedata[2][nodeCount]=sendDir;
nodedata[3][nodeCount]=sendDirectionInCommAfterFtoC;
- nodedata[1][nodeCount]= u < para->getParH(level)->intFCBorder.kFC;
nodeCount++;
}
@@ -715,6 +730,57 @@ namespace InterfaceDebugWriter
}
}
+ void writeInterfaceCFC_Recv(Parameter* para){
+ std::vector< UbTupleFloat3 > nodesVec;
+ int nodeNumberVec = 0;
+
+ // nodedata
+ std::vector< std::string > datanames = {"sparse index", "borderBulk", "recvDirection", "recvDirectionInCommAfterFtoC"};
+ // sendDirection: x = 2, y = 4, z = 8
+ // borderBulk: border = 1, bulk = 0
+ std::vector< std::vector<double>> nodedata;
+
+ for (int level = 0; level < para->getMaxLevel(); level++)
+ {
+ nodeNumberVec += (int) para->getParH(level)->intCF.kCF;
+ }
+
+ nodesVec.resize(nodeNumberVec);
+ nodedata.resize(datanames.size(), std::vector<double>(nodeNumberVec));
+
+ int nodeCount = 0;
+ for (int level = 0; level < para->getMaxLevel(); level++)
+ {
+ for(unsigned int u=0;u<para->getParH(level)->intCF.kCF;u++)
+ {
+ int pos = para->getParH(level)->intCF.ICellCFC[u];
+ nodedata[0][nodeCount]=pos;
+
+ // coordinate section
+ double x1 = para->getParH(level)->coordX_SP[pos];
+ double x2 = para->getParH(level)->coordY_SP[pos];
+ double x3 = para->getParH(level)->coordZ_SP[pos];
+ nodesVec[nodeCount]=( makeUbTuple( (float)(x1),(float)(x2),(float)(x3) ) );
+
+ // nodedata section
+ nodedata[1][nodeCount]= u < para->getParH(level)->intCFBorder.kCF;
+ int recvDir = 0.0;
+ int recvDirectionInCommAfterFtoC = 0.0;
+
+ checkForRecvNodeX(pos, recvDir, recvDirectionInCommAfterFtoC, para, level);
+ checkForRecvNodeY(pos, recvDir, recvDirectionInCommAfterFtoC, para, level);
+ checkForRecvNodeZ(pos, recvDir, recvDirectionInCommAfterFtoC, para, level);
+ nodedata[2][nodeCount]= recvDir;
+ nodedata[3][nodeCount]= recvDirectionInCommAfterFtoC;
+
+ nodeCount++;
+ }
+ std::string filenameVec = para->getFName()+"_writeInterfaceCFC_Recv_PID_" + std::to_string(vf::gpu::Communicator::getInstanz()->getPID()) + "_" + StringUtil::toString<int>(level);
+
+ WbWriterVtkXmlBinary::getInstance()->writeNodesWithNodeData(filenameVec,nodesVec, datanames, nodedata);
+ }
+ }
+
void addToNodesVector(const int level, const int pos, std::vector< UbTupleFloat3 >& nodesVec, Parameter* para){
double x1 = para->getParH(level)->coordX_SP[pos];
double x2 = para->getParH(level)->coordY_SP[pos];
@@ -878,7 +944,7 @@ namespace InterfaceDebugWriter
}
}
- // recv nodes are not in iCellFCC, because they are ghost nodes
+ // Recv are nodes ghost nodes and therefore they can't be iCellCFCs
std::string filenameVec = para->getFName()+"_writeRecvNodesStreams_PID_" + std::to_string(vf::gpu::Communicator::getInstanz()->getPID()) + "_" +StringUtil::toString<int>(level);
--
GitLab