From 237b057eba1f166ed31e223c44cff7ddfad4268f Mon Sep 17 00:00:00 2001
From: Anna Wellmann <a.wellmann@tu-bs.de>
Date: Tue, 26 Apr 2022 11:23:16 +0200
Subject: [PATCH] Fix bug in communication after fine to coarse
In copyEdgeNodes the nodes were copied to the wrong places of the send vector. The problem was that the full vector sendProcessNeighborHostAllNodes was used for copying instead of the shorter vector sendProcessNeighborHostAfterFtoC
---
.../Communication/ExchangeData27.cpp | 29 +++++++++++--------
.../Communication/ExchangeData27.h | 3 +-
src/gpu/VirtualFluids_GPU/LBM/Simulation.cpp | 16 +++++-----
3 files changed, 26 insertions(+), 22 deletions(-)
diff --git a/src/gpu/VirtualFluids_GPU/Communication/ExchangeData27.cpp b/src/gpu/VirtualFluids_GPU/Communication/ExchangeData27.cpp
index eb8f7aa81..6b651b150 100644
--- a/src/gpu/VirtualFluids_GPU/Communication/ExchangeData27.cpp
+++ b/src/gpu/VirtualFluids_GPU/Communication/ExchangeData27.cpp
@@ -70,29 +70,34 @@ void startNonBlockingMpiReceive(unsigned int numberOfSendProcessNeighbors, vf::g
}
-void copyEdgeNodes(std::vector<LBMSimulationParameter::EdgeNodePositions> &edgeNodes, std::vector<ProcessNeighbor27> &recvProcessNeighborHostAllNodes, std::vector<ProcessNeighbor27> &sendProcessNeighborHostAllNodes,
- std::vector<ProcessNeighbor27> &sendProcessNeighborHost)
+void copyEdgeNodes(std::vector<LBMSimulationParameter::EdgeNodePositions> &edgeNodes,
+ std::vector<ProcessNeighbor27> &recvProcessNeighborHost, std::vector<ProcessNeighbor27> &sendProcessNeighborHost)
{
- uint indexInSubdomainRecv = 0;
- uint indexInSubdomainSend = 0;
- uint numNodesInBufferRecv = 0;
- uint numNodesInBufferSend = 0;
+
+ int indexInSubdomainRecv = 0;
+ int indexInSubdomainSend = 0;
+ int numNodesInBufferRecv = 0;
+ int numNodesInBufferSend = 0;
+
#pragma omp parallel for
for (uint i = 0; i < edgeNodes.size(); i++) {
indexInSubdomainRecv = edgeNodes[i].indexOfProcessNeighborRecv;
indexInSubdomainSend = edgeNodes[i].indexOfProcessNeighborSend;
- numNodesInBufferRecv = recvProcessNeighborHostAllNodes[indexInSubdomainRecv].numberOfNodes;
- numNodesInBufferSend = sendProcessNeighborHostAllNodes[indexInSubdomainSend].numberOfNodes;
- if(edgeNodes[i].indexInSendBuffer >= sendProcessNeighborHost[indexInSubdomainSend].numberOfNodes){
+ numNodesInBufferRecv = recvProcessNeighborHost[indexInSubdomainRecv].numberOfNodes;
+ numNodesInBufferSend = sendProcessNeighborHost[indexInSubdomainSend].numberOfNodes;
+ if(edgeNodes[i].indexInSendBuffer >= numNodesInBufferSend){
// for reduced communication after fine to coarse: only copy send nodes which are not part of the reduced comm
continue;
}
- for (uint direction = 0; direction <= dirEND; direction++) {
- (sendProcessNeighborHostAllNodes[indexInSubdomainSend].f[0] + (direction * numNodesInBufferSend))[edgeNodes[i].indexInSendBuffer] = 1000;
- // (recvProcessNeighborHostAllNodes[indexInSubdomainRecv].f[0] + (direction * numNodesInBufferRecv))[edgeNodes[i].indexInRecvBuffer];
+
+ for (int direction = 0; direction <= (int) dirEND; direction++) {
+ (sendProcessNeighborHost[indexInSubdomainSend].f[0] + (direction * numNodesInBufferSend))[edgeNodes[i].indexInSendBuffer] =
+ (recvProcessNeighborHost[indexInSubdomainRecv].f[0] + (direction * numNodesInBufferRecv))[edgeNodes[i].indexInRecvBuffer];
}
}
}
+
+
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// X
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/src/gpu/VirtualFluids_GPU/Communication/ExchangeData27.h b/src/gpu/VirtualFluids_GPU/Communication/ExchangeData27.h
index 9601ace8b..d57a9dad9 100644
--- a/src/gpu/VirtualFluids_GPU/Communication/ExchangeData27.h
+++ b/src/gpu/VirtualFluids_GPU/Communication/ExchangeData27.h
@@ -22,8 +22,7 @@ extern "C" void scatterNodesFromRecvBufferGPU(Parameter *para, int level, int st
std::vector<ProcessNeighbor27> *recvProcessNeighborDev,
unsigned int numberOfRecvProcessNeighbors);
extern "C" void copyEdgeNodes(std::vector<LBMSimulationParameter::EdgeNodePositions> &edgeNodes,
- std::vector<ProcessNeighbor27> &recvProcessNeighborHostAllNodes,
- std::vector<ProcessNeighbor27> &sendProcessNeighborHostAllNodes,
+ std::vector<ProcessNeighbor27> &recvProcessNeighborHost,
std::vector<ProcessNeighbor27> &sendProcessNeighborHost);
// x
diff --git a/src/gpu/VirtualFluids_GPU/LBM/Simulation.cpp b/src/gpu/VirtualFluids_GPU/LBM/Simulation.cpp
index 284b9f239..baf268b25 100644
--- a/src/gpu/VirtualFluids_GPU/LBM/Simulation.cpp
+++ b/src/gpu/VirtualFluids_GPU/LBM/Simulation.cpp
@@ -391,14 +391,14 @@ void Simulation::init(SPtr<Parameter> para, SPtr<GridProvider> gridProvider, std
//InterfaceDebugWriter::writeInterfaceLinesDebugFC(para.get());
// writers for version with communication hiding
- if(para->getNumprocs() > 1 && para->getUseStreams()){
- // InterfaceDebugWriter::writeInterfaceFCC_Send(para.get());
- // InterfaceDebugWriter::writeInterfaceCFC_Recv(para.get());
- // InterfaceDebugWriter::writeSendNodesStream(para.get());
- // InterfaceDebugWriter::writeRecvNodesStream(para.get());
- EdgeNodeDebugWriter::writeEdgeNodesXZ_Send(para);
- EdgeNodeDebugWriter::writeEdgeNodesXZ_Recv(para);
- }
+ // if(para->getNumprocs() > 1 && para->getUseStreams()){
+ // InterfaceDebugWriter::writeInterfaceFCC_Send(para.get());
+ // InterfaceDebugWriter::writeInterfaceCFC_Recv(para.get());
+ // InterfaceDebugWriter::writeSendNodesStream(para.get());
+ // InterfaceDebugWriter::writeRecvNodesStream(para.get());
+ // EdgeNodeDebugWriter::writeEdgeNodesXZ_Send(para);
+ // EdgeNodeDebugWriter::writeEdgeNodesXZ_Recv(para);
+ // }
}
void Simulation::allocNeighborsOffsetsScalesAndBoundaries(SPtr<GridProvider> &gridProvider)
--
GitLab