From 0f8d111e26db73c7a7f4bfae37c6788578fb70e6 Mon Sep 17 00:00:00 2001
From: Anna Wellmann <a.wellmann@tu-bs.de>
Date: Thu, 2 Jun 2022 14:09:31 +0000
Subject: [PATCH] Make routine for finding edge nodes to free functions
---
src/gpu/VirtualFluids_GPU/LBM/Simulation.cpp | 3 +-
.../Parameter/EdgeNodeFinder.cpp | 113 ++++++++++++++++++
.../Parameter/EdgeNodeFinder.h | 65 ++++++++++
.../VirtualFluids_GPU/Parameter/Parameter.cpp | 96 ---------------
.../VirtualFluids_GPU/Parameter/Parameter.h | 11 +-
5 files changed, 182 insertions(+), 106 deletions(-)
create mode 100644 src/gpu/VirtualFluids_GPU/Parameter/EdgeNodeFinder.cpp
create mode 100644 src/gpu/VirtualFluids_GPU/Parameter/EdgeNodeFinder.h
diff --git a/src/gpu/VirtualFluids_GPU/LBM/Simulation.cpp b/src/gpu/VirtualFluids_GPU/LBM/Simulation.cpp
index a05554298..b4a0828fb 100644
--- a/src/gpu/VirtualFluids_GPU/LBM/Simulation.cpp
+++ b/src/gpu/VirtualFluids_GPU/LBM/Simulation.cpp
@@ -10,6 +10,7 @@
#include "Communication/ExchangeData27.h"
#include "Parameter/Parameter.h"
#include "Parameter/CudaStreamManager.h"
+#include "Parameter/EdgeNodeFinder.h"
#include "GPU/GPU_Interface.h"
#include "basics/utilities/UbFileOutputASCII.h"
//////////////////////////////////////////////////////////////////////////
@@ -335,7 +336,7 @@ void Simulation::init(SPtr<Parameter> para, SPtr<GridProvider> gridProvider, std
//////////////////////////////////////////////////////////////////////////
if (para->getDevices().size() > 2) {
output << "Find indices of edge nodes for multiGPU communication ...";
- para->findEdgeNodesCommMultiGPU();
+ vf::gpu::findEdgeNodesCommMultiGPU(para);
output << "done.\n";
}
//////////////////////////////////////////////////////////////////////////
diff --git a/src/gpu/VirtualFluids_GPU/Parameter/EdgeNodeFinder.cpp b/src/gpu/VirtualFluids_GPU/Parameter/EdgeNodeFinder.cpp
new file mode 100644
index 000000000..1134ee0c6
--- /dev/null
+++ b/src/gpu/VirtualFluids_GPU/Parameter/EdgeNodeFinder.cpp
@@ -0,0 +1,113 @@
+#include "EdgeNodeFinder.h"
+#include "Parameter.h"
+
+namespace vf::gpu
+{
+void findEdgeNodesCommMultiGPU(SPtr<Parameter> parameter)
+{
+ for (int level = 0; level <= parameter->getFine(); level++) {
+ findEdgeNodesXY(level, parameter);
+ findEdgeNodesXZ(level, parameter);
+ findEdgeNodesYZ(level, parameter);
+ }
+}
+} // namespace vf::gpu
+
+namespace
+{
+void findEdgeNodesXY(int level, SPtr<Parameter> parameter)
+{
+ int indexOfProcessNeighborSend;
+ int indexInSendBuffer;
+ for (uint i = 0; i < (unsigned int)(parameter->getNumberOfProcessNeighborsX(level, "recv")); i++) {
+ for (int j = 0; j < parameter->getParH(level)->recvProcessNeighborX[i].numberOfNodes; j++) {
+ int index = parameter->getParH(level)->recvProcessNeighborX[i].index[j];
+ bool foundIndex =
+ findIndexInSendNodesXY(level, index, indexOfProcessNeighborSend, indexInSendBuffer, parameter);
+ if (foundIndex) {
+ parameter->getParH(level)->edgeNodesXtoY.emplace_back(i, j, indexOfProcessNeighborSend,
+ indexInSendBuffer);
+ }
+ }
+ }
+}
+
+bool findIndexInSendNodesXY(int level, int index, int &indexOfProcessNeighborSend, int &indexInSendBuffer,
+ SPtr<Parameter> parameter)
+{
+ for (uint k = 0; k < (unsigned int)(parameter->getNumberOfProcessNeighborsY(level, "send")); k++) {
+ for (int l = 0; l < parameter->getParH(level)->sendProcessNeighborY[k].numberOfNodes; l++) {
+ if (parameter->getParH(level)->sendProcessNeighborY[k].index[l] == index) {
+ indexOfProcessNeighborSend = k;
+ indexInSendBuffer = l;
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
+void findEdgeNodesXZ(int level, SPtr<Parameter> parameter)
+{
+ int indexOfProcessNeighborSend;
+ int indexInSendBuffer;
+ for (uint i = 0; i < (unsigned int)(parameter->getNumberOfProcessNeighborsX(level, "recv")); i++) {
+ for (int j = 0; j < parameter->getParH(level)->recvProcessNeighborX[i].numberOfNodes; j++) {
+ int index = parameter->getParH(level)->recvProcessNeighborX[i].index[j];
+ bool foundIndex =
+ findIndexInSendNodesXZ(level, index, indexOfProcessNeighborSend, indexInSendBuffer, parameter);
+ if (foundIndex) {
+ parameter->getParH(level)->edgeNodesXtoZ.emplace_back(i, j, indexOfProcessNeighborSend,
+ indexInSendBuffer);
+ }
+ }
+ }
+}
+
+bool findIndexInSendNodesXZ(int level, int index, int &indexOfProcessNeighborSend, int &indexInSendBuffer,
+ SPtr<Parameter> parameter)
+{
+ for (uint k = 0; k < (unsigned int)(parameter->getNumberOfProcessNeighborsZ(level, "send")); k++) {
+ for (int l = 0; l < parameter->getParH(level)->sendProcessNeighborZ[k].numberOfNodes; l++) {
+ if (parameter->getParH(level)->sendProcessNeighborZ[k].index[l] == index) {
+ indexOfProcessNeighborSend = k;
+ indexInSendBuffer = l;
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
+void findEdgeNodesYZ(int level, SPtr<Parameter> parameter)
+{
+ int indexOfProcessNeighborSend;
+ int indexInSendBuffer;
+ for (uint i = 0; i < (unsigned int)(parameter->getNumberOfProcessNeighborsY(level, "recv")); i++) {
+ for (int j = 0; j < parameter->getParH(level)->recvProcessNeighborY[i].numberOfNodes; j++) {
+ int index = parameter->getParH(level)->recvProcessNeighborY[i].index[j];
+ bool foundIndex =
+ findIndexInSendNodesYZ(level, index, indexOfProcessNeighborSend, indexInSendBuffer, parameter);
+ if (foundIndex) {
+ parameter->getParH(level)->edgeNodesYtoZ.emplace_back(i, j, indexOfProcessNeighborSend,
+ indexInSendBuffer);
+ }
+ }
+ }
+}
+
+bool findIndexInSendNodesYZ(int level, int index, int &indexOfProcessNeighborSend, int &indexInSendBuffer,
+ SPtr<Parameter> parameter)
+{
+ for (uint k = 0; k < (unsigned int)(parameter->getNumberOfProcessNeighborsZ(level, "send")); k++) {
+ for (int l = 0; l < parameter->getParH(level)->sendProcessNeighborZ[k].numberOfNodes; l++) {
+ if (parameter->getParH(level)->sendProcessNeighborZ[k].index[l] == index) {
+ indexOfProcessNeighborSend = k;
+ indexInSendBuffer = l;
+ return true;
+ }
+ }
+ }
+ return false;
+}
+} // namespace
\ No newline at end of file
diff --git a/src/gpu/VirtualFluids_GPU/Parameter/EdgeNodeFinder.h b/src/gpu/VirtualFluids_GPU/Parameter/EdgeNodeFinder.h
new file mode 100644
index 000000000..a526c2840
--- /dev/null
+++ b/src/gpu/VirtualFluids_GPU/Parameter/EdgeNodeFinder.h
@@ -0,0 +1,65 @@
+//=======================================================================================
+// ____ ____ __ ______ __________ __ __ __ __
+// \ \ | | | | | _ \ |___ ___| | | | | / \ | |
+// \ \ | | | | | |_) | | | | | | | / \ | |
+// \ \ | | | | | _ / | | | | | | / /\ \ | |
+// \ \ | | | | | | \ \ | | | \__/ | / ____ \ | |____
+// \ \ | | |__| |__| \__\ |__| \________/ /__/ \__\ |_______|
+// \ \ | | ________________________________________________________________
+// \ \ | | | ______________________________________________________________|
+// \ \| | | | __ __ __ __ ______ _______
+// \ | | |_____ | | | | | | | | | _ \ / _____)
+// \ | | _____| | | | | | | | | | | \ \ \_______
+// \ | | | | |_____ | \_/ | | | | |_/ / _____ |
+// \ _____| |__| |________| \_______/ |__| |______/ (_______/
+//
+// This file is part of VirtualFluids. VirtualFluids is free software: you can
+// redistribute it and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation, either version 3 of
+// the License, or (at your option) any later version.
+//
+// VirtualFluids is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+// for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with VirtualFluids (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
+//
+//! \file EdgeNodeFinder.h
+//! \ingroup Parameter
+//! \author Anna Wellmann
+//! \brief functions for finding edge nodes in the multi-gpu implementation
+//! \details Edge nodes are nodes, which are part of the communication in multiple directions
+//! \ref master thesis of Anna Wellmann (p. 54-57)
+
+//=======================================================================================
+#ifndef GPU_EDGENODES_H
+#define GPU_EDGENODES_H
+
+#include "Core/DataTypes.h"
+#include "basics/PointerDefinitions.h"
+
+class Parameter;
+
+namespace vf::gpu
+{
+
+void findEdgeNodesCommMultiGPU(SPtr<Parameter> parameter);
+}
+
+// anonymous namespace
+namespace
+{
+static void findEdgeNodesXY(int level, SPtr<Parameter> parameter);
+static bool findIndexInSendNodesXY(int level, int index, int &indexOfProcessNeighborSend, int &indexInSendBuffer,
+ SPtr<Parameter> parameter);
+static void findEdgeNodesXZ(int level, SPtr<Parameter> parameter);
+static bool findIndexInSendNodesXZ(int level, int index, int &indexOfProcessNeighborSend, int &indexInSendBuffer,
+ SPtr<Parameter> parameter);
+static void findEdgeNodesYZ(int level, SPtr<Parameter> parameter);
+static bool findIndexInSendNodesYZ(int level, int index, int &indexOfProcessNeighborSend, int &indexInSendBuffer,
+ SPtr<Parameter> parameter);
+} // namespace
+
+#endif
\ No newline at end of file
diff --git a/src/gpu/VirtualFluids_GPU/Parameter/Parameter.cpp b/src/gpu/VirtualFluids_GPU/Parameter/Parameter.cpp
index d63fb1d3f..0187cc364 100644
--- a/src/gpu/VirtualFluids_GPU/Parameter/Parameter.cpp
+++ b/src/gpu/VirtualFluids_GPU/Parameter/Parameter.cpp
@@ -2587,102 +2587,6 @@ bool Parameter::getKernelNeedsFluidNodeIndicesToRun()
return this->kernelNeedsFluidNodeIndicesToRun;
}
-void Parameter::findEdgeNodesCommMultiGPU()
-{
- for (uint level = 0; level < parH.size(); level++) {
- findEdgeNodesXY(level);
- findEdgeNodesXZ(level);
- findEdgeNodesYZ(level);
- }
-}
-
-void Parameter::findEdgeNodesXY(int level)
-{
- int indexOfProcessNeighborSend;
- int indexInSendBuffer;
- for (uint i = 0; i < (unsigned int)(this->getNumberOfProcessNeighborsX(level, "recv")); i++) {
- for (int j = 0; j < parH[level]->recvProcessNeighborX[i].numberOfNodes; j++) {
- int index = parH[level]->recvProcessNeighborX[i].index[j];
- bool foundIndex = findIndexInSendNodesXY(level, index, indexOfProcessNeighborSend, indexInSendBuffer);
- if (foundIndex) {
- this->parH[level]->edgeNodesXtoY.emplace_back(i, j, indexOfProcessNeighborSend, indexInSendBuffer);
- }
- }
- }
-}
-
-bool Parameter::findIndexInSendNodesXY(int level, int index, int &indexOfProcessNeighborSend, int &indexInSendBuffer)
-{
- for (uint k = 0; k < (unsigned int)(this->getNumberOfProcessNeighborsY(level, "send")); k++) {
- for (int l = 0; l < parH[level]->sendProcessNeighborY[k].numberOfNodes; l++) {
- if (parH[level]->sendProcessNeighborY[k].index[l] == index) {
- indexOfProcessNeighborSend = k;
- indexInSendBuffer = l;
- return true;
- }
- }
- }
- return false;
-}
-
-void Parameter::findEdgeNodesXZ(int level)
-{
- int indexOfProcessNeighborSend;
- int indexInSendBuffer;
- for (uint i = 0; i < (unsigned int)(this->getNumberOfProcessNeighborsX(level, "recv")); i++) {
- for (int j = 0; j < parH[level]->recvProcessNeighborX[i].numberOfNodes; j++) {
- int index = parH[level]->recvProcessNeighborX[i].index[j];
- bool foundIndex = findIndexInSendNodesXZ(level, index, indexOfProcessNeighborSend, indexInSendBuffer);
- if (foundIndex) {
- this->parH[level]->edgeNodesXtoZ.emplace_back(i, j, indexOfProcessNeighborSend, indexInSendBuffer);
- }
- }
- }
-}
-
-bool Parameter::findIndexInSendNodesXZ(int level, int index, int &indexOfProcessNeighborSend, int &indexInSendBuffer)
-{
- for (uint k = 0; k < (unsigned int)(this->getNumberOfProcessNeighborsZ(level, "send")); k++) {
- for (int l = 0; l < parH[level]->sendProcessNeighborZ[k].numberOfNodes; l++) {
- if (parH[level]->sendProcessNeighborZ[k].index[l] == index) {
- indexOfProcessNeighborSend = k;
- indexInSendBuffer = l;
- return true;
- }
- }
- }
- return false;
-}
-
-void Parameter::findEdgeNodesYZ(int level)
-{
- int indexOfProcessNeighborSend;
- int indexInSendBuffer;
- for (uint i = 0; i < (unsigned int)(this->getNumberOfProcessNeighborsY(level, "recv")); i++) {
- for (int j = 0; j < parH[level]->recvProcessNeighborY[i].numberOfNodes; j++) {
- int index = parH[level]->recvProcessNeighborY[i].index[j];
- bool foundIndex = findIndexInSendNodesYZ(level, index, indexOfProcessNeighborSend, indexInSendBuffer);
- if (foundIndex) {
- this->parH[level]->edgeNodesYtoZ.emplace_back(i, j, indexOfProcessNeighborSend, indexInSendBuffer);
- }
- }
- }
-}
-
-bool Parameter::findIndexInSendNodesYZ(int level, int index, int &indexOfProcessNeighborSend, int &indexInSendBuffer)
-{
- for (uint k = 0; k < (unsigned int)(this->getNumberOfProcessNeighborsZ(level, "send")); k++) {
- for (int l = 0; l < parH[level]->sendProcessNeighborZ[k].numberOfNodes; l++) {
- if (parH[level]->sendProcessNeighborZ[k].index[l] == index) {
- indexOfProcessNeighborSend = k;
- indexInSendBuffer = l;
- return true;
- }
- }
- }
- return false;
-}
-
void Parameter::initProcessNeighborsAfterFtoCX(int level)
{
this->getParH(level)->sendProcessNeighborsAfterFtoCX.resize(this->getParH(level)->sendProcessNeighborX.size());
diff --git a/src/gpu/VirtualFluids_GPU/Parameter/Parameter.h b/src/gpu/VirtualFluids_GPU/Parameter/Parameter.h
index 6646cdbfb..161fae53b 100644
--- a/src/gpu/VirtualFluids_GPU/Parameter/Parameter.h
+++ b/src/gpu/VirtualFluids_GPU/Parameter/Parameter.h
@@ -610,7 +610,9 @@ public:
bool getCalcParticle();
bool getWriteVeloASCIIfiles();
bool getCalcPlaneConc();
+ //! \returns index of finest level
int getFine();
+ //! \returns index of coarsest level
int getCoarse();
int getParticleBasicLevel();
int getParticleInitLevel();
@@ -933,16 +935,7 @@ public:
void initProcessNeighborsAfterFtoCY(int level);
void initProcessNeighborsAfterFtoCZ(int level);
- void findEdgeNodesCommMultiGPU();
bool useReducedCommunicationAfterFtoC{ true };
-
-private:
- void findEdgeNodesXY(int level);
- bool findIndexInSendNodesXY(int level, int index, int &indexOfProcessNeighborSend, int &indexInSendBuffer);
- void findEdgeNodesXZ(int level);
- bool findIndexInSendNodesXZ(int level, int index, int &indexOfProcessNeighborSend, int &indexInSendBuffer);
- void findEdgeNodesYZ(int level);
- bool findIndexInSendNodesYZ(int level, int index, int &indexOfProcessNeighborSend, int &indexInSendBuffer);
};
#endif
--
GitLab