From eff8595dde28198765cd2a87eccd1b5db0f2b9bb Mon Sep 17 00:00:00 2001
From: Sven Marcus <s.marcus@outlook.de>
Date: Thu, 24 Sep 2020 17:22:27 +0200
Subject: [PATCH] Adds a new constructor to SetKernelBlockVisitor with num
processes
visit method throws exception if required memory determined from simulation grid exceeds available physical memory
---
.../Visitors/SetKernelBlockVisitor.cpp | 165 ++++++++++++------
.../Visitors/SetKernelBlockVisitor.h | 41 +++--
2 files changed, 138 insertions(+), 68 deletions(-)
diff --git a/src/cpu/VirtualFluidsCore/Visitors/SetKernelBlockVisitor.cpp b/src/cpu/VirtualFluidsCore/Visitors/SetKernelBlockVisitor.cpp
index a2987ee43..69f344301 100644
--- a/src/cpu/VirtualFluidsCore/Visitors/SetKernelBlockVisitor.cpp
+++ b/src/cpu/VirtualFluidsCore/Visitors/SetKernelBlockVisitor.cpp
@@ -1,3 +1,4 @@
+#include "MemoryUtil.h"
#include "SetKernelBlockVisitor.h"
#include "Grid3DSystem.h"
#include "LBMSystem.h"
@@ -8,70 +9,122 @@
#include "LBMKernel.h"
//////////////////////////////////////////////////////////////////////////
-SetKernelBlockVisitor::SetKernelBlockVisitor(SPtr<LBMKernel> kernel, LBMReal nue, double availMem, double needMem, SetKernelBlockVisitor::Action action) : Block3DVisitor(0, Grid3DSystem::MAXLEVEL), kernel(kernel), nue(nue), action(action), dataSetFlag(true)
+SetKernelBlockVisitor::SetKernelBlockVisitor(SPtr<LBMKernel> kernel, LBMReal nue, double availMem, double needMem,
+ SetKernelBlockVisitor::Action action) : Block3DVisitor(0,
+ Grid3DSystem::MAXLEVEL),
+ kernel(kernel), nue(nue),
+ action(action), dataSetFlag(true)
{
- if (needMem > availMem)
- {
- throw UbException(UB_EXARGS, "SetKernelBlockVisitor: Not enough memory!!!");
- }
+ if (needMem > availMem)
+ {
+ throw UbException(UB_EXARGS, "SetKernelBlockVisitor: Not enough memory!!!");
+ }
}
+
+SetKernelBlockVisitor::SetKernelBlockVisitor(SPtr<LBMKernel> kernel,
+ LBMReal nue,
+ int &numberOfProcesses,
+ SetKernelBlockVisitor::Action action) : Block3DVisitor(0,
+ Grid3DSystem::MAXLEVEL),
+ kernel(std::move(kernel)),
+ nue(nue),
+ action(action),
+ dataSetFlag(true),
+ numberOfProcesses(
+ numberOfProcesses)
+{}
+
//////////////////////////////////////////////////////////////////////////
void SetKernelBlockVisitor::visit(SPtr<Grid3D> grid, SPtr<Block3D> block)
{
- if(kernel && (block->getRank() == grid->getRank()))
- {
- LBMReal collFactor = LBMSystem::calcCollisionFactor(nue, block->getLevel());
- kernel->setCollisionFactor(collFactor);
- kernel->setIndex(block->getX1(), block->getX2(), block->getX3());
- kernel->setDeltaT(LBMSystem::getDeltaT(block->getLevel()));
- kernel->setBlock(block);
- UbTupleInt3 blockNX = grid->getBlockNX();
- kernel->setNX(std::array<int,3>{{val<1>(blockNX), val<2>(blockNX), val<3>(blockNX)}});
- SPtr<LBMKernel> newKernel = kernel->clone();
-
- switch (action)
- {
- case SetKernelBlockVisitor::NewKernel:
- block->setKernel(newKernel);
- break;
- case SetKernelBlockVisitor::ChangeKernel:
- {
- SPtr<DataSet3D> dataSet = block->getKernel()->getDataSet();
- if (!dataSet)
- {
- UB_THROW(UbException(UB_EXARGS, "It is not possible to change a DataSet in kernel! Old DataSet is not exist!"));
- }
-
- newKernel->setDataSet(dataSet);
-
- SPtr<BCProcessor> bcProc = block->getKernel()->getBCProcessor();
- if (!bcProc)
- {
- UB_THROW(UbException(UB_EXARGS, "It is not possible to change a BCProcessor in kernel! Old BCProcessor is not exist!"));
- }
- newKernel->setBCProcessor(bcProc);
- block->setKernel(newKernel);
- }
- break;
-
- case SetKernelBlockVisitor::ChangeKernelWithData:
- {
- SPtr<BCProcessor> bcProc = block->getKernel()->getBCProcessor();
- if (!bcProc)
- {
- UB_THROW(UbException(UB_EXARGS, "It is not possible to change a BCProcessor in kernel! Old BCProcessor is not exist!"));
- }
- newKernel->setBCProcessor(bcProc);
- block->setKernel(newKernel);
- }
- break;
- }
-
- }
+ throwExceptionIfNotEnoughMemory(grid);
+
+ if (kernel && (block->getRank() == grid->getRank()))
+ {
+ LBMReal collFactor = LBMSystem::calcCollisionFactor(nue, block->getLevel());
+ kernel->setCollisionFactor(collFactor);
+ kernel->setIndex(block->getX1(), block->getX2(), block->getX3());
+ kernel->setDeltaT(LBMSystem::getDeltaT(block->getLevel()));
+ kernel->setBlock(block);
+ UbTupleInt3 blockNX = grid->getBlockNX();
+ kernel->setNX(std::array<int, 3>{{val<1>(blockNX), val<2>(blockNX), val<3>(blockNX)}});
+ SPtr<LBMKernel> newKernel = kernel->clone();
+
+ switch (action)
+ {
+ case SetKernelBlockVisitor::NewKernel:
+ block->setKernel(newKernel);
+ break;
+ case SetKernelBlockVisitor::ChangeKernel:
+ {
+ SPtr<DataSet3D> dataSet = block->getKernel()->getDataSet();
+ if (!dataSet)
+ {
+ UB_THROW(UbException(UB_EXARGS,
+ "It is not possible to change a DataSet in kernel! Old DataSet is not exist!"));
+ }
+
+ newKernel->setDataSet(dataSet);
+
+ SPtr<BCProcessor> bcProc = block->getKernel()->getBCProcessor();
+ if (!bcProc)
+ {
+ UB_THROW(UbException(UB_EXARGS,
+ "It is not possible to change a BCProcessor in kernel! Old BCProcessor is not exist!"));
+ }
+ newKernel->setBCProcessor(bcProc);
+ block->setKernel(newKernel);
+ }
+ break;
+
+ case SetKernelBlockVisitor::ChangeKernelWithData:
+ {
+ SPtr<BCProcessor> bcProc = block->getKernel()->getBCProcessor();
+ if (!bcProc)
+ {
+ UB_THROW(UbException(UB_EXARGS,
+ "It is not possible to change a BCProcessor in kernel! Old BCProcessor is not exist!"));
+ }
+ newKernel->setBCProcessor(bcProc);
+ block->setKernel(newKernel);
+ }
+ break;
+ }
+
+ }
}
+
void SetKernelBlockVisitor::setNoDataSetFlag(bool flag)
{
- dataSetFlag = flag;
+ dataSetFlag = flag;
+}
+
+void SetKernelBlockVisitor::throwExceptionIfNotEnoughMemory(const SPtr<Grid3D> &grid)
+{
+ auto availableMemory = Utilities::getTotalPhysMem();
+ auto requiredMemory = getRequiredPhysicalMemory(grid);
+ if (requiredMemory > availableMemory)
+ throw UbException(UB_EXARGS, "SetKernelBlockVisitor: Not enough memory!!!");
+}
+
+double SetKernelBlockVisitor::getRequiredPhysicalMemory(const SPtr<Grid3D> &grid) const
+{
+ unsigned long long numberOfNodesPerBlockWithGhostLayer;
+ auto numberOfBlocks = (unsigned long long) grid->getNumberOfBlocks();
+ auto blockNx = grid->getBlockNX();
+ int ghostLayer = 3;
+
+ numberOfNodesPerBlockWithGhostLayer = numberOfBlocks
+ * (val<1>(blockNx) + ghostLayer)
+ * (val<2>(blockNx) + ghostLayer)
+ * (val<3>(blockNx) + ghostLayer);
+
+ auto needMemAll = double(numberOfNodesPerBlockWithGhostLayer
+ * (27 * sizeof(double)
+ + sizeof(int)
+ + sizeof(float) * 4));
+
+ return needMemAll / double(numberOfProcesses);
}
diff --git a/src/cpu/VirtualFluidsCore/Visitors/SetKernelBlockVisitor.h b/src/cpu/VirtualFluidsCore/Visitors/SetKernelBlockVisitor.h
index 0d5a2cbc9..44890499f 100644
--- a/src/cpu/VirtualFluidsCore/Visitors/SetKernelBlockVisitor.h
+++ b/src/cpu/VirtualFluidsCore/Visitors/SetKernelBlockVisitor.h
@@ -7,30 +7,47 @@
#include "LBMSystem.h"
class Grid3D;
+
class Block3D;
+
class LBMKernel;
class SetKernelBlockVisitor : public Block3DVisitor
{
public:
- enum Action { NewKernel, ChangeKernel, ChangeKernelWithData};
+ enum Action
+ {
+ NewKernel, ChangeKernel, ChangeKernelWithData
+ };
+
+ //SetKernelBlockVisitor(LBMKernel3DPtr kernel, LBMReal nue);
+
+ //SetKernelBlockVisitor(LBMKernel3DPtr kernel, LBMReal nue, double availMem, double needMem);
- //SetKernelBlockVisitor(LBMKernel3DPtr kernel, LBMReal nue);
-
- //SetKernelBlockVisitor(LBMKernel3DPtr kernel, LBMReal nue, double availMem, double needMem);
+ SetKernelBlockVisitor(SPtr<LBMKernel> kernel, LBMReal nue, double availMem, double needMem,
+ SetKernelBlockVisitor::Action action = SetKernelBlockVisitor::NewKernel);
- SetKernelBlockVisitor(SPtr<LBMKernel> kernel, LBMReal nue, double availMem, double needMem, SetKernelBlockVisitor::Action action = SetKernelBlockVisitor::NewKernel);
- virtual ~SetKernelBlockVisitor() {}
+ SetKernelBlockVisitor(SPtr<LBMKernel> kernel, LBMReal nue, int &numberOfProcesses,
+ SetKernelBlockVisitor::Action action = SetKernelBlockVisitor::NewKernel);
- void visit(SPtr<Grid3D> grid, SPtr<Block3D> block) override;
+ virtual ~SetKernelBlockVisitor()
+ {}
- void setNoDataSetFlag(bool flag);
+ void visit(SPtr<Grid3D> grid, SPtr<Block3D> block) override;
+
+ void setNoDataSetFlag(bool flag);
private:
- SPtr<LBMKernel> kernel;
- LBMReal nue;
- Action action;
- bool dataSetFlag;
+ SPtr<LBMKernel> kernel;
+ LBMReal nue;
+ Action action;
+ bool dataSetFlag;
+
+ int numberOfProcesses{1};
+
+ double getRequiredPhysicalMemory(const SPtr<Grid3D> &grid) const;
+
+ void throwExceptionIfNotEnoughMemory(const SPtr<Grid3D> &grid);
};
#endif
--
GitLab