From 574968b1c655ab5d7c9a704dbc3ebfbe53ad27f1 Mon Sep 17 00:00:00 2001
From: Konstantin Kutscher <kutscher@irmb.tu-bs.de>
Date: Sat, 10 Feb 2018 00:40:01 +0100
Subject: [PATCH] -fixed F16 setup -add directions into SpongeLayerBlockVisitor
-add mq selection
---
.../Applications/DLR-F16-Solid/f16-solid.cfg | 80 +-
source/Applications/DLR-F16-Solid/f16.cpp | 376 +--
source/CMakeLists.txt | 13 +-
source/VirtualFluids.h | 1 +
.../MPIIOMigrationCoProcessor.cpp | 2401 +++++++++++----
.../CoProcessors/MPIIOMigrationCoProcessor.h | 156 +-
.../CoProcessors/MPIIORestartCoProcessor.cpp | 2659 ++++++++++++-----
.../CoProcessors/MPIIORestartCoProcessor.h | 167 +-
.../WriteMQFromSelectionCoProcessor.cpp | 237 ++
.../WriteMQFromSelectionCoProcessor.h | 57 +
.../WriteMacroscopicQuantitiesCoProcessor.h | 5 +-
.../Interactors/Interactor3D.cpp | 12 +-
.../Visitors/GenBlocksGridVisitor.cpp | 20 +-
.../Visitors/SpongeLayerBlockVisitor.cpp | 58 +-
.../Visitors/SpongeLayerBlockVisitor.h | 7 +-
15 files changed, 4629 insertions(+), 1620 deletions(-)
create mode 100644 source/VirtualFluidsCore/CoProcessors/WriteMQFromSelectionCoProcessor.cpp
create mode 100644 source/VirtualFluidsCore/CoProcessors/WriteMQFromSelectionCoProcessor.h
diff --git a/source/Applications/DLR-F16-Solid/f16-solid.cfg b/source/Applications/DLR-F16-Solid/f16-solid.cfg
index ad1a2468f..944a53c02 100644
--- a/source/Applications/DLR-F16-Solid/f16-solid.cfg
+++ b/source/Applications/DLR-F16-Solid/f16-solid.cfg
@@ -1,70 +1,62 @@
-pathOut = d:/temp/DLR-F16-Solid2
+pathOut = d:/temp/DLR-F16-Solid6
pathGeo = d:/Projects/SFB880/DLR-F16/Geometry
-fngFileWhole = F16_broad_full_coarse.stl
+#fngFileWhole = F16_broad_full.stl
+#fngFileWhole = F16_broad_full_coarse.stl
+#fngFileWhole = F16withTapeMeshedFineBig.stl
+#fngFileWhole = F16withTapeMeshedFine.stl
+#fngFileWhole = f16-ascii.stl
+#fngFileWhole = f16.stl
+fngFileWhole = F16_broad_Quad_full.stl
+#fngFileWhole = F16_broad_noTape_full_bin.stl
pathReInit = /work/koskuche/DLR-F16_L7
stepReInit = 10000
numOfThreads = 4
-availMem = 15e9
+availMem = 3.5e9
logToFile = false
#x1min x1max x2min x2max x3min x3max [m]
-#deltaXfine = 0.00001171875
-#deltaXfine = 0.00075
-#boundingBox = -0.90 1.20 0.035 0.065 -0.65 0.65
-#boundingBox = -0.90 2.1 0.035 0.065 -0.66 0.66
-boundingBox = -0.12 1.1 0.0 0.03 -0.24 0.24
-#boundingBox = 0.1 0.15 -0.002 0.0302 -0.01 0.01
-#boundingBox = -0.01 0.0 -0.06 0.06 0.01 0.03
-blockNx = 10 5 10
-
-#deltaXfine = 13393e-9
-#boundingBox = -0.90 1.19145087 0.035 0.06928608 -0.65 0.65
-#blockNx = 10 10 10
-
-#boundingBox = -0.90 1.19998697917 0.035 6.49869791667e-2 -0.65 0.65
-#deltaXfine = 1.30208333333e-5
-#blockNx = 9 9 9
-
-refineLevel = 1
-
-#deltaXfine = 0.001
-#deltaXfine = 0.006 #level 0
+boundingBox = -0.90 2.1 0.0 0.03 -0.66 0.66
+
+blockNx = 10 10 10
+
+refineLevel = 0
+
deltaXfine = 0.003
-#deltaXfine = 0.0015 #level 0
-#deltaXfine = 0.00075 #level 1
-#deltaXfine = 0.000375 #level 2
-#deltaXfine = 0.0001875 #level 3
-#deltaXfine = 0.00009375 #level 4
-#deltaXfine = 0.000046875 #level 5
-#deltaXfine = 0.0000234375 #level 6
-
-startDistance = -1.0
-#refineDistance = 12
-#refineDistance = 1.2
-refineDistance = 0.6
-#refineDistance = 0.3
+#deltaXfine = 0.00075 #level 0
+#deltaXfine = 0.000375 #level 1
+#deltaXfine = 0.0001875 #level 2
+#deltaXfine = 0.00009375 #level 3
+#deltaXfine = 0.000046875 #level 4
+#deltaXfine = 0.0000234375 #level 5
+
+#startDistance = -0.0007
+#refineDistance = 0.0008
+
+startDistance = -1.0e-3
+refineDistance = 0.6e-3
writeBlocks = true
newStart = true
-restartStep = 40000
+restartStep = 50
+
+cpStep = 50
+cpStart = 50
-cpStep = 400000
-cpStart = 400000
+outTimeStep = 10
+outTimeStart = 10
-outTimeStep = 400000
-outTimeStart = 0
+endTime = 100
-endTime = 40000
#Cp
pcpStart = 1000000
pcpStop = 1000000
-timeAvStart = 10000
+timeAvStart = 0
timeAvStop = 40000
nupsStep = 1000 1000 10000000
diff --git a/source/Applications/DLR-F16-Solid/f16.cpp b/source/Applications/DLR-F16-Solid/f16.cpp
index b62a4d442..5b855d5d2 100644
--- a/source/Applications/DLR-F16-Solid/f16.cpp
+++ b/source/Applications/DLR-F16-Solid/f16.cpp
@@ -75,13 +75,13 @@ void run(string configname)
//the geometry is in mm
- double g_minX1 = boundingBox[0];
- double g_minX2 = boundingBox[2];
- double g_minX3 = boundingBox[4];
-
- double g_maxX1 = boundingBox[1];
- double g_maxX2 = boundingBox[3];
- double g_maxX3 = boundingBox[5];
+ double g_minX1 = boundingBox[0];//*1000.0;
+ double g_minX2 = boundingBox[2];//*1000.0;
+ double g_minX3 = boundingBox[4];//*1000.0;
+ double g_maxX1 = boundingBox[1];//*1000.0;
+ double g_maxX2 = boundingBox[3];//*1000.0;
+ double g_maxX3 = boundingBox[5];//*1000.0;
+ //deltaXfine *=1000.0;
//////////////////////////////////////////////////////////////////////////
double deltaXcoarse = deltaXfine*(double)(1<<refineLevel);
@@ -111,7 +111,7 @@ void run(string configname)
double uLB = uReal * unitConverter.getFactorVelocityWToLb();
double nuLB = nuReal * unitConverter.getFactorViscosityWToLb();
- double lLB = lReal*1000.0/deltaXcoarse;
+ double lLB = lReal/deltaXcoarse;
//double nuLB = (uLB*lLB)/Re; //0.005;
//double nuLB = 0.005;
@@ -150,20 +150,24 @@ void run(string configname)
bcVisitor.addBC(velBCAdapter);
bcVisitor.addBC(outflowBCAdapter);
+ SPtr<BCProcessor> bcProc;
+ bcProc = SPtr<BCProcessor>(new BCProcessor());
+
//SPtr<LBMKernel> kernel = SPtr<LBMKernel>(new CompressibleCumulantLBMKernel());
//dynamicPointerCast<CompressibleCumulantLBMKernel>(kernel)->setRelaxationParameter(CompressibleCumulantLBMKernel::NORMAL);
SPtr<LBMKernel> kernel = SPtr<LBMKernel>(new CompressibleCumulant4thOrderViscosityLBMKernel());
//dynamicPointerCast<CompressibleCumulant4thOrderViscosityLBMKernel>(kernel)->setBulkViscosity(nuLB*2.0e3);
-
- SPtr<BCProcessor> bcProc;
- bcProc = SPtr<BCProcessor>(new BCProcessor());
kernel->setBCProcessor(bcProc);
+
+ SPtr<LBMKernel> spKernel = SPtr<LBMKernel>(new CompressibleCumulantLBMKernel());
+ spKernel->setBCProcessor(bcProc);
//////////////////////////////////////////////////////////////////////////
//restart
SPtr<UbScheduler> rSch(new UbScheduler(cpStep, cpStart));
SPtr<MPIIORestartCoProcessor> restartCoProcessor(new MPIIORestartCoProcessor(grid, rSch, pathOut, comm));
+ //SPtr<MPIIOMigrationCoProcessor> restartCoProcessor(new MPIIOMigrationCoProcessor(grid, rSch, pathOut, comm));
restartCoProcessor->setLBMKernel(kernel);
restartCoProcessor->setBCProcessor(bcProc);
//////////////////////////////////////////////////////////////////////////
@@ -223,8 +227,8 @@ void run(string configname)
fngMeshWhole = SPtr<GbTriFaceMesh3D>(GbTriFaceMesh3DCreator::getInstance()->readMeshFromSTLFile2(pathGeo+"/"+fngFileWhole, "fngMeshWhole", GbTriFaceMesh3D::KDTREE_SAHPLIT, false));
if (myid==0) UBLOG(logINFO, "Read fngFileWhole:end");
fngMeshWhole->rotate(0.0, 0.5, 0.0);
- //fngMeshWhole->scale(1e-3,1e-3,1e-3);
- //fngMeshWhole->translate(-150.0,-50.0,-1.28);
+ //fngMeshWhole->scale(1e3,1e3,1e3);
+ //fngMeshWhole->translate(1.932008e-5-149.867,-0.03-49.95,-0.0172298-1.32814);
if (myid==0) GbSystem3D::writeGeoObject(fngMeshWhole.get(), pathOut+"/geo/fngMeshWhole", WbWriterVtkXmlBinary::getInstance());
if (myid==0)
@@ -235,7 +239,7 @@ void run(string configname)
//////////////////////////////////////////////////////////////////////////
SPtr<Interactor3D> fngIntrWhole;
//fngIntrWhole = SPtr<D3Q27Interactor>(new D3Q27Interactor(fngMeshWhole, grid, noSlipBCAdapter, Interactor3D::SOLID));//, Interactor3D::POINTS));
- fngIntrWhole = SPtr<D3Q27TriFaceMeshInteractor>(new D3Q27TriFaceMeshInteractor(fngMeshWhole, grid, noSlipBCAdapter, Interactor3D::SOLID, Interactor3D::POINTS));
+ fngIntrWhole = SPtr<D3Q27TriFaceMeshInteractor>(new D3Q27TriFaceMeshInteractor(fngMeshWhole, grid, noSlipBCAdapter, Interactor3D::SOLID));//, Interactor3D::POINTS));
if (refineLevel>0 && myid==0 && writeBlocks)
{
@@ -243,77 +247,82 @@ void run(string configname)
int rank = grid->getRank();
grid->setRank(0);
+
int level;
//level 1
level = 1;
if (refineLevel - level >= 0)
{
- SPtr<GbObject3D> refCylinderL2(new GbCylinder3D(0.015, 0.0, 0.0, 0.015, 0.1, 0.0, 0.040));
- GbSystem3D::writeGeoObject(refCylinderL2.get(), pathOut + "/geo/refCylinderL2", WbWriterVtkXmlBinary::getInstance());
- RefineCrossAndInsideGbObjectBlockVisitor refVisitorCylinderL2(refCylinderL2, level);
- grid->accept(refVisitorCylinderL2);
-
- //SPtr<GbObject3D> refBoxL2(new GbCuboid3D(0.015, 0.0, -0.04, 1.100, 0.1, 0.04));
- SPtr<GbObject3D> refBoxL2(new GbCuboid3D(0.015, 0.0, -0.04, 0.5, 0.1, 0.04));
- if (myid==0) GbSystem3D::writeGeoObject(refBoxL2.get(), pathOut+"/geo/refBoxL2", WbWriterVtkXmlASCII::getInstance());
- RefineCrossAndInsideGbObjectBlockVisitor refVisitorBoxL2(refBoxL2, level);
- grid->accept(refVisitorBoxL2);
+ SPtr<GbObject3D> refCylinderL1(new GbCylinder3D(0.015, -0.03, 0.0, 0.015, 0.06, 0.0, 0.03));
+ GbSystem3D::writeGeoObject(refCylinderL1.get(), pathOut + "/geo/refCylinderL1", WbWriterVtkXmlBinary::getInstance());
+ RefineCrossAndInsideGbObjectBlockVisitor refVisitorCylinderL1(refCylinderL1, level);
+ grid->accept(refVisitorCylinderL1);
+
+ SPtr<GbObject3D> refBoxL1(new GbCuboid3D(0.015, -0.03, -0.03, 1.100, 0.06, 0.03));
+ //SPtr<GbObject3D> refBoxL1(new GbCuboid3D(0.015, 0.0, -0.04, 0.5, 0.1, 0.04));
+ if (myid==0) GbSystem3D::writeGeoObject(refBoxL1.get(), pathOut+"/geo/refBoxL1", WbWriterVtkXmlASCII::getInstance());
+ RefineCrossAndInsideGbObjectBlockVisitor refVisitorBoxL1(refBoxL1, level);
+ grid->accept(refVisitorBoxL1);
}
//level 2
level = 2;
if (refineLevel - level >= 0)
{
- SPtr<GbObject3D> refCylinderL3(new GbCylinder3D(15.0, 0.0, 0.0, 15.0, 100.0, 0.0, 30.0));
- GbSystem3D::writeGeoObject(refCylinderL3.get(), pathOut + "/geo/refCylinderL3", WbWriterVtkXmlBinary::getInstance());
- RefineCrossAndInsideGbObjectBlockVisitor refVisitorCylinderL3(refCylinderL3, level);
- grid->accept(refVisitorCylinderL3);
+ //SPtr<GbObject3D> refCylinderL2(new GbCylinder3D(0.015, -0.03, 0.0, 0.015, 0.06, 0.0, 0.03));
+ //GbSystem3D::writeGeoObject(refCylinderL2.get(), pathOut + "/geo/refCylinderL2", WbWriterVtkXmlBinary::getInstance());
+ //RefineCrossAndInsideGbObjectBlockVisitor refVisitorCylinderL2(refCylinderL2, level);
+ //grid->accept(refVisitorCylinderL2);
- SPtr<GbObject3D> refBoxL3(new GbCuboid3D(15.0, 0.0, -30.0, 700.0, 100.0, 30.0));
- if (myid==0) GbSystem3D::writeGeoObject(refBoxL3.get(), pathOut+"/geo/refBoxL3", WbWriterVtkXmlASCII::getInstance());
- RefineCrossAndInsideGbObjectBlockVisitor refVisitorBoxL3(refBoxL3, level);
- grid->accept(refVisitorBoxL3);
+ SPtr<GbObject3D> refBoxL2(new GbCuboid3D(0.15, -0.03, -0.015, 0.7, 0.06, 0.015));
+ if (myid==0) GbSystem3D::writeGeoObject(refBoxL2.get(), pathOut+"/geo/refBoxL2", WbWriterVtkXmlASCII::getInstance());
+ RefineCrossAndInsideGbObjectBlockVisitor refVisitorBoxL2(refBoxL2, level);
+ grid->accept(refVisitorBoxL2);
+
+ //dynamicPointerCast<D3Q27TriFaceMeshInteractor>(fngIntrWhole)->refineBlockGridToLevel(level, startDistance, 0.01);
}
//level 3
level = 3;
if (refineLevel - level >= 0)
{
- SPtr<GbObject3D> refCylinderL4(new GbCylinder3D(15.0, 0.0, 0.0, 15.0, 100.0, 0.0, 25.0));
- GbSystem3D::writeGeoObject(refCylinderL4.get(), pathOut + "/geo/refCylinderL4", WbWriterVtkXmlBinary::getInstance());
- RefineCrossAndInsideGbObjectBlockVisitor refVisitorCylinderL4(refCylinderL4, level);
- grid->accept(refVisitorCylinderL4);
+ //SPtr<GbObject3D> refCylinderL3(new GbCylinder3D(0.015, -0.03, 0.0, 0.015, 0.06, 0.0, 0.025));
+ //GbSystem3D::writeGeoObject(refCylinderL3.get(), pathOut + "/geo/refCylinderL3", WbWriterVtkXmlBinary::getInstance());
+ //RefineCrossAndInsideGbObjectBlockVisitor refVisitorCylinderL3(refCylinderL3, level);
+ //grid->accept(refVisitorCylinderL3);
- SPtr<GbObject3D> refBoxL4(new GbCuboid3D(15.0, 0.0, -25.0, 400.0, 100.0, 25.0));
- if (myid==0) GbSystem3D::writeGeoObject(refBoxL4.get(), pathOut+"/geo/refBoxL4", WbWriterVtkXmlASCII::getInstance());
- RefineCrossAndInsideGbObjectBlockVisitor refVisitorBoxL4(refBoxL4, level);
- grid->accept(refVisitorBoxL4);
+ SPtr<GbObject3D> refBoxL3(new GbCuboid3D(0.15, -0.03, -0.010, 0.4, 0.06, 0.012));
+ if (myid==0) GbSystem3D::writeGeoObject(refBoxL3.get(), pathOut+"/geo/refBoxL3", WbWriterVtkXmlASCII::getInstance());
+ RefineCrossAndInsideGbObjectBlockVisitor refVisitorBoxL3(refBoxL3, level);
+ grid->accept(refVisitorBoxL3);
+
+ //dynamicPointerCast<D3Q27TriFaceMeshInteractor>(fngIntrWhole)->refineBlockGridToLevel(level, startDistance, 0.005);
}
//level 4
level = 4;
if (refineLevel - level >= 0)
{
- SPtr<GbObject3D> refBoxL5(new GbCuboid3D(120.0, 0.0, -9.0, 320.0, 100.0, 18.0));
- if (myid==0) GbSystem3D::writeGeoObject(refBoxL5.get(), pathOut+"/geo/refBoxL5", WbWriterVtkXmlASCII::getInstance());
- RefineCrossAndInsideGbObjectBlockVisitor refVisitorBoxL5(refBoxL5, level);
- grid->accept(refVisitorBoxL5);
- }
+ SPtr<GbObject3D> refBoxL4(new GbCuboid3D(0.15, -0.03, -0.005, 0.32, 0.06, 0.01));
+ if (myid==0) GbSystem3D::writeGeoObject(refBoxL4.get(), pathOut+"/geo/refBoxL4", WbWriterVtkXmlASCII::getInstance());
+ RefineCrossAndInsideGbObjectBlockVisitor refVisitorBoxL4(refBoxL4, level);
+ grid->accept(refVisitorBoxL4);
- //level 5
- level = 5;
- if (refineLevel - level >= 0)
- {
- dynamicPointerCast<D3Q27TriFaceMeshInteractor>(fngIntrWhole)->refineBlockGridToLevel(refineLevel, startDistance, refineDistance);
+ //dynamicPointerCast<D3Q27TriFaceMeshInteractor>(fngIntrWhole)->refineBlockGridToLevel(level, startDistance, 0.0016);
}
- grid->setRank(rank);
- {
- WriteBlocksCoProcessor ppblocks(grid, SPtr<UbScheduler>(new UbScheduler(1)), pathOut, WbWriterVtkXmlBinary::getInstance(), comm);
- ppblocks.process(0);
- }
+
+ ////level 5
+ //level = 5;
+ //if (refineLevel - level >= 0)
+ //{
+ // dynamicPointerCast<D3Q27TriFaceMeshInteractor>(fngIntrWhole)->refineBlockGridToLevel(level, startDistance, refineDistance);
+ //}
+
+ dynamicPointerCast<D3Q27TriFaceMeshInteractor>(fngIntrWhole)->refineBlockGridToLevel(refineLevel, startDistance, refineDistance);
+ //
/////delete solid blocks
if (myid==0) UBLOG(logINFO, "deleteSolidBlocks - start");
@@ -328,18 +337,41 @@ void run(string configname)
fngIntrWhole->removeSolidBlocks();
fngIntrWhole->removeBcBlocks();
+ //SPtr<GbObject3D> delBox(new GbCuboid3D(0.03, -0.03, -0.010, 0.2, 0.06, 0.012));
+ //if (myid==0) GbSystem3D::writeGeoObject(delBox.get(), pathOut+"/geo/delBox", WbWriterVtkXmlASCII::getInstance());
+ //SPtr<D3Q27Interactor> delBoxInter(new D3Q27Interactor(delBox, grid, noSlipBCAdapter, Interactor3D::SOLID));
+ //SetSolidBlockVisitor v(delBoxInter, BlockType::SOLID);
+ //grid->accept(v);
+ //std::vector<SPtr<Block3D>>& sb = delBoxInter->getSolidBlockSet();
+ //for (SPtr<Block3D> block : sb)
+ //{
+ // grid->deleteBlock(block);
+ //}
+ //delBoxInter->removeSolidBlocks();
+ //delBoxInter->removeBcBlocks();
+
if (myid==0) UBLOG(logINFO, "deleteSolidBlocks - end");
- //////////////////////////////////////
+ ////////////////////////////////////////
+ {
+ WriteBlocksCoProcessor ppblocks(grid, SPtr<UbScheduler>(new UbScheduler(1)), pathOut, WbWriterVtkXmlBinary::getInstance(), comm);
+ ppblocks.process(0);
+ }
+
+ grid->setRank(rank);
RatioBlockVisitor ratioVisitor(refineLevel);
CheckRatioBlockVisitor checkRatio(refineLevel);
int count = 0;
do {
+ if (myid==0) UBLOG(logINFO, "ratioVisitor - strat");
grid->accept(ratioVisitor);
+ if (myid==0) UBLOG(logINFO, "ratioVisitor - end");
+ if (myid==0) UBLOG(logINFO, "checkRatio - strat");
checkRatio.resetState();
grid->accept(checkRatio);
+ if (myid==0) UBLOG(logINFO, "checkRatio - end");
if (myid==0) UBLOG(logINFO, "count = "<<count++<<" state = "<<checkRatio.getState());
} while (!checkRatio.getState());
@@ -442,6 +474,7 @@ void run(string configname)
UBLOG(logINFO, "Available memory per process = "<<availMem<<" bytes");
}
+ restartCoProcessor->writeBlocks(0);
SetKernelBlockVisitor kernelVisitor(kernel, nuLB, availMem, needMem);
grid->accept(kernelVisitor);
@@ -512,30 +545,25 @@ void run(string configname)
////sponge layer
////////////////////////////////////////////////////////////////////////////
- GbCuboid3DPtr spongeLayerX1max(new GbCuboid3D(g_maxX1-8.0*blockLength, g_minX2-blockLength, g_minX3-blockLength, g_maxX1+blockLength, g_maxX2+blockLength, g_maxX3+blockLength));
+ GbCuboid3DPtr spongeLayerX1max(new GbCuboid3D(g_maxX1-0.7, g_minX2-blockLength, g_minX3-blockLength, g_maxX1, g_maxX2+blockLength, g_maxX3+blockLength));
if (myid==0) GbSystem3D::writeGeoObject(spongeLayerX1max.get(), pathOut+"/geo/spongeLayerX1max", WbWriterVtkXmlASCII::getInstance());
- SpongeLayerBlockVisitor slVisitorX1max;
- slVisitorX1max.setBoundingBox(spongeLayerX1max);
- SPtr<LBMKernel> spKernel = SPtr<LBMKernel>(new CompressibleCumulantLBMKernel());
- dynamicPointerCast<CompressibleCumulantLBMKernel>(spKernel)->setRelaxationParameter(CompressibleCumulantLBMKernel::NORMAL);
- spKernel->setBCProcessor(bcProc);
- slVisitorX1max.setKernel(spKernel);
+ SpongeLayerBlockVisitor slVisitorX1max(spongeLayerX1max, spKernel, D3Q27System::E);
grid->accept(slVisitorX1max);
- //GbCuboid3DPtr spongeLayerX1min(new GbCuboid3D(g_minX1-blockLength, g_minX2-blockLength, g_minX3-blockLength, g_minX1+75, g_maxX2+blockLength, g_maxX3+blockLength));
- //if (myid==0) GbSystem3D::writeGeoObject(spongeLayerX1min.get(), pathOut+"/geo/spongeLayerX1min", WbWriterVtkXmlASCII::getInstance());
- //SpongeLayerBlockVisitor slVisitorX1min(spongeLayerX1min);
- //grid->accept(slVisitorX1min);
+ GbCuboid3DPtr spongeLayerX1min(new GbCuboid3D(g_minX1-blockLength, g_minX2-blockLength, g_minX3-blockLength, g_minX1+0.25, g_maxX2+blockLength, g_maxX3+blockLength));
+ if (myid==0) GbSystem3D::writeGeoObject(spongeLayerX1min.get(), pathOut+"/geo/spongeLayerX1min", WbWriterVtkXmlASCII::getInstance());
+ SpongeLayerBlockVisitor slVisitorX1min(spongeLayerX1min, spKernel,D3Q27System::W);
+ grid->accept(slVisitorX1min);
- //GbCuboid3DPtr spongeLayerX3min(new GbCuboid3D(g_minX1-blockLength, g_minX2-blockLength, g_minX3-blockLength, g_maxX1+blockLength, g_maxX2+blockLength, g_minX3+75));
- //if (myid==0) GbSystem3D::writeGeoObject(spongeLayerX3min.get(), pathOut+"/geo/spongeLayerX3min", WbWriterVtkXmlASCII::getInstance());
- //SpongeLayerBlockVisitor slVisitorX3min(spongeLayerX3min);
- //grid->accept(slVisitorX3min);
+ GbCuboid3DPtr spongeLayerX3min(new GbCuboid3D(g_minX1+0.25, g_minX2-blockLength, g_minX3-blockLength, g_maxX1-0.7, g_maxX2+blockLength, g_minX3+0.25));
+ if (myid==0) GbSystem3D::writeGeoObject(spongeLayerX3min.get(), pathOut+"/geo/spongeLayerX3min", WbWriterVtkXmlASCII::getInstance());
+ SpongeLayerBlockVisitor slVisitorX3min(spongeLayerX3min, spKernel, D3Q27System::B);
+ grid->accept(slVisitorX3min);
- //GbCuboid3DPtr spongeLayerX3max(new GbCuboid3D(g_minX1-blockLength, g_minX2-blockLength, g_maxX3-75, g_maxX1+blockLength, g_maxX2+blockLength, g_maxX3+blockLength));
- //if (myid==0) GbSystem3D::writeGeoObject(spongeLayerX3max.get(), pathOut+"/geo/spongeLayerX3max", WbWriterVtkXmlASCII::getInstance());
- //SpongeLayerBlockVisitor slVisitorX3max(spongeLayerX3max);
- //grid->accept(slVisitorX3max);
+ GbCuboid3DPtr spongeLayerX3max(new GbCuboid3D(g_minX1+0.25, g_minX2-blockLength, g_maxX3-0.25, g_maxX1-0.7, g_maxX2+blockLength, g_maxX3+blockLength));
+ if (myid==0) GbSystem3D::writeGeoObject(spongeLayerX3max.get(), pathOut+"/geo/spongeLayerX3max", WbWriterVtkXmlASCII::getInstance());
+ SpongeLayerBlockVisitor slVisitorX3max(spongeLayerX3max, spKernel, D3Q27System::T);
+ grid->accept(slVisitorX3max);
/////////////////////////////////////////////////////////////////////////////
if (myid==0) UBLOG(logINFO, "Preprozess - end");
@@ -554,14 +582,9 @@ void run(string configname)
////sponge layer
////////////////////////////////////////////////////////////////////////////
- GbCuboid3DPtr spongeLayerX1max(new GbCuboid3D(g_maxX1-8.0*blockLength, g_minX2-blockLength, g_minX3-blockLength, g_maxX1+blockLength, g_maxX2+blockLength, g_maxX3+blockLength));
+ GbCuboid3DPtr spongeLayerX1max(new GbCuboid3D(g_maxX1-0.7, g_minX2-blockLength, g_minX3-blockLength, g_maxX1+blockLength, g_maxX2+blockLength, g_maxX3+blockLength));
if (myid==0) GbSystem3D::writeGeoObject(spongeLayerX1max.get(), pathOut+"/geo/spongeLayerX1max", WbWriterVtkXmlASCII::getInstance());
- SpongeLayerBlockVisitor slVisitorX1max;
- slVisitorX1max.setBoundingBox(spongeLayerX1max);
- SPtr<LBMKernel> spKernel = SPtr<LBMKernel>(new CompressibleCumulantLBMKernel());
- dynamicPointerCast<CompressibleCumulantLBMKernel>(spKernel)->setRelaxationParameter(CompressibleCumulantLBMKernel::NORMAL);
- spKernel->setBCProcessor(bcProc);
- slVisitorX1max.setKernel(spKernel);
+ SpongeLayerBlockVisitor slVisitorX1max(spongeLayerX1max, spKernel, D3Q27System::E);
grid->accept(slVisitorX1max);
}
@@ -578,6 +601,10 @@ void run(string configname)
SPtr<WriteMacroscopicQuantitiesCoProcessor> writeMQCoProcessor(new WriteMacroscopicQuantitiesCoProcessor(grid, stepSch, pathOut, WbWriterVtkXmlBinary::getInstance(), conv, comm));
+ SPtr<GbObject3D> bbBox(new GbCuboid3D(g_minX1-blockLength, (g_maxX2-g_minX2)/2.0, g_minX3-blockLength, g_maxX1+blockLength, (g_maxX2-g_minX2)/2.0+deltaXfine, g_maxX3+blockLength));
+ if (myid==0) GbSystem3D::writeGeoObject(bbBox.get(), pathOut+"/geo/bbBox", WbWriterVtkXmlASCII::getInstance());
+ SPtr<WriteMQFromSelectionCoProcessor> writeMQSelectCoProcessor(new WriteMQFromSelectionCoProcessor(grid, stepSch, bbBox, pathOut, WbWriterVtkXmlBinary::getInstance(), conv, comm));
+
SPtr<UbScheduler> tavSch(new UbScheduler(1, timeAvStart, timeAvStop));
SPtr<TimeAveragedValuesCoProcessor> tav(new TimeAveragedValuesCoProcessor(grid, pathOut, WbWriterVtkXmlBinary::getInstance(), tavSch, comm,
TimeAveragedValuesCoProcessor::Density | TimeAveragedValuesCoProcessor::Velocity | TimeAveragedValuesCoProcessor::Fluctuations));
@@ -589,11 +616,12 @@ void run(string configname)
//TimeseriesCoProcessor tsp1(grid, stepMV, mic1, pathOut+"/mic/mic1", comm);
omp_set_num_threads(numOfThreads);
- SPtr<Calculator> calculator(new BasicCalculator(grid, tavSch, endTime));
+ SPtr<Calculator> calculator(new BasicCalculator(grid, stepSch, endTime));
calculator->addCoProcessor(nupsCoProcessor);
calculator->addCoProcessor(restartCoProcessor);
+ calculator->addCoProcessor(writeMQSelectCoProcessor);
calculator->addCoProcessor(writeMQCoProcessor);
- calculator->addCoProcessor(tav);
+ //calculator->addCoProcessor(tav);
if (myid==0) UBLOG(logINFO, "Simulation-start");
@@ -621,96 +649,96 @@ void run(string configname)
}
-void test_run()
-{
- try
- {
-
- SPtr<Communicator> comm = MPICommunicator::getInstance();
- int myid = comm->getProcessID();
-
- if (myid==0)
- {
- UBLOG(logINFO, "PID = "<<myid<<" Point 1");
- UBLOG(logINFO, "PID = "<<myid<<" Total Physical Memory (RAM): "<<Utilities::getTotalPhysMem());
- UBLOG(logINFO, "PID = "<<myid<<" Physical Memory currently used: "<<Utilities::getPhysMemUsed());
- UBLOG(logINFO, "PID = "<<myid<<" Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
- }
-
- double g_minX1 = 0;
- double g_minX2 = 0;
- double g_minX3 = 0;
-
- double g_maxX1 = 5;
- double g_maxX2 = 5;
- double g_maxX3 = 5;
-
- int blockNx[3] ={ 5, 5, 5 };
-
- string pathOut = "d:/temp/DLR-F16-Solid-test";
-
- double deltaX = 1;
- double rhoLB = 0.0;
- double uLB = 0.0866025;
- double nuLB = 0.001; //4.33013e-06;
- SPtr<LBMUnitConverter> conv = SPtr<LBMUnitConverter>(new LBMUnitConverter());
- ////////////////////////////////////////////////////////////////////////
- //Grid
- //////////////////////////////////////////////////////////////////////////
- SPtr<Grid3D> grid(new Grid3D(comm));
- grid->setDeltaX(deltaX);
- grid->setBlockNX(blockNx[0], blockNx[1], blockNx[2]);
- grid->setPeriodicX1(false);
- grid->setPeriodicX2(false);
- grid->setPeriodicX3(false);
-
- SPtr<GbObject3D> gridCube(new GbCuboid3D(g_minX1, g_minX2, g_minX3, g_maxX1, g_maxX2, g_maxX3));
- if (myid==0) GbSystem3D::writeGeoObject(gridCube.get(), pathOut+"/geo/gridCube", WbWriterVtkXmlASCII::getInstance());
- GenBlocksGridVisitor genBlocks(gridCube);
- grid->accept(genBlocks);
-
- WriteBlocksCoProcessor ppblocks(grid, SPtr<UbScheduler>(new UbScheduler(1)), pathOut, WbWriterVtkXmlBinary::getInstance(), comm);
- ppblocks.process(0);
-
- SPtr<LBMKernel> kernel = SPtr<LBMKernel>(new CompressibleCumulant4thOrderViscosityLBMKernel());
- kernel->setNX(std::array<int, 3>{ {blockNx[0], blockNx[1], blockNx[2]}});
- SPtr<BCProcessor> bcProc;
- bcProc = SPtr<BCProcessor>(new BCProcessor());
- kernel->setBCProcessor(bcProc);
-
- SetKernelBlockVisitor kernelVisitor(kernel, nuLB, 1e9, 12);
- grid->accept(kernelVisitor);
-
- //initialization of distributions
- InitDistributionsBlockVisitor initVisitor1;
- initVisitor1.setVx1(0.001);
- grid->accept(initVisitor1);
-
- SPtr<UbScheduler> stepSch(new UbScheduler(1));
- SPtr<WriteMacroscopicQuantitiesCoProcessor> writeMQCoProcessor(new WriteMacroscopicQuantitiesCoProcessor(grid, stepSch, pathOut, WbWriterVtkXmlBinary::getInstance(), conv, comm));
-
- //omp_set_num_threads(numOfThreads);
- SPtr<Calculator> calculator(new BasicCalculator(grid, stepSch, 2));
- calculator->addCoProcessor(writeMQCoProcessor);
-
-
- if (myid==0) UBLOG(logINFO, "Simulation-start");
- calculator->calculate();
- if (myid==0) UBLOG(logINFO, "Simulation-end");
- }
- catch (std::exception& e)
- {
- cerr<<e.what()<<endl<<flush;
- }
- catch (std::string& s)
- {
- cerr<<s<<endl;
- }
- catch (...)
- {
- cerr<<"unknown exception"<<endl;
- }
-}
+//void test_run()
+//{
+// try
+// {
+//
+// SPtr<Communicator> comm = MPICommunicator::getInstance();
+// int myid = comm->getProcessID();
+//
+// if (myid==0)
+// {
+// UBLOG(logINFO, "PID = "<<myid<<" Point 1");
+// UBLOG(logINFO, "PID = "<<myid<<" Total Physical Memory (RAM): "<<Utilities::getTotalPhysMem());
+// UBLOG(logINFO, "PID = "<<myid<<" Physical Memory currently used: "<<Utilities::getPhysMemUsed());
+// UBLOG(logINFO, "PID = "<<myid<<" Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+// }
+//
+// double g_minX1 = 0;
+// double g_minX2 = 0;
+// double g_minX3 = 0;
+//
+// double g_maxX1 = 5;
+// double g_maxX2 = 5;
+// double g_maxX3 = 5;
+//
+// int blockNx[3] ={ 5, 5, 5 };
+//
+// string pathOut = "d:/temp/DLR-F16-Solid-test";
+//
+// double deltaX = 1;
+// double rhoLB = 0.0;
+// double uLB = 0.0866025;
+// double nuLB = 0.001; //4.33013e-06;
+// SPtr<LBMUnitConverter> conv = SPtr<LBMUnitConverter>(new LBMUnitConverter());
+// ////////////////////////////////////////////////////////////////////////
+// //Grid
+// //////////////////////////////////////////////////////////////////////////
+// SPtr<Grid3D> grid(new Grid3D(comm));
+// grid->setDeltaX(deltaX);
+// grid->setBlockNX(blockNx[0], blockNx[1], blockNx[2]);
+// grid->setPeriodicX1(false);
+// grid->setPeriodicX2(false);
+// grid->setPeriodicX3(false);
+//
+// SPtr<GbObject3D> gridCube(new GbCuboid3D(g_minX1, g_minX2, g_minX3, g_maxX1, g_maxX2, g_maxX3));
+// if (myid==0) GbSystem3D::writeGeoObject(gridCube.get(), pathOut+"/geo/gridCube", WbWriterVtkXmlASCII::getInstance());
+// GenBlocksGridVisitor genBlocks(gridCube);
+// grid->accept(genBlocks);
+//
+// WriteBlocksCoProcessor ppblocks(grid, SPtr<UbScheduler>(new UbScheduler(1)), pathOut, WbWriterVtkXmlBinary::getInstance(), comm);
+// ppblocks.process(0);
+//
+// SPtr<LBMKernel> kernel = SPtr<LBMKernel>(new CompressibleCumulant4thOrderViscosityLBMKernel());
+// kernel->setNX(std::array<int, 3>{ {blockNx[0], blockNx[1], blockNx[2]}});
+// SPtr<BCProcessor> bcProc;
+// bcProc = SPtr<BCProcessor>(new BCProcessor());
+// kernel->setBCProcessor(bcProc);
+//
+// SetKernelBlockVisitor kernelVisitor(kernel, nuLB, 1e9, 12);
+// grid->accept(kernelVisitor);
+//
+// //initialization of distributions
+// InitDistributionsBlockVisitor initVisitor1;
+// initVisitor1.setVx1(0.001);
+// grid->accept(initVisitor1);
+//
+// SPtr<UbScheduler> stepSch(new UbScheduler(1));
+// SPtr<WriteMacroscopicQuantitiesCoProcessor> writeMQCoProcessor(new WriteMacroscopicQuantitiesCoProcessor(grid, stepSch, pathOut, WbWriterVtkXmlBinary::getInstance(), conv, comm));
+//
+// //omp_set_num_threads(numOfThreads);
+// SPtr<Calculator> calculator(new BasicCalculator(grid, stepSch, 2));
+// calculator->addCoProcessor(writeMQCoProcessor);
+//
+//
+// if (myid==0) UBLOG(logINFO, "Simulation-start");
+// calculator->calculate();
+// if (myid==0) UBLOG(logINFO, "Simulation-end");
+// }
+// catch (std::exception& e)
+// {
+// cerr<<e.what()<<endl<<flush;
+// }
+// catch (std::string& s)
+// {
+// cerr<<s<<endl;
+// }
+// catch (...)
+// {
+// cerr<<"unknown exception"<<endl;
+// }
+//}
int main(int argc, char* argv[])
{
diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt
index b3618d463..994e2c6ae 100644
--- a/source/CMakeLists.txt
+++ b/source/CMakeLists.txt
@@ -33,11 +33,11 @@ SET(USE_METIS ON CACHE BOOL "include METIS library support")
SET(USE_MPI ON CACHE BOOL "include MPI library support")
SET(USE_VTK OFF CACHE BOOL "include VTK library support")
SET(USE_CATALYST OFF CACHE BOOL "include Paraview Catalyst support")
-SET(USE_BOOST OFF CACHE BOOL "include Paraview Catalyst support")
+SET(USE_BOOST OFF CACHE BOOL "include Boost support")
#SET(USE_PYTHON OFF CACHE BOOL "include Python scripting support")
#SET(USE_FETOL OFF CACHE BOOL "include FETOL library support")
SET(USE_INTEL OFF CACHE BOOL "include Intel compiler support")
-#SET(USE_GCC OFF CACHE BOOL "include gcc compiler support")
+SET(USE_GCC OFF CACHE BOOL "include gcc compiler support")
SET(USE_HLRN_LUSTRE OFF CACHE BOOL "include HLRN Lustre support")
#CAB
@@ -118,14 +118,19 @@ IF(${USE_BOOST})
LIST(APPEND CAB_ADDTIONAL_COMPILER_FLAGS -DVF_BOOST)
ENDIF()
+IF(${USE_HLRN_LUSTRE})
+ LIST(APPEND CAB_ADDTIONAL_COMPILER_FLAGS -DHLRN_LUSTRE)
+ENDIF()
+
IF(${USE_INTEL})
SET(CAB_ADDITIONAL_LINK_FLAGS ${CAB_ADDITIONAL_LINK_FLAGS} -parallel)
ENDIF()
-IF(${USE_HLRN_LUSTRE})
- SET(APPEND CAB_ADDTIONAL_COMPILER_FLAGS -DHLRN_LUSTRE)
+IF(${USE_GCC})
+ SET(CAB_ADDITIONAL_LINK_FLAGS ${CAB_ADDITIONAL_LINK_FLAGS} -lgomp)
ENDIF()
+
# IF(${USE_PYTHON})
# FIND_PACKAGE(PythonLibs)
# INCLUDE_DIRECTORIES(${PYTHON_INCLUDE_DIR})
diff --git a/source/VirtualFluids.h b/source/VirtualFluids.h
index 09701541e..42e746115 100644
--- a/source/VirtualFluids.h
+++ b/source/VirtualFluids.h
@@ -145,6 +145,7 @@
#include <CoProcessors/AdjustForcingCoProcessor.h>
#include <CoProcessors/CalculateForcesCoProcessor.h>
#include <CoProcessors/WriteMacroscopicQuantitiesCoProcessor.h>
+#include <CoProcessors/WriteMQFromSelectionCoProcessor.h>
#include <CoProcessors/WriteBoundaryConditionsCoProcessor.h>
//#include <CoProcessors/PathLineCoProcessor.h>
//#include <CoProcessors/PathLineCoProcessorMcpart.h>
diff --git a/source/VirtualFluidsCore/CoProcessors/MPIIOMigrationCoProcessor.cpp b/source/VirtualFluidsCore/CoProcessors/MPIIOMigrationCoProcessor.cpp
index 7a540fac0..1a2b427bb 100644
--- a/source/VirtualFluidsCore/CoProcessors/MPIIOMigrationCoProcessor.cpp
+++ b/source/VirtualFluidsCore/CoProcessors/MPIIOMigrationCoProcessor.cpp
@@ -24,14 +24,12 @@ MPIIOMigrationCoProcessor::MPIIOMigrationCoProcessor(SPtr<Grid3D> grid, SPtr<UbS
SPtr<Communicator> comm) :
CoProcessor(grid, s),
path(path),
- comm(comm),
- mpiTypeFreeFlag(false)
+ comm(comm)
{
- UbSystem::makeDirectory(path+"/mpi_io_cp");
+ UbSystem::makeDirectory(path + "/mpi_io_cp");
- memset(&dataSetParamStr, 0, sizeof(dataSetParamStr));
memset(&boundCondParamStr, 0, sizeof(boundCondParamStr));
-
+
//------------------------- define MPI types ---------------------------------
MPI_Datatype typesGP[3] = { MPI_DOUBLE, MPI_INT, MPI_CHAR };
@@ -40,50 +38,55 @@ MPIIOMigrationCoProcessor::MPIIOMigrationCoProcessor(SPtr<Grid3D> grid, SPtr<UbS
offsetsGP[0] = 0;
MPI_Type_get_extent(MPI_DOUBLE, &lbGP, &extentGP);
- offsetsGP[1] = blocksGP[0]*extentGP;
+ offsetsGP[1] = blocksGP[0] * extentGP;
MPI_Type_get_extent(MPI_INT, &lbGP, &extentGP);
- offsetsGP[2] = offsetsGP[1]+blocksGP[1]*extentGP;
+ offsetsGP[2] = offsetsGP[1] + blocksGP[1] * extentGP;
MPI_Type_create_struct(3, blocksGP, offsetsGP, typesGP, &gridParamType);
MPI_Type_commit(&gridParamType);
//-----------------------------------------------------------------------
- MPI_Type_contiguous(40, MPI_INT, &dataSetParamType);
- MPI_Type_commit(&dataSetParamType);
-
- //-----------------------------------------------------------------------
-
MPI_Datatype typesBlock[2] = { MPI_INT, MPI_CHAR };
int blocksBlock[2] = { 13, 1 };
MPI_Aint offsetsBlock[2], lbBlock, extentBlock;
offsetsBlock[0] = 0;
MPI_Type_get_extent(MPI_INT, &lbBlock, &extentBlock);
- offsetsBlock[1] = blocksBlock[0]*extentBlock;
+ offsetsBlock[1] = blocksBlock[0] * extentBlock;
MPI_Type_create_struct(2, blocksBlock, offsetsBlock, typesBlock, &block3dType);
MPI_Type_commit(&block3dType);
//-----------------------------------------------------------------------
+ MPI_Type_contiguous(7, MPI_INT, &dataSetParamType);
+ MPI_Type_commit(&dataSetParamType);
+
+ //-----------------------------------------------------------------------
+
MPI_Datatype typesDataSet[3] = { MPI_DOUBLE, MPI_INT, MPI_CHAR };
int blocksDataSet[3] = { 2, 2, 2 };
MPI_Aint offsetsDatatSet[3], lbDataSet, extentDataSet;
offsetsDatatSet[0] = 0;
MPI_Type_get_extent(MPI_DOUBLE, &lbDataSet, &extentDataSet);
- offsetsDatatSet[1] = blocksDataSet[0]*extentDataSet;
+ offsetsDatatSet[1] = blocksDataSet[0] * extentDataSet;
MPI_Type_get_extent(MPI_INT, &lbDataSet, &extentDataSet);
- offsetsDatatSet[2] = offsetsDatatSet[1]+blocksDataSet[1]*extentDataSet;
+ offsetsDatatSet[2] = offsetsDatatSet[1] + blocksDataSet[1] * extentDataSet;
MPI_Type_create_struct(3, blocksDataSet, offsetsDatatSet, typesDataSet, &dataSetType);
MPI_Type_commit(&dataSetType);
//-----------------------------------------------------------------------
+ MPI_Type_contiguous(1, MPI_INT, &dataSetSmallType);
+ MPI_Type_commit(&dataSetSmallType);
+
+ //-----------------------------------------------------------------------
+
MPI_Type_contiguous(4, MPI_INT, &boundCondParamType);
MPI_Type_commit(&boundCondParamType);
@@ -95,10 +98,10 @@ MPIIOMigrationCoProcessor::MPIIOMigrationCoProcessor(SPtr<Grid3D> grid, SPtr<UbS
offsetsBC[0] = 0;
MPI_Type_get_extent(MPI_LONG_LONG_INT, &lbBC, &extentBC);
- offsetsBC[1] = blocksBC[0]*extentBC;
+ offsetsBC[1] = blocksBC[0] * extentBC;
MPI_Type_get_extent(MPI_FLOAT, &lbBC, &extentBC);
- offsetsBC[2] = offsetsBC[1]+blocksBC[1]*extentBC;
+ offsetsBC[2] = offsetsBC[1] + blocksBC[1] * extentBC;
MPI_Type_create_struct(3, blocksBC, offsetsBC, typesBC, &boundCondType);
MPI_Type_commit(&boundCondType);
@@ -107,6 +110,10 @@ MPIIOMigrationCoProcessor::MPIIOMigrationCoProcessor(SPtr<Grid3D> grid, SPtr<UbS
MPI_Type_contiguous(3, MPI_INT, &boundCondTypeAdd);
MPI_Type_commit(&boundCondTypeAdd);
+ //---------------------------------------
+
+ MPI_Type_contiguous(6, MPI_CHAR, &arrayPresenceType);
+ MPI_Type_commit(&arrayPresenceType);
}
//////////////////////////////////////////////////////////////////////////
@@ -116,15 +123,11 @@ MPIIOMigrationCoProcessor::~MPIIOMigrationCoProcessor()
MPI_Type_free(&dataSetParamType);
MPI_Type_free(&block3dType);
MPI_Type_free(&dataSetType);
+ MPI_Type_free(&dataSetSmallType);
MPI_Type_free(&boundCondParamType);
MPI_Type_free(&boundCondType);
MPI_Type_free(&boundCondTypeAdd);
-
- if (mpiTypeFreeFlag)
- {
- MPI_Type_free(&dataSetDoubleType);
- MPI_Type_free(&bcindexmatrixType);
- }
+ MPI_Type_free(&arrayPresenceType);
}
//////////////////////////////////////////////////////////////////////////
@@ -132,46 +135,91 @@ void MPIIOMigrationCoProcessor::process(double step)
{
if (scheduler->isDue(step))
{
- if (comm->isRoot()) UBLOG(logINFO, "MPIIOMigrationCoProcessor save step: "<<step);
+ if (comm->isRoot()) UBLOG(logINFO, "MPIIOMigrationCoProcessor save step: " << step);
if (comm->isRoot()) UBLOG(logINFO, "Save check point - start");
/*if (comm->isRoot())*/ clearAllFiles((int)step);
+
writeBlocks((int)step);
writeDataSet((int)step);
writeBoundaryConds((int)step);
+
if (comm->isRoot()) UBLOG(logINFO, "Save check point - end");
}
}
-//////////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////
void MPIIOMigrationCoProcessor::clearAllFiles(int step)
{
- MPI_File file_handler1, file_handler2, file_handler3;
+ MPI_File file_handler;
MPI_Info info = MPI_INFO_NULL;
MPI_Offset new_size = 0;
UbSystem::makeDirectory(path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step));
+
std::string filename1 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpBlocks.bin";
- //MPI_File_delete(filename1.c_str(), info);
- int rc1 = MPI_File_open(MPI_COMM_WORLD, filename1.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &file_handler1);
+ int rc1 = MPI_File_open(MPI_COMM_WORLD, filename1.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &file_handler);
if (rc1 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename1);
- MPI_File_set_size(file_handler1, new_size);
- //MPI_File_sync(file_handler1);
- MPI_File_close(&file_handler1);
+ MPI_File_set_size(file_handler, new_size);
+ MPI_File_close(&file_handler);
std::string filename2 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSet.bin";
- //MPI_File_delete(filename2.c_str(), info);
- int rc2 = MPI_File_open(MPI_COMM_WORLD, filename2.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler2);
+ int rc2 = MPI_File_open(MPI_COMM_WORLD, filename2.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
if (rc2 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename2);
- MPI_File_set_size(file_handler2, new_size);
- //MPI_File_sync(file_handler2);
- MPI_File_close(&file_handler2);
+ MPI_File_set_size(file_handler, new_size);
+ MPI_File_close(&file_handler);
- std::string filename3 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpBC.bin";
- //MPI_File_delete(filename3.c_str(), info);
- int rc3 = MPI_File_open(MPI_COMM_WORLD, filename3.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler3);
+ std::string filename3 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpArrays.bin";
+ int rc3 = MPI_File_open(MPI_COMM_WORLD, filename3.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
if (rc3 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename3);
- MPI_File_set_size(file_handler3, new_size);
- //MPI_File_sync(file_handler3);
- MPI_File_close(&file_handler3);
+ MPI_File_set_size(file_handler, new_size);
+ MPI_File_close(&file_handler);
+
+ std::string filename4 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageDensityArray.bin";
+ MPI_File_delete(filename4.c_str(), info);
+ //int rc4 = MPI_File_open(MPI_COMM_WORLD, filename4.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ //if (rc4 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename4);
+ //MPI_File_set_size(file_handler, new_size);
+ //MPI_File_close(&file_handler);
+
+ std::string filename5 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageVelocityArray.bin";
+ MPI_File_delete(filename5.c_str(), info);
+ //int rc5 = MPI_File_open(MPI_COMM_WORLD, filename5.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ //if (rc5 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename5);
+ //MPI_File_set_size(file_handler, new_size);
+ //MPI_File_close(&file_handler);
+
+ std::string filename6 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageFluktuationsArray.bin";
+ MPI_File_delete(filename6.c_str(), info);
+ //int rc6 = MPI_File_open(MPI_COMM_WORLD, filename6.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ //if (rc6 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename6);
+ //MPI_File_set_size(file_handler, new_size);
+ //MPI_File_close(&file_handler);
+
+ std::string filename7 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageTripleArray.bin";
+ MPI_File_delete(filename7.c_str(), info);
+ //int rc7 = MPI_File_open(MPI_COMM_WORLD, filename7.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ //if (rc7 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename7);
+ //MPI_File_set_size(file_handler, new_size);
+ //MPI_File_close(&file_handler);
+
+ std::string filename8 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpShearStressValArray.bin";
+ MPI_File_delete(filename8.c_str(), info);
+ //int rc8 = MPI_File_open(MPI_COMM_WORLD, filename8.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ //if (rc8 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename8);
+ //MPI_File_set_size(file_handler, new_size);
+ //MPI_File_close(&file_handler);
+
+ std::string filename9 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpRelaxationFactor.bin";
+ MPI_File_delete(filename9.c_str(), info);
+ //int rc9 = MPI_File_open(MPI_COMM_WORLD, filename9.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ //if (rc9 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename9);
+ //MPI_File_set_size(file_handler, new_size);
+ //MPI_File_close(&file_handler);
+
+ std::string filename10 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpBC.bin";
+ int rc10 = MPI_File_open(MPI_COMM_WORLD, filename10.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc10 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename10);
+ MPI_File_set_size(file_handler, new_size);
+ MPI_File_close(&file_handler);
}
//////////////////////////////////////////////////////////////////////////
void MPIIOMigrationCoProcessor::writeBlocks(int step)
@@ -179,7 +227,7 @@ void MPIIOMigrationCoProcessor::writeBlocks(int step)
int rank, size;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
//MPI_Comm_size(MPI_COMM_WORLD, &size);
- size=1;
+ size = 1;
grid->deleteBlockIDs();
RenumberBlockVisitor renumber;
@@ -187,8 +235,8 @@ void MPIIOMigrationCoProcessor::writeBlocks(int step)
if (comm->isRoot())
{
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeBlocks start collect data rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeBlocks start collect data rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
int blocksCount = 0; // quantity of all the blocks in the grid, max 2147483648 blocks!
@@ -196,7 +244,7 @@ void MPIIOMigrationCoProcessor::writeBlocks(int step)
int maxInitLevel = this->grid->getFinestInitializedLevel();
std::vector<SPtr<Block3D>> blocksVector[25]; // max 25 levels
- for (int level = minInitLevel; level<=maxInitLevel; level++)
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
{
//grid->getBlocks(level, rank, blockVector[level]);
grid->getBlocks(level, blocksVector[level]);
@@ -259,9 +307,9 @@ void MPIIOMigrationCoProcessor::writeBlocks(int step)
Block3d* block3dArray = new Block3d[blocksCount];
int ic = 0;
- for (int level = minInitLevel; level<=maxInitLevel; level++)
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
{
- for(SPtr<Block3D> block : blocksVector[level]) // all the blocks of the current level
+ for (SPtr<Block3D> block : blocksVector[level]) // all the blocks of the current level
{
// save data describing the block
block3dArray[ic].x1 = block->getX1();
@@ -285,8 +333,8 @@ void MPIIOMigrationCoProcessor::writeBlocks(int step)
if (comm->isRoot())
{
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeBlocks start MPI IO rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeBlocks start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
// write to the file
@@ -299,33 +347,33 @@ void MPIIOMigrationCoProcessor::writeBlocks(int step)
// if (comm->isRoot())
// {
- UbSystem::makeDirectory(path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step));
- std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpBlocks.bin";
- int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &file_handler);
- if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
+ UbSystem::makeDirectory(path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step));
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpBlocks.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
// }
double start, finish;
- MPI_Offset write_offset = (MPI_Offset)(size*sizeof(int));
-
+ MPI_Offset write_offset = (MPI_Offset)(size * sizeof(int));
+
if (comm->isRoot())
{
start = MPI_Wtime();
-
+
// each process writes the quantity of it's blocks
MPI_File_write_at(file_handler, 0/*rank*sizeof(int)*/, &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
// each process writes parameters of the grid
MPI_File_write_at(file_handler, write_offset, gridParameters, 1, gridParamType, MPI_STATUS_IGNORE);
// each process writes it's blocks
- MPI_File_write_at(file_handler, (MPI_Offset)(write_offset +sizeof(GridParam)), &block3dArray[0], blocksCount, block3dType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(GridParam)), &block3dArray[0], blocksCount, block3dType, MPI_STATUS_IGNORE);
//MPI_File_sync(file_handler);
}
MPI_File_close(&file_handler);
-
+
if (comm->isRoot())
{
finish = MPI_Wtime();
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeBlocks time: "<<finish-start<<" s");
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeBlocks time: " << finish - start << " s");
}
delete[] block3dArray;
@@ -343,26 +391,29 @@ void MPIIOMigrationCoProcessor::writeDataSet(int step)
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
int maxInitLevel = this->grid->getFinestInitializedLevel();
- for (int level = minInitLevel; level<=maxInitLevel; level++)
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
{
grid->getBlocks(level, rank, blocksVector[level]);
blocksCount += static_cast<int>(blocksVector[level].size());
}
+ dataSetParam dataSetParamStr1, dataSetParamStr2, dataSetParamStr3;
DataSet* dataSetArray = new DataSet[blocksCount];
std::vector<double> doubleValuesArray; // double-values (arrays of f's) in all blocks
if (comm->isRoot())
{
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeDataSet start collect data rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeDataSet start collect data rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
+ DSArraysPresence arrPresence;
bool firstBlock = true;
+ int doubleCountInBlock = 0;
int ic = 0;
- for (int level = minInitLevel; level<=maxInitLevel; level++)
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
{
- for(SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
+ for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
{
dataSetArray[ic].globalID = block->getGlobalID(); // id of the block needed to find it while regenerating the grid
dataSetArray[ic].ghostLayerWidth = block->getKernel()->getGhostLayerWidth();
@@ -371,144 +422,89 @@ void MPIIOMigrationCoProcessor::writeDataSet(int step)
dataSetArray[ic].compressible = block->getKernel()->getCompressible();
dataSetArray[ic].withForcing = block->getKernel()->getWithForcing();
+ SPtr< D3Q27EsoTwist3DSplittedVector > D3Q27EsoTwist3DSplittedVectorPtr = dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(block->getKernel()->getDataSet()->getFdistributions());
+ CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr localDistributions = D3Q27EsoTwist3DSplittedVectorPtr->getLocalDistributions();
+ CbArray4D <LBMReal, IndexerX4X3X2X1>::CbArray4DPtr nonLocalDistributions = D3Q27EsoTwist3DSplittedVectorPtr->getNonLocalDistributions();
+ CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr zeroDistributions = D3Q27EsoTwist3DSplittedVectorPtr->getZeroDistributions();
+
if (firstBlock)// && block->getKernel()) // when first (any) valid block...
{
- SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > averageValuesArray3D = block->getKernel()->getDataSet()->getAverageValues();
- if (averageValuesArray3D)
- {
- dataSetParamStr.nx[0][0] = static_cast<int>(averageValuesArray3D->getNX1());
- dataSetParamStr.nx[0][1] = static_cast<int>(averageValuesArray3D->getNX2());
- dataSetParamStr.nx[0][2] = static_cast<int>(averageValuesArray3D->getNX3());
- dataSetParamStr.nx[0][3] = static_cast<int>(averageValuesArray3D->getNX4());
- }
-
- SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > AverageVelocityArray3DPtr = block->getKernel()->getDataSet()->getAverageVelocity();
- if (AverageVelocityArray3DPtr)
- {
- dataSetParamStr.nx[1][0] = static_cast<int>(AverageVelocityArray3DPtr->getNX1());
- dataSetParamStr.nx[1][1] = static_cast<int>(AverageVelocityArray3DPtr->getNX2());
- dataSetParamStr.nx[1][2] = static_cast<int>(AverageVelocityArray3DPtr->getNX3());
- dataSetParamStr.nx[1][3] = static_cast<int>(AverageVelocityArray3DPtr->getNX4());
- }
-
- SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > AverageFluctArray3DPtr = block->getKernel()->getDataSet()->getAverageFluctuations();
- if (AverageFluctArray3DPtr)
- {
- dataSetParamStr.nx[2][0] = static_cast<int>(AverageFluctArray3DPtr->getNX1());
- dataSetParamStr.nx[2][1] = static_cast<int>(AverageFluctArray3DPtr->getNX2());
- dataSetParamStr.nx[2][2] = static_cast<int>(AverageFluctArray3DPtr->getNX3());
- dataSetParamStr.nx[2][3] = static_cast<int>(AverageFluctArray3DPtr->getNX4());
- }
-
- SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > AverageTripleArray3DPtr = block->getKernel()->getDataSet()->getAverageTriplecorrelations();
- if (AverageTripleArray3DPtr)
- {
- dataSetParamStr.nx[3][0] = static_cast<int>(AverageTripleArray3DPtr->getNX1());
- dataSetParamStr.nx[3][1] = static_cast<int>(AverageTripleArray3DPtr->getNX2());
- dataSetParamStr.nx[3][2] = static_cast<int>(AverageTripleArray3DPtr->getNX3());
- dataSetParamStr.nx[3][3] = static_cast<int>(AverageTripleArray3DPtr->getNX4());
- }
-
- SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > ShearStressValArray3DPtr = block->getKernel()->getDataSet()->getShearStressValues();
- if (ShearStressValArray3DPtr)
- {
- dataSetParamStr.nx[4][0] = static_cast<int>(ShearStressValArray3DPtr->getNX1());
- dataSetParamStr.nx[4][1] = static_cast<int>(ShearStressValArray3DPtr->getNX2());
- dataSetParamStr.nx[4][2] = static_cast<int>(ShearStressValArray3DPtr->getNX3());
- dataSetParamStr.nx[4][3] = static_cast<int>(ShearStressValArray3DPtr->getNX4());
- }
-
- SPtr< CbArray3D<LBMReal, IndexerX3X2X1> > relaxationFactor3DPtr = block->getKernel()->getDataSet()->getRelaxationFactor();
- if (relaxationFactor3DPtr)
- {
- dataSetParamStr.nx[5][0] = static_cast<int>(relaxationFactor3DPtr->getNX1());
- dataSetParamStr.nx[5][1] = static_cast<int>(relaxationFactor3DPtr->getNX2());
- dataSetParamStr.nx[5][2] = static_cast<int>(relaxationFactor3DPtr->getNX3());
- dataSetParamStr.nx[5][3] = 1;
- }
-
- SPtr< D3Q27EsoTwist3DSplittedVector > D3Q27EsoTwist3DSplittedVectorPtr = dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(block->getKernel()->getDataSet()->getFdistributions());
- CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr localDistributions = D3Q27EsoTwist3DSplittedVectorPtr->getLocalDistributions();
if (localDistributions)
{
- dataSetParamStr.nx[6][0] = static_cast<int>(localDistributions->getNX1());
- dataSetParamStr.nx[6][1] = static_cast<int>(localDistributions->getNX2());
- dataSetParamStr.nx[6][2] = static_cast<int>(localDistributions->getNX3());
- dataSetParamStr.nx[6][3] = static_cast<int>(localDistributions->getNX4());
+ dataSetParamStr1.nx[0] = static_cast<int>(localDistributions->getNX1());
+ dataSetParamStr1.nx[1] = static_cast<int>(localDistributions->getNX2());
+ dataSetParamStr1.nx[2] = static_cast<int>(localDistributions->getNX3());
+ dataSetParamStr1.nx[3] = static_cast<int>(localDistributions->getNX4());
}
- CbArray4D <LBMReal, IndexerX4X3X2X1>::CbArray4DPtr nonLocalDistributions = D3Q27EsoTwist3DSplittedVectorPtr->getNonLocalDistributions();
if (nonLocalDistributions)
{
- dataSetParamStr.nx[7][0] = static_cast<int>(nonLocalDistributions->getNX1());
- dataSetParamStr.nx[7][1] = static_cast<int>(nonLocalDistributions->getNX2());
- dataSetParamStr.nx[7][2] = static_cast<int>(nonLocalDistributions->getNX3());
- dataSetParamStr.nx[7][3] = static_cast<int>(nonLocalDistributions->getNX4());
+ dataSetParamStr2.nx[0] = static_cast<int>(nonLocalDistributions->getNX1());
+ dataSetParamStr2.nx[1] = static_cast<int>(nonLocalDistributions->getNX2());
+ dataSetParamStr2.nx[2] = static_cast<int>(nonLocalDistributions->getNX3());
+ dataSetParamStr2.nx[3] = static_cast<int>(nonLocalDistributions->getNX4());
}
-
- CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr zeroDistributions = D3Q27EsoTwist3DSplittedVectorPtr->getZeroDistributions();
if (zeroDistributions)
{
- dataSetParamStr.nx[8][0] = static_cast<int>(zeroDistributions->getNX1());
- dataSetParamStr.nx[8][1] = static_cast<int>(zeroDistributions->getNX2());
- dataSetParamStr.nx[8][2] = static_cast<int>(zeroDistributions->getNX3());
- dataSetParamStr.nx[8][3] = 1;
+ dataSetParamStr3.nx[0] = static_cast<int>(zeroDistributions->getNX1());
+ dataSetParamStr3.nx[1] = static_cast<int>(zeroDistributions->getNX2());
+ dataSetParamStr3.nx[2] = static_cast<int>(zeroDistributions->getNX3());
+ dataSetParamStr3.nx[3] = 1;
}
// ... than save some parameters that are equal in all blocks
- dataSetParamStr.nx1 = static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX1());
- dataSetParamStr.nx2 = static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX2());
- dataSetParamStr.nx3 = static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX3());
+ dataSetParamStr1.nx1 = dataSetParamStr2.nx1 = dataSetParamStr3.nx1 = static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX1());
+ dataSetParamStr1.nx2 = dataSetParamStr2.nx2 = dataSetParamStr3.nx2 = static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX2());
+ dataSetParamStr1.nx3 = dataSetParamStr2.nx3 = dataSetParamStr3.nx3 = static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX3());
- firstBlock = false;
+ doubleCountInBlock = dataSetParamStr1.nx[0] * dataSetParamStr1.nx[1] * dataSetParamStr1.nx[2] * dataSetParamStr1.nx[3] +
+ dataSetParamStr2.nx[0] * dataSetParamStr2.nx[1] * dataSetParamStr2.nx[2] * dataSetParamStr2.nx[3] +
+ dataSetParamStr3.nx[0] * dataSetParamStr3.nx[1] * dataSetParamStr3.nx[2] * dataSetParamStr3.nx[3];
- // how many elements are in all arrays of DataSet (equal in all blocks)
- int doubleCount = 0, temp;
- for (int i = 0; i < 9; i++) // 9 arrays ( averageValues, averageVelocity, averageFluktuations,
- { // averageTriplecorrelations, shearStressValues, relaxationFactor, 3 * fdistributions
- temp = 1;
- for (int ii = 0; ii < 4; ii++) // 4 values (nx1, nx2, nx3, nx4)
- temp *= dataSetParamStr.nx[i][ii];
- doubleCount += temp;
- }
- dataSetParamStr.doubleCountInBlock = doubleCount;
- }
+ SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > averageDensityArray = block->getKernel()->getDataSet()->getAverageDencity();
+ if (averageDensityArray)
+ arrPresence.isAverageDensityArrayPresent = true;
+ else
+ arrPresence.isAverageDensityArrayPresent = false;
- SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > averageValuesArray3D = block->getKernel()->getDataSet()->getAverageValues();
- if (averageValuesArray3D&&(dataSetParamStr.nx[0][0]>0)&&(dataSetParamStr.nx[0][1]>0)&&(dataSetParamStr.nx[0][2]>0)&&(dataSetParamStr.nx[0][3]>0))
- doubleValuesArray.insert(doubleValuesArray.end(), averageValuesArray3D->getDataVector().begin(), averageValuesArray3D->getDataVector().end());
+ SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > AverageVelocityArray3DPtr = block->getKernel()->getDataSet()->getAverageVelocity();
+ if (AverageVelocityArray3DPtr)
+ arrPresence.isAverageVelocityArrayPresent = true;
+ else
+ arrPresence.isAverageVelocityArrayPresent = false;
- SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > AverageVelocityArray3DPtr = block->getKernel()->getDataSet()->getAverageVelocity();
- if (AverageVelocityArray3DPtr&&(dataSetParamStr.nx[1][0]>0)&&(dataSetParamStr.nx[1][1]>0)&&(dataSetParamStr.nx[1][2]>0)&&(dataSetParamStr.nx[1][3]>0))
- doubleValuesArray.insert(doubleValuesArray.end(), AverageVelocityArray3DPtr->getDataVector().begin(), AverageVelocityArray3DPtr->getDataVector().end());
+ SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > AverageFluctArray3DPtr = block->getKernel()->getDataSet()->getAverageFluctuations();
+ if (AverageFluctArray3DPtr)
+ arrPresence.isAverageFluktuationsArrayPresent = true;
+ else
+ arrPresence.isAverageFluktuationsArrayPresent = false;
- SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > AverageFluctArray3DPtr = block->getKernel()->getDataSet()->getAverageFluctuations();
- if (AverageFluctArray3DPtr&&(dataSetParamStr.nx[2][0]>0)&&(dataSetParamStr.nx[2][1]>0)&&(dataSetParamStr.nx[2][2]>0)&&(dataSetParamStr.nx[2][3]>0))
- doubleValuesArray.insert(doubleValuesArray.end(), AverageFluctArray3DPtr->getDataVector().begin(), AverageFluctArray3DPtr->getDataVector().end());
+ SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > AverageTripleArray3DPtr = block->getKernel()->getDataSet()->getAverageTriplecorrelations();
+ if (AverageTripleArray3DPtr)
+ arrPresence.isAverageTripleArrayPresent = true;
+ else
+ arrPresence.isAverageTripleArrayPresent = false;
- SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > AverageTripleArray3DPtr = block->getKernel()->getDataSet()->getAverageTriplecorrelations();
- if (AverageTripleArray3DPtr&&(dataSetParamStr.nx[3][0]>0)&&(dataSetParamStr.nx[3][1]>0)&&(dataSetParamStr.nx[3][2]>0)&&(dataSetParamStr.nx[3][3]>0))
- doubleValuesArray.insert(doubleValuesArray.end(), AverageTripleArray3DPtr->getDataVector().begin(), AverageTripleArray3DPtr->getDataVector().end());
+ SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > ShearStressValArray3DPtr = block->getKernel()->getDataSet()->getShearStressValues();
+ if (ShearStressValArray3DPtr)
+ arrPresence.isShearStressValArrayPresent = true;
+ else
+ arrPresence.isShearStressValArrayPresent = false;
- SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > ShearStressValArray3DPtr = block->getKernel()->getDataSet()->getShearStressValues();
- if (ShearStressValArray3DPtr&&(dataSetParamStr.nx[4][0]>0)&&(dataSetParamStr.nx[4][1]>0)&&(dataSetParamStr.nx[4][2]>0)&&(dataSetParamStr.nx[4][3]>0))
- doubleValuesArray.insert(doubleValuesArray.end(), ShearStressValArray3DPtr->getDataVector().begin(), ShearStressValArray3DPtr->getDataVector().end());
+ SPtr< CbArray3D<LBMReal, IndexerX3X2X1> > relaxationFactor3DPtr = block->getKernel()->getDataSet()->getRelaxationFactor();
+ if (relaxationFactor3DPtr)
+ arrPresence.isRelaxationFactorPresent = true;
+ else
+ arrPresence.isRelaxationFactorPresent = false;
- SPtr< CbArray3D<LBMReal, IndexerX3X2X1> > RelaxationFactor3DPtr = block->getKernel()->getDataSet()->getRelaxationFactor();
- if (RelaxationFactor3DPtr&&(dataSetParamStr.nx[5][0]>0)&&(dataSetParamStr.nx[5][1]>0)&&(dataSetParamStr.nx[5][2]>0))
- doubleValuesArray.insert(doubleValuesArray.end(), RelaxationFactor3DPtr->getDataVector().begin(), RelaxationFactor3DPtr->getDataVector().end());
+ firstBlock = false;
+ }
- SPtr< D3Q27EsoTwist3DSplittedVector > D3Q27EsoTwist3DSplittedVectorPtr = dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(block->getKernel()->getDataSet()->getFdistributions());
- CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr localDistributions = D3Q27EsoTwist3DSplittedVectorPtr->getLocalDistributions();
- if (localDistributions&&(dataSetParamStr.nx[6][0]>0)&&(dataSetParamStr.nx[6][1]>0)&&(dataSetParamStr.nx[6][2]>0)&&(dataSetParamStr.nx[6][3]>0))
+ if (localDistributions && (dataSetParamStr1.nx[0]>0) && (dataSetParamStr1.nx[1]>0) && (dataSetParamStr1.nx[2]>0) && (dataSetParamStr1.nx[3]>0))
doubleValuesArray.insert(doubleValuesArray.end(), localDistributions->getDataVector().begin(), localDistributions->getDataVector().end());
-
- CbArray4D <LBMReal, IndexerX4X3X2X1>::CbArray4DPtr nonLocalDistributions = D3Q27EsoTwist3DSplittedVectorPtr->getNonLocalDistributions();
- if (nonLocalDistributions&&(dataSetParamStr.nx[7][0]>0)&&(dataSetParamStr.nx[7][1]>0)&&(dataSetParamStr.nx[7][2]>0)&&(dataSetParamStr.nx[7][3]>0))
+ if (nonLocalDistributions && (dataSetParamStr2.nx[0]>0) && (dataSetParamStr2.nx[1]>0) && (dataSetParamStr2.nx[2]>0) && (dataSetParamStr2.nx[3]>0))
doubleValuesArray.insert(doubleValuesArray.end(), nonLocalDistributions->getDataVector().begin(), nonLocalDistributions->getDataVector().end());
-
- CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr zeroDistributions = D3Q27EsoTwist3DSplittedVectorPtr->getZeroDistributions();
- if (zeroDistributions&&(dataSetParamStr.nx[8][0]>0)&&(dataSetParamStr.nx[8][1]>0)&&(dataSetParamStr.nx[8][2]>0))
+ if (zeroDistributions && (dataSetParamStr3.nx[0]>0) && (dataSetParamStr3.nx[1]>0) && (dataSetParamStr3.nx[2]>0))
doubleValuesArray.insert(doubleValuesArray.end(), zeroDistributions->getDataVector().begin(), zeroDistributions->getDataVector().end());
ic++;
@@ -516,14 +512,13 @@ void MPIIOMigrationCoProcessor::writeDataSet(int step)
}
// register new MPI-type depending on the block-specific information
- MPI_Type_contiguous(dataSetParamStr.doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
MPI_Type_commit(&dataSetDoubleType);
- mpiTypeFreeFlag = true;
if (comm->isRoot())
{
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeDataSet start MPI IO rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeDataSet start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
@@ -538,380 +533,1648 @@ void MPIIOMigrationCoProcessor::writeDataSet(int step)
// write to the file
MPI_File file_handler;
- std::string filename = path+"/mpi_io_cp/mpi_io_cp_"+UbSystem::toString(step)+"/cpDataSet.bin";
- int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE|MPI_MODE_WRONLY, info, &file_handler);
- if (rc!=MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file "+filename);
-
- size_t sizeofOneDataSet = sizeof(DataSet) + dataSetParamStr.doubleCountInBlock * sizeof(double);
- MPI_Offset write_offset = 0;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSet.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
- MPI_File_write_at(file_handler, write_offset, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)0, &dataSetParamStr1, 1, dataSetParamType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(sizeof(dataSetParam)), &dataSetParamStr2, 1, dataSetParamType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(2 * sizeof(dataSetParam)), &dataSetParamStr3, 1, dataSetParamType, MPI_STATUS_IGNORE);
+
+ MPI_Offset write_offset;
+ size_t sizeofOneDataSet = sizeof(DataSet) + doubleCountInBlock * sizeof(double);
for (int nb = 0; nb < blocksCount; nb++)
{
- write_offset = (MPI_Offset)(sizeof(dataSetParam) + dataSetArray[nb].globalID * sizeofOneDataSet);
+ write_offset = (MPI_Offset)(3 * sizeof(dataSetParam) + dataSetArray[nb].globalID * sizeofOneDataSet);
MPI_File_write_at(file_handler, write_offset, &dataSetArray[nb], 1, dataSetType, MPI_STATUS_IGNORE);
- MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(DataSet)), &doubleValuesArray[nb * dataSetParamStr.doubleCountInBlock], 1, dataSetDoubleType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(DataSet)), &doubleValuesArray[nb * doubleCountInBlock], 1, dataSetDoubleType, MPI_STATUS_IGNORE);
}
MPI_File_sync(file_handler);
MPI_File_close(&file_handler);
+ MPI_Type_free(&dataSetDoubleType);
+
if (comm->isRoot())
{
finish = MPI_Wtime();
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeDataSet time: "<<finish-start<<" s");
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeDataSet time: " << finish - start << " s");
}
delete[] dataSetArray;
+
+ MPI_File file_handler1;
+ std::string filename1 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpArrays.bin";
+ rc = MPI_File_open(MPI_COMM_WORLD, filename1.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler1);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename1);
+ MPI_File_write_at(file_handler1, (MPI_Offset)0, &arrPresence, 1, arrayPresenceType, MPI_STATUS_IGNORE);
+ MPI_File_sync(file_handler1);
+ MPI_File_close(&file_handler1);
+
+ if (arrPresence.isAverageDensityArrayPresent)
+ writeAverageDensityArray(step);
+
+ if (arrPresence.isAverageVelocityArrayPresent)
+ writeAverageVelocityArray(step);
+
+ if (arrPresence.isAverageFluktuationsArrayPresent)
+ writeAverageFluktuationsArray(step);
+
+ if (arrPresence.isAverageTripleArrayPresent)
+ writeAverageTripleArray(step);
+
+ if (arrPresence.isShearStressValArrayPresent)
+ writeShearStressValArray(step);
+
+ if (arrPresence.isRelaxationFactorPresent)
+ writeRelaxationFactor(step);
+
}
-void MPIIOMigrationCoProcessor::writeBoundaryConds(int step)
+void MPIIOMigrationCoProcessor::writeAverageDensityArray(int step)
{
int rank, size;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
- if (comm->isRoot())
- {
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeBoundaryConds start collect data rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
- }
-
- int blocksCount = 0; // quantity of blocks, that belong to this process
- size_t allBytesCount = 0; // quantity of bytes, that one process writes to the file
- size_t count_boundCond = 0; // how many BoundaryConditions in all blocks
- int count_indexContainer = 0; // how many indexContainer-values in all blocks
+ int blocksCount = 0; // quantity of blocks in the grid, max 2147483648 blocks!
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
int maxInitLevel = this->grid->getFinestInitializedLevel();
- for (int level = minInitLevel; level<=maxInitLevel; level++)
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
{
grid->getBlocks(level, rank, blocksVector[level]);
blocksCount += static_cast<int>(blocksVector[level].size());
}
- BCAdd* bcAddArray = new BCAdd[blocksCount];
- size_t* bytesCount = new size_t[blocksCount]; // quantity of bytes, that each block writes to the file
- std::vector<BoundaryCondition>* bcVector = new std::vector<BoundaryCondition>[blocksCount];
- std::vector<int>* bcindexmatrixVector = new std::vector<int>[blocksCount];
- std::vector<int>* indexContainerVector = new std::vector<int>[blocksCount];
+ DataSetSmall* dataSetSmallArray = new DataSetSmall[blocksCount];
+ std::vector<double> doubleValuesArray; // double-values of the AverageDensityArray in all blocks
+ dataSetParam dataSetParamStr;
- bool bcindexmatrixCountNotInit = true;
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeAverageDensityArray start collect data rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ bool firstBlock = true;
+ int doubleCountInBlock = 0;
int ic = 0;
- for (int level = minInitLevel; level<=maxInitLevel; level++)
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
{
- for(SPtr<Block3D> block : blocksVector[level]) // all the blocks of the current level
+ for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
{
- SPtr<BCArray3D> bcArr = block->getKernel()->getBCProcessor()->getBCArray();
+ dataSetSmallArray[ic].globalID = block->getGlobalID(); // id of the block needed to find it while regenerating the grid
- bcAddArray[ic].globalID = block->getGlobalID(); // id of the block needed to find it while regenerating the grid
- bcAddArray[ic].boundCond_count = 0; // how many BoundaryConditions in this block
- bcAddArray[ic].indexContainer_count = 0; // how many indexContainer-values in this block
- bytesCount[ic] = sizeof(BCAdd);
- bcVector[ic].resize(0);
- bcindexmatrixVector[ic].resize(0);
- indexContainerVector[ic].resize(0);
+ SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > averageDensityArray = block->getKernel()->getDataSet()->getAverageDencity();
- for (int bc = 0; bc<bcArr->getBCVectorSize(); bc++)
+ if (firstBlock) // when first (any) valid block...
{
- BoundaryCondition* bouCond = new BoundaryCondition();
- if (bcArr->bcvector[bc]==NULL)
- {
- memset(bouCond, 0, sizeof(BoundaryCondition));
- }
- else
- {
- bouCond->noslipBoundaryFlags = bcArr->bcvector[bc]->getNoSlipBoundary();
- bouCond->slipBoundaryFlags = bcArr->bcvector[bc]->getSlipBoundary();
- bouCond->velocityBoundaryFlags = bcArr->bcvector[bc]->getVelocityBoundary();
- bouCond->densityBoundaryFlags = bcArr->bcvector[bc]->getDensityBoundary();
- bouCond->wallModelBoundaryFlags = bcArr->bcvector[bc]->getWallModelBoundary();
- bouCond->bcVelocityX1 = bcArr->bcvector[bc]->getBoundaryVelocityX1();
- bouCond->bcVelocityX2 = bcArr->bcvector[bc]->getBoundaryVelocityX2();
- bouCond->bcVelocityX3 = bcArr->bcvector[bc]->getBoundaryVelocityX3();
- bouCond->bcDensity = bcArr->bcvector[bc]->getBoundaryDensity();
- bouCond->bcLodiDensity = bcArr->bcvector[bc]->getDensityLodiDensity();
- bouCond->bcLodiVelocityX1 = bcArr->bcvector[bc]->getDensityLodiVelocityX1();
- bouCond->bcLodiVelocityX2 = bcArr->bcvector[bc]->getDensityLodiVelocityX2();
- bouCond->bcLodiVelocityX3 = bcArr->bcvector[bc]->getDensityLodiVelocityX3();
- bouCond->bcLodiLentgh = bcArr->bcvector[bc]->getDensityLodiLength();
- bouCond->nx1 = bcArr->bcvector[bc]->nx1;
- bouCond->nx2 = bcArr->bcvector[bc]->nx2;
- bouCond->nx3 = bcArr->bcvector[bc]->nx3;
- for (int iq = 0; iq<26; iq++)
- bouCond->q[iq] = bcArr->bcvector[bc]->getQ(iq);
- bouCond->algorithmType = bcArr->bcvector[bc]->getBcAlgorithmType();
- }
+ //if (averageDensityArray)
+ //{
+ dataSetParamStr.nx1 = dataSetParamStr.nx2 = dataSetParamStr.nx3 = 0;
+ dataSetParamStr.nx[0] = static_cast<int>(averageDensityArray->getNX1());
+ dataSetParamStr.nx[1] = static_cast<int>(averageDensityArray->getNX2());
+ dataSetParamStr.nx[2] = static_cast<int>(averageDensityArray->getNX3());
+ dataSetParamStr.nx[3] = static_cast<int>(averageDensityArray->getNX4());
+ doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ //}
+ //else
+ // break;
- bcVector[ic].push_back(*bouCond);
- bcAddArray[ic].boundCond_count++;
- count_boundCond++;
- bytesCount[ic] += sizeof(BoundaryCondition);
- }
-
- if (bcindexmatrixCountNotInit)
- {
- boundCondParamStr.nx1 = static_cast<int>(bcArr->bcindexmatrix.getNX1());
- boundCondParamStr.nx2 = static_cast<int>(bcArr->bcindexmatrix.getNX2());
- boundCondParamStr.nx3 = static_cast<int>(bcArr->bcindexmatrix.getNX3());
- boundCondParamStr.bcindexmatrixCount = static_cast<int>(bcArr->bcindexmatrix.getDataVector().size());
- bcindexmatrixCountNotInit = false;
+ firstBlock = false;
}
- bcindexmatrixVector[ic].insert(bcindexmatrixVector[ic].begin(), bcArr->bcindexmatrix.getDataVector().begin(), bcArr->bcindexmatrix.getDataVector().end());
- bytesCount[ic] += boundCondParamStr.bcindexmatrixCount * sizeof(int);
-
- indexContainerVector[ic].insert(indexContainerVector[ic].begin(), bcArr->indexContainer.begin(), bcArr->indexContainer.end());
- bcAddArray[ic].indexContainer_count = static_cast<int>(bcArr->indexContainer.size());
- count_indexContainer += bcAddArray[ic].indexContainer_count;
- bytesCount[ic] += bcAddArray[ic].indexContainer_count * sizeof(int);
- allBytesCount += bytesCount[ic];
+ if (averageDensityArray && (dataSetParamStr.nx[0] > 0) && (dataSetParamStr.nx[1] > 0) && (dataSetParamStr.nx[2] > 0) && (dataSetParamStr.nx[3] > 0))
+ doubleValuesArray.insert(doubleValuesArray.end(), averageDensityArray->getDataVector().begin(), averageDensityArray->getDataVector().end());
ic++;
}
}
- MPI_Type_contiguous(boundCondParamStr.bcindexmatrixCount, MPI_INT, &bcindexmatrixType);
- MPI_Type_commit(&bcindexmatrixType);
- mpiTypeFreeFlag = true;
+ // register new MPI-types depending on the block-specific information
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_commit(&dataSetDoubleType);
if (comm->isRoot())
{
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeBoundaryConds start MPI IO rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeAverageDensityArray start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
if (comm->isRoot()) start = MPI_Wtime();
MPI_Info info = MPI_INFO_NULL;
- //MPI_Info_create (&info);
- //MPI_Info_set(info,"romio_cb_write","enable");
- //MPI_Info_set(info,"cb_buffer_size","4194304");
- //MPI_Info_set(info,"striping_unit","4194304");
MPI_File file_handler;
- std::string filename = path+"/mpi_io_cp/mpi_io_cp_"+UbSystem::toString(step)+"/cpBC.bin";
- int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE|MPI_MODE_WRONLY, info, &file_handler);
- if (rc!=MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file "+filename);
-
- MPI_Offset write_offset = (MPI_Offset)(sizeof(boundCondParam) + grid->getNumberOfBlocks() * sizeof(size_t));
- size_t next_file_offset = 0;
- if (size > 1)
- {
- if (rank == 0)
- {
- next_file_offset = write_offset + allBytesCount;
- MPI_Send(&next_file_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
- }
- else
- {
- MPI_Recv(&write_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
- next_file_offset = write_offset + allBytesCount;
- if (rank < size - 1)
- MPI_Send(&next_file_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
- }
- }
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageDensityArray.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
- MPI_File_write_at(file_handler, 0, &boundCondParamStr, 1, boundCondParamType, MPI_STATUS_IGNORE);
+ // each process writes common parameters of a dataSet
+ MPI_File_write_at(file_handler, 0, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
- MPI_Offset write_offsetIndex;
+ MPI_Offset write_offset;
+ size_t sizeofOneDataSet = sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double);
for (int nb = 0; nb < blocksCount; nb++)
{
- write_offsetIndex = (MPI_Offset)(sizeof(boundCondParam) + bcAddArray[nb].globalID * sizeof(size_t));
- MPI_File_write_at(file_handler, write_offsetIndex, &write_offset, 1, MPI_LONG_LONG_INT, MPI_STATUS_IGNORE);
-
- MPI_File_write_at(file_handler, write_offset, &bcAddArray[nb], 1, boundCondTypeAdd, MPI_STATUS_IGNORE);
- if (bcVector[nb].size() > 0)
- MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(BCAdd)), &bcVector[nb][0], bcAddArray[nb].boundCond_count, boundCondType, MPI_STATUS_IGNORE);
-
- if (bcindexmatrixVector[nb].size() > 0)
- MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(BCAdd) + bcAddArray[nb].boundCond_count * sizeof(BoundaryCondition)),
- &bcindexmatrixVector[nb][0], 1, bcindexmatrixType, MPI_STATUS_IGNORE);
-
- if (indexContainerVector[nb].size() > 0)
- MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(BCAdd) + bcAddArray[nb].boundCond_count * sizeof(BoundaryCondition) + boundCondParamStr.bcindexmatrixCount * sizeof(int)),
- &indexContainerVector[nb][0], bcAddArray[nb].indexContainer_count, MPI_INT, MPI_STATUS_IGNORE);
-
- write_offset += bytesCount[nb];
+ write_offset = (MPI_Offset)(sizeof(dataSetParam) + dataSetSmallArray[nb].globalID * sizeofOneDataSet);
+ MPI_File_write_at(file_handler, write_offset, &dataSetSmallArray[nb], 1, dataSetSmallType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(DataSetSmall)), &doubleValuesArray[nb * doubleCountInBlock], 1, dataSetDoubleType, MPI_STATUS_IGNORE);
}
MPI_File_sync(file_handler);
MPI_File_close(&file_handler);
+ MPI_Type_free(&dataSetDoubleType);
+
if (comm->isRoot())
{
finish = MPI_Wtime();
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeBoundaryConds time: "<<finish-start<<" s");
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeAverageDensityArray time: " << finish - start << " s");
}
- delete[] bcAddArray;
- delete[] bytesCount;
- delete[] bcVector;
- delete[] bcindexmatrixVector;
- delete[] indexContainerVector;
-}
-
-//------------------------------------------- READ -----------------------------------------------
-void MPIIOMigrationCoProcessor::restart(int step)
-{
- if (comm->isRoot()) UBLOG(logINFO, "MPIIOMigrationCoProcessor restart step: "<<step);
- if (comm->isRoot()) UBLOG(logINFO, "Load check point - start");
- readBlocks(step);
-
- SPtr<Grid3DVisitor> metisVisitor(new MetisPartitioningGridVisitor(comm, MetisPartitioningGridVisitor::LevelBased, D3Q27System::BSW, MetisPartitioner::KWAY));
- grid->accept(metisVisitor);
-
- readDataSet(step);
- readBoundaryConds(step);
- if (comm->isRoot()) UBLOG(logINFO, "Load check point - end");
+ delete[] dataSetSmallArray;
}
-void MPIIOMigrationCoProcessor::readBlocks(int step)
+void MPIIOMigrationCoProcessor::writeAverageVelocityArray(int step)
{
int rank, size;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
- //MPI_Comm_size(MPI_COMM_WORLD, &size);
- size = 1;
-
- if (comm->isRoot())
- {
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::readBlocks start MPI IO rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
- }
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
- double start, finish;
- if (comm->isRoot()) start = MPI_Wtime();
+ int blocksCount = 0; // quantity of blocks in the grid, max 2147483648 blocks!
+
+ std::vector<SPtr<Block3D>> blocksVector[25];
+ int minInitLevel = this->grid->getCoarsestInitializedLevel();
+ int maxInitLevel = this->grid->getFinestInitializedLevel();
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ grid->getBlocks(level, rank, blocksVector[level]);
+ blocksCount += static_cast<int>(blocksVector[level].size());
+ }
+
+ DataSetSmall* dataSetSmallArray = new DataSetSmall[blocksCount];
+ std::vector<double> doubleValuesArray; // double-values (arrays of f's) in all blocks
+ dataSetParam dataSetParamStr;
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeAverageVelocityArray start collect data rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ bool firstBlock = true;
+ int doubleCountInBlock = 0;
+ int ic = 0;
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
+ {
+ dataSetSmallArray[ic].globalID = block->getGlobalID(); // id of the block needed to find it while regenerating the grid
+
+ SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > AverageVelocityArray3DPtr = block->getKernel()->getDataSet()->getAverageVelocity();
+
+ if (firstBlock) // when first (any) valid block...
+ {
+ //if (AverageVelocityArray3DPtr)
+ //{
+ dataSetParamStr.nx1 = dataSetParamStr.nx2 = dataSetParamStr.nx3 = 0;
+ dataSetParamStr.nx[0] = static_cast<int>(AverageVelocityArray3DPtr->getNX1());
+ dataSetParamStr.nx[1] = static_cast<int>(AverageVelocityArray3DPtr->getNX2());
+ dataSetParamStr.nx[2] = static_cast<int>(AverageVelocityArray3DPtr->getNX3());
+ dataSetParamStr.nx[3] = static_cast<int>(AverageVelocityArray3DPtr->getNX4());
+ doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ //}
+ //else
+ // break;
+
+ firstBlock = false;
+ }
+
+ if (AverageVelocityArray3DPtr && (dataSetParamStr.nx[0]>0) && (dataSetParamStr.nx[1]>0) && (dataSetParamStr.nx[2]>0) && (dataSetParamStr.nx[3]>0))
+ doubleValuesArray.insert(doubleValuesArray.end(), AverageVelocityArray3DPtr->getDataVector().begin(), AverageVelocityArray3DPtr->getDataVector().end());
+
+ ic++;
+ }
+ }
+
+ // register new MPI-types depending on the block-specific information
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_commit(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeAverageVelocityArray start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
+
+ MPI_Info info = MPI_INFO_NULL;
+
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageVelocityArray.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ // each process writes common parameters of a dataSet
+ MPI_File_write_at(file_handler, 0, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
+
+ MPI_Offset write_offset;
+ size_t sizeofOneDataSet = sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double);
+
+ for (int nb = 0; nb < blocksCount; nb++)
+ {
+ write_offset = (MPI_Offset)(sizeof(dataSetParam) + dataSetSmallArray[nb].globalID * sizeofOneDataSet);
+ MPI_File_write_at(file_handler, write_offset, &dataSetSmallArray[nb], 1, dataSetSmallType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(DataSetSmall)), &doubleValuesArray[nb * doubleCountInBlock], 1, dataSetDoubleType, MPI_STATUS_IGNORE);
+ }
+
+ MPI_File_sync(file_handler);
+ MPI_File_close(&file_handler);
+
+ MPI_Type_free(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeAverageVelocityArray time: " << finish - start << " s");
+ }
+
+ delete[] dataSetSmallArray;
+}
+
+void MPIIOMigrationCoProcessor::writeAverageFluktuationsArray(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+ int blocksCount = 0; // quantity of blocks in the grid, max 2147483648 blocks!
+
+ std::vector<SPtr<Block3D>> blocksVector[25];
+ int minInitLevel = this->grid->getCoarsestInitializedLevel();
+ int maxInitLevel = this->grid->getFinestInitializedLevel();
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ grid->getBlocks(level, rank, blocksVector[level]);
+ blocksCount += static_cast<int>(blocksVector[level].size());
+ }
+
+ DataSetSmall* dataSetSmallArray = new DataSetSmall[blocksCount];
+ std::vector<double> doubleValuesArray; // double-values (arrays of f's) in all blocks
+ dataSetParam dataSetParamStr;
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeAverageFluktuationsArray start collect data rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ bool firstBlock = true;
+ int doubleCountInBlock = 0;
+ int ic = 0;
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
+ {
+ dataSetSmallArray[ic].globalID = block->getGlobalID(); // id of the block needed to find it while regenerating the grid
+
+ SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > AverageFluctArray3DPtr = block->getKernel()->getDataSet()->getAverageFluctuations();
+
+ if (firstBlock) // when first (any) valid block...
+ {
+ //if (AverageFluctArray3DPtr)
+ //{
+ dataSetParamStr.nx1 = dataSetParamStr.nx2 = dataSetParamStr.nx3 = 0;
+ dataSetParamStr.nx[0] = static_cast<int>(AverageFluctArray3DPtr->getNX1());
+ dataSetParamStr.nx[1] = static_cast<int>(AverageFluctArray3DPtr->getNX2());
+ dataSetParamStr.nx[2] = static_cast<int>(AverageFluctArray3DPtr->getNX3());
+ dataSetParamStr.nx[3] = static_cast<int>(AverageFluctArray3DPtr->getNX4());
+ doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ //}
+ //else
+ // break;
+
+ firstBlock = false;
+ }
+
+ if (AverageFluctArray3DPtr && (dataSetParamStr.nx[0]>0) && (dataSetParamStr.nx[1]>0) && (dataSetParamStr.nx[2]>0) && (dataSetParamStr.nx[3]>0))
+ doubleValuesArray.insert(doubleValuesArray.end(), AverageFluctArray3DPtr->getDataVector().begin(), AverageFluctArray3DPtr->getDataVector().end());
+
+ ic++;
+ }
+ }
+
+ // register new MPI-types depending on the block-specific information
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_commit(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeAverageFluktuationsArray start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
+
+ MPI_Info info = MPI_INFO_NULL;
+#ifdef HLRN_LUSTRE
+ MPI_Info_create(&info);
+ MPI_Info_set(info, "striping_factor", "40");
+ MPI_Info_set(info, "striping_unit", "4M");
+#endif
+
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageFluktuationsArray.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ // each process writes common parameters of a dataSet
+ MPI_File_write_at(file_handler, 0, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
+
+ MPI_Offset write_offset;
+ size_t sizeofOneDataSet = sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double);
+
+ for (int nb = 0; nb < blocksCount; nb++)
+ {
+ write_offset = (MPI_Offset)(sizeof(dataSetParam) + dataSetSmallArray[nb].globalID * sizeofOneDataSet);
+ MPI_File_write_at(file_handler, write_offset, &dataSetSmallArray[nb], 1, dataSetSmallType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(DataSetSmall)), &doubleValuesArray[nb * doubleCountInBlock], 1, dataSetDoubleType, MPI_STATUS_IGNORE);
+ }
+
+ MPI_File_sync(file_handler);
+ MPI_File_close(&file_handler);
+
+ MPI_Type_free(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeAverageFluktuationsArray time: " << finish - start << " s");
+ }
+
+ delete[] dataSetSmallArray;
+}
+
+void MPIIOMigrationCoProcessor::writeAverageTripleArray(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+ int blocksCount = 0; // quantity of blocks in the grid, max 2147483648 blocks!
+
+ std::vector<SPtr<Block3D>> blocksVector[25];
+ int minInitLevel = this->grid->getCoarsestInitializedLevel();
+ int maxInitLevel = this->grid->getFinestInitializedLevel();
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ grid->getBlocks(level, rank, blocksVector[level]);
+ blocksCount += static_cast<int>(blocksVector[level].size());
+ }
+
+ DataSetSmall* dataSetSmallArray = new DataSetSmall[blocksCount];
+ std::vector<double> doubleValuesArray; // double-values (arrays of f's) in all blocks
+ dataSetParam dataSetParamStr;
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeAverageTripleArray start collect data rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ bool firstBlock = true;
+ int doubleCountInBlock = 0;
+ int ic = 0;
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
+ {
+ dataSetSmallArray[ic].globalID = block->getGlobalID(); // id of the block needed to find it while regenerating the grid
+
+ SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > AverageTripleArray3DPtr = block->getKernel()->getDataSet()->getAverageTriplecorrelations();
+
+ if (firstBlock) // when first (any) valid block...
+ {
+ //if (AverageTripleArray3DPtr)
+ //{
+ dataSetParamStr.nx1 = dataSetParamStr.nx2 = dataSetParamStr.nx3 = 0;
+ dataSetParamStr.nx[0] = static_cast<int>(AverageTripleArray3DPtr->getNX1());
+ dataSetParamStr.nx[1] = static_cast<int>(AverageTripleArray3DPtr->getNX2());
+ dataSetParamStr.nx[2] = static_cast<int>(AverageTripleArray3DPtr->getNX3());
+ dataSetParamStr.nx[3] = static_cast<int>(AverageTripleArray3DPtr->getNX4());
+ doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ //}
+ //else
+ // break;
+
+ firstBlock = false;
+ }
+
+ if (AverageTripleArray3DPtr && (dataSetParamStr.nx[0]>0) && (dataSetParamStr.nx[1]>0) && (dataSetParamStr.nx[2]>0) && (dataSetParamStr.nx[3]>0))
+ doubleValuesArray.insert(doubleValuesArray.end(), AverageTripleArray3DPtr->getDataVector().begin(), AverageTripleArray3DPtr->getDataVector().end());
+
+ ic++;
+ }
+ }
+
+ // register new MPI-types depending on the block-specific information
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_commit(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeAverageTripleArray start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
+
+ MPI_Info info = MPI_INFO_NULL;
+
+#ifdef HLRN_LUSTRE
+ MPI_Info_create(&info);
+ MPI_Info_set(info, "striping_factor", "40");
+ MPI_Info_set(info, "striping_unit", "4M");
+#endif
+
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageTripleArray.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ // each process writes common parameters of a dataSet
+ MPI_File_write_at(file_handler, 0, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
+
+ MPI_Offset write_offset;
+ size_t sizeofOneDataSet = sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double);
+
+ for (int nb = 0; nb < blocksCount; nb++)
+ {
+ write_offset = (MPI_Offset)(sizeof(dataSetParam) + dataSetSmallArray[nb].globalID * sizeofOneDataSet);
+ MPI_File_write_at(file_handler, write_offset, &dataSetSmallArray[nb], 1, dataSetSmallType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(DataSetSmall)), &doubleValuesArray[nb * doubleCountInBlock], 1, dataSetDoubleType, MPI_STATUS_IGNORE);
+ }
+
+ MPI_File_sync(file_handler);
+ MPI_File_close(&file_handler);
+
+ MPI_Type_free(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeAverageTripleArray time: " << finish - start << " s");
+ }
+
+ delete[] dataSetSmallArray;
+}
+
+void MPIIOMigrationCoProcessor::writeShearStressValArray(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+ int blocksCount = 0; // quantity of blocks in the grid, max 2147483648 blocks!
+
+ std::vector<SPtr<Block3D>> blocksVector[25];
+ int minInitLevel = this->grid->getCoarsestInitializedLevel();
+ int maxInitLevel = this->grid->getFinestInitializedLevel();
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ grid->getBlocks(level, rank, blocksVector[level]);
+ blocksCount += static_cast<int>(blocksVector[level].size());
+ }
+
+ DataSetSmall* dataSetSmallArray = new DataSetSmall[blocksCount];
+ std::vector<double> doubleValuesArray; // double-values (arrays of f's) in all blocks
+ dataSetParam dataSetParamStr;
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeShearStressValArray start collect data rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ bool firstBlock = true;
+ int doubleCountInBlock = 0;
+ int ic = 0;
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
+ {
+ dataSetSmallArray[ic].globalID = block->getGlobalID(); // id of the block needed to find it while regenerating the grid
+
+ SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > ShearStressValArray3DPtr = block->getKernel()->getDataSet()->getShearStressValues();
+
+ if (firstBlock) // when first (any) valid block...
+ {
+ //if (ShearStressValArray3DPtr)
+ //{
+ dataSetParamStr.nx1 = dataSetParamStr.nx2 = dataSetParamStr.nx3 = 0;
+ dataSetParamStr.nx[0] = static_cast<int>(ShearStressValArray3DPtr->getNX1());
+ dataSetParamStr.nx[1] = static_cast<int>(ShearStressValArray3DPtr->getNX2());
+ dataSetParamStr.nx[2] = static_cast<int>(ShearStressValArray3DPtr->getNX3());
+ dataSetParamStr.nx[3] = static_cast<int>(ShearStressValArray3DPtr->getNX4());
+ doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ //}
+ //else
+ // break;
+
+ firstBlock = false;
+ }
+
+ if (ShearStressValArray3DPtr && (dataSetParamStr.nx[0]>0) && (dataSetParamStr.nx[1]>0) && (dataSetParamStr.nx[2]>0) && (dataSetParamStr.nx[3]>0))
+ doubleValuesArray.insert(doubleValuesArray.end(), ShearStressValArray3DPtr->getDataVector().begin(), ShearStressValArray3DPtr->getDataVector().end());
+
+ ic++;
+ }
+ }
+
+ // register new MPI-types depending on the block-specific information
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_commit(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeShearStressValArray start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
+
+ MPI_Info info = MPI_INFO_NULL;
+
+#ifdef HLRN_LUSTRE
+ MPI_Info_create(&info);
+ MPI_Info_set(info, "striping_factor", "40");
+ MPI_Info_set(info, "striping_unit", "4M");
+#endif
+
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpShearStressValArray.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ // each process writes common parameters of a dataSet
+ MPI_File_write_at(file_handler, 0, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
+
+ MPI_Offset write_offset;
+ size_t sizeofOneDataSet = sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double);
+
+ for (int nb = 0; nb < blocksCount; nb++)
+ {
+ write_offset = (MPI_Offset)(sizeof(dataSetParam) + dataSetSmallArray[nb].globalID * sizeofOneDataSet);
+ MPI_File_write_at(file_handler, write_offset, &dataSetSmallArray[nb], 1, dataSetSmallType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(DataSetSmall)), &doubleValuesArray[nb * doubleCountInBlock], 1, dataSetDoubleType, MPI_STATUS_IGNORE);
+ }
+
+ MPI_File_sync(file_handler);
+ MPI_File_close(&file_handler);
+
+ MPI_Type_free(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeShearStressValArray time: " << finish - start << " s");
+ }
+
+ delete[] dataSetSmallArray;
+}
+
+void MPIIOMigrationCoProcessor::writeRelaxationFactor(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+ int blocksCount = 0; // quantity of blocks in the grid, max 2147483648 blocks!
+
+ std::vector<SPtr<Block3D>> blocksVector[25];
+ int minInitLevel = this->grid->getCoarsestInitializedLevel();
+ int maxInitLevel = this->grid->getFinestInitializedLevel();
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ grid->getBlocks(level, rank, blocksVector[level]);
+ blocksCount += static_cast<int>(blocksVector[level].size());
+ }
+
+ DataSetSmall* dataSetSmallArray = new DataSetSmall[blocksCount];
+ std::vector<double> doubleValuesArray; // double-values (arrays of f's) in all blocks
+ dataSetParam dataSetParamStr;
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeRelaxationFactor start collect data rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ bool firstBlock = true;
+ int doubleCountInBlock = 0;
+ int ic = 0;
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
+ {
+ dataSetSmallArray[ic].globalID = block->getGlobalID(); // id of the block needed to find it while regenerating the grid
+
+ SPtr< CbArray3D<LBMReal, IndexerX3X2X1> > relaxationFactor3DPtr = block->getKernel()->getDataSet()->getRelaxationFactor();
+
+ if (firstBlock) // when first (any) valid block...
+ {
+ //if (relaxationFactor3DPtr)
+ //{
+ dataSetParamStr.nx1 = dataSetParamStr.nx2 = dataSetParamStr.nx3 = 0;
+ dataSetParamStr.nx[0] = static_cast<int>(relaxationFactor3DPtr->getNX1());
+ dataSetParamStr.nx[1] = static_cast<int>(relaxationFactor3DPtr->getNX2());
+ dataSetParamStr.nx[2] = static_cast<int>(relaxationFactor3DPtr->getNX3());
+ dataSetParamStr.nx[3] = 1;
+ doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ //}
+ //else
+ // break;
+
+ firstBlock = false;
+ }
+
+ if (relaxationFactor3DPtr && (dataSetParamStr.nx[0]>0) && (dataSetParamStr.nx[1]>0) && (dataSetParamStr.nx[2]>0))
+ doubleValuesArray.insert(doubleValuesArray.end(), relaxationFactor3DPtr->getDataVector().begin(), relaxationFactor3DPtr->getDataVector().end());
+
+ ic++;
+ }
+ }
+
+ // register new MPI-types depending on the block-specific information
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_commit(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeRelaxationFactor start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
+
+ MPI_Info info = MPI_INFO_NULL;
+
+#ifdef HLRN_LUSTRE
+ MPI_Info_create(&info);
+ MPI_Info_set(info, "striping_factor", "40");
+ MPI_Info_set(info, "striping_unit", "4M");
+#endif
+
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpRelaxationFactor.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ // each process writes common parameters of a dataSet
+ MPI_File_write_at(file_handler, 0, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
+
+ MPI_Offset write_offset;
+ size_t sizeofOneDataSet = sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double);
+
+ for (int nb = 0; nb < blocksCount; nb++)
+ {
+ write_offset = (MPI_Offset)(sizeof(dataSetParam) + dataSetSmallArray[nb].globalID * sizeofOneDataSet);
+ MPI_File_write_at(file_handler, write_offset, &dataSetSmallArray[nb], 1, dataSetSmallType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(DataSetSmall)), &doubleValuesArray[nb * doubleCountInBlock], 1, dataSetDoubleType, MPI_STATUS_IGNORE);
+ }
+
+ MPI_File_sync(file_handler);
+ MPI_File_close(&file_handler);
+
+ MPI_Type_free(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeRelaxationFactor time: " << finish - start << " s");
+ }
+
+ delete[] dataSetSmallArray;
+}
+
+void MPIIOMigrationCoProcessor::writeBoundaryConds(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeBoundaryConds start collect data rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ int blocksCount = 0; // quantity of blocks, that belong to this process
+ size_t allBytesCount = 0; // quantity of bytes, that one process writes to the file
+ size_t count_boundCond = 0; // how many BoundaryConditions in all blocks
+ int count_indexContainer = 0; // how many indexContainer-values in all blocks
+
+ std::vector<SPtr<Block3D>> blocksVector[25];
+ int minInitLevel = this->grid->getCoarsestInitializedLevel();
+ int maxInitLevel = this->grid->getFinestInitializedLevel();
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ grid->getBlocks(level, rank, blocksVector[level]);
+ blocksCount += static_cast<int>(blocksVector[level].size());
+ }
+
+ BCAdd* bcAddArray = new BCAdd[blocksCount];
+ size_t* bytesCount = new size_t[blocksCount]; // quantity of bytes, that each block writes to the file
+ std::vector<BoundaryCondition>* bcVector = new std::vector<BoundaryCondition>[blocksCount];
+ std::vector<int>* bcindexmatrixVector = new std::vector<int>[blocksCount];
+ std::vector<int>* indexContainerVector = new std::vector<int>[blocksCount];
+
+ bool bcindexmatrixCountNotInit = true;
+ int ic = 0;
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ for (SPtr<Block3D> block : blocksVector[level]) // all the blocks of the current level
+ {
+ SPtr<BCArray3D> bcArr = block->getKernel()->getBCProcessor()->getBCArray();
+
+ bcAddArray[ic].globalID = block->getGlobalID(); // id of the block needed to find it while regenerating the grid
+ bcAddArray[ic].boundCond_count = 0; // how many BoundaryConditions in this block
+ bcAddArray[ic].indexContainer_count = 0; // how many indexContainer-values in this block
+ bytesCount[ic] = sizeof(BCAdd);
+ bcVector[ic].resize(0);
+ bcindexmatrixVector[ic].resize(0);
+ indexContainerVector[ic].resize(0);
+
+ for (int bc = 0; bc<bcArr->getBCVectorSize(); bc++)
+ {
+ BoundaryCondition* bouCond = new BoundaryCondition();
+ if (bcArr->bcvector[bc] == NULL)
+ {
+ memset(bouCond, 0, sizeof(BoundaryCondition));
+ }
+ else
+ {
+ bouCond->noslipBoundaryFlags = bcArr->bcvector[bc]->getNoSlipBoundary();
+ bouCond->slipBoundaryFlags = bcArr->bcvector[bc]->getSlipBoundary();
+ bouCond->velocityBoundaryFlags = bcArr->bcvector[bc]->getVelocityBoundary();
+ bouCond->densityBoundaryFlags = bcArr->bcvector[bc]->getDensityBoundary();
+ bouCond->wallModelBoundaryFlags = bcArr->bcvector[bc]->getWallModelBoundary();
+ bouCond->bcVelocityX1 = bcArr->bcvector[bc]->getBoundaryVelocityX1();
+ bouCond->bcVelocityX2 = bcArr->bcvector[bc]->getBoundaryVelocityX2();
+ bouCond->bcVelocityX3 = bcArr->bcvector[bc]->getBoundaryVelocityX3();
+ bouCond->bcDensity = bcArr->bcvector[bc]->getBoundaryDensity();
+ bouCond->bcLodiDensity = bcArr->bcvector[bc]->getDensityLodiDensity();
+ bouCond->bcLodiVelocityX1 = bcArr->bcvector[bc]->getDensityLodiVelocityX1();
+ bouCond->bcLodiVelocityX2 = bcArr->bcvector[bc]->getDensityLodiVelocityX2();
+ bouCond->bcLodiVelocityX3 = bcArr->bcvector[bc]->getDensityLodiVelocityX3();
+ bouCond->bcLodiLentgh = bcArr->bcvector[bc]->getDensityLodiLength();
+ bouCond->nx1 = bcArr->bcvector[bc]->nx1;
+ bouCond->nx2 = bcArr->bcvector[bc]->nx2;
+ bouCond->nx3 = bcArr->bcvector[bc]->nx3;
+ for (int iq = 0; iq<26; iq++)
+ bouCond->q[iq] = bcArr->bcvector[bc]->getQ(iq);
+ bouCond->algorithmType = bcArr->bcvector[bc]->getBcAlgorithmType();
+ }
+
+ bcVector[ic].push_back(*bouCond);
+ bcAddArray[ic].boundCond_count++;
+ count_boundCond++;
+ bytesCount[ic] += sizeof(BoundaryCondition);
+ }
+
+ if (bcindexmatrixCountNotInit)
+ {
+ boundCondParamStr.nx1 = static_cast<int>(bcArr->bcindexmatrix.getNX1());
+ boundCondParamStr.nx2 = static_cast<int>(bcArr->bcindexmatrix.getNX2());
+ boundCondParamStr.nx3 = static_cast<int>(bcArr->bcindexmatrix.getNX3());
+ boundCondParamStr.bcindexmatrixCount = static_cast<int>(bcArr->bcindexmatrix.getDataVector().size());
+ bcindexmatrixCountNotInit = false;
+ }
+ bcindexmatrixVector[ic].insert(bcindexmatrixVector[ic].begin(), bcArr->bcindexmatrix.getDataVector().begin(), bcArr->bcindexmatrix.getDataVector().end());
+ bytesCount[ic] += boundCondParamStr.bcindexmatrixCount * sizeof(int);
+
+ indexContainerVector[ic].insert(indexContainerVector[ic].begin(), bcArr->indexContainer.begin(), bcArr->indexContainer.end());
+ bcAddArray[ic].indexContainer_count = static_cast<int>(bcArr->indexContainer.size());
+ count_indexContainer += bcAddArray[ic].indexContainer_count;
+ bytesCount[ic] += bcAddArray[ic].indexContainer_count * sizeof(int);
+
+ allBytesCount += bytesCount[ic];
+
+ ic++;
+ }
+ }
+
+ MPI_Type_contiguous(boundCondParamStr.bcindexmatrixCount, MPI_INT, &bcindexmatrixType);
+ MPI_Type_commit(&bcindexmatrixType);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeBoundaryConds start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
+
+ MPI_Info info = MPI_INFO_NULL;
+ //MPI_Info_create (&info);
+ //MPI_Info_set(info,"romio_cb_write","enable");
+ //MPI_Info_set(info,"cb_buffer_size","4194304");
+ //MPI_Info_set(info,"striping_unit","4194304");
+
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpBC.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ MPI_Offset write_offset = (MPI_Offset)(sizeof(boundCondParam) + grid->getNumberOfBlocks() * sizeof(size_t));
+ size_t next_file_offset = 0;
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_file_offset = write_offset + allBytesCount;
+ MPI_Send(&next_file_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
+ }
+ else
+ {
+ MPI_Recv(&write_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+ next_file_offset = write_offset + allBytesCount;
+ if (rank < size - 1)
+ MPI_Send(&next_file_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
+ }
+ }
+
+ MPI_File_write_at(file_handler, 0, &boundCondParamStr, 1, boundCondParamType, MPI_STATUS_IGNORE);
+
+ MPI_Offset write_offsetIndex;
+
+ for (int nb = 0; nb < blocksCount; nb++)
+ {
+ write_offsetIndex = (MPI_Offset)(sizeof(boundCondParam) + bcAddArray[nb].globalID * sizeof(size_t));
+ MPI_File_write_at(file_handler, write_offsetIndex, &write_offset, 1, MPI_LONG_LONG_INT, MPI_STATUS_IGNORE);
+
+ MPI_File_write_at(file_handler, write_offset, &bcAddArray[nb], 1, boundCondTypeAdd, MPI_STATUS_IGNORE);
+ if (bcVector[nb].size() > 0)
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(BCAdd)), &bcVector[nb][0], bcAddArray[nb].boundCond_count, boundCondType, MPI_STATUS_IGNORE);
+
+ if (bcindexmatrixVector[nb].size() > 0)
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(BCAdd) + bcAddArray[nb].boundCond_count * sizeof(BoundaryCondition)),
+ &bcindexmatrixVector[nb][0], 1, bcindexmatrixType, MPI_STATUS_IGNORE);
+
+ if (indexContainerVector[nb].size() > 0)
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(BCAdd) + bcAddArray[nb].boundCond_count * sizeof(BoundaryCondition) + boundCondParamStr.bcindexmatrixCount * sizeof(int)),
+ &indexContainerVector[nb][0], bcAddArray[nb].indexContainer_count, MPI_INT, MPI_STATUS_IGNORE);
+
+ write_offset += bytesCount[nb];
+ }
+
+ MPI_File_sync(file_handler);
+ MPI_File_close(&file_handler);
+
+ MPI_Type_free(&bcindexmatrixType);
+
+ if (comm->isRoot())
+ {
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeBoundaryConds time: " << finish - start << " s");
+ }
+
+ delete[] bcAddArray;
+ delete[] bytesCount;
+ delete[] bcVector;
+ delete[] bcindexmatrixVector;
+ delete[] indexContainerVector;
+}
+
+//------------------------------------------- READ -----------------------------------------------
+void MPIIOMigrationCoProcessor::restart(int step)
+{
+ if (comm->isRoot()) UBLOG(logINFO, "MPIIOMigrationCoProcessor restart step: " << step);
+ if (comm->isRoot()) UBLOG(logINFO, "Load check point - start");
+
+ readBlocks(step);
+
+ SPtr<Grid3DVisitor> metisVisitor(new MetisPartitioningGridVisitor(comm, MetisPartitioningGridVisitor::LevelBased, D3Q27System::BSW, MetisPartitioner::KWAY));
+ grid->accept(metisVisitor);
+
+ readDataSet(step);
+ readBoundaryConds(step);
+
+ if (comm->isRoot()) UBLOG(logINFO, "Load check point - end");
+ //this->reconnect(grid);
+}
+
+void MPIIOMigrationCoProcessor::readBlocks(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ //MPI_Comm_size(MPI_COMM_WORLD, &size);
+ size = 1;
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readBlocks start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
+
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpBlocks.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ // read count of blocks
+ int blocksCount = 0;
+ //MPI_File_read_at(file_handler, rank*sizeof(int), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, 0, &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
+ Block3d* block3dArray = new Block3d[blocksCount];
+
+ GridParam* gridParameters = new GridParam;
+
+ // calculate the read offset
+ MPI_Offset read_offset = (MPI_Offset)(size * sizeof(int));
+
+ // read parameters of the grid
+ MPI_File_read_at(file_handler, read_offset, gridParameters, 1, gridParamType, MPI_STATUS_IGNORE);
+ // read all the blocks
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(GridParam)), &block3dArray[0], blocksCount, block3dType, MPI_STATUS_IGNORE);
+
+ MPI_File_close(&file_handler);
+
+ if (comm->isRoot())
+ {
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readBlocks time: " << finish - start << " s");
+ }
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readBlocks start of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ // clear the grid
+ std::vector<SPtr<Block3D>> blocksVector;
+ grid->getBlocks(0, blocksVector);
+ int del = 0;
+ for (SPtr<Block3D> block : blocksVector)
+ {
+ grid->deleteBlock(block);
+ del++;
+ }
+
+ // restore the grid
+ SPtr<CoordinateTransformation3D> trafo(new CoordinateTransformation3D());
+ trafo->Tx1 = gridParameters->trafoParams[0];
+ trafo->Tx2 = gridParameters->trafoParams[1];
+ trafo->Tx3 = gridParameters->trafoParams[2];
+ trafo->Sx1 = gridParameters->trafoParams[3];
+ trafo->Sx2 = gridParameters->trafoParams[4];
+ trafo->Sx3 = gridParameters->trafoParams[5];
+ trafo->alpha = gridParameters->trafoParams[6];
+ trafo->beta = gridParameters->trafoParams[7];
+ trafo->gamma = gridParameters->trafoParams[8];
+
+ trafo->toX1factorX1 = gridParameters->trafoParams[9];
+ trafo->toX1factorX2 = gridParameters->trafoParams[10];
+ trafo->toX1factorX3 = gridParameters->trafoParams[11];
+ trafo->toX1delta = gridParameters->trafoParams[12];
+ trafo->toX2factorX1 = gridParameters->trafoParams[13];
+ trafo->toX2factorX2 = gridParameters->trafoParams[14];
+ trafo->toX2factorX3 = gridParameters->trafoParams[15];
+ trafo->toX2delta = gridParameters->trafoParams[16];
+ trafo->toX3factorX1 = gridParameters->trafoParams[17];
+ trafo->toX3factorX2 = gridParameters->trafoParams[18];
+ trafo->toX3factorX3 = gridParameters->trafoParams[19];
+ trafo->toX3delta = gridParameters->trafoParams[20];
+
+ trafo->fromX1factorX1 = gridParameters->trafoParams[21];
+ trafo->fromX1factorX2 = gridParameters->trafoParams[22];
+ trafo->fromX1factorX3 = gridParameters->trafoParams[23];
+ trafo->fromX1delta = gridParameters->trafoParams[24];
+ trafo->fromX2factorX1 = gridParameters->trafoParams[25];
+ trafo->fromX2factorX2 = gridParameters->trafoParams[26];
+ trafo->fromX2factorX3 = gridParameters->trafoParams[27];
+ trafo->fromX2delta = gridParameters->trafoParams[28];
+ trafo->fromX3factorX1 = gridParameters->trafoParams[29];
+ trafo->fromX3factorX2 = gridParameters->trafoParams[30];
+ trafo->fromX3factorX3 = gridParameters->trafoParams[31];
+ trafo->fromX3delta = gridParameters->trafoParams[32];
+
+ trafo->active = gridParameters->active;
+ trafo->transformation = gridParameters->transformation;
+
+ grid->setCoordinateTransformator(trafo);
+
+ grid->setDeltaX(gridParameters->deltaX);
+ grid->setBlockNX(gridParameters->blockNx1, gridParameters->blockNx2, gridParameters->blockNx3);
+ grid->setNX1(gridParameters->nx1);
+ grid->setNX2(gridParameters->nx2);
+ grid->setNX3(gridParameters->nx3);
+ grid->setPeriodicX1(gridParameters->periodicX1);
+ grid->setPeriodicX2(gridParameters->periodicX2);
+ grid->setPeriodicX3(gridParameters->periodicX3);
+
+ // regenerate blocks
+ for (int n = 0; n<blocksCount; n++)
+ {
+ SPtr<Block3D> block(new Block3D(block3dArray[n].x1, block3dArray[n].x2, block3dArray[n].x3, block3dArray[n].level));
+ block->setActive(block3dArray[n].active);
+ block->setBundle(block3dArray[n].bundle);
+ block->setRank(block3dArray[n].rank);
+ block->setLocalRank(block3dArray[n].lrank);
+ block->setGlobalID(block3dArray[n].globalID);
+ block->setLocalID(block3dArray[n].localID);
+ block->setPart(block3dArray[n].part);
+ block->setLevel(block3dArray[n].level);
+ block->interpolationFlagCF = block3dArray[n].interpolationFlagCF;
+ block->interpolationFlagFC = block3dArray[n].interpolationFlagFC;
+
+ grid->addBlock(block);
+ }
+
+ delete gridParameters;
+ delete[] block3dArray;
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readBlocks end of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+}
+
+void MPIIOMigrationCoProcessor::readDataSet(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readDataSet start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
+
+ int blocksCount = 0; // quantity of the blocks, that belong to this process
+ dataSetParam dataSetParamStr1, dataSetParamStr2, dataSetParamStr3;
+
+ // read from the grid the blocks, that belong to this process
+ std::vector<SPtr<Block3D>> blocksVector[25];
+ int minInitLevel = this->grid->getCoarsestInitializedLevel();
+ int maxInitLevel = this->grid->getFinestInitializedLevel();
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ grid->getBlocks(level, rank, blocksVector[level]);
+ blocksCount += static_cast<int>(blocksVector[level].size());
+ }
+
+ DataSet* dataSetArray = new DataSet[blocksCount];
+
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSet.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ MPI_File_read_at(file_handler, (MPI_Offset)0, &dataSetParamStr1, 1, dataSetParamType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(sizeof(dataSetParam)), &dataSetParamStr2, 1, dataSetParamType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(2 * sizeof(dataSetParam)), &dataSetParamStr3, 1, dataSetParamType, MPI_STATUS_IGNORE);
+
+ double doubleCountInBlock = dataSetParamStr1.nx[0] * dataSetParamStr1.nx[1] * dataSetParamStr1.nx[2] * dataSetParamStr1.nx[3] +
+ dataSetParamStr2.nx[0] * dataSetParamStr2.nx[1] * dataSetParamStr2.nx[2] * dataSetParamStr2.nx[3] +
+ dataSetParamStr3.nx[0] * dataSetParamStr3.nx[1] * dataSetParamStr3.nx[2] * dataSetParamStr3.nx[3];
+ std::vector<double> doubleValuesArray(blocksCount * doubleCountInBlock); // double-values in all blocks
+
+ // define MPI_types depending on the block-specific information
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_commit(&dataSetDoubleType);
+
+ int ic = 0;
+ MPI_Offset read_offset;
+ size_t sizeofOneDataSet = size_t(sizeof(DataSet) + doubleCountInBlock * sizeof(double));
+
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
+ {
+ read_offset = (MPI_Offset)(3 * sizeof(dataSetParam) + block->getGlobalID() * sizeofOneDataSet);
+ MPI_File_read_at(file_handler, read_offset, &dataSetArray[ic], 1, dataSetType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(DataSet)), &doubleValuesArray[ic * doubleCountInBlock], 1, dataSetDoubleType, MPI_STATUS_IGNORE);
+ ic++;
+ }
+ }
+
+ MPI_File_close(&file_handler);
+
+ if (comm->isRoot())
+ {
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readDataSet time: " << finish - start << " s");
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readDataSet start of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ size_t index = 0, vectorSize = 0;
+ std::vector<double> vectorsOfValues1, vectorsOfValues2, vectorsOfValues3;
+
+ for (int n = 0; n < blocksCount; n++)
+ {
+ vectorSize = dataSetParamStr1.nx[0] * dataSetParamStr1.nx[1] * dataSetParamStr1.nx[2] * dataSetParamStr1.nx[3];
+ vectorsOfValues1.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + vectorSize);
+ index += vectorSize;
+
+ vectorSize = dataSetParamStr2.nx[0] * dataSetParamStr2.nx[1] * dataSetParamStr2.nx[2] * dataSetParamStr2.nx[3];
+ vectorsOfValues2.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + vectorSize);
+ index += vectorSize;
+
+ vectorSize = dataSetParamStr3.nx[0] * dataSetParamStr3.nx[1] * dataSetParamStr3.nx[2] * dataSetParamStr3.nx[3];
+ vectorsOfValues3.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + vectorSize);
+ index += vectorSize;
+
+ SPtr<DistributionArray3D> mFdistributions(new D3Q27EsoTwist3DSplittedVector());
+
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues1, dataSetParamStr1.nx[0], dataSetParamStr1.nx[1], dataSetParamStr1.nx[2], dataSetParamStr1.nx[3])));
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNonLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues2, dataSetParamStr2.nx[0], dataSetParamStr2.nx[1], dataSetParamStr2.nx[2], dataSetParamStr2.nx[3])));
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setZeroDistributions(CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(vectorsOfValues3, dataSetParamStr3.nx[0], dataSetParamStr3.nx[1], dataSetParamStr3.nx[2])));
+
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNX1(dataSetParamStr1.nx1);
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNX2(dataSetParamStr1.nx2);
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNX3(dataSetParamStr1.nx3);
+
+ // find the nesessary block and fill it
+ SPtr<Block3D> block = grid->getBlock(dataSetArray[n].globalID);
+ //std::cout << "rank="<<rank<<", dataSetArray[n].globalID=" << dataSetArray[n].globalID << std::endl;
+
+ SPtr<LBMKernel> kernel = this->lbmKernel->clone();
+ kernel->setGhostLayerWidth(dataSetArray[n].ghostLayerWidth);
+ kernel->setCollisionFactor(dataSetArray[n].collFactor);
+ kernel->setDeltaT(dataSetArray[n].deltaT);
+ kernel->setCompressible(dataSetArray[n].compressible);
+ kernel->setWithForcing(dataSetArray[n].withForcing);
+ SPtr<DataSet3D> dataSetPtr = SPtr<DataSet3D>(new DataSet3D());
+ dataSetPtr->setFdistributions(mFdistributions);
+ kernel->setDataSet(dataSetPtr);
+ block->setKernel(kernel);
+ }
+
+ MPI_Type_free(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readDataSet end of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ delete[] dataSetArray;
+
+ //-------------------------------------------------------------
+
+ DSArraysPresence arrPresence;
+ MPI_File file_handler1;
+ std::string filename1 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpArrays.bin";
+ rc = MPI_File_open(MPI_COMM_WORLD, filename1.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler1);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename1);
+ MPI_File_read_at(file_handler1, (MPI_Offset)0, &arrPresence, 1, arrayPresenceType, MPI_STATUS_IGNORE);
+ MPI_File_close(&file_handler1);
+
+ if (arrPresence.isAverageDensityArrayPresent)
+ readAverageDensityArray(step);
+
+ if (arrPresence.isAverageVelocityArrayPresent)
+ readAverageVelocityArray(step);
+
+ if (arrPresence.isAverageFluktuationsArrayPresent)
+ readAverageFluktuationsArray(step);
+
+ if (arrPresence.isAverageTripleArrayPresent)
+ readAverageTripleArray(step);
+
+ if (arrPresence.isShearStressValArrayPresent)
+ readShearStressValArray(step);
+
+ if (arrPresence.isRelaxationFactorPresent)
+ readRelaxationFactor(step);
+
+}
+
+void MPIIOMigrationCoProcessor::readAverageDensityArray(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readAverageDensityArray start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
MPI_File file_handler;
- std::string filename = path+"/mpi_io_cp/mpi_io_cp_"+UbSystem::toString(step)+"/cpBlocks.bin";
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageDensityArray.bin";
int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
- if (rc!=MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file "+filename);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
// read count of blocks
int blocksCount = 0;
- //MPI_File_read_at(file_handler, rank*sizeof(int), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, 0, &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
- Block3d* block3dArray = new Block3d[blocksCount];
-
- GridParam* gridParameters = new GridParam;
+ dataSetParam dataSetParamStr;
+ memset(&dataSetParamStr, 0, sizeof(dataSetParam));
+
+ // read from the grid the blocks, that belong to this process
+ std::vector<SPtr<Block3D>> blocksVector[25];
+ int minInitLevel = this->grid->getCoarsestInitializedLevel();
+ int maxInitLevel = this->grid->getFinestInitializedLevel();
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ grid->getBlocks(level, rank, blocksVector[level]);
+ blocksCount += static_cast<int>(blocksVector[level].size());
+ }
+
+ MPI_File_read_at(file_handler, (MPI_Offset)0, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
+
+ DataSetSmall* dataSetSmallArray = new DataSetSmall[blocksCount];
+ int doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ std::vector<double> doubleValuesArray(blocksCount * doubleCountInBlock); // double-values in all blocks
+
+ // define MPI_types depending on the block-specific information
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_commit(&dataSetDoubleType);
+
+ int ic = 0;
+ MPI_Offset read_offset;
+ size_t sizeofOneDataSet = size_t(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
+ {
+ read_offset = (MPI_Offset)(sizeof(dataSetParam) + block->getGlobalID() * sizeofOneDataSet);
+ MPI_File_read_at(file_handler, read_offset, &dataSetSmallArray[ic], 1, dataSetSmallType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(DataSetSmall)), &doubleValuesArray[ic * doubleCountInBlock], 1, dataSetDoubleType, MPI_STATUS_IGNORE);
+ ic++;
+ }
+ }
+
+ MPI_File_close(&file_handler);
+
+ if (comm->isRoot())
+ {
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readAverageDensityArray time: " << finish - start << " s");
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readAverageDensityArray start of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ size_t index = 0;
+ size_t nextVectorSize = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ std::vector<double> vectorsOfValues;
+ for (int n = 0; n < blocksCount; n++)
+ {
+ vectorsOfValues.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + nextVectorSize);
+ index += nextVectorSize;
+
+ // fill mAverageDensity arrays
+ SPtr<AverageValuesArray3D> mAverageDensity;
+ //if ((dataSetParamStr.nx[0]==0)&&(dataSetParamStr.nx[1]==0)&&(dataSetParamStr.nx[2]==0)&&(dataSetParamStr.nx[3]==0))
+ // mAverageDensity = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr();
+ //else
+ mAverageDensity = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
+
+ //std::cout << "rank=" << rank << ", dataSetArray[n].globalID=" << dataSetSmallArray[n].globalID << std::endl;
+ // find the nesessary block and fill it
+ SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].globalID);
+ block->kernel->getDataSet()->setAverageDencity(mAverageDensity);
+ }
+
+ MPI_Type_free(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readAverageDensityArray end of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ delete[] dataSetSmallArray;
+}
+
+void MPIIOMigrationCoProcessor::readAverageVelocityArray(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readAverageVelocityArray start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
+
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageVelocityArray.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ // read count of blocks
+ int blocksCount = 0;
+ dataSetParam dataSetParamStr;
+ memset(&dataSetParamStr, 0, sizeof(dataSetParam));
+
+ // read from the grid the blocks, that belong to this process
+ std::vector<SPtr<Block3D>> blocksVector[25];
+ int minInitLevel = this->grid->getCoarsestInitializedLevel();
+ int maxInitLevel = this->grid->getFinestInitializedLevel();
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ grid->getBlocks(level, rank, blocksVector[level]);
+ blocksCount += static_cast<int>(blocksVector[level].size());
+ }
+
+ MPI_File_read_at(file_handler, (MPI_Offset)0, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
+
+ DataSetSmall* dataSetSmallArray = new DataSetSmall[blocksCount];
+ int doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ std::vector<double> doubleValuesArray(blocksCount * doubleCountInBlock); // double-values in all blocks
+
+ // define MPI_types depending on the block-specific information
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_commit(&dataSetDoubleType);
+
+ int ic = 0;
+ MPI_Offset read_offset;
+ size_t sizeofOneDataSet = size_t(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
+ {
+ read_offset = (MPI_Offset)(sizeof(dataSetParam) + block->getGlobalID() * sizeofOneDataSet);
+ MPI_File_read_at(file_handler, read_offset, &dataSetSmallArray[ic], 1, dataSetSmallType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(DataSetSmall)), &doubleValuesArray[ic * doubleCountInBlock], 1, dataSetDoubleType, MPI_STATUS_IGNORE);
+ ic++;
+ }
+ }
+
+ MPI_File_close(&file_handler);
+
+ if (comm->isRoot())
+ {
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readAverageVelocityArray time: " << finish - start << " s");
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readAverageVelocityArray start of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ size_t index = 0;
+ size_t nextVectorSize = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ std::vector<double> vectorsOfValues;
+ for (int n = 0; n < blocksCount; n++)
+ {
+ vectorsOfValues.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + nextVectorSize);
+ index += nextVectorSize;
+
+ // fill mAverageVelocity array
+ SPtr<AverageValuesArray3D> mAverageVelocity;
+ //if ((dataSetParamStr.nx[0] == 0) && (dataSetParamStr.nx[1] == 0) && (dataSetParamStr.nx[2] == 0) && (dataSetParamStr.nx[3] == 0))
+ // mAverageVelocity = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr();
+ //else
+ mAverageVelocity = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
+
+ // find the nesessary block and fill it
+ SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].globalID);
+ block->kernel->getDataSet()->setAverageVelocity(mAverageVelocity);
+ }
+
+ MPI_Type_free(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readAverageVelocityArray end of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ delete[] dataSetSmallArray;
+}
+
+void MPIIOMigrationCoProcessor::readAverageFluktuationsArray(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readAverageFluktuationsArray start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
+
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageFluktuationsArray.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ // read count of blocks
+ int blocksCount = 0;
+ dataSetParam dataSetParamStr;
+ memset(&dataSetParamStr, 0, sizeof(dataSetParam));
+
+ // read from the grid the blocks, that belong to this process
+ std::vector<SPtr<Block3D>> blocksVector[25];
+ int minInitLevel = this->grid->getCoarsestInitializedLevel();
+ int maxInitLevel = this->grid->getFinestInitializedLevel();
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ grid->getBlocks(level, rank, blocksVector[level]);
+ blocksCount += static_cast<int>(blocksVector[level].size());
+ }
+
+ MPI_File_read_at(file_handler, (MPI_Offset)0, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
+
+ DataSetSmall* dataSetSmallArray = new DataSetSmall[blocksCount];
+ int doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ std::vector<double> doubleValuesArray(blocksCount * doubleCountInBlock); // double-values in all blocks
+
+ // define MPI_types depending on the block-specific information
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_commit(&dataSetDoubleType);
- // calculate the read offset
- MPI_Offset read_offset = (MPI_Offset)(size*sizeof(int));
+ int ic = 0;
+ MPI_Offset read_offset;
+ size_t sizeofOneDataSet = size_t(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
- // read parameters of the grid
- MPI_File_read_at(file_handler, read_offset, gridParameters, 1, gridParamType, MPI_STATUS_IGNORE);
- // read all the blocks
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset+sizeof(GridParam)), &block3dArray[0], blocksCount, block3dType, MPI_STATUS_IGNORE);
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
+ {
+ read_offset = (MPI_Offset)(sizeof(dataSetParam) + block->getGlobalID() * sizeofOneDataSet);
+ MPI_File_read_at(file_handler, read_offset, &dataSetSmallArray[ic], 1, dataSetSmallType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(DataSetSmall)), &doubleValuesArray[ic * doubleCountInBlock], 1, dataSetDoubleType, MPI_STATUS_IGNORE);
+ ic++;
+ }
+ }
MPI_File_close(&file_handler);
if (comm->isRoot())
{
finish = MPI_Wtime();
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::readBlocks time: "<<finish-start<<" s");
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readAverageFluktuationsArray time: " << finish - start << " s");
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readAverageFluktuationsArray start of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ size_t index = 0;
+ size_t nextVectorSize = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ std::vector<double> vectorsOfValues;
+ for (int n = 0; n < blocksCount; n++)
+ {
+ vectorsOfValues.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + nextVectorSize);
+ index += nextVectorSize;
+
+ // fill AverageFluktuations array
+ SPtr<AverageValuesArray3D> mAverageFluktuations;
+ //if ((dataSetParamStr.nx[0] == 0) && (dataSetParamStr.nx[1] == 0) && (dataSetParamStr.nx[2] == 0) && (dataSetParamStr.nx[3] == 0))
+ // mAverageFluktuations = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr();
+ //else
+ mAverageFluktuations = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
+
+ // find the nesessary block and fill it
+ SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].globalID);
+ block->kernel->getDataSet()->setAverageFluctuations(mAverageFluktuations);
}
+ MPI_Type_free(&dataSetDoubleType);
+
if (comm->isRoot())
{
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::readBlocks start of restore of data, rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readAverageFluktuationsArray end of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
- // clear the grid
- std::vector<SPtr<Block3D>> blocksVector;
- grid->getBlocks(0, blocksVector);
- int del = 0;
- for(SPtr<Block3D> block : blocksVector)
+ delete[] dataSetSmallArray;
+}
+
+void MPIIOMigrationCoProcessor::readAverageTripleArray(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+ if (comm->isRoot())
{
- grid->deleteBlock(block);
- del++;
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readAverageTripleArray start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
- // restore the grid
- SPtr<CoordinateTransformation3D> trafo(new CoordinateTransformation3D());
- trafo->Tx1 = gridParameters->trafoParams[0];
- trafo->Tx2 = gridParameters->trafoParams[1];
- trafo->Tx3 = gridParameters->trafoParams[2];
- trafo->Sx1 = gridParameters->trafoParams[3];
- trafo->Sx2 = gridParameters->trafoParams[4];
- trafo->Sx3 = gridParameters->trafoParams[5];
- trafo->alpha = gridParameters->trafoParams[6];
- trafo->beta = gridParameters->trafoParams[7];
- trafo->gamma = gridParameters->trafoParams[8];
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageTripleArray.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
- trafo->toX1factorX1 = gridParameters->trafoParams[9];
- trafo->toX1factorX2 = gridParameters->trafoParams[10];
- trafo->toX1factorX3 = gridParameters->trafoParams[11];
- trafo->toX1delta = gridParameters->trafoParams[12];
- trafo->toX2factorX1 = gridParameters->trafoParams[13];
- trafo->toX2factorX2 = gridParameters->trafoParams[14];
- trafo->toX2factorX3 = gridParameters->trafoParams[15];
- trafo->toX2delta = gridParameters->trafoParams[16];
- trafo->toX3factorX1 = gridParameters->trafoParams[17];
- trafo->toX3factorX2 = gridParameters->trafoParams[18];
- trafo->toX3factorX3 = gridParameters->trafoParams[19];
- trafo->toX3delta = gridParameters->trafoParams[20];
+ // read count of blocks
+ int blocksCount = 0;
+ dataSetParam dataSetParamStr;
+ memset(&dataSetParamStr, 0, sizeof(dataSetParam));
- trafo->fromX1factorX1 = gridParameters->trafoParams[21];
- trafo->fromX1factorX2 = gridParameters->trafoParams[22];
- trafo->fromX1factorX3 = gridParameters->trafoParams[23];
- trafo->fromX1delta = gridParameters->trafoParams[24];
- trafo->fromX2factorX1 = gridParameters->trafoParams[25];
- trafo->fromX2factorX2 = gridParameters->trafoParams[26];
- trafo->fromX2factorX3 = gridParameters->trafoParams[27];
- trafo->fromX2delta = gridParameters->trafoParams[28];
- trafo->fromX3factorX1 = gridParameters->trafoParams[29];
- trafo->fromX3factorX2 = gridParameters->trafoParams[30];
- trafo->fromX3factorX3 = gridParameters->trafoParams[31];
- trafo->fromX3delta = gridParameters->trafoParams[32];
+ // read from the grid the blocks, that belong to this process
+ std::vector<SPtr<Block3D>> blocksVector[25];
+ int minInitLevel = this->grid->getCoarsestInitializedLevel();
+ int maxInitLevel = this->grid->getFinestInitializedLevel();
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ grid->getBlocks(level, rank, blocksVector[level]);
+ blocksCount += static_cast<int>(blocksVector[level].size());
+ }
- trafo->active = gridParameters->active;
- trafo->transformation = gridParameters->transformation;
+ MPI_File_read_at(file_handler, (MPI_Offset)0, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
- grid->setCoordinateTransformator(trafo);
+ DataSetSmall* dataSetSmallArray = new DataSetSmall[blocksCount];
+ int doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ std::vector<double> doubleValuesArray(blocksCount * doubleCountInBlock); // double-values in all blocks
- grid->setDeltaX(gridParameters->deltaX);
- grid->setBlockNX(gridParameters->blockNx1, gridParameters->blockNx2, gridParameters->blockNx3);
- grid->setNX1(gridParameters->nx1);
- grid->setNX2(gridParameters->nx2);
- grid->setNX3(gridParameters->nx3);
- grid->setPeriodicX1(gridParameters->periodicX1);
- grid->setPeriodicX2(gridParameters->periodicX2);
- grid->setPeriodicX3(gridParameters->periodicX3);
+ // define MPI_types depending on the block-specific information
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_commit(&dataSetDoubleType);
- // regenerate blocks
- for (int n = 0; n<blocksCount; n++)
+ int ic = 0;
+ MPI_Offset read_offset;
+ size_t sizeofOneDataSet = size_t(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
{
- SPtr<Block3D> block(new Block3D(block3dArray[n].x1, block3dArray[n].x2, block3dArray[n].x3, block3dArray[n].level));
- block->setActive(block3dArray[n].active);
- block->setBundle(block3dArray[n].bundle);
- block->setRank(block3dArray[n].rank);
- block->setLocalRank(block3dArray[n].lrank);
- block->setGlobalID(block3dArray[n].globalID);
- block->setLocalID(block3dArray[n].localID);
- block->setPart(block3dArray[n].part);
- block->setLevel(block3dArray[n].level);
- block->interpolationFlagCF = block3dArray[n].interpolationFlagCF;
- block->interpolationFlagFC = block3dArray[n].interpolationFlagFC;
+ for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
+ {
+ read_offset = (MPI_Offset)(sizeof(dataSetParam) + block->getGlobalID() * sizeofOneDataSet);
+ MPI_File_read_at(file_handler, read_offset, &dataSetSmallArray[ic], 1, dataSetSmallType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(DataSetSmall)), &doubleValuesArray[ic * doubleCountInBlock], 1, dataSetDoubleType, MPI_STATUS_IGNORE);
+ ic++;
+ }
+ }
- grid->addBlock(block);
+ MPI_File_close(&file_handler);
+
+ if (comm->isRoot())
+ {
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readAverageTripleArray time: " << finish - start << " s");
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readAverageTripleArray start of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
- delete gridParameters;
- delete[] block3dArray;
+ size_t index = 0;
+ size_t nextVectorSize = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ std::vector<double> vectorsOfValues;
+ for (int n = 0; n < blocksCount; n++)
+ {
+ vectorsOfValues.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + nextVectorSize);
+ index += nextVectorSize;
+
+ // fill AverageTriplecorrelations array
+ SPtr<AverageValuesArray3D> mAverageTriplecorrelations;
+ //if ((dataSetParamStr.nx[0] == 0) && (dataSetParamStr.nx[1] == 0) && (dataSetParamStr.nx[2] == 0) && (dataSetParamStr.nx[3] == 0))
+ // mAverageTriplecorrelations = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr();
+ //else
+ mAverageTriplecorrelations = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
+
+ // find the nesessary block and fill it
+ SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].globalID);
+ block->kernel->getDataSet()->setAverageTriplecorrelations(mAverageTriplecorrelations);
+ }
+
+ MPI_Type_free(&dataSetDoubleType);
if (comm->isRoot())
{
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::readBlocks end of restore of data, rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readAverageTripleArray end of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
+
+ delete[] dataSetSmallArray;
}
-void MPIIOMigrationCoProcessor::readDataSet(int step)
+void MPIIOMigrationCoProcessor::readShearStressValArray(int step)
{
int rank, size;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
@@ -919,13 +2182,21 @@ void MPIIOMigrationCoProcessor::readDataSet(int step)
if (comm->isRoot())
{
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::readDataSet start MPI IO rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readShearStressValArray start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
if (comm->isRoot()) start = MPI_Wtime();
- int blocksCount = 0; // quantity of the blocks, that belong to this process
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpShearStressValArray.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ // read count of blocks
+ int blocksCount = 0;
+ dataSetParam dataSetParamStr;
+ memset(&dataSetParamStr, 0, sizeof(dataSetParam));
// read from the grid the blocks, that belong to this process
std::vector<SPtr<Block3D>> blocksVector[25];
@@ -937,32 +2208,27 @@ void MPIIOMigrationCoProcessor::readDataSet(int step)
blocksCount += static_cast<int>(blocksVector[level].size());
}
- DataSet* dataSetArray = new DataSet[blocksCount];
-
- MPI_File file_handler;
- std::string filename = path+"/mpi_io_cp/mpi_io_cp_"+UbSystem::toString(step)+"/cpDataSet.bin";
- int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
- if (rc!=MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file "+filename);
-
MPI_File_read_at(file_handler, (MPI_Offset)0, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
-
- std::vector<double> doubleValuesArray(blocksCount * dataSetParamStr.doubleCountInBlock); // double-values in all blocks
+
+ DataSetSmall* dataSetSmallArray = new DataSetSmall[blocksCount];
+ int doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ std::vector<double> doubleValuesArray(blocksCount * doubleCountInBlock); // double-values in all blocks
// define MPI_types depending on the block-specific information
- MPI_Type_contiguous(dataSetParamStr.doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
MPI_Type_commit(&dataSetDoubleType);
- mpiTypeFreeFlag = true;
int ic = 0;
MPI_Offset read_offset;
- size_t sizeofOneDataSet = size_t(sizeof(DataSet) + dataSetParamStr.doubleCountInBlock * sizeof(double));
+ size_t sizeofOneDataSet = size_t(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+
for (int level = minInitLevel; level <= maxInitLevel; level++)
{
- for(SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
+ for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
{
read_offset = (MPI_Offset)(sizeof(dataSetParam) + block->getGlobalID() * sizeofOneDataSet);
- MPI_File_read_at(file_handler, read_offset, &dataSetArray[ic], 1, dataSetType, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(DataSet)), &doubleValuesArray[ic * dataSetParamStr.doubleCountInBlock], 1, dataSetDoubleType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, read_offset, &dataSetSmallArray[ic], 1, dataSetSmallType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(DataSetSmall)), &doubleValuesArray[ic * doubleCountInBlock], 1, dataSetDoubleType, MPI_STATUS_IGNORE);
ic++;
}
}
@@ -972,99 +2238,140 @@ void MPIIOMigrationCoProcessor::readDataSet(int step)
if (comm->isRoot())
{
finish = MPI_Wtime();
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::readDataSet time: "<<finish-start<<" s");
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::readDataSet start of restore of data, rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readShearStressValArray time: " << finish - start << " s");
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readShearStressValArray start of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
-
- size_t index = 0, nextVectorSize = 0;
- std::vector<double> vectorsOfValues[9];
- for (int n = 0; n<blocksCount; n++)
+
+ size_t index = 0;
+ size_t nextVectorSize = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ std::vector<double> vectorsOfValues;
+ for (int n = 0; n < blocksCount; n++)
{
- for (int b = 0; b<9; b++) // assign approciate vectors to 9 dataSet arrays
- {
- nextVectorSize = dataSetParamStr.nx[b][0]* dataSetParamStr.nx[b][1]* dataSetParamStr.nx[b][2]* dataSetParamStr.nx[b][3];
- vectorsOfValues[b].assign(doubleValuesArray.data()+index, doubleValuesArray.data()+index+nextVectorSize);
- index += nextVectorSize;
- }
+ vectorsOfValues.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + nextVectorSize);
+ index += nextVectorSize;
- // fill dataSet arrays
- SPtr<AverageValuesArray3D> mAverageValues;
- if ((dataSetParamStr.nx[0][0]==0)&&(dataSetParamStr.nx[0][1]==0)&&(dataSetParamStr.nx[0][2]==0)&&(dataSetParamStr.nx[0][3]==0))
- mAverageValues = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr();
- else
- mAverageValues = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues[0], dataSetParamStr.nx[0][0], dataSetParamStr.nx[0][1], dataSetParamStr.nx[0][2], dataSetParamStr.nx[0][3]));
+ // fill ShearStressValuesArray array
+ SPtr<ShearStressValuesArray3D> mShearStressValues;
+ //if ((dataSetParamStr.nx[0] == 0) && (dataSetParamStr.nx[1] == 0) && (dataSetParamStr.nx[2] == 0) && (dataSetParamStr.nx[3] == 0))
+ // mShearStressValues = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr();
+ //else
+ mShearStressValues = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
- SPtr<AverageValuesArray3D> mAverageVelocity;
- if ((dataSetParamStr.nx[1][0]==0)&&(dataSetParamStr.nx[1][1]==0)&&(dataSetParamStr.nx[1][2]==0)&&(dataSetParamStr.nx[1][3]==0))
- mAverageVelocity = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr();
- else
- mAverageVelocity = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues[1], dataSetParamStr.nx[1][0], dataSetParamStr.nx[1][1], dataSetParamStr.nx[1][2], dataSetParamStr.nx[1][3]));
+ // find the nesessary block and fill it
+ SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].globalID);
+ block->kernel->getDataSet()->setShearStressValues(mShearStressValues);
+ }
- SPtr<AverageValuesArray3D> mAverageFluktuations;
- if ((dataSetParamStr.nx[2][0]==0)&&(dataSetParamStr.nx[2][1]==0)&&(dataSetParamStr.nx[2][2]==0)&&(dataSetParamStr.nx[2][3]==0))
- mAverageFluktuations = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr();
- else
- mAverageFluktuations = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues[2], dataSetParamStr.nx[2][0], dataSetParamStr.nx[2][1], dataSetParamStr.nx[2][2], dataSetParamStr.nx[2][3]));
+ MPI_Type_free(&dataSetDoubleType);
- SPtr<AverageValuesArray3D> mAverageTriplecorrelations;
- if ((dataSetParamStr.nx[3][0]==0)&&(dataSetParamStr.nx[3][1]==0)&&(dataSetParamStr.nx[3][2]==0)&&(dataSetParamStr.nx[3][3]==0))
- mAverageTriplecorrelations = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr();
- else
- mAverageTriplecorrelations = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues[3], dataSetParamStr.nx[3][0], dataSetParamStr.nx[3][1], dataSetParamStr.nx[3][2], dataSetParamStr.nx[3][3]));
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readShearStressValArray end of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
- SPtr<ShearStressValuesArray3D> mShearStressValues;
- if ((dataSetParamStr.nx[4][0]==0)&&(dataSetParamStr.nx[4][1]==0)&&(dataSetParamStr.nx[4][2]==0)&&(dataSetParamStr.nx[4][3]==0))
- mShearStressValues = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr();
- else
- mShearStressValues = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues[4], dataSetParamStr.nx[4][0], dataSetParamStr.nx[4][1], dataSetParamStr.nx[4][2], dataSetParamStr.nx[4][3]));
+ delete[] dataSetSmallArray;
+}
- SPtr<RelaxationFactorArray3D> mRelaxationFactor;
- if ((dataSetParamStr.nx[5][0]==0)&&(dataSetParamStr.nx[5][1]==0)&&(dataSetParamStr.nx[5][2]==0))
- mRelaxationFactor = CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr();
- else
- mRelaxationFactor = CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(vectorsOfValues[5], dataSetParamStr.nx[5][0], dataSetParamStr.nx[5][1], dataSetParamStr.nx[5][2]));
+void MPIIOMigrationCoProcessor::readRelaxationFactor(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
- SPtr<DistributionArray3D> mFdistributions(new D3Q27EsoTwist3DSplittedVector());
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readRelaxationFactor start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues[6], dataSetParamStr.nx[6][0], dataSetParamStr.nx[6][1], dataSetParamStr.nx[6][2], dataSetParamStr.nx[6][3])));
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNonLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues[7], dataSetParamStr.nx[7][0], dataSetParamStr.nx[7][1], dataSetParamStr.nx[7][2], dataSetParamStr.nx[7][3])));
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setZeroDistributions(CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(vectorsOfValues[8], dataSetParamStr.nx[8][0], dataSetParamStr.nx[8][1], dataSetParamStr.nx[8][2])));
-
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNX1(dataSetParamStr.nx1);
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNX2(dataSetParamStr.nx2);
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNX3(dataSetParamStr.nx3);
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpRelaxationFactor.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
- SPtr<DataSet3D> dataSetPtr = SPtr<DataSet3D>(new DataSet3D());
- dataSetPtr->setAverageValues(mAverageValues);
- dataSetPtr->setAverageVelocity(mAverageVelocity);
- dataSetPtr->setAverageFluctuations(mAverageFluktuations);
- dataSetPtr->setAverageTriplecorrelations(mAverageTriplecorrelations);
- dataSetPtr->setShearStressValues(mShearStressValues);
- dataSetPtr->setRelaxationFactor(mRelaxationFactor);
- dataSetPtr->setFdistributions(mFdistributions);
+ // read count of blocks
+ int blocksCount = 0;
+ dataSetParam dataSetParamStr;
+ memset(&dataSetParamStr, 0, sizeof(dataSetParam));
+
+ // read from the grid the blocks, that belong to this process
+ std::vector<SPtr<Block3D>> blocksVector[25];
+ int minInitLevel = this->grid->getCoarsestInitializedLevel();
+ int maxInitLevel = this->grid->getFinestInitializedLevel();
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ grid->getBlocks(level, rank, blocksVector[level]);
+ blocksCount += static_cast<int>(blocksVector[level].size());
+ }
+
+ MPI_File_read_at(file_handler, (MPI_Offset)0, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
+
+ DataSetSmall* dataSetSmallArray = new DataSetSmall[blocksCount];
+ int doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ std::vector<double> doubleValuesArray(blocksCount * doubleCountInBlock); // double-values in all blocks
+
+ // define MPI_types depending on the block-specific information
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_commit(&dataSetDoubleType);
+
+ int ic = 0;
+ MPI_Offset read_offset;
+ size_t sizeofOneDataSet = size_t(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
+ {
+ read_offset = (MPI_Offset)(sizeof(dataSetParam) + block->getGlobalID() * sizeofOneDataSet);
+ MPI_File_read_at(file_handler, read_offset, &dataSetSmallArray[ic], 1, dataSetSmallType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(DataSetSmall)), &doubleValuesArray[ic * doubleCountInBlock], 1, dataSetDoubleType, MPI_STATUS_IGNORE);
+ ic++;
+ }
+ }
+
+ MPI_File_close(&file_handler);
+
+ if (comm->isRoot())
+ {
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readRelaxationFactor time: " << finish - start << " s");
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readRelaxationFactor start of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ size_t index = 0;
+ size_t nextVectorSize = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ std::vector<double> vectorsOfValues;
+ for (int n = 0; n < blocksCount; n++)
+ {
+ vectorsOfValues.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + nextVectorSize);
+ index += nextVectorSize;
+
+ // fill RelaxationFactor array
+ SPtr<RelaxationFactorArray3D> mRelaxationFactor;
+ //if ((dataSetParamStr.nx[0] == 0) && (dataSetParamStr.nx[1] == 0) && (dataSetParamStr.nx[2] == 0))
+ // mRelaxationFactor = CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr();
+ //else
+ mRelaxationFactor = CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2]));
// find the nesessary block and fill it
- SPtr<Block3D> block = grid->getBlock(dataSetArray[n].globalID);
- UbTupleInt3 blockNX = grid->getBlockNX();
- this->lbmKernel->setNX(std::array<int, 3>{ {val<1>(blockNX), val<2>(blockNX), val<3>(blockNX)}});
- SPtr<LBMKernel> kernel = this->lbmKernel->clone();
- kernel->setGhostLayerWidth(dataSetArray[n].ghostLayerWidth);
- kernel->setCollisionFactor(dataSetArray[n].collFactor);
- kernel->setDeltaT(dataSetArray[n].deltaT);
- kernel->setCompressible(dataSetArray[n].compressible);
- kernel->setWithForcing(dataSetArray[n].withForcing);
- kernel->setDataSet(dataSetPtr);
- block->setKernel(kernel);
+ SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].globalID);
+ block->kernel->getDataSet()->setRelaxationFactor(mRelaxationFactor);
}
- delete[] dataSetArray;
+ MPI_Type_free(&dataSetDoubleType);
if (comm->isRoot())
{
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::readDataSet end of restore of data, rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readRelaxationFactor end of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
+
+ delete[] dataSetSmallArray;
}
void MPIIOMigrationCoProcessor::readBoundaryConds(int step)
@@ -1075,20 +2382,20 @@ void MPIIOMigrationCoProcessor::readBoundaryConds(int step)
if (comm->isRoot())
{
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::readBoundaryConds start MPI IO rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readBoundaryConds start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
if (comm->isRoot()) start = MPI_Wtime();
MPI_File file_handler;
- std::string filename = path+"/mpi_io_cp/mpi_io_cp_"+UbSystem::toString(step)+"/cpBC.bin";
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpBC.bin";
int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
- if (rc!=MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file "+filename);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
int blocksCount = 0; // quantity of the blocks, that belong to this process
-
+
// read from the grid the blocks, that belong to this process
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
@@ -1120,13 +2427,12 @@ void MPIIOMigrationCoProcessor::readBoundaryConds(int step)
MPI_File_read_at(file_handler, (MPI_Offset)0, &boundCondParamStr, 1, boundCondParamType, MPI_STATUS_IGNORE);
MPI_Type_contiguous(boundCondParamStr.bcindexmatrixCount, MPI_INT, &bcindexmatrixType);
MPI_Type_commit(&bcindexmatrixType);
- mpiTypeFreeFlag = true;
int ic = 0;
MPI_Offset read_offset1, read_offset2;
for (int level = minInitLevel; level <= maxInitLevel; level++)
{
- for(SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
+ for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
{
read_offset1 = (MPI_Offset)(sizeof(boundCondParam) + block->getGlobalID() * sizeof(size_t));
@@ -1156,7 +2462,7 @@ void MPIIOMigrationCoProcessor::readBoundaryConds(int step)
for (size_t ibc = 0; ibc<bcAddArray[ic].boundCond_count; ibc++)
{
SPtr<BoundaryConditions> bc;
- if (memcmp(&bcArray[ibc], nullBouCond, sizeof(BoundaryCondition))==0)
+ if (memcmp(&bcArray[ibc], nullBouCond, sizeof(BoundaryCondition)) == 0)
bc = SPtr<BoundaryConditions>();
else
{
@@ -1192,13 +2498,11 @@ void MPIIOMigrationCoProcessor::readBoundaryConds(int step)
for (int b2 = 0; b2 < bcAddArray[ic].indexContainer_count; b2++)
indexContainerV.push_back(intArray2[b2]);
-
+
CbArray3D<int, IndexerX3X2X1> bcim(bcindexmatrixV, boundCondParamStr.nx1, boundCondParamStr.nx2, boundCondParamStr.nx3);
SPtr<Block3D> block1 = grid->getBlock(bcAddArray[ic].globalID);
- //SPtr<BCProcessor> bcProc(new BCProcessor());
SPtr<BCProcessor> bcProc = bcProcessor->clone(block1->getKernel());
- //SPtr<BCArray3D> bcArr = bcProc->getBCArray();
SPtr<BCArray3D> bcArr(new BCArray3D());
bcArr->bcindexmatrix = bcim;
bcArr->bcvector = bcVector;
@@ -1207,23 +2511,26 @@ void MPIIOMigrationCoProcessor::readBoundaryConds(int step)
block1->getKernel()->setBCProcessor(bcProc);
- delete bcArray;
- delete intArray1;
- delete intArray2;
+ delete bcArray;
+ delete intArray1;
+ delete intArray2;
- ic++;
+ ic++;
}
}
MPI_File_close(&file_handler);
+ MPI_Type_free(&bcindexmatrixType);
+
delete nullBouCond;
if (comm->isRoot())
{
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::readBoundaryConds end of restore of data, rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ UBLOG(logINFO, "MPIIOMigrationCoProcessor::readBoundaryConds end of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
}
+
//////////////////////////////////////////////////////////////////////////
void MPIIOMigrationCoProcessor::setLBMKernel(SPtr<LBMKernel> kernel)
{
diff --git a/source/VirtualFluidsCore/CoProcessors/MPIIOMigrationCoProcessor.h b/source/VirtualFluidsCore/CoProcessors/MPIIOMigrationCoProcessor.h
index e7d61504d..ad87d6ba4 100644
--- a/source/VirtualFluidsCore/CoProcessors/MPIIOMigrationCoProcessor.h
+++ b/source/VirtualFluidsCore/CoProcessors/MPIIOMigrationCoProcessor.h
@@ -1,5 +1,5 @@
-#ifndef _MPIIORestart21CoProcessor_H_
-#define _MPIIORestart21CoProcessor_H_
+#ifndef _MPIIOMigrationCoProcessor_H_
+#define _MPIIOMigrationCoProcessor_H_
#include <mpi.h>
#include <PointerDefinitions.h>
@@ -15,14 +15,13 @@ class LBMKernel;
//! \class MPIWriteBlocksCoProcessor
//! \brief Writes the grid each timestep into the files and reads the grip from the files before regenerating
-//! \author Alena Karanchuk
-class MPIIOMigrationCoProcessor: public CoProcessor
+class MPIIOMigrationCoProcessor : public CoProcessor
{
//! \struct GridParam
//! \brief Structure describes parameters of the grid
//! \details The structure is nessasary to restore the grid correctly
struct GridParam
- {
+ {
double trafoParams[33];
double deltaX;
int blockNx1;
@@ -36,47 +35,46 @@ class MPIIOMigrationCoProcessor: public CoProcessor
bool periodicX3;
bool active;
bool transformation;
- };
+ };
//! \struct Block3d
//! \brief Structure contains information of the block
//! \details The structure is used to write the data describing the block in the grid when saving the grid
//! and to read it when restoring the grid
struct Block3d
- {
- int x1;
- int x2;
- int x3;
+ {
+ int x1;
+ int x2;
+ int x3;
int bundle;
- int rank;
- int lrank;
- int part;
- int globalID;
- int localID;
- int level;
- int interpolationFlagCF;
- int interpolationFlagFC;
- int counter;
- bool active;
- };
+ int rank;
+ int lrank;
+ int part;
+ int globalID;
+ int localID;
+ int level;
+ int interpolationFlagCF;
+ int interpolationFlagFC;
+ int counter;
+ bool active;
+ };
//! \struct dataSetParam
//! \brief Structure describes parameters of the dataSet that are equal in all blocks
//! \details The structure used to store some parameters needed to restore dataSet arrays
struct dataSetParam
{
- int nx1;
+ int nx1;
int nx2;
int nx3;
- int nx[9][4]; // 9 arrays x (nx1, nx2, nx3, nx4)
- int doubleCountInBlock; // how many double-values are in all arrays dataSet in one (any) block
+ int nx[4]; //nx1, nx2, nx3, nx4
};
//! \struct dataSet
//! \brief Structure containes information identifying the block
//! \details The structure is used to find the needed block in the grid when restoring a dataSet
struct DataSet
- {
+ {
double collFactor;
double deltaT;
int globalID;
@@ -84,32 +82,44 @@ class MPIIOMigrationCoProcessor: public CoProcessor
bool compressible;
bool withForcing;
};
-
+
+ //! \struct dataSetSmall
+ //! \brief Structure containes information identifying the block
+ //! \details The structure is used to find the needed block in the grid when restoring a dataSet arrays
+ struct DataSetSmall
+ {
+ int globalID;
+ //int x1;
+ //int x2;
+ //int x3;
+ //int level;
+ };
+
//! \struct BoundaryCondition
//! \brief Structure containes information about boundary conditions of the block
//! \details The structure is used to write data describing boundary conditions of the blocks when saving the grid
//! and to read it when restoring the grid
struct BoundaryCondition
- {
- long long noslipBoundaryFlags; // MPI_LONG_LONG
- long long slipBoundaryFlags;
- long long velocityBoundaryFlags;
- long long densityBoundaryFlags;
- long long wallModelBoundaryFlags;
-
- float bcVelocityX1;
- float bcVelocityX2;
- float bcVelocityX3;
- float bcDensity;
-
- float bcLodiDensity;
- float bcLodiVelocityX1;
- float bcLodiVelocityX2;
- float bcLodiVelocityX3;
- float bcLodiLentgh;
-
- float nx1,nx2,nx3;
- float q[26]; // MPI_FLOAT
+ {
+ long long noslipBoundaryFlags; // MPI_LONG_LONG
+ long long slipBoundaryFlags;
+ long long velocityBoundaryFlags;
+ long long densityBoundaryFlags;
+ long long wallModelBoundaryFlags;
+
+ float bcVelocityX1;
+ float bcVelocityX2;
+ float bcVelocityX3;
+ float bcDensity;
+
+ float bcLodiDensity;
+ float bcLodiVelocityX1;
+ float bcLodiVelocityX2;
+ float bcLodiVelocityX3;
+ float bcLodiLentgh;
+
+ float nx1, nx2, nx3;
+ float q[26]; // MPI_FLOAT
char algorithmType;
};
@@ -119,7 +129,7 @@ class MPIIOMigrationCoProcessor: public CoProcessor
//! \details The structure used to store some parameters needed to restore boundaryConditions arrays
struct boundCondParam
{
- int nx1;
+ int nx1;
int nx2;
int nx3;
int bcindexmatrixCount; // how many bcindexmatrix-values in one (any) block
@@ -131,16 +141,26 @@ class MPIIOMigrationCoProcessor: public CoProcessor
//! \details The structure is used to find the needed block in the grid when restoring a dataSet
//! and to set common parameters
struct BCAdd
- {
+ {
int globalID;
- //int x1; // to find the right block
- //int x2;
- //int x3;
- //int level;
+ //int x1; // to find the right block
+ //int x2;
+ //int x3;
+ //int level;
int boundCond_count; // how many BoundaryCondition-structures are in this block
int indexContainer_count; // how many indexContainer-values are in this block
};
+ struct DSArraysPresence
+ {
+ bool isAverageDensityArrayPresent;
+ bool isAverageVelocityArrayPresent;
+ bool isAverageFluktuationsArrayPresent;
+ bool isAverageTripleArrayPresent;
+ bool isShearStressValArrayPresent;
+ bool isRelaxationFactorPresent;
+ };
+
public:
MPIIOMigrationCoProcessor(SPtr<Grid3D> grid, SPtr<UbScheduler> s, const std::string& path, SPtr<Communicator> comm);
virtual ~MPIIOMigrationCoProcessor();
@@ -148,17 +168,30 @@ public:
void process(double step);
//! Reads the grid from the files before grid reconstruction
void restart(int step);
- //! Writes the blocks of the grid into the file outputBlocks.bin
+ //! Writes the blocks of the grid into the file cpBlocks.bin
void writeBlocks(int step);
- //! Writes the datasets of the blocks into the file outputDataSet.bin
+ //! Writes the datasets of the blocks into the file cpDataSet.bin
void writeDataSet(int step);
- //! Writes the boundary conditions of the blocks into the file outputBoundCond.bin
+ void writeAverageDensityArray(int step);
+ void writeAverageVelocityArray(int step);
+ void writeAverageFluktuationsArray(int step);
+ void writeAverageTripleArray(int step);
+ void writeShearStressValArray(int step);
+ void writeRelaxationFactor(int step);
+ //! Writes the boundary conditions of the blocks into the file cpBC.bin
void writeBoundaryConds(int step);
- //! Reads the blocks of the grid from the file outputBlocks.bin
+
+ //! Reads the blocks of the grid from the file cpBlocks.bin
void readBlocks(int step);
- //! Reads the datasets of the blocks from the file outputDataSet.bin
+ //! Reads the datasets of the blocks from the file cpDataSet.bin
void readDataSet(int step);
- //! Reads the boundary conditions of the blocks from the file outputBoundCond.bin
+ void readAverageDensityArray(int step);
+ void readAverageVelocityArray(int step);
+ void readAverageFluktuationsArray(int step);
+ void readAverageTripleArray(int step);
+ void readShearStressValArray(int step);
+ void readRelaxationFactor(int step);
+ //! Reads the boundary conditions of the blocks from the file cpBC.bin
void readBoundaryConds(int step);
//! The function sets LBMKernel
void setLBMKernel(SPtr<LBMKernel> kernel);
@@ -170,11 +203,12 @@ public:
protected:
std::string path;
SPtr<Communicator> comm;
- bool mpiTypeFreeFlag;
private:
- MPI_Datatype gridParamType, block3dType, dataSetParamType, dataSetType, dataSetDoubleType, boundCondParamType, boundCondType, boundCondTypeAdd, bcindexmatrixType;
- dataSetParam dataSetParamStr;
+ MPI_Datatype gridParamType, block3dType, arrayPresenceType;
+ MPI_Datatype dataSetParamType, dataSetType, dataSetSmallType, dataSetDoubleType;
+ MPI_Datatype boundCondParamType, boundCondType, boundCondTypeAdd, bcindexmatrixType;
+
boundCondParam boundCondParamStr;
SPtr<LBMKernel> lbmKernel;
SPtr<BCProcessor> bcProcessor;
diff --git a/source/VirtualFluidsCore/CoProcessors/MPIIORestartCoProcessor.cpp b/source/VirtualFluidsCore/CoProcessors/MPIIORestartCoProcessor.cpp
index deb081cbe..0188db222 100644
--- a/source/VirtualFluidsCore/CoProcessors/MPIIORestartCoProcessor.cpp
+++ b/source/VirtualFluidsCore/CoProcessors/MPIIORestartCoProcessor.cpp
@@ -26,12 +26,10 @@ MPIIORestartCoProcessor::MPIIORestartCoProcessor(SPtr<Grid3D> grid, SPtr<UbSched
SPtr<Communicator> comm) :
CoProcessor(grid, s),
path(path),
- comm(comm),
- mpiTypeFreeFlag(false)
+ comm(comm)
{
- UbSystem::makeDirectory(path+"/mpi_io_cp");
+ UbSystem::makeDirectory(path + "/mpi_io_cp");
- memset(&dataSetParamStr, 0, sizeof(dataSetParamStr));
memset(&boundCondParamStr, 0, sizeof(boundCondParamStr));
//------------------------- define MPI types ---------------------------------
@@ -42,12 +40,12 @@ MPIIORestartCoProcessor::MPIIORestartCoProcessor(SPtr<Grid3D> grid, SPtr<UbSched
offsetsGP[0] = 0;
MPI_Type_get_extent(MPI_DOUBLE, &lbGP, &extentGP);
- offsetsGP[1] = blocksGP[0]*extentGP;
+ offsetsGP[1] = blocksGP[0] * extentGP;
MPI_Type_get_extent(MPI_INT, &lbGP, &extentGP);
- offsetsGP[2] = offsetsGP[1]+blocksGP[1]*extentGP;
+ offsetsGP[2] = offsetsGP[1] + blocksGP[1] * extentGP;
- MPI_Type_create_struct (3, blocksGP, offsetsGP, typesGP, &gridParamType);
+ MPI_Type_create_struct(3, blocksGP, offsetsGP, typesGP, &gridParamType);
MPI_Type_commit(&gridParamType);
//-----------------------------------------------------------------------
@@ -58,14 +56,14 @@ MPIIORestartCoProcessor::MPIIORestartCoProcessor(SPtr<Grid3D> grid, SPtr<UbSched
offsetsBlock[0] = 0;
MPI_Type_get_extent(MPI_INT, &lbBlock, &extentBlock);
- offsetsBlock[1] = blocksBlock[0]*extentBlock;
+ offsetsBlock[1] = blocksBlock[0] * extentBlock;
MPI_Type_create_struct(2, blocksBlock, offsetsBlock, typesBlock, &block3dType);
MPI_Type_commit(&block3dType);
//-----------------------------------------------------------------------
- MPI_Type_contiguous(40, MPI_INT, &dataSetParamType);
+ MPI_Type_contiguous(7, MPI_INT, &dataSetParamType);
MPI_Type_commit(&dataSetParamType);
//-----------------------------------------------------------------------
@@ -76,16 +74,21 @@ MPIIORestartCoProcessor::MPIIORestartCoProcessor(SPtr<Grid3D> grid, SPtr<UbSched
offsetsDatatSet[0] = 0;
MPI_Type_get_extent(MPI_DOUBLE, &lbDataSet, &extentDataSet);
- offsetsDatatSet[1] = blocksDataSet[0]*extentDataSet;
+ offsetsDatatSet[1] = blocksDataSet[0] * extentDataSet;
MPI_Type_get_extent(MPI_INT, &lbDataSet, &extentDataSet);
- offsetsDatatSet[2] = offsetsDatatSet[1]+blocksDataSet[1]*extentDataSet;
+ offsetsDatatSet[2] = offsetsDatatSet[1] + blocksDataSet[1] * extentDataSet;
MPI_Type_create_struct(3, blocksDataSet, offsetsDatatSet, typesDataSet, &dataSetType);
MPI_Type_commit(&dataSetType);
//-----------------------------------------------------------------------
+ MPI_Type_contiguous(4, MPI_INT, &dataSetSmallType);
+ MPI_Type_commit(&dataSetSmallType);
+
+ //-----------------------------------------------------------------------
+
MPI_Type_contiguous(4, MPI_INT, &boundCondParamType);
MPI_Type_commit(&boundCondParamType);
@@ -97,10 +100,10 @@ MPIIORestartCoProcessor::MPIIORestartCoProcessor(SPtr<Grid3D> grid, SPtr<UbSched
offsetsBC[0] = 0;
MPI_Type_get_extent(MPI_LONG_LONG_INT, &lbBC, &extentBC);
- offsetsBC[1] = blocksBC[0]*extentBC;
+ offsetsBC[1] = blocksBC[0] * extentBC;
MPI_Type_get_extent(MPI_FLOAT, &lbBC, &extentBC);
- offsetsBC[2] = offsetsBC[1]+blocksBC[1]*extentBC;
+ offsetsBC[2] = offsetsBC[1] + blocksBC[1] * extentBC;
MPI_Type_create_struct(3, blocksBC, offsetsBC, typesBC, &boundCondType);
MPI_Type_commit(&boundCondType);
@@ -115,6 +118,11 @@ MPIIORestartCoProcessor::MPIIORestartCoProcessor(SPtr<Grid3D> grid, SPtr<UbSched
MPI_Type_contiguous(6, MPI_INT, &boundCondTypeAdd);
MPI_Type_commit(&boundCondTypeAdd);
+ //---------------------------------------
+
+ MPI_Type_contiguous(6, MPI_CHAR, &arrayPresenceType);
+ MPI_Type_commit(&arrayPresenceType);
+
}
//////////////////////////////////////////////////////////////////////////
MPIIORestartCoProcessor::~MPIIORestartCoProcessor()
@@ -123,16 +131,12 @@ MPIIORestartCoProcessor::~MPIIORestartCoProcessor()
MPI_Type_free(&block3dType);
MPI_Type_free(&dataSetParamType);
MPI_Type_free(&dataSetType);
+ MPI_Type_free(&dataSetSmallType);
MPI_Type_free(&boundCondParamType);
MPI_Type_free(&boundCondType);
MPI_Type_free(&boundCondType1000);
MPI_Type_free(&boundCondTypeAdd);
-
- if (mpiTypeFreeFlag)
- {
- MPI_Type_free(&dataSetDoubleType);
- MPI_Type_free(&bcindexmatrixType);
- }
+ MPI_Type_free(&arrayPresenceType);
}
//////////////////////////////////////////////////////////////////////////
@@ -140,49 +144,91 @@ void MPIIORestartCoProcessor::process(double step)
{
if (scheduler->isDue(step))
{
- if (comm->isRoot()) UBLOG(logINFO, "MPIIORestartCoProcessor save step: "<<step);
+ if (comm->isRoot()) UBLOG(logINFO, "MPIIORestartCoProcessor save step: " << step);
if (comm->isRoot()) UBLOG(logINFO, "Save check point - start");
/*if (comm->isRoot())*/ clearAllFiles((int)step);
+
writeBlocks((int)step);
writeDataSet((int)step);
writeBoundaryConds((int)step);
+
if (comm->isRoot()) UBLOG(logINFO, "Save check point - end");
-
- //readDataSet((int)step);
- //readBoundaryConds((int)step);
}
}
//////////////////////////////////////////////////////////////////////////
void MPIIORestartCoProcessor::clearAllFiles(int step)
{
- MPI_File file_handler1, file_handler2, file_handler3;
+ MPI_File file_handler;
MPI_Info info = MPI_INFO_NULL;
MPI_Offset new_size = 0;
UbSystem::makeDirectory(path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step));
+
std::string filename1 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpBlocks.bin";
- //MPI_File_delete(filename1.c_str(), info);
- int rc1 = MPI_File_open(MPI_COMM_WORLD, filename1.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &file_handler1);
+ int rc1 = MPI_File_open(MPI_COMM_WORLD, filename1.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &file_handler);
if (rc1 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename1);
- MPI_File_set_size(file_handler1, new_size);
- //MPI_File_sync(file_handler1);
- MPI_File_close(&file_handler1);
+ MPI_File_set_size(file_handler, new_size);
+ MPI_File_close(&file_handler);
std::string filename2 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSet.bin";
- //MPI_File_delete(filename2.c_str(), info);
- int rc2 = MPI_File_open(MPI_COMM_WORLD, filename2.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler2);
+ int rc2 = MPI_File_open(MPI_COMM_WORLD, filename2.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
if (rc2 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename2);
- MPI_File_set_size(file_handler2, new_size);
- //MPI_File_sync(file_handler2);
- MPI_File_close(&file_handler2);
+ MPI_File_set_size(file_handler, new_size);
+ MPI_File_close(&file_handler);
- std::string filename3 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpBC.bin";
- //MPI_File_delete(filename3.c_str(), info);
- int rc3 = MPI_File_open(MPI_COMM_WORLD, filename3.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler3);
+ std::string filename3 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpArrays.bin";
+ int rc3 = MPI_File_open(MPI_COMM_WORLD, filename3.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
if (rc3 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename3);
- MPI_File_set_size(file_handler3, new_size);
- //MPI_File_sync(file_handler3);
- MPI_File_close(&file_handler3);
+ MPI_File_set_size(file_handler, new_size);
+ MPI_File_close(&file_handler);
+
+ std::string filename4 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageDensityArray.bin";
+ MPI_File_delete(filename4.c_str(), info);
+ //int rc4 = MPI_File_open(MPI_COMM_WORLD, filename4.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ //if (rc4 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename4);
+ //MPI_File_set_size(file_handler, new_size);
+ //MPI_File_close(&file_handler);
+
+ std::string filename5 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageVelocityArray.bin";
+ MPI_File_delete(filename5.c_str(), info);
+ //int rc5 = MPI_File_open(MPI_COMM_WORLD, filename5.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ //if (rc5 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename5);
+ //MPI_File_set_size(file_handler, new_size);
+ //MPI_File_close(&file_handler);
+
+ std::string filename6 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageFluktuationsArray.bin";
+ MPI_File_delete(filename6.c_str(), info);
+ //int rc6 = MPI_File_open(MPI_COMM_WORLD, filename6.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ //if (rc6 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename6);
+ //MPI_File_set_size(file_handler, new_size);
+ //MPI_File_close(&file_handler);
+
+ std::string filename7 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageTripleArray.bin";
+ MPI_File_delete(filename7.c_str(), info);
+ //int rc7 = MPI_File_open(MPI_COMM_WORLD, filename7.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ //if (rc7 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename7);
+ //MPI_File_set_size(file_handler, new_size);
+ //MPI_File_close(&file_handler);
+
+ std::string filename8 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpShearStressValArray.bin";
+ MPI_File_delete(filename8.c_str(), info);
+ //int rc8 = MPI_File_open(MPI_COMM_WORLD, filename8.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ //if (rc8 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename8);
+ //MPI_File_set_size(file_handler, new_size);
+ //MPI_File_close(&file_handler);
+
+ std::string filename9 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpRelaxationFactor.bin";
+ MPI_File_delete(filename9.c_str(), info);
+ //int rc9 = MPI_File_open(MPI_COMM_WORLD, filename9.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ //if (rc9 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename9);
+ //MPI_File_set_size(file_handler, new_size);
+ //MPI_File_close(&file_handler);
+
+ std::string filename10 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpBC.bin";
+ int rc10 = MPI_File_open(MPI_COMM_WORLD, filename10.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc10 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename10);
+ MPI_File_set_size(file_handler, new_size);
+ MPI_File_close(&file_handler);
}
//////////////////////////////////////////////////////////////////////////
void MPIIORestartCoProcessor::writeBlocks(int step)
@@ -190,12 +236,12 @@ void MPIIORestartCoProcessor::writeBlocks(int step)
int rank, size;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
//MPI_Comm_size(MPI_COMM_WORLD, &size);
- size=1;
+ size = 1;
if (comm->isRoot())
{
- UBLOG(logINFO, "MPIIORestartCoProcessor::writeBlocks start collect data rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeBlocks start collect data rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
int blocksCount = 0; // quantity of blocks in the grid, max 2147483648 blocks!
@@ -203,7 +249,7 @@ void MPIIORestartCoProcessor::writeBlocks(int step)
int maxInitLevel = this->grid->getFinestInitializedLevel();
std::vector<SPtr<Block3D>> blocksVector[25]; // max 25 levels
- for (int level = minInitLevel; level<=maxInitLevel; level++)
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
{
//grid->getBlocks(level, rank, blockVector[level]);
grid->getBlocks(level, blocksVector[level]);
@@ -266,9 +312,9 @@ void MPIIORestartCoProcessor::writeBlocks(int step)
Block3d* block3dArray = new Block3d[blocksCount];
int ic = 0;
- for (int level = minInitLevel; level<=maxInitLevel; level++)
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
{
- for(SPtr<Block3D> block : blocksVector[level]) // all the blocks of the current level
+ for (SPtr<Block3D> block : blocksVector[level]) // all the blocks of the current level
{
// save data describing the block
block3dArray[ic].x1 = block->getX1();
@@ -292,8 +338,8 @@ void MPIIORestartCoProcessor::writeBlocks(int step)
if (comm->isRoot())
{
- UBLOG(logINFO, "MPIIORestartCoProcessor::writeBlocks start MPI IO rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeBlocks start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
MPI_File file_handler;
@@ -319,20 +365,20 @@ void MPIIORestartCoProcessor::writeBlocks(int step)
start = MPI_Wtime();
// each process writes the quantity of it's blocks
- MPI_File_write_at(file_handler, (MPI_Offset)(rank*sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(rank * sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
// each process writes parameters of the grid
MPI_File_write_at(file_handler, write_offset, gridParameters, 1, gridParamType, MPI_STATUS_IGNORE);
// each process writes it's blocks
- MPI_File_write_at(file_handler, (MPI_Offset)(write_offset +sizeof(GridParam)), &block3dArray[0], blocksCount, block3dType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(GridParam)), &block3dArray[0], blocksCount, block3dType, MPI_STATUS_IGNORE);
}
MPI_File_sync(file_handler);
MPI_File_close(&file_handler);
-
+
if (comm->isRoot())
{
finish = MPI_Wtime();
- UBLOG(logINFO, "MPIIORestartCoProcessor::writeBlocks time: "<<finish-start<<" s");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeBlocks time: " << finish - start << " s");
}
delete[] block3dArray;
@@ -350,201 +396,122 @@ void MPIIORestartCoProcessor::writeDataSet(int step)
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
int maxInitLevel = this->grid->getFinestInitializedLevel();
- for (int level = minInitLevel; level<=maxInitLevel; level++)
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
{
grid->getBlocks(level, rank, blocksVector[level]);
blocksCount += static_cast<int>(blocksVector[level].size());
}
+ dataSetParam dataSetParamStr1, dataSetParamStr2, dataSetParamStr3;
DataSet* dataSetArray = new DataSet[blocksCount];
std::vector<double> doubleValuesArray; // double-values (arrays of f's) in all blocks
if (comm->isRoot())
{
- UBLOG(logINFO, "MPIIORestartCoProcessor::writeDataSet start collect data rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeDataSet start collect data rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
+ DSArraysPresence arrPresence;
bool firstBlock = true;
+ int doubleCountInBlock = 0;
int ic = 0;
- for (int level = minInitLevel; level<=maxInitLevel; level++)
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
{
- for(SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
+ for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
{
dataSetArray[ic].x1 = block->getX1(); // coordinates of the block needed to find it while regenerating the grid
dataSetArray[ic].x2 = block->getX2();
dataSetArray[ic].x3 = block->getX3();
dataSetArray[ic].level = block->getLevel();
- //if (block->getKernel())
- //{
dataSetArray[ic].ghostLayerWidth = block->getKernel()->getGhostLayerWidth();
dataSetArray[ic].collFactor = block->getKernel()->getCollisionFactor();
dataSetArray[ic].deltaT = block->getKernel()->getDeltaT();
dataSetArray[ic].compressible = block->getKernel()->getCompressible();
dataSetArray[ic].withForcing = block->getKernel()->getWithForcing();
- //}
- //else
- //{
- // dataSetArray[ic].ghostLayerWidth = 0;
- // dataSetArray[ic].collFactor = 0.0;
- // dataSetArray[ic].deltaT = 0.0;
- // dataSetArray[ic].compressible = false;
- // dataSetArray[ic].withForcing = false;
- //}
- //std::cout << "ic="<<ic<<"-"<<dataSetArray[ic].x1 << "," << dataSetArray[ic].x2 << "," << dataSetArray[ic].x3 << "," << dataSetArray[ic].level << "," << dataSetArray[ic].ghostLayerWidth;
- //std::cout << dataSetArray[ic].collFactor<<","<<dataSetArray[ic].deltaT<<","<<dataSetArray[ic].compressible<<","<<dataSetArray[ic].withForcing<<std::endl;
- //dataSetArrayGW[ic].x1 = dataSetArray[ic].x1;
- //dataSetArrayGW[ic].x2 = dataSetArray[ic].x2;
- //dataSetArrayGW[ic].x3 = dataSetArray[ic].x3;
- //dataSetArrayGW[ic].level = dataSetArray[ic].level;
- //dataSetArrayGW[ic].ghostLayerWidth = dataSetArray[ic].ghostLayerWidth;
- //dataSetArrayGW[ic].collFactor = dataSetArray[ic].collFactor;
- //dataSetArrayGW[ic].deltaT = dataSetArray[ic].deltaT;
- //dataSetArrayGW[ic].compressible = dataSetArray[ic].compressible;
- //dataSetArrayGW[ic].withForcing = dataSetArray[ic].withForcing;
-
- if (firstBlock /*&& block->getKernel()*/) // when first (any) valid block...
- {
- SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > averageDensityArray = block->getKernel()->getDataSet()->getAverageDencity();
- if (averageDensityArray)
- {
- dataSetParamStr.nx[0][0] = static_cast<int>(averageDensityArray->getNX1());
- dataSetParamStr.nx[0][1] = static_cast<int>(averageDensityArray->getNX2());
- dataSetParamStr.nx[0][2] = static_cast<int>(averageDensityArray->getNX3());
- dataSetParamStr.nx[0][3] = static_cast<int>(averageDensityArray->getNX4());
- }
-
- SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > AverageVelocityArray3DPtr = block->getKernel()->getDataSet()->getAverageVelocity();
- if (AverageVelocityArray3DPtr)
- {
- dataSetParamStr.nx[1][0] = static_cast<int>(AverageVelocityArray3DPtr->getNX1());
- dataSetParamStr.nx[1][1] = static_cast<int>(AverageVelocityArray3DPtr->getNX2());
- dataSetParamStr.nx[1][2] = static_cast<int>(AverageVelocityArray3DPtr->getNX3());
- dataSetParamStr.nx[1][3] = static_cast<int>(AverageVelocityArray3DPtr->getNX4());
- }
-
- SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > AverageFluctArray3DPtr = block->getKernel()->getDataSet()->getAverageFluctuations();
- if (AverageFluctArray3DPtr)
- {
- dataSetParamStr.nx[2][0] = static_cast<int>(AverageFluctArray3DPtr->getNX1());
- dataSetParamStr.nx[2][1] = static_cast<int>(AverageFluctArray3DPtr->getNX2());
- dataSetParamStr.nx[2][2] = static_cast<int>(AverageFluctArray3DPtr->getNX3());
- dataSetParamStr.nx[2][3] = static_cast<int>(AverageFluctArray3DPtr->getNX4());
- }
-
- SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > AverageTripleArray3DPtr = block->getKernel()->getDataSet()->getAverageTriplecorrelations();
- if (AverageTripleArray3DPtr)
- {
- dataSetParamStr.nx[3][0] = static_cast<int>(AverageTripleArray3DPtr->getNX1());
- dataSetParamStr.nx[3][1] = static_cast<int>(AverageTripleArray3DPtr->getNX2());
- dataSetParamStr.nx[3][2] = static_cast<int>(AverageTripleArray3DPtr->getNX3());
- dataSetParamStr.nx[3][3] = static_cast<int>(AverageTripleArray3DPtr->getNX4());
- }
-
- SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > ShearStressValArray3DPtr = block->getKernel()->getDataSet()->getShearStressValues();
- if (ShearStressValArray3DPtr)
- {
- dataSetParamStr.nx[4][0] = static_cast<int>(ShearStressValArray3DPtr->getNX1());
- dataSetParamStr.nx[4][1] = static_cast<int>(ShearStressValArray3DPtr->getNX2());
- dataSetParamStr.nx[4][2] = static_cast<int>(ShearStressValArray3DPtr->getNX3());
- dataSetParamStr.nx[4][3] = static_cast<int>(ShearStressValArray3DPtr->getNX4());
- }
- SPtr< CbArray3D<LBMReal, IndexerX3X2X1> > relaxationFactor3DPtr = block->getKernel()->getDataSet()->getRelaxationFactor();
- if (relaxationFactor3DPtr)
- {
- dataSetParamStr.nx[5][0] = static_cast<int>(relaxationFactor3DPtr->getNX1());
- dataSetParamStr.nx[5][1] = static_cast<int>(relaxationFactor3DPtr->getNX2());
- dataSetParamStr.nx[5][2] = static_cast<int>(relaxationFactor3DPtr->getNX3());
- dataSetParamStr.nx[5][3] = 1;
- }
+ SPtr< D3Q27EsoTwist3DSplittedVector > D3Q27EsoTwist3DSplittedVectorPtr = dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(block->getKernel()->getDataSet()->getFdistributions());
+ CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr localDistributions = D3Q27EsoTwist3DSplittedVectorPtr->getLocalDistributions();
+ CbArray4D <LBMReal, IndexerX4X3X2X1>::CbArray4DPtr nonLocalDistributions = D3Q27EsoTwist3DSplittedVectorPtr->getNonLocalDistributions();
+ CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr zeroDistributions = D3Q27EsoTwist3DSplittedVectorPtr->getZeroDistributions();
- SPtr< D3Q27EsoTwist3DSplittedVector > D3Q27EsoTwist3DSplittedVectorPtr = dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(block->getKernel()->getDataSet()->getFdistributions());
- CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr localDistributions = D3Q27EsoTwist3DSplittedVectorPtr->getLocalDistributions();
+ if (firstBlock) // when first (any) valid block...
+ {
if (localDistributions)
{
- dataSetParamStr.nx[6][0] = static_cast<int>(localDistributions->getNX1());
- dataSetParamStr.nx[6][1] = static_cast<int>(localDistributions->getNX2());
- dataSetParamStr.nx[6][2] = static_cast<int>(localDistributions->getNX3());
- dataSetParamStr.nx[6][3] = static_cast<int>(localDistributions->getNX4());
+ dataSetParamStr1.nx[0] = static_cast<int>(localDistributions->getNX1());
+ dataSetParamStr1.nx[1] = static_cast<int>(localDistributions->getNX2());
+ dataSetParamStr1.nx[2] = static_cast<int>(localDistributions->getNX3());
+ dataSetParamStr1.nx[3] = static_cast<int>(localDistributions->getNX4());
}
-
- CbArray4D <LBMReal, IndexerX4X3X2X1>::CbArray4DPtr nonLocalDistributions = D3Q27EsoTwist3DSplittedVectorPtr->getNonLocalDistributions();
if (nonLocalDistributions)
{
- dataSetParamStr.nx[7][0] = static_cast<int>(nonLocalDistributions->getNX1());
- dataSetParamStr.nx[7][1] = static_cast<int>(nonLocalDistributions->getNX2());
- dataSetParamStr.nx[7][2] = static_cast<int>(nonLocalDistributions->getNX3());
- dataSetParamStr.nx[7][3] = static_cast<int>(nonLocalDistributions->getNX4());
+ dataSetParamStr2.nx[0] = static_cast<int>(nonLocalDistributions->getNX1());
+ dataSetParamStr2.nx[1] = static_cast<int>(nonLocalDistributions->getNX2());
+ dataSetParamStr2.nx[2] = static_cast<int>(nonLocalDistributions->getNX3());
+ dataSetParamStr2.nx[3] = static_cast<int>(nonLocalDistributions->getNX4());
}
-
- CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr zeroDistributions = D3Q27EsoTwist3DSplittedVectorPtr->getZeroDistributions();
if (zeroDistributions)
{
- dataSetParamStr.nx[8][0] = static_cast<int>(zeroDistributions->getNX1());
- dataSetParamStr.nx[8][1] = static_cast<int>(zeroDistributions->getNX2());
- dataSetParamStr.nx[8][2] = static_cast<int>(zeroDistributions->getNX3());
- dataSetParamStr.nx[8][3] = 1;
+ dataSetParamStr3.nx[0] = static_cast<int>(zeroDistributions->getNX1());
+ dataSetParamStr3.nx[1] = static_cast<int>(zeroDistributions->getNX2());
+ dataSetParamStr3.nx[2] = static_cast<int>(zeroDistributions->getNX3());
+ dataSetParamStr3.nx[3] = 1;
}
// ... than save some parameters that are equal in all dataSets
- dataSetParamStr.nx1 = static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX1());
- dataSetParamStr.nx2 = static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX2());
- dataSetParamStr.nx3 = static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX3());
+ dataSetParamStr1.nx1 = dataSetParamStr2.nx1 = dataSetParamStr3.nx1 = static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX1());
+ dataSetParamStr1.nx2 = dataSetParamStr2.nx2 = dataSetParamStr3.nx2 = static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX2());
+ dataSetParamStr1.nx3 = dataSetParamStr2.nx3 = dataSetParamStr3.nx3 = static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX3());
- firstBlock = false;
+ doubleCountInBlock = dataSetParamStr1.nx[0] * dataSetParamStr1.nx[1] * dataSetParamStr1.nx[2] * dataSetParamStr1.nx[3] +
+ dataSetParamStr2.nx[0] * dataSetParamStr2.nx[1] * dataSetParamStr2.nx[2] * dataSetParamStr2.nx[3] +
+ dataSetParamStr3.nx[0] * dataSetParamStr3.nx[1] * dataSetParamStr3.nx[2] * dataSetParamStr3.nx[3];
- // how many elements are in all arrays of DataSet (equal in all blocks)
- int doubleCount = 0, temp;
- for (int i = 0; i<9; i++) // 9 arrays ( averageValues, averageVelocity, averageFluktuations,
- { // averageTriplecorrelations, shearStressValues, relaxationFactor, 3 * fdistributions
- temp = 1;
- for (int ii = 0; ii < 4; ii++)
- {
- temp *= dataSetParamStr.nx[i][ii];
- //std::cout << ",dataSetParamStr.nx[" << i << "][" << ii << "]" << "=" << dataSetParamStr.nx[i][ii];
- }
- doubleCount += temp;
- }
- dataSetParamStr.doubleCountInBlock = doubleCount;
- }
- //std::cout << ",doubleCountInBlock="<<dataSetParamStr.doubleCountInBlock<< "," << dataSetParamStr.nx1 << "," << dataSetParamStr.nx2 << "," << dataSetParamStr.nx3 << std::endl;
+ SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > averageDensityArray = block->getKernel()->getDataSet()->getAverageDencity();
+ if (averageDensityArray)
+ arrPresence.isAverageDensityArrayPresent = true;
+ else
+ arrPresence.isAverageDensityArrayPresent = false;
- SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > averageValuesArray3D = block->getKernel()->getDataSet()->getAverageDencity();
- if (averageValuesArray3D &&(dataSetParamStr.nx[0][0]>0)&&(dataSetParamStr.nx[0][1]>0)&&(dataSetParamStr.nx[0][2]>0)&&(dataSetParamStr.nx[0][3]>0))
- doubleValuesArray.insert(doubleValuesArray.end(), averageValuesArray3D->getDataVector().begin(), averageValuesArray3D->getDataVector().end());
+ SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > AverageVelocityArray3DPtr = block->getKernel()->getDataSet()->getAverageVelocity();
+ if (AverageVelocityArray3DPtr)
+ arrPresence.isAverageVelocityArrayPresent = true;
+ else
+ arrPresence.isAverageVelocityArrayPresent = false;
- SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > AverageVelocityArray3DPtr = block->getKernel()->getDataSet()->getAverageVelocity();
- if (AverageVelocityArray3DPtr&&(dataSetParamStr.nx[1][0]>0)&&(dataSetParamStr.nx[1][1]>0)&&(dataSetParamStr.nx[1][2]>0)&&(dataSetParamStr.nx[1][3]>0))
- doubleValuesArray.insert(doubleValuesArray.end(), AverageVelocityArray3DPtr->getDataVector().begin(), AverageVelocityArray3DPtr->getDataVector().end());
+ SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > AverageFluctArray3DPtr = block->getKernel()->getDataSet()->getAverageFluctuations();
+ if (AverageFluctArray3DPtr)
+ arrPresence.isAverageFluktuationsArrayPresent = true;
+ else
+ arrPresence.isAverageFluktuationsArrayPresent = false;
- SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > AverageFluctArray3DPtr = block->getKernel()->getDataSet()->getAverageFluctuations();
- if (AverageFluctArray3DPtr&&(dataSetParamStr.nx[2][0]>0)&&(dataSetParamStr.nx[2][1]>0)&&(dataSetParamStr.nx[2][2]>0)&&(dataSetParamStr.nx[2][3]>0))
- doubleValuesArray.insert(doubleValuesArray.end(), AverageFluctArray3DPtr->getDataVector().begin(), AverageFluctArray3DPtr->getDataVector().end());
+ SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > AverageTripleArray3DPtr = block->getKernel()->getDataSet()->getAverageTriplecorrelations();
+ if (AverageTripleArray3DPtr)
+ arrPresence.isAverageTripleArrayPresent = true;
+ else
+ arrPresence.isAverageTripleArrayPresent = false;
- SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > AverageTripleArray3DPtr = block->getKernel()->getDataSet()->getAverageTriplecorrelations();
- if (AverageTripleArray3DPtr&&(dataSetParamStr.nx[3][0]>0)&&(dataSetParamStr.nx[3][1]>0)&&(dataSetParamStr.nx[3][2]>0)&&(dataSetParamStr.nx[3][3]>0))
- doubleValuesArray.insert(doubleValuesArray.end(), AverageTripleArray3DPtr->getDataVector().begin(), AverageTripleArray3DPtr->getDataVector().end());
+ SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > ShearStressValArray3DPtr = block->getKernel()->getDataSet()->getShearStressValues();
+ if (ShearStressValArray3DPtr)
+ arrPresence.isShearStressValArrayPresent = true;
+ else
+ arrPresence.isShearStressValArrayPresent = false;
- SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > ShearStressValArray3DPtr = block->getKernel()->getDataSet()->getShearStressValues();
- if (ShearStressValArray3DPtr&&(dataSetParamStr.nx[4][0]>0)&&(dataSetParamStr.nx[4][1]>0)&&(dataSetParamStr.nx[4][2]>0)&&(dataSetParamStr.nx[4][3]>0))
- doubleValuesArray.insert(doubleValuesArray.end(), ShearStressValArray3DPtr->getDataVector().begin(), ShearStressValArray3DPtr->getDataVector().end());
+ SPtr< CbArray3D<LBMReal, IndexerX3X2X1> > relaxationFactor3DPtr = block->getKernel()->getDataSet()->getRelaxationFactor();
+ if (relaxationFactor3DPtr)
+ arrPresence.isRelaxationFactorPresent = true;
+ else
+ arrPresence.isRelaxationFactorPresent = false;
- SPtr< CbArray3D<LBMReal, IndexerX3X2X1> > RelaxationFactor3DPtr = block->getKernel()->getDataSet()->getRelaxationFactor();
- if (RelaxationFactor3DPtr&&(dataSetParamStr.nx[5][0]>0)&&(dataSetParamStr.nx[5][1]>0)&&(dataSetParamStr.nx[5][2]>0))
- doubleValuesArray.insert(doubleValuesArray.end(), RelaxationFactor3DPtr->getDataVector().begin(), RelaxationFactor3DPtr->getDataVector().end());
+ firstBlock = false;
+ }
- SPtr< D3Q27EsoTwist3DSplittedVector > D3Q27EsoTwist3DSplittedVectorPtr = dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(block->getKernel()->getDataSet()->getFdistributions());
- CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr localDistributions = D3Q27EsoTwist3DSplittedVectorPtr->getLocalDistributions();
- if (localDistributions&&(dataSetParamStr.nx[6][0]>0)&&(dataSetParamStr.nx[6][1]>0)&&(dataSetParamStr.nx[6][2]>0)&&(dataSetParamStr.nx[6][3]>0))
+ if (localDistributions && (dataSetParamStr1.nx[0]>0) && (dataSetParamStr1.nx[1]>0) && (dataSetParamStr1.nx[2]>0) && (dataSetParamStr1.nx[3]>0))
doubleValuesArray.insert(doubleValuesArray.end(), localDistributions->getDataVector().begin(), localDistributions->getDataVector().end());
-
- CbArray4D <LBMReal, IndexerX4X3X2X1>::CbArray4DPtr nonLocalDistributions = D3Q27EsoTwist3DSplittedVectorPtr->getNonLocalDistributions();
- if (nonLocalDistributions&&(dataSetParamStr.nx[7][0]>0)&&(dataSetParamStr.nx[7][1]>0)&&(dataSetParamStr.nx[7][2]>0)&&(dataSetParamStr.nx[7][3]>0))
+ if (nonLocalDistributions && (dataSetParamStr2.nx[0]>0) && (dataSetParamStr2.nx[1]>0) && (dataSetParamStr2.nx[2]>0) && (dataSetParamStr2.nx[3]>0))
doubleValuesArray.insert(doubleValuesArray.end(), nonLocalDistributions->getDataVector().begin(), nonLocalDistributions->getDataVector().end());
-
- CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr zeroDistributions = D3Q27EsoTwist3DSplittedVectorPtr->getZeroDistributions();
- if (zeroDistributions&&(dataSetParamStr.nx[8][0]>0)&&(dataSetParamStr.nx[8][1]>0)&&(dataSetParamStr.nx[8][2]>0))
+ if (zeroDistributions && (dataSetParamStr3.nx[0]>0) && (dataSetParamStr3.nx[1]>0) && (dataSetParamStr3.nx[2]>0))
doubleValuesArray.insert(doubleValuesArray.end(), zeroDistributions->getDataVector().begin(), zeroDistributions->getDataVector().end());
ic++;
@@ -552,16 +519,13 @@ void MPIIORestartCoProcessor::writeDataSet(int step)
}
// register new MPI-types depending on the block-specific information
- MPI_Type_contiguous(dataSetParamStr.doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
MPI_Type_commit(&dataSetDoubleType);
- mpiTypeFreeFlag = true;
- //doubleValuesArrayGW.assign(doubleValuesArray.begin(), doubleValuesArray.end());
-
if (comm->isRoot())
{
- UBLOG(logINFO, "MPIIORestartCoProcessor::writeDataSet start MPI IO rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeDataSet start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
// write to the file
@@ -572,17 +536,17 @@ void MPIIORestartCoProcessor::writeDataSet(int step)
if (size>1)
{
- if (rank==0)
+ if (rank == 0)
{
- next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSet)+ dataSetParamStr.doubleCountInBlock*sizeof(double));
+ next_write_offset = write_offset + 3 * sizeof(dataSetParam) + blocksCount * (sizeof(DataSet) + doubleCountInBlock * sizeof(double));
MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
}
else
{
- MPI_Recv(&write_offset, 1, MPI_LONG_LONG_INT, rank-1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
- next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSet)+ dataSetParamStr.doubleCountInBlock*sizeof(double));
- if (rank<size-1)
- MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, rank+1, 5, MPI_COMM_WORLD);
+ MPI_Recv(&write_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+ next_write_offset = write_offset + 3 * sizeof(dataSetParam) + blocksCount * (sizeof(DataSet) + doubleCountInBlock * sizeof(double));
+ if (rank<size - 1)
+ MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
}
}
@@ -591,198 +555,167 @@ void MPIIORestartCoProcessor::writeDataSet(int step)
MPI_Info info = MPI_INFO_NULL;
-#ifdef HLRN
+#ifdef HLRN_LUSTRE
MPI_Info_create(&info);
MPI_Info_set(info, "striping_factor", "40");
MPI_Info_set(info, "striping_unit", "4M");
#endif
MPI_File file_handler;
- std::string filename = path+"/mpi_io_cp/mpi_io_cp_"+UbSystem::toString(step)+"/cpDataSet.bin";
- int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE| MPI_MODE_WRONLY, info, &file_handler);
- if (rc!=MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file "+filename);
-
- //std::cout << "writeDataSet rank=" << rank << ",blocksCount=" << blocksCount;
- //std::cout << ", rank*sizeof(int)=" << (MPI_Offset)(rank * sizeof(int)) << ", write_offset=" << write_offset << std::endl;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSet.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
// each process writes the quantity of it's blocks
- MPI_File_write_at(file_handler, (MPI_Offset)(rank*sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(rank * sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
// each process writes common parameters of a dataSet
- MPI_File_write_at(file_handler, write_offset, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, write_offset, &dataSetParamStr1, 1, dataSetParamType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam)), &dataSetParamStr2, 1, dataSetParamType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + 2 * sizeof(dataSetParam)), &dataSetParamStr3, 1, dataSetParamType, MPI_STATUS_IGNORE);
// each process writes data identifying blocks
- MPI_File_write_at(file_handler, (MPI_Offset)(write_offset+sizeof(dataSetParam)), dataSetArray, blocksCount, dataSetType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + 3 * sizeof(dataSetParam)), dataSetArray, blocksCount, dataSetType, MPI_STATUS_IGNORE);
// each process writes the dataSet arrays
- MPI_File_write_at(file_handler, (MPI_Offset)(write_offset+sizeof(dataSetParam)+blocksCount*sizeof(DataSet)), &doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
+ if (doubleValuesArray.size() > 0)
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + 3 * sizeof(dataSetParam) + blocksCount * sizeof(DataSet)), &doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
+
MPI_File_sync(file_handler);
-
- //int blockC;
- //MPI_File_read_at(file_handler, (MPI_Offset)(rank * sizeof(int)), &blockC, 1, MPI_INT, MPI_STATUS_IGNORE);
- //std::cout << "readDataSet rank=" << rank << ", blockC=" << blockC << std::endl;
-
MPI_File_close(&file_handler);
+ MPI_Type_free(&dataSetDoubleType);
+
+ delete[] dataSetArray;
+
if (comm->isRoot())
{
finish = MPI_Wtime();
- UBLOG(logINFO, "MPIIORestartCoProcessor::writeDataSet time: "<<finish-start<<" s");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeDataSet time: " << finish - start << " s");
}
- delete[] dataSetArray;
+ MPI_File file_handler1;
+ std::string filename1 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpArrays.bin";
+ rc = MPI_File_open(MPI_COMM_WORLD, filename1.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler1);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename1);
+ MPI_File_write_at(file_handler1, (MPI_Offset)0, &arrPresence, 1, arrayPresenceType, MPI_STATUS_IGNORE);
+ MPI_File_sync(file_handler1);
+ MPI_File_close(&file_handler1);
+
+ if (arrPresence.isAverageDensityArrayPresent)
+ writeAverageDensityArray(step);
+
+ if (arrPresence.isAverageVelocityArrayPresent)
+ writeAverageVelocityArray(step);
+
+ if (arrPresence.isAverageFluktuationsArrayPresent)
+ writeAverageFluktuationsArray(step);
+
+ if (arrPresence.isAverageTripleArrayPresent)
+ writeAverageTripleArray(step);
+
+ if (arrPresence.isShearStressValArrayPresent)
+ writeShearStressValArray(step);
+
+ if (arrPresence.isRelaxationFactorPresent)
+ writeRelaxationFactor(step);
+
}
-void MPIIORestartCoProcessor::writeBoundaryConds(int step)
+void MPIIORestartCoProcessor::writeAverageDensityArray(int step)
{
int rank, size;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
- if (comm->isRoot())
- {
- UBLOG(logINFO, "MPIIORestartCoProcessor::writeBoundaryConds start collect data rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
- }
-
- int blocksCount = 0; // quantity of blocks in the grid, max 2147483648 blocks!
- size_t count_boundCond = 0; // how many BoundaryConditions in all blocks
- int count_indexContainer = 0; // how many indexContainer-values in all blocks
- size_t byteCount = 0; // how many bytes writes this process in the file
+ int blocksCount = 0; // quantity of blocks in the grid, max 2147483648 blocks!
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
int maxInitLevel = this->grid->getFinestInitializedLevel();
- for (int level = minInitLevel; level<=maxInitLevel; level++)
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
{
grid->getBlocks(level, rank, blocksVector[level]);
blocksCount += static_cast<int>(blocksVector[level].size());
}
- BCAdd* bcAddArray = new BCAdd[blocksCount];
- std::vector<BoundaryCondition> bcVector;
- std::vector<int> bcindexmatrixV;
- std::vector<int> indexContainerV;
- bool bcindexmatrixCountNotInit = true;
+ DataSetSmall* dataSetSmallArray = new DataSetSmall[blocksCount];
+ std::vector<double> doubleValuesArray; // double-values of the AverageDensityArray in all blocks
+ dataSetParam dataSetParamStr;
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeAverageDensityArray start collect data rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ bool firstBlock = true;
+ int doubleCountInBlock = 0;
int ic = 0;
- for (int level = minInitLevel; level<=maxInitLevel; level++)
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
{
- for(SPtr<Block3D> block : blocksVector[level]) // all the blocks of the current level
+ for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
{
- SPtr<BCArray3D> bcArr = block->getKernel()->getBCProcessor()->getBCArray();
+ dataSetSmallArray[ic].x1 = block->getX1(); // coordinates of the block needed to find it while regenerating the grid
+ dataSetSmallArray[ic].x2 = block->getX2();
+ dataSetSmallArray[ic].x3 = block->getX3();
+ dataSetSmallArray[ic].level = block->getLevel();
- bcAddArray[ic].x1 = block->getX1(); // coordinates of the block needed to find it while regenerating the grid
- bcAddArray[ic].x2 = block->getX2();
- bcAddArray[ic].x3 = block->getX3();
- bcAddArray[ic].level = block->getLevel();
- bcAddArray[ic].boundCond_count = 0; // how many BoundaryConditions in this block
- bcAddArray[ic].indexContainer_count = 0; // how many indexContainer-values in this block
+ SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > averageDensityArray = block->getKernel()->getDataSet()->getAverageDencity();
- for (int bc = 0; bc<bcArr->getBCVectorSize(); bc++)
+ if (firstBlock) // when first (any) valid block...
{
- BoundaryCondition* bouCond = new BoundaryCondition();
- if (bcArr->bcvector[bc]==NULL)
- {
- memset(bouCond, 0, sizeof(BoundaryCondition));
- }
- else
- {
- bouCond->noslipBoundaryFlags = bcArr->bcvector[bc]->getNoSlipBoundary();
- bouCond->slipBoundaryFlags = bcArr->bcvector[bc]->getSlipBoundary();
- bouCond->velocityBoundaryFlags = bcArr->bcvector[bc]->getVelocityBoundary();
- bouCond->densityBoundaryFlags = bcArr->bcvector[bc]->getDensityBoundary();
- bouCond->wallModelBoundaryFlags = bcArr->bcvector[bc]->getWallModelBoundary();
- bouCond->bcVelocityX1 = bcArr->bcvector[bc]->getBoundaryVelocityX1();
- bouCond->bcVelocityX2 = bcArr->bcvector[bc]->getBoundaryVelocityX2();
- bouCond->bcVelocityX3 = bcArr->bcvector[bc]->getBoundaryVelocityX3();
- bouCond->bcDensity = bcArr->bcvector[bc]->getBoundaryDensity();
- bouCond->bcLodiDensity = bcArr->bcvector[bc]->getDensityLodiDensity();
- bouCond->bcLodiVelocityX1 = bcArr->bcvector[bc]->getDensityLodiVelocityX1();
- bouCond->bcLodiVelocityX2 = bcArr->bcvector[bc]->getDensityLodiVelocityX2();
- bouCond->bcLodiVelocityX3 = bcArr->bcvector[bc]->getDensityLodiVelocityX3();
- bouCond->bcLodiLentgh = bcArr->bcvector[bc]->getDensityLodiLength();
- bouCond->nx1 = bcArr->bcvector[bc]->nx1;
- bouCond->nx2 = bcArr->bcvector[bc]->nx2;
- bouCond->nx3 = bcArr->bcvector[bc]->nx3;
- for (int iq = 0; iq<26; iq++)
- bouCond->q[iq] = bcArr->bcvector[bc]->getQ(iq);
- bouCond->algorithmType = bcArr->bcvector[bc]->getBcAlgorithmType();
- }
- //std::cout << "writeBoundaryConds noslipBoundaryFlags="<< bouCond->noslipBoundaryFlags << std::endl;
- bcVector.push_back(*bouCond);
- //bcVectorGW.push_back(*bouCond);
- //if (bcVector[count_boundCond].noslipBoundaryFlags != bcVectorGW[count_boundCond].noslipBoundaryFlags)
- // std::cout << "bcVector[count_boundCond].noslipBoundaryFlags != bcVectorGW[count_boundCond].noslipBoundaryFlags!!!" << std::endl;
- bcAddArray[ic].boundCond_count++;
- count_boundCond++;
- }
+ //if (averageDensityArray)
+ //{
+ dataSetParamStr.nx1 = dataSetParamStr.nx2 = dataSetParamStr.nx3 = 0;
+ dataSetParamStr.nx[0] = static_cast<int>(averageDensityArray->getNX1());
+ dataSetParamStr.nx[1] = static_cast<int>(averageDensityArray->getNX2());
+ dataSetParamStr.nx[2] = static_cast<int>(averageDensityArray->getNX3());
+ dataSetParamStr.nx[3] = static_cast<int>(averageDensityArray->getNX4());
+ doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ //}
+ //else
+ // break;
- // the quantity of elements in the bcindexmatrix array (CbArray3D<int, IndexerX3X2X1>) in bcArray(BCArray3D) is always equal,
- // this will be the size of the "write-read-block" in MPI_write_.../MPI_read-functions when writing/reading BoundConds
- if (bcindexmatrixCountNotInit)
- {
- boundCondParamStr.nx1 = static_cast<int>(bcArr->bcindexmatrix.getNX1());
- boundCondParamStr.nx2 = static_cast<int>(bcArr->bcindexmatrix.getNX2());
- boundCondParamStr.nx3 = static_cast<int>(bcArr->bcindexmatrix.getNX3());
- boundCondParamStr.bcindexmatrixCount = static_cast<int>(bcArr->bcindexmatrix.getDataVector().size());
- bcindexmatrixCountNotInit = false;
+ firstBlock = false;
}
- bcindexmatrixV.insert(bcindexmatrixV.end(), bcArr->bcindexmatrix.getDataVector().begin(), bcArr->bcindexmatrix.getDataVector().end());
- indexContainerV.insert(indexContainerV.end(), bcArr->indexContainer.begin(), bcArr->indexContainer.end());
- bcAddArray[ic].indexContainer_count = static_cast<int>(bcArr->indexContainer.size());
- count_indexContainer += bcAddArray[ic].indexContainer_count;
+ if (averageDensityArray && (dataSetParamStr.nx[0] > 0) && (dataSetParamStr.nx[1] > 0) && (dataSetParamStr.nx[2] > 0) && (dataSetParamStr.nx[3] > 0))
+ doubleValuesArray.insert(doubleValuesArray.end(), averageDensityArray->getDataVector().begin(), averageDensityArray->getDataVector().end());
ic++;
}
}
- //bcindexmatrixVGW.assign(bcindexmatrixV.begin(), bcindexmatrixV.end());
- //indexContainerVGW.assign(indexContainerV.begin(), indexContainerV.end());
-
- MPI_Type_contiguous(boundCondParamStr.bcindexmatrixCount, MPI_INT, &bcindexmatrixType);
- MPI_Type_commit(&bcindexmatrixType);
- mpiTypeFreeFlag = true;
+ // register new MPI-types depending on the block-specific information
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_commit(&dataSetDoubleType);
- //how many "big blocks" of BLOCK_SIZE size can by formed
- int bcBlockCount = (int)(count_boundCond/BLOCK_SIZE);
- if (bcBlockCount * BLOCK_SIZE<count_boundCond)
- bcBlockCount += 1;
- for (int i = (int)count_boundCond; i<bcBlockCount * BLOCK_SIZE; i++)
+ if (comm->isRoot())
{
- BoundaryCondition* bouCond = new BoundaryCondition();
- memset(bouCond, 0, sizeof(BoundaryCondition));
- bcVector.push_back(*bouCond);
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeAverageDensityArray start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
- byteCount = bcBlockCount * BLOCK_SIZE * sizeof(BoundaryCondition) + blocksCount * sizeof(BCAdd) + sizeof(int) * (blocksCount * boundCondParamStr.bcindexmatrixCount + count_indexContainer);
-
// write to the file
// all processes calculate their offsets (quantity of bytes that the process is going to write)
// and notify the next process (with the rank = rank + 1)
- MPI_Offset write_offset = (MPI_Offset)(size * (3 * sizeof(int) + sizeof(boundCondParam)));
+ MPI_Offset write_offset = (MPI_Offset)(size * sizeof(int));
size_t next_write_offset = 0;
- if (size>1)
+ if (size > 1)
{
- if (rank==0)
+ if (rank == 0)
{
- next_write_offset = write_offset + byteCount;
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount*(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
}
else
{
- MPI_Recv(&write_offset, 1, MPI_LONG_LONG_INT, rank-1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
- next_write_offset = write_offset + byteCount;
- if (rank<size-1)
- MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, rank+1, 5, MPI_COMM_WORLD);
+ MPI_Recv(&write_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount*(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+ if (rank<size - 1)
+ MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
}
}
- if (comm->isRoot())
- {
- UBLOG(logINFO, "MPIIORestartCoProcessor::writeBoundaryConds start MPI IO rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
- }
-
double start, finish;
if (comm->isRoot()) start = MPI_Wtime();
@@ -795,206 +728,1615 @@ void MPIIORestartCoProcessor::writeBoundaryConds(int step)
#endif
MPI_File file_handler;
- std::string filename = path+"/mpi_io_cp/mpi_io_cp_"+UbSystem::toString(step)+"/cpBC.bin";
- int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE|MPI_MODE_WRONLY, info, &file_handler);
- if (rc!=MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file "+filename);
-
- MPI_Offset write_offset1 = (MPI_Offset)(rank * (3 * sizeof(int) + sizeof(boundCondParam)));
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageDensityArray.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
// each process writes the quantity of it's blocks
- MPI_File_write_at(file_handler, write_offset1, &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
- // each process writes the quantity of "big blocks" of BLOCK_SIZE of boundary conditions
- MPI_File_write_at(file_handler, (MPI_Offset)(write_offset1+sizeof(int)), &bcBlockCount, 1, MPI_INT, MPI_STATUS_IGNORE);
- // each process writes the quantity of indexContainer elements in all blocks
- MPI_File_write_at(file_handler, (MPI_Offset)(write_offset1+2*sizeof(int)), &count_indexContainer, 1, MPI_INT, MPI_STATUS_IGNORE);
- // each process writes the quantity of bcindexmatrix elements in every block
- MPI_File_write_at(file_handler, (MPI_Offset)(write_offset1+3*sizeof(int)), &boundCondParamStr, 1, boundCondParamType, MPI_STATUS_IGNORE);
-
- //std::cout << "rank=" << rank << ",(rank*write_offset1)=" << rank*write_offset1<< ",blocksCount=" << blocksCount ;
- //std::cout << ", " << rank*write_offset1 + sizeof(int) << ",bcBlockCount=" << bcBlockCount;
- //std::cout << ", " << rank*write_offset1 + 2 * sizeof(int) << ",count_indexContainer=" << count_indexContainer;
- //std::cout << ", " << rank*write_offset1 + 3 * sizeof(int) << ",boundCondParamStr=" << boundCondParamStr.bcindexmatrixCount << std::endl;
+ MPI_File_write_at(file_handler, (MPI_Offset)(rank * sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
+ // each process writes common parameters of a dataSet
+ MPI_File_write_at(file_handler, write_offset, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
+ // each process writes data identifying blocks
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount, dataSetSmallType, MPI_STATUS_IGNORE);
+ // each process writes the dataSet arrays
+ if (doubleValuesArray.size() > 0)
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmall)), &doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
- // each process writes data identifying the blocks
- MPI_File_write_at(file_handler, write_offset, bcAddArray, blocksCount, boundCondTypeAdd, MPI_STATUS_IGNORE);
- // each process writes boundary conditions
- if (bcVector.size()>0)
- MPI_File_write_at(file_handler, (MPI_Offset)(write_offset+blocksCount*sizeof(BCAdd)), &bcVector[0], bcBlockCount, boundCondType1000, MPI_STATUS_IGNORE);
- // each process writes bcindexmatrix values
- if (bcindexmatrixV.size()>0)
- MPI_File_write_at(file_handler, (MPI_Offset)(write_offset+blocksCount*sizeof(BCAdd)+bcBlockCount*BLOCK_SIZE*sizeof(BoundaryCondition)), &bcindexmatrixV[0], blocksCount, bcindexmatrixType, MPI_STATUS_IGNORE);
- // each process writes indexContainer values
- if (indexContainerV.size()>0)
- MPI_File_write_at(file_handler, (MPI_Offset)(write_offset+blocksCount*sizeof(BCAdd)+bcBlockCount*BLOCK_SIZE*sizeof(BoundaryCondition)+blocksCount*boundCondParamStr.bcindexmatrixCount*sizeof(int)), &indexContainerV[0], count_indexContainer, MPI_INT, MPI_STATUS_IGNORE);
MPI_File_sync(file_handler);
-
- //std::cout <<"rank="<<rank<<",blocksCount="<< blocksCount<<", "<< bcBlockCount<<", "<< count_indexContainer<<", "<< bcindexmatrixCount << std::endl;
- //std::cout <<"rank="<<rank<<",write_offset="<< write_offset <<", "<< write_offset + blocksCount * sizeof(BCAdd) <<", "<< write_offset + blocksCount * sizeof(BCAdd) + bcBlockCount*BLOCK_SIZE * sizeof(BoundaryCondition) <<", "<< write_offset + blocksCount * sizeof(BCAdd) + bcBlockCount*BLOCK_SIZE * sizeof(BoundaryCondition) + blocksCount*bcindexmatrixCount * sizeof(int)<< std::endl;
-
MPI_File_close(&file_handler);
+ MPI_Type_free(&dataSetDoubleType);
+
if (comm->isRoot())
{
finish = MPI_Wtime();
- UBLOG(logINFO, "MPIIORestartCoProcessor::writeBoundaryConds time: "<<finish-start<<" s");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeAverageDensityArray time: " << finish - start << " s");
}
- delete[] bcAddArray;
-}
-
-//------------------------------------------- READ -----------------------------------------------
-void MPIIORestartCoProcessor::restart(int step)
-{
- if (comm->isRoot()) UBLOG(logINFO, "MPIIORestartCoProcessor restart step: "<<step);
- if (comm->isRoot()) UBLOG(logINFO, "Load check point - start");
- readBlocks(step);
- readDataSet(step);
- readBoundaryConds(step);
- if (comm->isRoot()) UBLOG(logINFO, "Load check point - end");
+ delete[] dataSetSmallArray;
}
-void MPIIORestartCoProcessor::readBlocks(int step)
+void MPIIORestartCoProcessor::writeAverageVelocityArray(int step)
{
int rank, size;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
- //MPI_Comm_size(MPI_COMM_WORLD, &size);
- size = 1;
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
- if (comm->isRoot())
+ int blocksCount = 0; // quantity of blocks in the grid, max 2147483648 blocks!
+
+ std::vector<SPtr<Block3D>> blocksVector[25];
+ int minInitLevel = this->grid->getCoarsestInitializedLevel();
+ int maxInitLevel = this->grid->getFinestInitializedLevel();
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
{
- UBLOG(logINFO, "MPIIORestartCoProcessor::readBlocks start MPI IO rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ grid->getBlocks(level, rank, blocksVector[level]);
+ blocksCount += static_cast<int>(blocksVector[level].size());
}
- double start, finish;
+ DataSetSmall* dataSetSmallArray = new DataSetSmall[blocksCount];
+ std::vector<double> doubleValuesArray; // double-values (arrays of f's) in all blocks
+ dataSetParam dataSetParamStr;
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeAverageVelocityArray start collect data rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ bool firstBlock = true;
+ int doubleCountInBlock = 0;
+ int ic = 0;
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
+ {
+ dataSetSmallArray[ic].x1 = block->getX1(); // coordinates of the block needed to find it while regenerating the grid
+ dataSetSmallArray[ic].x2 = block->getX2();
+ dataSetSmallArray[ic].x3 = block->getX3();
+ dataSetSmallArray[ic].level = block->getLevel();
+
+ SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > AverageVelocityArray3DPtr = block->getKernel()->getDataSet()->getAverageVelocity();
+
+ if (firstBlock) // when first (any) valid block...
+ {
+ //if (AverageVelocityArray3DPtr)
+ //{
+ dataSetParamStr.nx1 = dataSetParamStr.nx2 = dataSetParamStr.nx3 = 0;
+ dataSetParamStr.nx[0] = static_cast<int>(AverageVelocityArray3DPtr->getNX1());
+ dataSetParamStr.nx[1] = static_cast<int>(AverageVelocityArray3DPtr->getNX2());
+ dataSetParamStr.nx[2] = static_cast<int>(AverageVelocityArray3DPtr->getNX3());
+ dataSetParamStr.nx[3] = static_cast<int>(AverageVelocityArray3DPtr->getNX4());
+ doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ //}
+ //else
+ // break;
+
+ firstBlock = false;
+ }
+
+ if (AverageVelocityArray3DPtr && (dataSetParamStr.nx[0]>0) && (dataSetParamStr.nx[1]>0) && (dataSetParamStr.nx[2]>0) && (dataSetParamStr.nx[3]>0))
+ doubleValuesArray.insert(doubleValuesArray.end(), AverageVelocityArray3DPtr->getDataVector().begin(), AverageVelocityArray3DPtr->getDataVector().end());
+
+ ic++;
+ }
+ }
+
+ // register new MPI-types depending on the block-specific information
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_commit(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeAverageVelocityArray start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ // write to the file
+ // all processes calculate their offsets (quantity of bytes that the process is going to write)
+ // and notify the next process (with the rank = rank + 1)
+ MPI_Offset write_offset = (MPI_Offset)(size * sizeof(int));
+ size_t next_write_offset = 0;
+
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount*(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+ MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
+ }
+ else
+ {
+ MPI_Recv(&write_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount*(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+ if (rank<size - 1)
+ MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
+ }
+ }
+
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
+
+ MPI_Info info = MPI_INFO_NULL;
+
+#ifdef HLRN_LUSTRE
+ MPI_Info_create(&info);
+ MPI_Info_set(info, "striping_factor", "40");
+ MPI_Info_set(info, "striping_unit", "4M");
+#endif
+
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageVelocityArray.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ // each process writes the quantity of it's blocks
+ MPI_File_write_at(file_handler, (MPI_Offset)(rank * sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
+ // each process writes common parameters of a dataSet
+ MPI_File_write_at(file_handler, write_offset, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
+ // each process writes data identifying blocks
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount, dataSetSmallType, MPI_STATUS_IGNORE);
+ // each process writes the dataSet arrays
+ if (doubleValuesArray.size() > 0)
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmall)), &doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
+
+ MPI_File_sync(file_handler);
+ MPI_File_close(&file_handler);
+
+ MPI_Type_free(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeAverageVelocityArray time: " << finish - start << " s");
+ }
+
+ delete[] dataSetSmallArray;
+}
+
+void MPIIORestartCoProcessor::writeAverageFluktuationsArray(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+ int blocksCount = 0; // quantity of blocks in the grid, max 2147483648 blocks!
+
+ std::vector<SPtr<Block3D>> blocksVector[25];
+ int minInitLevel = this->grid->getCoarsestInitializedLevel();
+ int maxInitLevel = this->grid->getFinestInitializedLevel();
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ grid->getBlocks(level, rank, blocksVector[level]);
+ blocksCount += static_cast<int>(blocksVector[level].size());
+ }
+
+ DataSetSmall* dataSetSmallArray = new DataSetSmall[blocksCount];
+ std::vector<double> doubleValuesArray; // double-values (arrays of f's) in all blocks
+ dataSetParam dataSetParamStr;
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeAverageFluktuationsArray start collect data rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ bool firstBlock = true;
+ int doubleCountInBlock = 0;
+ int ic = 0;
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
+ {
+ dataSetSmallArray[ic].x1 = block->getX1(); // coordinates of the block needed to find it while regenerating the grid
+ dataSetSmallArray[ic].x2 = block->getX2();
+ dataSetSmallArray[ic].x3 = block->getX3();
+ dataSetSmallArray[ic].level = block->getLevel();
+
+ SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > AverageFluctArray3DPtr = block->getKernel()->getDataSet()->getAverageFluctuations();
+
+ if (firstBlock) // when first (any) valid block...
+ {
+ //if (AverageFluctArray3DPtr)
+ //{
+ dataSetParamStr.nx1 = dataSetParamStr.nx2 = dataSetParamStr.nx3 = 0;
+ dataSetParamStr.nx[0] = static_cast<int>(AverageFluctArray3DPtr->getNX1());
+ dataSetParamStr.nx[1] = static_cast<int>(AverageFluctArray3DPtr->getNX2());
+ dataSetParamStr.nx[2] = static_cast<int>(AverageFluctArray3DPtr->getNX3());
+ dataSetParamStr.nx[3] = static_cast<int>(AverageFluctArray3DPtr->getNX4());
+ doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ //}
+ //else
+ // break;
+
+ firstBlock = false;
+ }
+
+ if (AverageFluctArray3DPtr && (dataSetParamStr.nx[0]>0) && (dataSetParamStr.nx[1]>0) && (dataSetParamStr.nx[2]>0) && (dataSetParamStr.nx[3]>0))
+ doubleValuesArray.insert(doubleValuesArray.end(), AverageFluctArray3DPtr->getDataVector().begin(), AverageFluctArray3DPtr->getDataVector().end());
+
+ ic++;
+ }
+ }
+
+ // register new MPI-types depending on the block-specific information
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_commit(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeAverageFluktuationsArray start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ // write to the file
+ // all processes calculate their offsets (quantity of bytes that the process is going to write)
+ // and notify the next process (with the rank = rank + 1)
+ MPI_Offset write_offset = (MPI_Offset)(size * sizeof(int));
+ size_t next_write_offset = 0;
+
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount*(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+ MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
+ }
+ else
+ {
+ MPI_Recv(&write_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount*(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+ if (rank<size - 1)
+ MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
+ }
+ }
+
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
+
+ MPI_Info info = MPI_INFO_NULL;
+
+#ifdef HLRN_LUSTRE
+ MPI_Info_create(&info);
+ MPI_Info_set(info, "striping_factor", "40");
+ MPI_Info_set(info, "striping_unit", "4M");
+#endif
+
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageFluktuationsArray.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ // each process writes the quantity of it's blocks
+ MPI_File_write_at(file_handler, (MPI_Offset)(rank * sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
+ // each process writes common parameters of a dataSet
+ MPI_File_write_at(file_handler, write_offset, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
+ // each process writes data identifying blocks
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount, dataSetSmallType, MPI_STATUS_IGNORE);
+ // each process writes the dataSet arrays
+ if (doubleValuesArray.size() > 0)
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmall)), &doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
+
+ MPI_File_sync(file_handler);
+ MPI_File_close(&file_handler);
+
+ MPI_Type_free(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeAverageFluktuationsArray time: " << finish - start << " s");
+ }
+
+ delete[] dataSetSmallArray;
+}
+
+void MPIIORestartCoProcessor::writeAverageTripleArray(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+ int blocksCount = 0; // quantity of blocks in the grid, max 2147483648 blocks!
+
+ std::vector<SPtr<Block3D>> blocksVector[25];
+ int minInitLevel = this->grid->getCoarsestInitializedLevel();
+ int maxInitLevel = this->grid->getFinestInitializedLevel();
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ grid->getBlocks(level, rank, blocksVector[level]);
+ blocksCount += static_cast<int>(blocksVector[level].size());
+ }
+
+ DataSetSmall* dataSetSmallArray = new DataSetSmall[blocksCount];
+ std::vector<double> doubleValuesArray; // double-values (arrays of f's) in all blocks
+ dataSetParam dataSetParamStr;
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeAverageTripleArray start collect data rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ bool firstBlock = true;
+ int doubleCountInBlock = 0;
+ int ic = 0;
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
+ {
+ dataSetSmallArray[ic].x1 = block->getX1(); // coordinates of the block needed to find it while regenerating the grid
+ dataSetSmallArray[ic].x2 = block->getX2();
+ dataSetSmallArray[ic].x3 = block->getX3();
+ dataSetSmallArray[ic].level = block->getLevel();
+
+ SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > AverageTripleArray3DPtr = block->getKernel()->getDataSet()->getAverageTriplecorrelations();
+
+ if (firstBlock) // when first (any) valid block...
+ {
+ //if (AverageTripleArray3DPtr)
+ //{
+ dataSetParamStr.nx1 = dataSetParamStr.nx2 = dataSetParamStr.nx3 = 0;
+ dataSetParamStr.nx[0] = static_cast<int>(AverageTripleArray3DPtr->getNX1());
+ dataSetParamStr.nx[1] = static_cast<int>(AverageTripleArray3DPtr->getNX2());
+ dataSetParamStr.nx[2] = static_cast<int>(AverageTripleArray3DPtr->getNX3());
+ dataSetParamStr.nx[3] = static_cast<int>(AverageTripleArray3DPtr->getNX4());
+ doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ //}
+ //else
+ // break;
+
+ firstBlock = false;
+ }
+
+ if (AverageTripleArray3DPtr && (dataSetParamStr.nx[0]>0) && (dataSetParamStr.nx[1]>0) && (dataSetParamStr.nx[2]>0) && (dataSetParamStr.nx[3]>0))
+ doubleValuesArray.insert(doubleValuesArray.end(), AverageTripleArray3DPtr->getDataVector().begin(), AverageTripleArray3DPtr->getDataVector().end());
+
+ ic++;
+ }
+ }
+
+ // register new MPI-types depending on the block-specific information
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_commit(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeAverageTripleArray start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ // write to the file
+ // all processes calculate their offsets (quantity of bytes that the process is going to write)
+ // and notify the next process (with the rank = rank + 1)
+ MPI_Offset write_offset = (MPI_Offset)(size * sizeof(int));
+ size_t next_write_offset = 0;
+
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount*(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+ MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
+ }
+ else
+ {
+ MPI_Recv(&write_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount*(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+ if (rank<size - 1)
+ MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
+ }
+ }
+
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
+
+ MPI_Info info = MPI_INFO_NULL;
+
+#ifdef HLRN_LUSTRE
+ MPI_Info_create(&info);
+ MPI_Info_set(info, "striping_factor", "40");
+ MPI_Info_set(info, "striping_unit", "4M");
+#endif
+
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageTripleArray.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ // each process writes the quantity of it's blocks
+ MPI_File_write_at(file_handler, (MPI_Offset)(rank * sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
+ // each process writes common parameters of a dataSet
+ MPI_File_write_at(file_handler, write_offset, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
+ // each process writes data identifying blocks
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount, dataSetSmallType, MPI_STATUS_IGNORE);
+ // each process writes the dataSet arrays
+ if (doubleValuesArray.size() > 0)
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmall)), &doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
+
+ MPI_File_sync(file_handler);
+ MPI_File_close(&file_handler);
+
+ MPI_Type_free(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeAverageTripleArray time: " << finish - start << " s");
+ }
+
+ delete[] dataSetSmallArray;
+}
+
+void MPIIORestartCoProcessor::writeShearStressValArray(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+ int blocksCount = 0; // quantity of blocks in the grid, max 2147483648 blocks!
+
+ std::vector<SPtr<Block3D>> blocksVector[25];
+ int minInitLevel = this->grid->getCoarsestInitializedLevel();
+ int maxInitLevel = this->grid->getFinestInitializedLevel();
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ grid->getBlocks(level, rank, blocksVector[level]);
+ blocksCount += static_cast<int>(blocksVector[level].size());
+ }
+
+ DataSetSmall* dataSetSmallArray = new DataSetSmall[blocksCount];
+ std::vector<double> doubleValuesArray; // double-values (arrays of f's) in all blocks
+ dataSetParam dataSetParamStr;
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeShearStressValArray start collect data rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ bool firstBlock = true;
+ int doubleCountInBlock = 0;
+ int ic = 0;
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
+ {
+ dataSetSmallArray[ic].x1 = block->getX1(); // coordinates of the block needed to find it while regenerating the grid
+ dataSetSmallArray[ic].x2 = block->getX2();
+ dataSetSmallArray[ic].x3 = block->getX3();
+ dataSetSmallArray[ic].level = block->getLevel();
+
+ SPtr< CbArray4D<LBMReal, IndexerX4X3X2X1> > ShearStressValArray3DPtr = block->getKernel()->getDataSet()->getShearStressValues();
+
+ if (firstBlock) // when first (any) valid block...
+ {
+ //if (ShearStressValArray3DPtr)
+ //{
+ dataSetParamStr.nx1 = dataSetParamStr.nx2 = dataSetParamStr.nx3 = 0;
+ dataSetParamStr.nx[0] = static_cast<int>(ShearStressValArray3DPtr->getNX1());
+ dataSetParamStr.nx[1] = static_cast<int>(ShearStressValArray3DPtr->getNX2());
+ dataSetParamStr.nx[2] = static_cast<int>(ShearStressValArray3DPtr->getNX3());
+ dataSetParamStr.nx[3] = static_cast<int>(ShearStressValArray3DPtr->getNX4());
+ doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ //}
+ //else
+ // break;
+
+ firstBlock = false;
+ }
+
+ if (ShearStressValArray3DPtr && (dataSetParamStr.nx[0]>0) && (dataSetParamStr.nx[1]>0) && (dataSetParamStr.nx[2]>0) && (dataSetParamStr.nx[3]>0))
+ doubleValuesArray.insert(doubleValuesArray.end(), ShearStressValArray3DPtr->getDataVector().begin(), ShearStressValArray3DPtr->getDataVector().end());
+
+ ic++;
+ }
+ }
+
+ // register new MPI-types depending on the block-specific information
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_commit(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeShearStressValArray start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ // write to the file
+ // all processes calculate their offsets (quantity of bytes that the process is going to write)
+ // and notify the next process (with the rank = rank + 1)
+ MPI_Offset write_offset = (MPI_Offset)(size * sizeof(int));
+ size_t next_write_offset = 0;
+
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount*(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+ MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
+ }
+ else
+ {
+ MPI_Recv(&write_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount*(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+ if (rank<size - 1)
+ MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
+ }
+ }
+
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
+
+ MPI_Info info = MPI_INFO_NULL;
+
+#ifdef HLRN_LUSTRE
+ MPI_Info_create(&info);
+ MPI_Info_set(info, "striping_factor", "40");
+ MPI_Info_set(info, "striping_unit", "4M");
+#endif
+
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpShearStressValArray.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ // each process writes the quantity of it's blocks
+ MPI_File_write_at(file_handler, (MPI_Offset)(rank * sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
+ // each process writes common parameters of a dataSet
+ MPI_File_write_at(file_handler, write_offset, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
+ // each process writes data identifying blocks
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount, dataSetSmallType, MPI_STATUS_IGNORE);
+ // each process writes the dataSet arrays
+ if (doubleValuesArray.size() > 0)
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmall)), &doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
+
+ MPI_File_sync(file_handler);
+ MPI_File_close(&file_handler);
+
+ MPI_Type_free(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeShearStressValArray time: " << finish - start << " s");
+ }
+
+ delete[] dataSetSmallArray;
+}
+
+void MPIIORestartCoProcessor::writeRelaxationFactor(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+ int blocksCount = 0; // quantity of blocks in the grid, max 2147483648 blocks!
+
+ std::vector<SPtr<Block3D>> blocksVector[25];
+ int minInitLevel = this->grid->getCoarsestInitializedLevel();
+ int maxInitLevel = this->grid->getFinestInitializedLevel();
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ grid->getBlocks(level, rank, blocksVector[level]);
+ blocksCount += static_cast<int>(blocksVector[level].size());
+ }
+
+ DataSetSmall* dataSetSmallArray = new DataSetSmall[blocksCount];
+ std::vector<double> doubleValuesArray; // double-values (arrays of f's) in all blocks
+ dataSetParam dataSetParamStr;
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeRelaxationFactor start collect data rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ bool firstBlock = true;
+ int doubleCountInBlock = 0;
+ int ic = 0;
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
+ {
+ dataSetSmallArray[ic].x1 = block->getX1(); // coordinates of the block needed to find it while regenerating the grid
+ dataSetSmallArray[ic].x2 = block->getX2();
+ dataSetSmallArray[ic].x3 = block->getX3();
+ dataSetSmallArray[ic].level = block->getLevel();
+
+ SPtr< CbArray3D<LBMReal, IndexerX3X2X1> > RelaxationFactor3DPtr = block->getKernel()->getDataSet()->getRelaxationFactor();
+
+ if (firstBlock) // when first (any) valid block...
+ {
+ //if (relaxationFactor3DPtr)
+ //{
+ dataSetParamStr.nx1 = dataSetParamStr.nx2 = dataSetParamStr.nx3 = 0;
+ dataSetParamStr.nx[0] = static_cast<int>(RelaxationFactor3DPtr->getNX1());
+ dataSetParamStr.nx[1] = static_cast<int>(RelaxationFactor3DPtr->getNX2());
+ dataSetParamStr.nx[2] = static_cast<int>(RelaxationFactor3DPtr->getNX3());
+ dataSetParamStr.nx[3] = 1;
+ doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ //}
+ //else
+ // break;
+
+ firstBlock = false;
+ }
+
+ if (RelaxationFactor3DPtr && (dataSetParamStr.nx[0]>0) && (dataSetParamStr.nx[1]>0) && (dataSetParamStr.nx[2]>0))
+ doubleValuesArray.insert(doubleValuesArray.end(), RelaxationFactor3DPtr->getDataVector().begin(), RelaxationFactor3DPtr->getDataVector().end());
+
+ ic++;
+ }
+ }
+
+ // register new MPI-types depending on the block-specific information
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_commit(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeRelaxationFactor start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ // write to the file
+ // all processes calculate their offsets (quantity of bytes that the process is going to write)
+ // and notify the next process (with the rank = rank + 1)
+ MPI_Offset write_offset = (MPI_Offset)(size * sizeof(int));
+ size_t next_write_offset = 0;
+
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount*(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+ MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
+ }
+ else
+ {
+ MPI_Recv(&write_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount*(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+ if (rank<size - 1)
+ MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
+ }
+ }
+
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
+
+ MPI_Info info = MPI_INFO_NULL;
+
+#ifdef HLRN_LUSTRE
+ MPI_Info_create(&info);
+ MPI_Info_set(info, "striping_factor", "40");
+ MPI_Info_set(info, "striping_unit", "4M");
+#endif
+
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpRelaxationFactor.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ // each process writes the quantity of it's blocks
+ MPI_File_write_at(file_handler, (MPI_Offset)(rank * sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
+ // each process writes common parameters of a dataSet
+ MPI_File_write_at(file_handler, write_offset, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
+ // each process writes data identifying blocks
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount, dataSetSmallType, MPI_STATUS_IGNORE);
+ // each process writes the dataSet arrays
+ if (doubleValuesArray.size() > 0)
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmall)), &doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
+
+ MPI_File_sync(file_handler);
+ MPI_File_close(&file_handler);
+
+ MPI_Type_free(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeRelaxationFactor time: " << finish - start << " s");
+ }
+
+ delete[] dataSetSmallArray;
+}
+
+void MPIIORestartCoProcessor::writeBoundaryConds(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeBoundaryConds start collect data rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ int blocksCount = 0; // quantity of blocks in the grid, max 2147483648 blocks!
+ size_t count_boundCond = 0; // how many BoundaryConditions in all blocks
+ int count_indexContainer = 0; // how many indexContainer-values in all blocks
+ size_t byteCount = 0; // how many bytes writes this process in the file
+
+ std::vector<SPtr<Block3D>> blocksVector[25];
+ int minInitLevel = this->grid->getCoarsestInitializedLevel();
+ int maxInitLevel = this->grid->getFinestInitializedLevel();
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ grid->getBlocks(level, rank, blocksVector[level]);
+ blocksCount += static_cast<int>(blocksVector[level].size());
+ }
+
+ BCAdd* bcAddArray = new BCAdd[blocksCount];
+ std::vector<BoundaryCondition> bcVector;
+ std::vector<int> bcindexmatrixV;
+ std::vector<int> indexContainerV;
+ bool bcindexmatrixCountNotInit = true;
+
+ int ic = 0;
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ for (SPtr<Block3D> block : blocksVector[level]) // all the blocks of the current level
+ {
+ SPtr<BCArray3D> bcArr = block->getKernel()->getBCProcessor()->getBCArray();
+
+ bcAddArray[ic].x1 = block->getX1(); // coordinates of the block needed to find it while regenerating the grid
+ bcAddArray[ic].x2 = block->getX2();
+ bcAddArray[ic].x3 = block->getX3();
+ bcAddArray[ic].level = block->getLevel();
+ bcAddArray[ic].boundCond_count = 0; // how many BoundaryConditions in this block
+ bcAddArray[ic].indexContainer_count = 0; // how many indexContainer-values in this block
+
+ for (int bc = 0; bc<bcArr->getBCVectorSize(); bc++)
+ {
+ BoundaryCondition* bouCond = new BoundaryCondition();
+ if (bcArr->bcvector[bc] == NULL)
+ {
+ memset(bouCond, 0, sizeof(BoundaryCondition));
+ }
+ else
+ {
+ bouCond->noslipBoundaryFlags = bcArr->bcvector[bc]->getNoSlipBoundary();
+ bouCond->slipBoundaryFlags = bcArr->bcvector[bc]->getSlipBoundary();
+ bouCond->velocityBoundaryFlags = bcArr->bcvector[bc]->getVelocityBoundary();
+ bouCond->densityBoundaryFlags = bcArr->bcvector[bc]->getDensityBoundary();
+ bouCond->wallModelBoundaryFlags = bcArr->bcvector[bc]->getWallModelBoundary();
+ bouCond->bcVelocityX1 = bcArr->bcvector[bc]->getBoundaryVelocityX1();
+ bouCond->bcVelocityX2 = bcArr->bcvector[bc]->getBoundaryVelocityX2();
+ bouCond->bcVelocityX3 = bcArr->bcvector[bc]->getBoundaryVelocityX3();
+ bouCond->bcDensity = bcArr->bcvector[bc]->getBoundaryDensity();
+ bouCond->bcLodiDensity = bcArr->bcvector[bc]->getDensityLodiDensity();
+ bouCond->bcLodiVelocityX1 = bcArr->bcvector[bc]->getDensityLodiVelocityX1();
+ bouCond->bcLodiVelocityX2 = bcArr->bcvector[bc]->getDensityLodiVelocityX2();
+ bouCond->bcLodiVelocityX3 = bcArr->bcvector[bc]->getDensityLodiVelocityX3();
+ bouCond->bcLodiLentgh = bcArr->bcvector[bc]->getDensityLodiLength();
+ bouCond->nx1 = bcArr->bcvector[bc]->nx1;
+ bouCond->nx2 = bcArr->bcvector[bc]->nx2;
+ bouCond->nx3 = bcArr->bcvector[bc]->nx3;
+ for (int iq = 0; iq<26; iq++)
+ bouCond->q[iq] = bcArr->bcvector[bc]->getQ(iq);
+ bouCond->algorithmType = bcArr->bcvector[bc]->getBcAlgorithmType();
+ }
+
+ bcVector.push_back(*bouCond);
+ bcAddArray[ic].boundCond_count++;
+ count_boundCond++;
+ }
+
+ // the quantity of elements in the bcindexmatrix array (CbArray3D<int, IndexerX3X2X1>) in bcArray(BCArray3D) is always equal,
+ // this will be the size of the "write-read-block" in MPI_write_.../MPI_read-functions when writing/reading BoundConds
+ if (bcindexmatrixCountNotInit)
+ {
+ boundCondParamStr.nx1 = static_cast<int>(bcArr->bcindexmatrix.getNX1());
+ boundCondParamStr.nx2 = static_cast<int>(bcArr->bcindexmatrix.getNX2());
+ boundCondParamStr.nx3 = static_cast<int>(bcArr->bcindexmatrix.getNX3());
+ boundCondParamStr.bcindexmatrixCount = static_cast<int>(bcArr->bcindexmatrix.getDataVector().size());
+ bcindexmatrixCountNotInit = false;
+ }
+ bcindexmatrixV.insert(bcindexmatrixV.end(), bcArr->bcindexmatrix.getDataVector().begin(), bcArr->bcindexmatrix.getDataVector().end());
+
+ indexContainerV.insert(indexContainerV.end(), bcArr->indexContainer.begin(), bcArr->indexContainer.end());
+ bcAddArray[ic].indexContainer_count = static_cast<int>(bcArr->indexContainer.size());
+ count_indexContainer += bcAddArray[ic].indexContainer_count;
+
+ ic++;
+ }
+ }
+
+
+ MPI_Type_contiguous(boundCondParamStr.bcindexmatrixCount, MPI_INT, &bcindexmatrixType);
+ MPI_Type_commit(&bcindexmatrixType);
+
+ //how many "big blocks" of BLOCK_SIZE size can by formed
+ int bcBlockCount = (int)(count_boundCond / BLOCK_SIZE);
+ if (bcBlockCount * BLOCK_SIZE<count_boundCond)
+ bcBlockCount += 1;
+ for (int i = (int)count_boundCond; i<bcBlockCount * BLOCK_SIZE; i++)
+ {
+ BoundaryCondition* bouCond = new BoundaryCondition();
+ memset(bouCond, 0, sizeof(BoundaryCondition));
+ bcVector.push_back(*bouCond);
+ }
+
+ byteCount = bcBlockCount * BLOCK_SIZE * sizeof(BoundaryCondition) + blocksCount * sizeof(BCAdd) + sizeof(int) * (blocksCount * boundCondParamStr.bcindexmatrixCount + count_indexContainer);
+
+ // write to the file
+ // all processes calculate their offsets (quantity of bytes that the process is going to write)
+ // and notify the next process (with the rank = rank + 1)
+ MPI_Offset write_offset = (MPI_Offset)(size * (3 * sizeof(int) + sizeof(boundCondParam)));
+ size_t next_write_offset = 0;
+
+ if (size>1)
+ {
+ if (rank == 0)
+ {
+ next_write_offset = write_offset + byteCount;
+ MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
+ }
+ else
+ {
+ MPI_Recv(&write_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+ next_write_offset = write_offset + byteCount;
+ if (rank<size - 1)
+ MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
+ }
+ }
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeBoundaryConds start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
+
+ MPI_Info info = MPI_INFO_NULL;
+
+#ifdef HLRN_LUSTRE
+ MPI_Info_create(&info);
+ MPI_Info_set(info, "striping_factor", "40");
+ MPI_Info_set(info, "striping_unit", "4M");
+#endif
+
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpBC.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ MPI_Offset write_offset1 = (MPI_Offset)(rank * (3 * sizeof(int) + sizeof(boundCondParam)));
+
+ // each process writes the quantity of it's blocks
+ MPI_File_write_at(file_handler, write_offset1, &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
+ // each process writes the quantity of "big blocks" of BLOCK_SIZE of boundary conditions
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset1 + sizeof(int)), &bcBlockCount, 1, MPI_INT, MPI_STATUS_IGNORE);
+ // each process writes the quantity of indexContainer elements in all blocks
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset1 + 2 * sizeof(int)), &count_indexContainer, 1, MPI_INT, MPI_STATUS_IGNORE);
+ // each process writes the quantity of bcindexmatrix elements in every block
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset1 + 3 * sizeof(int)), &boundCondParamStr, 1, boundCondParamType, MPI_STATUS_IGNORE);
+
+ // each process writes data identifying the blocks
+ MPI_File_write_at(file_handler, write_offset, bcAddArray, blocksCount, boundCondTypeAdd, MPI_STATUS_IGNORE);
+ // each process writes boundary conditions
+ if (bcVector.size()>0)
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + blocksCount * sizeof(BCAdd)), &bcVector[0], bcBlockCount, boundCondType1000, MPI_STATUS_IGNORE);
+ // each process writes bcindexmatrix values
+ if (bcindexmatrixV.size()>0)
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + blocksCount * sizeof(BCAdd) + bcBlockCount*BLOCK_SIZE * sizeof(BoundaryCondition)), &bcindexmatrixV[0], blocksCount, bcindexmatrixType, MPI_STATUS_IGNORE);
+ // each process writes indexContainer values
+ if (indexContainerV.size()>0)
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + blocksCount * sizeof(BCAdd) + bcBlockCount*BLOCK_SIZE * sizeof(BoundaryCondition) + blocksCount*boundCondParamStr.bcindexmatrixCount * sizeof(int)), &indexContainerV[0], count_indexContainer, MPI_INT, MPI_STATUS_IGNORE);
+
+ MPI_File_sync(file_handler);
+ MPI_File_close(&file_handler);
+
+ MPI_Type_free(&bcindexmatrixType);
+
+ if (comm->isRoot())
+ {
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIORestartCoProcessor::writeBoundaryConds time: " << finish - start << " s");
+ }
+
+ delete[] bcAddArray;
+}
+
+//------------------------------------------- READ -----------------------------------------------
+void MPIIORestartCoProcessor::restart(int step)
+{
+ if (comm->isRoot()) UBLOG(logINFO, "MPIIORestartCoProcessor restart step: " << step);
+ if (comm->isRoot()) UBLOG(logINFO, "Load check point - start");
+
+ readBlocks(step);
+ readDataSet(step);
+ readBoundaryConds(step);
+
+ if (comm->isRoot()) UBLOG(logINFO, "Load check point - end");
+}
+
+void MPIIORestartCoProcessor::readBlocks(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ //MPI_Comm_size(MPI_COMM_WORLD, &size);
+ size = 1;
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readBlocks start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
+
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpBlocks.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ // read count of blocks
+ int blocksCount = 0;
+ //MPI_File_read_at(file_handler, rank*sizeof(int), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, 0, &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
+ Block3d* block3dArray = new Block3d[blocksCount];
+
+ // calculate the read offset
+ MPI_Offset read_offset = (MPI_Offset)(size * sizeof(int));
+
+ GridParam* gridParameters = new GridParam;
+
+ // read parameters of the grid
+ MPI_File_read_at(file_handler, read_offset, gridParameters, 1, gridParamType, MPI_STATUS_IGNORE);
+ // read all the blocks
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(GridParam)), &block3dArray[0], blocksCount, block3dType, MPI_STATUS_IGNORE);
+
+ MPI_File_close(&file_handler);
+
+ if (comm->isRoot())
+ {
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readBlocks time: " << finish - start << " s");
+ }
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readBlocks start of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ // clear the grid
+ std::vector<SPtr<Block3D>> blocksVector;
+ grid->getBlocks(0, blocksVector);
+ int del = 0;
+ for (SPtr<Block3D> block : blocksVector)
+ {
+ grid->deleteBlock(block);
+ del++;
+ }
+
+ // restore the grid
+ SPtr<CoordinateTransformation3D> trafo(new CoordinateTransformation3D());
+ trafo->Tx1 = gridParameters->trafoParams[0];
+ trafo->Tx2 = gridParameters->trafoParams[1];
+ trafo->Tx3 = gridParameters->trafoParams[2];
+ trafo->Sx1 = gridParameters->trafoParams[3];
+ trafo->Sx2 = gridParameters->trafoParams[4];
+ trafo->Sx3 = gridParameters->trafoParams[5];
+ trafo->alpha = gridParameters->trafoParams[6];
+ trafo->beta = gridParameters->trafoParams[7];
+ trafo->gamma = gridParameters->trafoParams[8];
+
+ trafo->toX1factorX1 = gridParameters->trafoParams[9];
+ trafo->toX1factorX2 = gridParameters->trafoParams[10];
+ trafo->toX1factorX3 = gridParameters->trafoParams[11];
+ trafo->toX1delta = gridParameters->trafoParams[12];
+ trafo->toX2factorX1 = gridParameters->trafoParams[13];
+ trafo->toX2factorX2 = gridParameters->trafoParams[14];
+ trafo->toX2factorX3 = gridParameters->trafoParams[15];
+ trafo->toX2delta = gridParameters->trafoParams[16];
+ trafo->toX3factorX1 = gridParameters->trafoParams[17];
+ trafo->toX3factorX2 = gridParameters->trafoParams[18];
+ trafo->toX3factorX3 = gridParameters->trafoParams[19];
+ trafo->toX3delta = gridParameters->trafoParams[20];
+
+ trafo->fromX1factorX1 = gridParameters->trafoParams[21];
+ trafo->fromX1factorX2 = gridParameters->trafoParams[22];
+ trafo->fromX1factorX3 = gridParameters->trafoParams[23];
+ trafo->fromX1delta = gridParameters->trafoParams[24];
+ trafo->fromX2factorX1 = gridParameters->trafoParams[25];
+ trafo->fromX2factorX2 = gridParameters->trafoParams[26];
+ trafo->fromX2factorX3 = gridParameters->trafoParams[27];
+ trafo->fromX2delta = gridParameters->trafoParams[28];
+ trafo->fromX3factorX1 = gridParameters->trafoParams[29];
+ trafo->fromX3factorX2 = gridParameters->trafoParams[30];
+ trafo->fromX3factorX3 = gridParameters->trafoParams[31];
+ trafo->fromX3delta = gridParameters->trafoParams[32];
+
+ trafo->active = gridParameters->active;
+ trafo->transformation = gridParameters->transformation;
+
+ grid->setCoordinateTransformator(trafo);
+
+ grid->setDeltaX(gridParameters->deltaX);
+ grid->setBlockNX(gridParameters->blockNx1, gridParameters->blockNx2, gridParameters->blockNx3);
+ grid->setNX1(gridParameters->nx1);
+ grid->setNX2(gridParameters->nx2);
+ grid->setNX3(gridParameters->nx3);
+ grid->setPeriodicX1(gridParameters->periodicX1);
+ grid->setPeriodicX2(gridParameters->periodicX2);
+ grid->setPeriodicX3(gridParameters->periodicX3);
+
+ // regenerate blocks
+ for (int n = 0; n<blocksCount; n++)
+ {
+ SPtr<Block3D> block(new Block3D(block3dArray[n].x1, block3dArray[n].x2, block3dArray[n].x3, block3dArray[n].level));
+ block->setActive(block3dArray[n].active);
+ block->setBundle(block3dArray[n].bundle);
+ block->setRank(block3dArray[n].rank);
+ block->setLocalRank(block3dArray[n].lrank);
+ block->setGlobalID(block3dArray[n].globalID);
+ block->setLocalID(block3dArray[n].localID);
+ block->setPart(block3dArray[n].part);
+ block->setLevel(block3dArray[n].level);
+ block->interpolationFlagCF = block3dArray[n].interpolationFlagCF;
+ block->interpolationFlagFC = block3dArray[n].interpolationFlagFC;
+
+ grid->addBlock(block);
+ }
+
+ delete gridParameters;
+ delete[] block3dArray;
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readBlocks end of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+}
+
+void MPIIORestartCoProcessor::readDataSet(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readDataSet start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
+
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSet.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ // calculate the read offset
+ MPI_Offset read_offset = (MPI_Offset)(size * sizeof(int));
+ size_t next_read_offset = 0;
+
+ // read count of blocks
+ int blocksCount = 0;
+ dataSetParam dataSetParamStr1, dataSetParamStr2, dataSetParamStr3;
+
+ MPI_File_read_at(file_handler, (MPI_Offset)(rank * sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, read_offset, &dataSetParamStr1, 1, dataSetParamType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam)), &dataSetParamStr2, 1, dataSetParamType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + 2 * sizeof(dataSetParam)), &dataSetParamStr3, 1, dataSetParamType, MPI_STATUS_IGNORE);
+
+ DataSet* dataSetArray = new DataSet[blocksCount];
+ double doubleCountInBlock = dataSetParamStr1.nx[0] * dataSetParamStr1.nx[1] * dataSetParamStr1.nx[2] * dataSetParamStr1.nx[3] +
+ dataSetParamStr2.nx[0] * dataSetParamStr2.nx[1] * dataSetParamStr2.nx[2] * dataSetParamStr2.nx[3] +
+ dataSetParamStr3.nx[0] * dataSetParamStr3.nx[1] * dataSetParamStr3.nx[2] * dataSetParamStr3.nx[3];
+ std::vector<double> doubleValuesArray(blocksCount * doubleCountInBlock); // double-values in all blocks
+
+ // define MPI_types depending on the block-specific information
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_commit(&dataSetDoubleType);
+
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_read_offset = read_offset + 3 * sizeof(dataSetParam) + blocksCount * (sizeof(DataSet) + doubleCountInBlock * sizeof(double));
+ MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
+ }
+ else
+ {
+ MPI_Recv(&read_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+ next_read_offset = read_offset + 3 * sizeof(dataSetParam) + blocksCount * (sizeof(DataSet) + doubleCountInBlock * sizeof(double));
+ if (rank < size - 1)
+ MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
+ }
+ }
+
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + 3 * sizeof(dataSetParam)), dataSetArray, blocksCount, dataSetType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + 3 * sizeof(dataSetParam) + blocksCount * sizeof(DataSet)), &doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
+ MPI_File_close(&file_handler);
+
+ if (comm->isRoot())
+ {
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readDataSet time: " << finish - start << " s");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readDataSet start of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ size_t index = 0, vectorSize = 0;
+ std::vector<double> vectorsOfValues1, vectorsOfValues2, vectorsOfValues3;
+
+ for (int n = 0; n < blocksCount; n++)
+ {
+ vectorSize = dataSetParamStr1.nx[0] * dataSetParamStr1.nx[1] * dataSetParamStr1.nx[2] * dataSetParamStr1.nx[3];
+ vectorsOfValues1.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + vectorSize);
+ index += vectorSize;
+
+ vectorSize = dataSetParamStr2.nx[0] * dataSetParamStr2.nx[1] * dataSetParamStr2.nx[2] * dataSetParamStr2.nx[3];
+ vectorsOfValues2.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + vectorSize);
+ index += vectorSize;
+
+ vectorSize = dataSetParamStr3.nx[0] * dataSetParamStr3.nx[1] * dataSetParamStr3.nx[2] * dataSetParamStr3.nx[3];
+ vectorsOfValues3.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + vectorSize);
+ index += vectorSize;
+
+ SPtr<DistributionArray3D> mFdistributions(new D3Q27EsoTwist3DSplittedVector());
+
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues1, dataSetParamStr1.nx[0], dataSetParamStr1.nx[1], dataSetParamStr1.nx[2], dataSetParamStr1.nx[3])));
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNonLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues2, dataSetParamStr2.nx[0], dataSetParamStr2.nx[1], dataSetParamStr2.nx[2], dataSetParamStr2.nx[3])));
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setZeroDistributions(CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(vectorsOfValues3, dataSetParamStr3.nx[0], dataSetParamStr3.nx[1], dataSetParamStr3.nx[2])));
+
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNX1(dataSetParamStr1.nx1);
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNX2(dataSetParamStr1.nx2);
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNX3(dataSetParamStr1.nx3);
+
+ // find the nesessary block and fill it
+ SPtr<Block3D> block = grid->getBlock(dataSetArray[n].x1, dataSetArray[n].x2, dataSetArray[n].x3, dataSetArray[n].level);
+ SPtr<LBMKernel> kernel = this->lbmKernel->clone();
+ kernel->setGhostLayerWidth(dataSetArray[n].ghostLayerWidth);
+ kernel->setCollisionFactor(dataSetArray[n].collFactor);
+ kernel->setDeltaT(dataSetArray[n].deltaT);
+ kernel->setCompressible(dataSetArray[n].compressible);
+ kernel->setWithForcing(dataSetArray[n].withForcing);
+ SPtr<DataSet3D> dataSetPtr = SPtr<DataSet3D>(new DataSet3D());
+ dataSetPtr->setFdistributions(mFdistributions);
+ kernel->setDataSet(dataSetPtr);
+ block->setKernel(kernel);
+ }
+
+ MPI_Type_free(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readDataSet end of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ delete[] dataSetArray;
+
+ //-------------------------------------------------------------
+
+ DSArraysPresence arrPresence;
+ MPI_File file_handler1;
+ std::string filename1 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpArrays.bin";
+ rc = MPI_File_open(MPI_COMM_WORLD, filename1.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler1);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename1);
+ MPI_File_read_at(file_handler1, (MPI_Offset)0, &arrPresence, 1, arrayPresenceType, MPI_STATUS_IGNORE);
+ MPI_File_close(&file_handler1);
+
+ if (arrPresence.isAverageDensityArrayPresent)
+ readAverageDensityArray(step);
+
+ if (arrPresence.isAverageVelocityArrayPresent)
+ readAverageVelocityArray(step);
+
+ if (arrPresence.isAverageFluktuationsArrayPresent)
+ readAverageFluktuationsArray(step);
+
+ if (arrPresence.isAverageTripleArrayPresent)
+ readAverageTripleArray(step);
+
+ if (arrPresence.isShearStressValArrayPresent)
+ readShearStressValArray(step);
+
+ if (arrPresence.isRelaxationFactorPresent)
+ readRelaxationFactor(step);
+
+}
+
+void MPIIORestartCoProcessor::readAverageDensityArray(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageDensityArray start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
+
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageDensityArray.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ // read count of blocks
+ int blocksCount = 0;
+ dataSetParam dataSetParamStr;
+ memset(&dataSetParamStr, 0, sizeof(dataSetParam));
+
+ MPI_File_read_at(file_handler, (MPI_Offset)(rank * sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(size * sizeof(int)), &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
+
+ DataSetSmall* dataSetSmallArray = new DataSetSmall[blocksCount];
+ int doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ std::vector<double> doubleValuesArray(blocksCount * doubleCountInBlock); // double-values in all blocks
+
+ // define MPI_types depending on the block-specific information
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_commit(&dataSetDoubleType);
+
+ // calculate the read offset
+ MPI_Offset read_offset = (MPI_Offset)(size * sizeof(int));
+ size_t next_read_offset = 0;
+
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount*(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+ MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
+ }
+ else
+ {
+ MPI_Recv(&read_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount*(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+ if (rank < size - 1)
+ MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
+ }
+ }
+
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount, dataSetSmallType, MPI_STATUS_IGNORE);
+ if (doubleCountInBlock > 0)
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmall)), &doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
+ MPI_File_close(&file_handler);
+
+ if (comm->isRoot())
+ {
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageDensityArray time: " << finish - start << " s");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageDensityArray start of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ size_t index = 0;
+ size_t nextVectorSize = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ std::vector<double> vectorsOfValues;
+ for (int n = 0; n < blocksCount; n++)
+ {
+ vectorsOfValues.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + nextVectorSize);
+ index += nextVectorSize;
+
+ // fill mAverageDensity arrays
+ SPtr<AverageValuesArray3D> mAverageDensity;
+ //if ((dataSetParamStr.nx[0]==0)&&(dataSetParamStr.nx[1]==0)&&(dataSetParamStr.nx[2]==0)&&(dataSetParamStr.nx[3]==0))
+ // mAverageDensity = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr();
+ //else
+ mAverageDensity = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
+
+ // find the nesessary block and fill it
+ SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].x1, dataSetSmallArray[n].x2, dataSetSmallArray[n].x3, dataSetSmallArray[n].level);
+ block->kernel->getDataSet()->setAverageDencity(mAverageDensity);
+ }
+
+ MPI_Type_free(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageDensityArray end of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ delete[] dataSetSmallArray;
+}
+
+void MPIIORestartCoProcessor::readAverageVelocityArray(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageVelocityArray start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
+
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageVelocityArray.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ // read count of blocks
+ int blocksCount = 0;
+ dataSetParam dataSetParamStr;
+ MPI_File_read_at(file_handler, (MPI_Offset)(rank * sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(size * sizeof(int)), &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
+
+ DataSetSmall* dataSetSmallArray = new DataSetSmall[blocksCount];
+ int doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ std::vector<double> doubleValuesArray(blocksCount * doubleCountInBlock); // double-values in all blocks
+
+ // define MPI_types depending on the block-specific information
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_commit(&dataSetDoubleType);
+
+ // calculate the read offset
+ MPI_Offset read_offset = (MPI_Offset)(size * sizeof(int));
+ size_t next_read_offset = 0;
+
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount*(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+ MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
+ }
+ else
+ {
+ MPI_Recv(&read_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount*(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+ if (rank < size - 1)
+ MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
+ }
+ }
+
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount, dataSetSmallType, MPI_STATUS_IGNORE);
+ if (doubleCountInBlock > 0)
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmall)), &doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
+ MPI_File_close(&file_handler);
+
+ if (comm->isRoot())
+ {
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageVelocityArray time: " << finish - start << " s");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageVelocityArray start of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ size_t index = 0;
+ size_t nextVectorSize = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ std::vector<double> vectorsOfValues;
+ for (int n = 0; n < blocksCount; n++)
+ {
+ vectorsOfValues.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + nextVectorSize);
+ index += nextVectorSize;
+
+ // fill mAverageVelocity array
+ SPtr<AverageValuesArray3D> mAverageVelocity;
+ //if ((dataSetParamStr.nx[0] == 0) && (dataSetParamStr.nx[1] == 0) && (dataSetParamStr.nx[2] == 0) && (dataSetParamStr.nx[3] == 0))
+ // mAverageVelocity = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr();
+ //else
+ mAverageVelocity = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
+
+ // find the nesessary block and fill it
+ SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].x1, dataSetSmallArray[n].x2, dataSetSmallArray[n].x3, dataSetSmallArray[n].level);
+ block->kernel->getDataSet()->setAverageVelocity(mAverageVelocity);
+ }
+
+ MPI_Type_free(&dataSetDoubleType);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageVelocityArray end of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ delete[] dataSetSmallArray;
+}
+
+void MPIIORestartCoProcessor::readAverageFluktuationsArray(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageFluktuationsArray start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+ double start, finish;
if (comm->isRoot()) start = MPI_Wtime();
MPI_File file_handler;
- std::string filename = path+"/mpi_io_cp/mpi_io_cp_"+UbSystem::toString(step)+"/cpBlocks.bin";
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageFluktuationsArray.bin";
int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
- if (rc!=MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file "+filename);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
// read count of blocks
int blocksCount = 0;
- //MPI_File_read_at(file_handler, rank*sizeof(int), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, 0, &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
- Block3d* block3dArray = new Block3d[blocksCount];
+ dataSetParam dataSetParamStr;
+ MPI_File_read_at(file_handler, (MPI_Offset)(rank * sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(size * sizeof(int)), &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
+
+ DataSetSmall* dataSetSmallArray = new DataSetSmall[blocksCount];
+ int doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ std::vector<double> doubleValuesArray(blocksCount * doubleCountInBlock); // double-values in all blocks
+
+ // define MPI_types depending on the block-specific information
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_commit(&dataSetDoubleType);
// calculate the read offset
MPI_Offset read_offset = (MPI_Offset)(size * sizeof(int));
+ size_t next_read_offset = 0;
- GridParam* gridParameters = new GridParam;
-
- // read parameters of the grid
- MPI_File_read_at(file_handler, read_offset, gridParameters, 1, gridParamType, MPI_STATUS_IGNORE);
- // read all the blocks
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset+sizeof(GridParam)), &block3dArray[0], blocksCount, block3dType, MPI_STATUS_IGNORE);
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount*(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+ MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
+ }
+ else
+ {
+ MPI_Recv(&read_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount*(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+ if (rank < size - 1)
+ MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
+ }
+ }
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount, dataSetSmallType, MPI_STATUS_IGNORE);
+ if (doubleCountInBlock > 0)
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmall)), &doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
MPI_File_close(&file_handler);
if (comm->isRoot())
{
finish = MPI_Wtime();
- UBLOG(logINFO, "MPIIORestartCoProcessor::readBlocks time: "<<finish-start<<" s");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageFluktuationsArray time: " << finish - start << " s");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageFluktuationsArray start of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ size_t index = 0;
+ size_t nextVectorSize = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ std::vector<double> vectorsOfValues;
+ for (int n = 0; n < blocksCount; n++)
+ {
+ vectorsOfValues.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + nextVectorSize);
+ index += nextVectorSize;
+
+ // fill AverageFluktuations array
+ SPtr<AverageValuesArray3D> mAverageFluktuations;
+ //if ((dataSetParamStr.nx[0] == 0) && (dataSetParamStr.nx[1] == 0) && (dataSetParamStr.nx[2] == 0) && (dataSetParamStr.nx[3] == 0))
+ // mAverageFluktuations = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr();
+ //else
+ mAverageFluktuations = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
+
+ // find the nesessary block and fill it
+ SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].x1, dataSetSmallArray[n].x2, dataSetSmallArray[n].x3, dataSetSmallArray[n].level);
+ block->kernel->getDataSet()->setAverageFluctuations(mAverageFluktuations);
}
+ MPI_Type_free(&dataSetDoubleType);
+
if (comm->isRoot())
{
- UBLOG(logINFO, "MPIIORestartCoProcessor::readBlocks start of restore of data, rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageFluktuationsArray end of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
- // clear the grid
- std::vector<SPtr<Block3D>> blocksVector;
- grid->getBlocks(0, blocksVector);
- int del = 0;
- for(SPtr<Block3D> block : blocksVector)
+ delete[] dataSetSmallArray;
+}
+
+void MPIIORestartCoProcessor::readAverageTripleArray(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+ if (comm->isRoot())
{
- grid->deleteBlock(block);
- del++;
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageTripleArray start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
- // restore the grid
- SPtr<CoordinateTransformation3D> trafo(new CoordinateTransformation3D());
- trafo->Tx1 = gridParameters->trafoParams[0];
- trafo->Tx2 = gridParameters->trafoParams[1];
- trafo->Tx3 = gridParameters->trafoParams[2];
- trafo->Sx1 = gridParameters->trafoParams[3];
- trafo->Sx2 = gridParameters->trafoParams[4];
- trafo->Sx3 = gridParameters->trafoParams[5];
- trafo->alpha = gridParameters->trafoParams[6];
- trafo->beta = gridParameters->trafoParams[7];
- trafo->gamma = gridParameters->trafoParams[8];
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageTripleArray.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
- trafo->toX1factorX1 = gridParameters->trafoParams[9];
- trafo->toX1factorX2 = gridParameters->trafoParams[10];
- trafo->toX1factorX3 = gridParameters->trafoParams[11];
- trafo->toX1delta = gridParameters->trafoParams[12];
- trafo->toX2factorX1 = gridParameters->trafoParams[13];
- trafo->toX2factorX2 = gridParameters->trafoParams[14];
- trafo->toX2factorX3 = gridParameters->trafoParams[15];
- trafo->toX2delta = gridParameters->trafoParams[16];
- trafo->toX3factorX1 = gridParameters->trafoParams[17];
- trafo->toX3factorX2 = gridParameters->trafoParams[18];
- trafo->toX3factorX3 = gridParameters->trafoParams[19];
- trafo->toX3delta = gridParameters->trafoParams[20];
+ // read count of blocks
+ int blocksCount = 0;
+ dataSetParam dataSetParamStr;
+ MPI_File_read_at(file_handler, (MPI_Offset)(rank * sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(size * sizeof(int)), &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
- trafo->fromX1factorX1 = gridParameters->trafoParams[21];
- trafo->fromX1factorX2 = gridParameters->trafoParams[22];
- trafo->fromX1factorX3 = gridParameters->trafoParams[23];
- trafo->fromX1delta = gridParameters->trafoParams[24];
- trafo->fromX2factorX1 = gridParameters->trafoParams[25];
- trafo->fromX2factorX2 = gridParameters->trafoParams[26];
- trafo->fromX2factorX3 = gridParameters->trafoParams[27];
- trafo->fromX2delta = gridParameters->trafoParams[28];
- trafo->fromX3factorX1 = gridParameters->trafoParams[29];
- trafo->fromX3factorX2 = gridParameters->trafoParams[30];
- trafo->fromX3factorX3 = gridParameters->trafoParams[31];
- trafo->fromX3delta = gridParameters->trafoParams[32];
+ DataSetSmall* dataSetSmallArray = new DataSetSmall[blocksCount];
+ int doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ std::vector<double> doubleValuesArray(blocksCount * doubleCountInBlock); // double-values in all blocks
- trafo->active = gridParameters->active;
- trafo->transformation = gridParameters->transformation;
+ // define MPI_types depending on the block-specific information
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_commit(&dataSetDoubleType);
- grid->setCoordinateTransformator(trafo);
+ // calculate the read offset
+ MPI_Offset read_offset = (MPI_Offset)(size * sizeof(int));
+ size_t next_read_offset = 0;
- grid->setDeltaX(gridParameters->deltaX);
- grid->setBlockNX(gridParameters->blockNx1, gridParameters->blockNx2, gridParameters->blockNx3);
- grid->setNX1(gridParameters->nx1);
- grid->setNX2(gridParameters->nx2);
- grid->setNX3(gridParameters->nx3);
- grid->setPeriodicX1(gridParameters->periodicX1);
- grid->setPeriodicX2(gridParameters->periodicX2);
- grid->setPeriodicX3(gridParameters->periodicX3);
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount*(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+ MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
+ }
+ else
+ {
+ MPI_Recv(&read_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount*(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+ if (rank < size - 1)
+ MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
+ }
+ }
- // regenerate blocks
- for (int n = 0; n<blocksCount; n++)
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount, dataSetSmallType, MPI_STATUS_IGNORE);
+ if (doubleCountInBlock > 0)
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmall)), &doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
+ MPI_File_close(&file_handler);
+
+ if (comm->isRoot())
{
- SPtr<Block3D> block(new Block3D(block3dArray[n].x1, block3dArray[n].x2, block3dArray[n].x3, block3dArray[n].level));
- block->setActive(block3dArray[n].active);
- block->setBundle(block3dArray[n].bundle);
- block->setRank(block3dArray[n].rank);
- block->setLocalRank(block3dArray[n].lrank);
- block->setGlobalID(block3dArray[n].globalID);
- block->setLocalID(block3dArray[n].localID);
- block->setPart(block3dArray[n].part);
- block->setLevel(block3dArray[n].level);
- block->interpolationFlagCF = block3dArray[n].interpolationFlagCF;
- block->interpolationFlagFC = block3dArray[n].interpolationFlagFC;
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageTripleArray time: " << finish - start << " s");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageTripleArray start of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
- grid->addBlock(block);
+ size_t index = 0;
+ size_t nextVectorSize = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ std::vector<double> vectorsOfValues;
+ for (int n = 0; n < blocksCount; n++)
+ {
+ vectorsOfValues.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + nextVectorSize);
+ index += nextVectorSize;
+
+ // fill AverageTriplecorrelations array
+ SPtr<AverageValuesArray3D> mAverageTriplecorrelations;
+ //if ((dataSetParamStr.nx[0] == 0) && (dataSetParamStr.nx[1] == 0) && (dataSetParamStr.nx[2] == 0) && (dataSetParamStr.nx[3] == 0))
+ // mAverageTriplecorrelations = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr();
+ //else
+ mAverageTriplecorrelations = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
+
+ // find the nesessary block and fill it
+ SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].x1, dataSetSmallArray[n].x2, dataSetSmallArray[n].x3, dataSetSmallArray[n].level);
+ block->kernel->getDataSet()->setAverageTriplecorrelations(mAverageTriplecorrelations);
}
- delete gridParameters;
- delete[] block3dArray;
+ MPI_Type_free(&dataSetDoubleType);
if (comm->isRoot())
{
- UBLOG(logINFO, "MPIIORestartCoProcessor::readBlocks end of restore of data, rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageTripleArray end of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
+
+ delete[] dataSetSmallArray;
}
-void MPIIORestartCoProcessor::readDataSet(int step)
+void MPIIORestartCoProcessor::readShearStressValArray(int step)
{
int rank, size;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
@@ -1002,209 +2344,190 @@ void MPIIORestartCoProcessor::readDataSet(int step)
if (comm->isRoot())
{
- UBLOG(logINFO, "MPIIORestartCoProcessor::readDataSet start MPI IO rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readShearStressValArray start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
if (comm->isRoot()) start = MPI_Wtime();
MPI_File file_handler;
- std::string filename = path+"/mpi_io_cp/mpi_io_cp_"+UbSystem::toString(step)+"/cpDataSet.bin";
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpShearStressValArray.bin";
int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
- if (rc!=MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file "+filename);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
// read count of blocks
int blocksCount = 0;
- MPI_File_read_at(file_handler, (MPI_Offset)(rank*sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, (MPI_Offset)(size*sizeof(int)), &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
- //std::cout <<"MPIIORestartCoProcessor::readDataSet rank=" << rank <<", dataSetParamStr.doubleCountInBlock="<< dataSetParamStr.doubleCountInBlock << std::endl;
- //std::cout << ",dataSetParamStr.nx[6][0]" << "=" << dataSetParamStr.nx[6][0] << "," << dataSetParamStr.nx[6][1] << "," << dataSetParamStr.nx[6][2] << "," << dataSetParamStr.nx[6][3];
- //std::cout << ",doubleCountInBlock=" << dataSetParamStr.doubleCountInBlock << "," << dataSetParamStr.nx1 << "," << dataSetParamStr.nx2 << "," << dataSetParamStr.nx3 << std::endl;
+ dataSetParam dataSetParamStr;
+ MPI_File_read_at(file_handler, (MPI_Offset)(rank * sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(size * sizeof(int)), &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
+
+ DataSetSmall* dataSetSmallArray = new DataSetSmall[blocksCount];
+ int doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ std::vector<double> doubleValuesArray(blocksCount * doubleCountInBlock); // double-values in all blocks
- DataSet* dataSetArray = new DataSet[blocksCount];
- std::vector<double> doubleValuesArray(blocksCount * dataSetParamStr.doubleCountInBlock); // double-values in all blocks
-
// define MPI_types depending on the block-specific information
- MPI_Type_contiguous(dataSetParamStr.doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
MPI_Type_commit(&dataSetDoubleType);
- mpiTypeFreeFlag = true;
- //std::cout << "MPIIORestartCoProcessor::readDataSet rank=" << rank << " 123=" << dataSetParamStr.doubleCountInBlock << std::endl;
// calculate the read offset
MPI_Offset read_offset = (MPI_Offset)(size * sizeof(int));
size_t next_read_offset = 0;
- if(size > 1)
+ if (size > 1)
{
- if(rank == 0)
- {
- next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSet) + dataSetParamStr.doubleCountInBlock * sizeof(double));
- MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
- }
- else
- {
- MPI_Recv(&read_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
- next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSet) + dataSetParamStr.doubleCountInBlock * sizeof(double));
- if(rank < size - 1)
- MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
- }
+ if (rank == 0)
+ {
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount*(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+ MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
+ }
+ else
+ {
+ MPI_Recv(&read_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount*(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+ if (rank < size - 1)
+ MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
+ }
}
- /*int chunkFlag = 0;
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount, dataSetSmallType, MPI_STATUS_IGNORE);
+ if (doubleCountInBlock > 0)
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmall)), &doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
+ MPI_File_close(&file_handler);
- if (rank == 0)
+ if (comm->isRoot())
{
- MPI_File_read_at(file_handler, read_offset, dataSetArray, blocksCount, dataSetType, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset+blocksCount*sizeof(DataSet)), &doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
-
- for (int i=1; i<size; i+=chunk)
- {
- for (int j=i; j<i+chunk; j++)
- {
- if (j < size)
- {
- MPI_Send(&chunkFlag, 1, MPI_INT, j, 77, MPI_COMM_WORLD);
- //UBLOG(logINFO, "j= "<<j);
- }
- }
- for (int j=i; j<i+chunk; j++)
- {
- if (j < size)
- {
- MPI_Recv(&chunkFlag, 1, MPI_INT, j, 77, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
- }
- }
- }
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readShearStressValArray time: " << finish - start << " s");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readShearStressValArray start of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
- else
+
+ size_t index = 0;
+ size_t nextVectorSize = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ std::vector<double> vectorsOfValues;
+ for (int n = 0; n < blocksCount; n++)
{
- MPI_Recv(&chunkFlag, 1, MPI_INT, 0, 77, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, read_offset, dataSetArray, blocksCount, dataSetType, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset+blocksCount*sizeof(DataSet)), &doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
- MPI_Send(&chunkFlag, 1, MPI_INT, 0, 77, MPI_COMM_WORLD);
- //UBLOG(logINFO, "read rank= "<<rank);
- }*/
+ vectorsOfValues.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + nextVectorSize);
+ index += nextVectorSize;
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset+sizeof(dataSetParam)), dataSetArray, blocksCount, dataSetType, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset+sizeof(dataSetParam)+blocksCount*sizeof(DataSet)), &doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
- MPI_File_close(&file_handler);
+ // fill ShearStressValuesArray array
+ SPtr<ShearStressValuesArray3D> mShearStressValues;
+ //if ((dataSetParamStr.nx[0] == 0) && (dataSetParamStr.nx[1] == 0) && (dataSetParamStr.nx[2] == 0) && (dataSetParamStr.nx[3] == 0))
+ // mShearStressValues = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr();
+ //else
+ mShearStressValues = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
+
+ // find the nesessary block and fill it
+ SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].x1, dataSetSmallArray[n].x2, dataSetSmallArray[n].x3, dataSetSmallArray[n].level);
+ block->kernel->getDataSet()->setShearStressValues(mShearStressValues);
+ }
- //for (int ch = 0; ch < blocksCount; ch++)
- //{
- // if ((dataSetArrayGW[ch].x1 != dataSetArray[ch].x1) ||
- // (dataSetArrayGW[ch].x2 != dataSetArray[ch].x2) ||
- // (dataSetArrayGW[ch].x3 != dataSetArray[ch].x3) ||
- // (dataSetArrayGW[ch].level != dataSetArray[ch].level) ||
- // (dataSetArrayGW[ch].ghostLayerWidth != dataSetArray[ch].ghostLayerWidth) ||
- // (dataSetArrayGW[ch].collFactor != dataSetArray[ch].collFactor) ||
- // (dataSetArrayGW[ch].deltaT != dataSetArray[ch].deltaT) ||
- // (dataSetArrayGW[ch].compressible != dataSetArray[ch].compressible) ||
- // (dataSetArrayGW[ch].withForcing != dataSetArray[ch].withForcing))
- // std::cout << "dataSetArrayGW != rank" << rank << ", !!!!!====="<< std::endl;
- //}
- //std::cout << "doubleValuesArrayGW.size" << doubleValuesArrayGW.size() << ", " << doubleValuesArray.size() << std::endl;
- //for (int vl = 0; vl < doubleValuesArrayGW.size(); vl++)
- // if(doubleValuesArrayGW[vl] != doubleValuesArray[vl])
- // std::cout << "doubleValuesArrayGW != rank" << rank << ", !!!!!====="<< std::endl;
+ MPI_Type_free(&dataSetDoubleType);
if (comm->isRoot())
{
- finish = MPI_Wtime();
- UBLOG(logINFO, "MPIIORestartCoProcessor::readDataSet time: "<<finish-start<<" s");
- UBLOG(logINFO, "MPIIORestartCoProcessor::readDataSet start of restore of data, rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readShearStressValArray end of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
- size_t index = 0, nextVectorSize = 0;
- std::vector<double> vectorsOfValues[9];
- for (int n = 0; n<blocksCount; n++)
- {
- for (int b = 0; b<9; b++) // assign approciate vectors for 9 dataSet arrays
- {
- nextVectorSize = dataSetParamStr.nx[b][0]* dataSetParamStr.nx[b][1]* dataSetParamStr.nx[b][2]* dataSetParamStr.nx[b][3];
- vectorsOfValues[b].assign(doubleValuesArray.data()+index, doubleValuesArray.data()+index+nextVectorSize);
- index += nextVectorSize;
- }
+ delete[] dataSetSmallArray;
+}
- // fill dataSet arrays
- SPtr<AverageValuesArray3D> mAverageDensity;
- if ((dataSetParamStr.nx[0][0]==0)&&(dataSetParamStr.nx[0][1]==0)&&(dataSetParamStr.nx[0][2]==0)&&(dataSetParamStr.nx[0][3]==0))
- mAverageDensity = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr();
- else
- mAverageDensity = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues[0], dataSetParamStr.nx[0][0], dataSetParamStr.nx[0][1], dataSetParamStr.nx[0][2], dataSetParamStr.nx[0][3]));
+void MPIIORestartCoProcessor::readRelaxationFactor(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
- SPtr<AverageValuesArray3D> mAverageVelocity;
- if ((dataSetParamStr.nx[1][0]==0)&&(dataSetParamStr.nx[1][1]==0)&&(dataSetParamStr.nx[1][2]==0)&&(dataSetParamStr.nx[1][3]==0))
- mAverageVelocity = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr();
- else
- mAverageVelocity = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues[1], dataSetParamStr.nx[1][0], dataSetParamStr.nx[1][1], dataSetParamStr.nx[1][2], dataSetParamStr.nx[1][3]));
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readRelaxationFactor start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+ double start, finish;
+ if (comm->isRoot()) start = MPI_Wtime();
- SPtr<AverageValuesArray3D> mAverageFluktuations;
- if ((dataSetParamStr.nx[2][0]==0)&&(dataSetParamStr.nx[2][1]==0)&&(dataSetParamStr.nx[2][2]==0)&&(dataSetParamStr.nx[2][3]==0))
- mAverageFluktuations = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr();
- else
- mAverageFluktuations = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues[2], dataSetParamStr.nx[2][0], dataSetParamStr.nx[2][1], dataSetParamStr.nx[2][2], dataSetParamStr.nx[2][3]));
+ MPI_File file_handler;
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpRelaxationFactor.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
- SPtr<AverageValuesArray3D> mAverageTriplecorrelations;
- if ((dataSetParamStr.nx[3][0]==0)&&(dataSetParamStr.nx[3][1]==0)&&(dataSetParamStr.nx[3][2]==0)&&(dataSetParamStr.nx[3][3]==0))
- mAverageTriplecorrelations = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr();
- else
- mAverageTriplecorrelations = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues[3], dataSetParamStr.nx[3][0], dataSetParamStr.nx[3][1], dataSetParamStr.nx[3][2], dataSetParamStr.nx[3][3]));
+ // read count of blocks
+ int blocksCount = 0;
+ dataSetParam dataSetParamStr;
+ MPI_File_read_at(file_handler, (MPI_Offset)(rank * sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(size * sizeof(int)), &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
+
+ DataSetSmall* dataSetSmallArray = new DataSetSmall[blocksCount];
+ int doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ std::vector<double> doubleValuesArray(blocksCount * doubleCountInBlock); // double-values in all blocks
+
+ // define MPI_types depending on the block-specific information
+ MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
+ MPI_Type_commit(&dataSetDoubleType);
- SPtr<ShearStressValuesArray3D> mShearStressValues;
- if ((dataSetParamStr.nx[4][0]==0)&&(dataSetParamStr.nx[4][1]==0)&&(dataSetParamStr.nx[4][2]==0)&&(dataSetParamStr.nx[4][3]==0))
- mShearStressValues = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr();
- else
- mShearStressValues = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues[4], dataSetParamStr.nx[4][0], dataSetParamStr.nx[4][1], dataSetParamStr.nx[4][2], dataSetParamStr.nx[4][3]));
+ // calculate the read offset
+ MPI_Offset read_offset = (MPI_Offset)(size * sizeof(int));
+ size_t next_read_offset = 0;
- SPtr<RelaxationFactorArray3D> mRelaxationFactor;
- if ((dataSetParamStr.nx[5][0]==0)&&(dataSetParamStr.nx[5][1]==0)&&(dataSetParamStr.nx[5][2]==0))
- mRelaxationFactor = CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr();
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount*(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+ MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
+ }
else
- mRelaxationFactor = CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(vectorsOfValues[5], dataSetParamStr.nx[5][0], dataSetParamStr.nx[5][1], dataSetParamStr.nx[5][2]));
+ {
+ MPI_Recv(&read_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount*(sizeof(DataSetSmall) + doubleCountInBlock * sizeof(double));
+ if (rank < size - 1)
+ MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
+ }
+ }
- //SPtr<DistributionArray3D> mFdistributions(new D3Q27EsoTwist3DSplittedVector(dataSetParamStr.nx1, dataSetParamStr.nx2, dataSetParamStr.nx3, -999.0));
- SPtr<DistributionArray3D> mFdistributions(new D3Q27EsoTwist3DSplittedVector());
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount, dataSetSmallType, MPI_STATUS_IGNORE);
+ if (doubleCountInBlock > 0)
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmall)), &doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
+ MPI_File_close(&file_handler);
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues[6], dataSetParamStr.nx[6][0], dataSetParamStr.nx[6][1], dataSetParamStr.nx[6][2], dataSetParamStr.nx[6][3])));
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNonLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues[7], dataSetParamStr.nx[7][0], dataSetParamStr.nx[7][1], dataSetParamStr.nx[7][2], dataSetParamStr.nx[7][3])));
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setZeroDistributions(CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(vectorsOfValues[8], dataSetParamStr.nx[8][0], dataSetParamStr.nx[8][1], dataSetParamStr.nx[8][2])));
+ if (comm->isRoot())
+ {
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readRelaxationFactor time: " << finish - start << " s");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readRelaxationFactor start of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNX1(dataSetParamStr.nx1);
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNX2(dataSetParamStr.nx2);
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNX3(dataSetParamStr.nx3);
+ size_t index = 0;
+ size_t nextVectorSize = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ std::vector<double> vectorsOfValues;
+ for (int n = 0; n < blocksCount; n++)
+ {
+ vectorsOfValues.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + nextVectorSize);
+ index += nextVectorSize;
- SPtr<DataSet3D> dataSetPtr = SPtr<DataSet3D>(new DataSet3D());
- dataSetPtr->setAverageDencity(mAverageDensity);
- dataSetPtr->setAverageVelocity(mAverageVelocity);
- dataSetPtr->setAverageFluctuations(mAverageFluktuations);
- dataSetPtr->setAverageTriplecorrelations(mAverageTriplecorrelations);
- dataSetPtr->setShearStressValues(mShearStressValues);
- dataSetPtr->setRelaxationFactor(mRelaxationFactor);
- dataSetPtr->setFdistributions(mFdistributions);
+ // fill RelaxationFactor array
+ SPtr<RelaxationFactorArray3D> mRelaxationFactor;
+ //if ((dataSetParamStr.nx[0] == 0) && (dataSetParamStr.nx[1] == 0) && (dataSetParamStr.nx[2] == 0))
+ // mRelaxationFactor = CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr();
+ //else
+ mRelaxationFactor = CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2]));
// find the nesessary block and fill it
- SPtr<Block3D> block = grid->getBlock(dataSetArray[n].x1, dataSetArray[n].x2, dataSetArray[n].x3, dataSetArray[n].level);
- UbTupleInt3 blockNX = grid->getBlockNX();
- this->lbmKernel->setNX(std::array<int, 3>{ {val<1>(blockNX), val<2>(blockNX), val<3>(blockNX)}});
- SPtr<LBMKernel> kernel = this->lbmKernel->clone();
- kernel->setGhostLayerWidth(dataSetArray[n].ghostLayerWidth);
- kernel->setCollisionFactor(dataSetArray[n].collFactor);
- kernel->setDeltaT(dataSetArray[n].deltaT);
- kernel->setCompressible(dataSetArray[n].compressible);
- kernel->setWithForcing(dataSetArray[n].withForcing);
- kernel->setDataSet(dataSetPtr);
- block->setKernel(kernel);
- //block->getKernel()->setDataSet(dataSetPtr);
+ SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].x1, dataSetSmallArray[n].x2, dataSetSmallArray[n].x3, dataSetSmallArray[n].level);
+ block->kernel->getDataSet()->setRelaxationFactor(mRelaxationFactor);
}
- delete[] dataSetArray;
+ MPI_Type_free(&dataSetDoubleType);
if (comm->isRoot())
{
- UBLOG(logINFO, "MPIIORestartCoProcessor::readDataSet end of restore of data, rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readRelaxationFactor end of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
+
+ delete[] dataSetSmallArray;
}
void MPIIORestartCoProcessor::readBoundaryConds(int step)
@@ -1215,16 +2538,16 @@ void MPIIORestartCoProcessor::readBoundaryConds(int step)
if (comm->isRoot())
{
- UBLOG(logINFO, "MPIIORestartCoProcessor::readBoundaryConds start MPI IO rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readBoundaryConds start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
if (comm->isRoot()) start = MPI_Wtime();
MPI_File file_handler;
- std::string filename = path+"/mpi_io_cp/mpi_io_cp_"+UbSystem::toString(step)+"/cpBC.bin";
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpBC.bin";
int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
- if (rc!=MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file "+filename);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
int blocksCount = 0;
int dataCount1000 = 0;
@@ -1234,21 +2557,14 @@ void MPIIORestartCoProcessor::readBoundaryConds(int step)
// read count of blocks
MPI_File_read_at(file_handler, read_offset1, &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
// read count of big BoundaryCondition blocks
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset1+sizeof(int)), &dataCount1000, 1, MPI_INT, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset1 + sizeof(int)), &dataCount1000, 1, MPI_INT, MPI_STATUS_IGNORE);
// read count of indexContainer values in all blocks
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset1+2*sizeof(int)), &dataCount2, 1, MPI_INT, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset1 + 2 * sizeof(int)), &dataCount2, 1, MPI_INT, MPI_STATUS_IGNORE);
// read count of bcindexmatrix values in every block
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset1+3*sizeof(int)), &boundCondParamStr, 1, boundCondParamType, MPI_STATUS_IGNORE);
-
- //std::cout << "rank=" << rank << ",(rank*read_offset1)=" << rank*read_offset1 << ",blocksCount=" << blocksCount;
- //std::cout << ", " << rank*read_offset1 + sizeof(int) << ",bcBlockCount=" << dataCount1000;
- //std::cout << ", " << rank*read_offset1 + 2 * sizeof(int) << ",count_indexContainer=" << dataCount2;
- //std::cout << ", " << rank*read_offset1 + 3 * sizeof(int) << ",boundCondParamStr=" << boundCondParamStr.bcindexmatrixCount << std::endl;
- //std::cout << "readrank=" << rank << ",blocksCount=" << blocksCount << ", " << dataCount1000 << ", " << dataCount2 << ", " << boundCondParamStr.bcindexmatrixCount << std::endl;
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset1 + 3 * sizeof(int)), &boundCondParamStr, 1, boundCondParamType, MPI_STATUS_IGNORE);
MPI_Type_contiguous(boundCondParamStr.bcindexmatrixCount, MPI_INT, &bcindexmatrixType);
MPI_Type_commit(&bcindexmatrixType);
- mpiTypeFreeFlag = true;
size_t dataCount = dataCount1000 * BLOCK_SIZE;
BCAdd* bcAddArray = new BCAdd[blocksCount];
@@ -1263,35 +2579,33 @@ void MPIIORestartCoProcessor::readBoundaryConds(int step)
if (size>1)
{
- if (rank==0)
+ if (rank == 0)
{
- next_read_offset = read_offset+blocksCount*sizeof(BCAdd)+dataCount*sizeof(BoundaryCondition)+(blocksCount * boundCondParamStr.bcindexmatrixCount + dataCount2)*sizeof(int);
+ next_read_offset = read_offset + blocksCount * sizeof(BCAdd) + dataCount * sizeof(BoundaryCondition) + (blocksCount * boundCondParamStr.bcindexmatrixCount + dataCount2) * sizeof(int);
MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
}
else
{
- MPI_Recv(&read_offset, 1, MPI_LONG_LONG_INT, rank-1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
- next_read_offset = read_offset+blocksCount*sizeof(BCAdd)+dataCount*sizeof(BoundaryCondition)+(blocksCount * boundCondParamStr.bcindexmatrixCount + dataCount2)*sizeof(int);
- if (rank<size-1)
- MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, rank+1, 5, MPI_COMM_WORLD);
+ MPI_Recv(&read_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+ next_read_offset = read_offset + blocksCount * sizeof(BCAdd) + dataCount * sizeof(BoundaryCondition) + (blocksCount * boundCondParamStr.bcindexmatrixCount + dataCount2) * sizeof(int);
+ if (rank<size - 1)
+ MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
}
}
- //std::cout << "readrank=" << rank << ",read_offset=" << read_offset << ", " << read_offset + blocksCount * sizeof(BCAdd) << ", " << read_offset + blocksCount * sizeof(BCAdd) + dataCount * sizeof(BoundaryCondition) << ", " << read_offset + blocksCount * sizeof(BCAdd) + dataCount * sizeof(BoundaryCondition) + blocksCount * bcindexmatrixCount * sizeof(int) << std::endl;
MPI_File_read_at(file_handler, read_offset, bcAddArray, blocksCount, boundCondTypeAdd, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset+blocksCount*sizeof(BCAdd)), &bcArray[0], dataCount1000, boundCondType1000, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset+blocksCount*sizeof(BCAdd)+dataCount*sizeof(BoundaryCondition)), &intArray1[0], blocksCount, bcindexmatrixType, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset+blocksCount*sizeof(BCAdd)+dataCount*sizeof(BoundaryCondition)+blocksCount * boundCondParamStr.bcindexmatrixCount*sizeof(int)), &intArray2[0], dataCount2, MPI_INT, MPI_STATUS_IGNORE);
- //MPI_File_sync(file_handler);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + blocksCount * sizeof(BCAdd)), &bcArray[0], dataCount1000, boundCondType1000, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + blocksCount * sizeof(BCAdd) + dataCount * sizeof(BoundaryCondition)), &intArray1[0], blocksCount, bcindexmatrixType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + blocksCount * sizeof(BCAdd) + dataCount * sizeof(BoundaryCondition) + blocksCount * boundCondParamStr.bcindexmatrixCount * sizeof(int)), &intArray2[0], dataCount2, MPI_INT, MPI_STATUS_IGNORE);
MPI_File_close(&file_handler);
if (comm->isRoot())
{
finish = MPI_Wtime();
- UBLOG(logINFO, "MPIIORestartCoProcessor::readBoundaryConds time: "<<finish-start<<" s");
- UBLOG(logINFO, "MPIIORestartCoProcessor::readBoundaryConds start of restore of data, rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readBoundaryConds time: " << finish - start << " s");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readBoundaryConds start of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
int index = 0, index1 = 0, index2 = 0;
@@ -1308,7 +2622,7 @@ void MPIIORestartCoProcessor::readBoundaryConds(int step)
for (size_t ibc = 0; ibc<bcAddArray[n].boundCond_count; ibc++)
{
SPtr<BoundaryConditions> bc;
- if (memcmp(&bcArray[index], nullBouCond, sizeof(BoundaryCondition))==0)
+ if (memcmp(&bcArray[index], nullBouCond, sizeof(BoundaryCondition)) == 0)
bc = SPtr<BoundaryConditions>();
else
{
@@ -1334,20 +2648,6 @@ void MPIIORestartCoProcessor::readBoundaryConds(int step)
for (int iq = 0; iq<26; iq++)
bc->setQ(bcArray[index].q[iq], iq);
bc->setBcAlgorithmType(bcArray[index].algorithmType);
-
- //if (bcVectorGW[index].nx1 != bc->nx1)
- // std::cout << "readBoundaryConds nx1 !!!!===" << bcVectorGW[index].nx1 << " ---- " << bc->nx1 << std::endl;
- //if (bcVectorGW[index].nx2 != bc->nx2)
- // std::cout << "readBoundaryConds nx2 !!!!===" << bcVectorGW[index].nx2 << " ---- " << bc->nx2 << std::endl;
- //if (bcVectorGW[index].nx3 != bc->nx3)
- // std::cout << "readBoundaryConds nx3 !!!!===" << bcVectorGW[index].nx3 << " ---- " << bc->nx3 << std::endl;
- //if (bcVectorGW[index].algorithmType != bc->algorithmType)
- // std::cout << "readBoundaryConds algorithmType !!!!===" << bcVectorGW[index].algorithmType << " ---- " << bc->algorithmType << std::endl;
- //for (int iq = 0; iq<26; iq++)
- // if (bcVectorGW[index].q[iq] != bc->q[iq])
- // std::cout << "readBoundaryConds q !!!!===" /*<< bcVectorGW[index].q << " ---- " << bc->q*/ << std::endl;
- //std::cout << "readBoundaryConds SPtr<BoundaryConditions> !!!!===" <<std::endl;
-
}
bcVector.push_back(bc);
@@ -1355,32 +2655,21 @@ void MPIIORestartCoProcessor::readBoundaryConds(int step)
}
for (int b1 = 0; b1 < boundCondParamStr.bcindexmatrixCount; b1++)
- {
- //if (bcindexmatrixVGW[index1] != intArray1[index1])
- // std::cout << "readBoundaryConds bcindexmatrixVGW !!!!===" << std::endl;
bcindexmatrixV.push_back(intArray1[index1++]);
- }
+
for (int b2 = 0; b2 < bcAddArray[n].indexContainer_count; b2++)
- {
- //if (indexContainerVGW[index2] != intArray2[index2])
- // std::cout << "readBoundaryConds indexContainerVGW !!!!===" << std::endl;
indexContainerV.push_back(intArray2[index2++]);
- }
CbArray3D<int, IndexerX3X2X1> bcim(bcindexmatrixV, boundCondParamStr.nx1, boundCondParamStr.nx2, boundCondParamStr.nx3);
SPtr<Block3D> block = grid->getBlock(bcAddArray[n].x1, bcAddArray[n].x2, bcAddArray[n].x3, bcAddArray[n].level);
- //if(!block) std::cout << "readBoundaryConds can't find the block!!!" << std::endl;
SPtr<BCProcessor> bcProc = bcProcessor->clone(block->getKernel());
- //if(!bcProc) std::cout << "readBoundaryConds can't find the bcProc!!!" << std::endl;
SPtr<BCArray3D> bcArr(new BCArray3D());
bcArr->bcindexmatrix = bcim;
bcArr->bcvector = bcVector;
bcArr->indexContainer = indexContainerV;
bcProc->setBCArray(bcArr);
-
- //if (!(block->getKernel()))
- // std::cout << "readBoundaryConds kernel=" << block->getKernel() <<" "<< bcAddArray[n].x1 << " " << bcAddArray[n].x2 << " " << bcAddArray[n].x3 << std::endl;
+
block->getKernel()->setBCProcessor(bcProc);
}
@@ -1389,11 +2678,13 @@ void MPIIORestartCoProcessor::readBoundaryConds(int step)
delete[] bcAddArray;
delete[] intArray1;
delete[] intArray2;
-
+
+ MPI_Type_free(&bcindexmatrixType);
+
if (comm->isRoot())
{
- UBLOG(logINFO, "MPIIORestartCoProcessor::readBoundaryConds end of restore of data, rank = "<<rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "<<Utilities::getPhysMemUsedByMe()/1073741824.0<<" GB");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readBoundaryConds end of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
}
//////////////////////////////////////////////////////////////////////////
diff --git a/source/VirtualFluidsCore/CoProcessors/MPIIORestartCoProcessor.h b/source/VirtualFluidsCore/CoProcessors/MPIIORestartCoProcessor.h
index f6758f9d8..fa95bed83 100644
--- a/source/VirtualFluidsCore/CoProcessors/MPIIORestartCoProcessor.h
+++ b/source/VirtualFluidsCore/CoProcessors/MPIIORestartCoProcessor.h
@@ -1,5 +1,5 @@
-#ifndef _MPIIORestart11CoProcessor_H_
-#define _MPIIORestart11CoProcessor_H_
+#ifndef _MPIIORestartCoProcessor_H_
+#define _MPIIORestartCoProcessor_H_
#include <mpi.h>
#include <PointerDefinitions.h>
@@ -15,14 +15,13 @@ class LBMKernel;
//! \class MPIWriteBlocksCoProcessor
//! \brief Writes the grid each timestep into the files and reads the grip from the files before regenerating
-//! \author Alena Karanchuk
-class MPIIORestartCoProcessor: public CoProcessor
+class MPIIORestartCoProcessor : public CoProcessor
{
//! \struct GridParam
//! \brief Structure describes parameters of the grid
//! \details The structure is nessasary to restore the grid correctly
struct GridParam
- {
+ {
double trafoParams[33];
double deltaX;
int blockNx1;
@@ -36,83 +35,93 @@ class MPIIORestartCoProcessor: public CoProcessor
bool periodicX3;
bool active;
bool transformation;
- };
+ };
//! \struct Block3d
//! \brief Structure contains information of the block
//! \details The structure is used to write the data describing the block in the grid when saving the grid
//! and to read it when restoring the grid
struct Block3d
- {
- int x1;
- int x2;
- int x3;
- int bundle;
- int rank;
- int lrank;
- int part;
- int globalID;
- int localID;
- int level;
- int interpolationFlagCF;
- int interpolationFlagFC;
- int counter;
- bool active;
- };
+ {
+ int x1;
+ int x2;
+ int x3;
+ int bundle;
+ int rank;
+ int lrank;
+ int part;
+ int globalID;
+ int localID;
+ int level;
+ int interpolationFlagCF;
+ int interpolationFlagFC;
+ int counter;
+ bool active;
+ };
//! \struct dataSetParam
- //! \brief Structure describes parameters of the dataSet that are equal in all blocks
+ //! \brief Structure describes parameters of one array of the dataSet that are equal in all blocks
//! \details The structure used to store some parameters needed to restore dataSet arrays
struct dataSetParam
{
- int nx1;
+ int nx1;
int nx2;
int nx3;
- int nx[9][4]; // 9 arrays x (nx1, nx2, nx3, nx4)
- int doubleCountInBlock; // how many double-values are in all arrays dataSet in one (any) block
+ int nx[4]; // nx1, nx2, nx3, nx4
};
//! \struct dataSet
//! \brief Structure containes information identifying the block
//! \details The structure is used to find the needed block in the grid when restoring a dataSet
struct DataSet
- {
+ {
double collFactor;
double deltaT;
- int x1; // to find the right block
- int x2;
- int x3;
- int level;
+ int x1;
+ int x2;
+ int x3;
+ int level;
int ghostLayerWidth;
bool compressible;
bool withForcing;
};
-
+
+ //! \struct dataSetSmall
+ //! \brief Structure containes information identifying the block
+ //! \details The structure is used to find the needed block in the grid when restoring a dataSet arrays
+ struct DataSetSmall
+ {
+ int x1;
+ int x2;
+ int x3;
+ int level;
+ };
+
//! \struct BoundaryCondition
//! \brief Structure containes information about boundary conditions of the block
//! \details The structure is used to write data describing boundary conditions of the blocks when saving the grid
//! and to read it when restoring the grid
struct BoundaryCondition
- {
- long long noslipBoundaryFlags; // MPI_LONG_LONG
- long long slipBoundaryFlags;
- long long velocityBoundaryFlags;
- long long densityBoundaryFlags;
- long long wallModelBoundaryFlags;
-
- float bcVelocityX1;
- float bcVelocityX2;
- float bcVelocityX3;
- float bcDensity;
-
- float bcLodiDensity;
- float bcLodiVelocityX1;
- float bcLodiVelocityX2;
- float bcLodiVelocityX3;
- float bcLodiLentgh;
-
- float nx1,nx2,nx3;
- float q[26]; // MPI_FLOAT
+ {
+ long long noslipBoundaryFlags; // MPI_LONG_LONG
+ long long slipBoundaryFlags;
+ long long velocityBoundaryFlags;
+ long long densityBoundaryFlags;
+ long long wallModelBoundaryFlags;
+
+ float bcVelocityX1;
+ float bcVelocityX2;
+ float bcVelocityX3;
+ float bcDensity;
+
+ float bcLodiDensity;
+ float bcLodiVelocityX1;
+ float bcLodiVelocityX2;
+ float bcLodiVelocityX3;
+ float bcLodiLentgh;
+
+ float nx1, nx2, nx3;
+ float q[26]; // MPI_FLOAT
char algorithmType;
};
@@ -127,22 +136,32 @@ class MPIIORestartCoProcessor: public CoProcessor
int nx3;
int bcindexmatrixCount; // how many bcindexmatrix-values in one (any) block
};
-
+
//! \struct BCAdd
//! \brief Structure containes information identifying the block
//! and some parameters of the arrays of boundary conditions that are equal in all blocks
//! \details The structure is used to find the needed block in the grid when restoring a dataSet
//! and to set common parameters
struct BCAdd
- {
- int x1; // to find the right block
- int x2;
- int x3;
- int level;
+ {
+ int x1; // to find the right block
+ int x2;
+ int x3;
+ int level;
int boundCond_count; // how many BoundaryCondition-structures are in this block
int indexContainer_count; // how many indexContainer-values are in this block
};
+ struct DSArraysPresence
+ {
+ bool isAverageDensityArrayPresent;
+ bool isAverageVelocityArrayPresent;
+ bool isAverageFluktuationsArrayPresent;
+ bool isAverageTripleArrayPresent;
+ bool isShearStressValArrayPresent;
+ bool isRelaxationFactorPresent;
+ };
+
public:
MPIIORestartCoProcessor(SPtr<Grid3D> grid, SPtr<UbScheduler> s, const std::string& path, SPtr<Communicator> comm);
virtual ~MPIIORestartCoProcessor();
@@ -150,17 +169,30 @@ public:
void process(double step);
//! Reads the grid from the files before grid reconstruction
void restart(int step);
- //! Writes the blocks of the grid into the file outputBlocks.bin
+ //! Writes the blocks of the grid into the file cpBlocks.bin
void writeBlocks(int step);
- //! Writes the datasets of the blocks into the file outputDataSet.bin
+ //! Writes the datasets of the blocks into the file cpDataSet.bin
void writeDataSet(int step);
- //! Writes the boundary conditions of the blocks into the file outputBoundCond.bin
+ void writeAverageDensityArray(int step);
+ void writeAverageVelocityArray(int step);
+ void writeAverageFluktuationsArray(int step);
+ void writeAverageTripleArray(int step);
+ void writeShearStressValArray(int step);
+ void writeRelaxationFactor(int step);
+ //! Writes the boundary conditions of the blocks into the file cpBC.bin
void writeBoundaryConds(int step);
- //! Reads the blocks of the grid from the file outputBlocks.bin
+
+ //! Reads the blocks of the grid from the file cpBlocks.bin
void readBlocks(int step);
- //! Reads the datasets of the blocks from the file outputDataSet.bin
+ //! Reads the datasets of the blocks from the file cpDataSet.bin
void readDataSet(int step);
- //! Reads the boundary conditions of the blocks from the file outputBoundCond.bin
+ void readAverageDensityArray(int step);
+ void readAverageVelocityArray(int step);
+ void readAverageFluktuationsArray(int step);
+ void readAverageTripleArray(int step);
+ void readShearStressValArray(int step);
+ void readRelaxationFactor(int step);
+ //! Reads the boundary conditions of the blocks from the file cpBC.bin
void readBoundaryConds(int step);
//! The function sets LBMKernel
void setLBMKernel(SPtr<LBMKernel> kernel);
@@ -172,11 +204,12 @@ public:
protected:
std::string path;
SPtr<Communicator> comm;
- bool mpiTypeFreeFlag;
private:
- MPI_Datatype gridParamType, block3dType, dataSetParamType, dataSetType, dataSetDoubleType, boundCondParamType, boundCondType, boundCondType1000, boundCondTypeAdd, bcindexmatrixType;
- dataSetParam dataSetParamStr;
+ MPI_Datatype gridParamType, block3dType, arrayPresenceType;
+ MPI_Datatype dataSetParamType, dataSetType, dataSetSmallType, dataSetDoubleType;
+ MPI_Datatype boundCondParamType, boundCondType, boundCondType1000, boundCondTypeAdd, bcindexmatrixType;
+
boundCondParam boundCondParamStr;
SPtr<LBMKernel> lbmKernel;
SPtr<BCProcessor> bcProcessor;
diff --git a/source/VirtualFluidsCore/CoProcessors/WriteMQFromSelectionCoProcessor.cpp b/source/VirtualFluidsCore/CoProcessors/WriteMQFromSelectionCoProcessor.cpp
new file mode 100644
index 000000000..1d3221ddf
--- /dev/null
+++ b/source/VirtualFluidsCore/CoProcessors/WriteMQFromSelectionCoProcessor.cpp
@@ -0,0 +1,237 @@
+#include "WriteMQFromSelectionCoProcessor.h"
+#include "LBMKernel.h"
+#include "BCProcessor.h"
+#include <vector>
+#include <string>
+
+#include "basics/writer/WbWriterVtkXmlASCII.h"
+#include "DataSet3D.h"
+#include "UbScheduler.h"
+#include "Grid3D.h"
+#include "Communicator.h"
+#include "LBMUnitConverter.h"
+#include "Block3D.h"
+#include "BCArray3D.h"
+#include "GbObject3D.h"
+
+WriteMQFromSelectionCoProcessor::WriteMQFromSelectionCoProcessor()
+{
+
+}
+//////////////////////////////////////////////////////////////////////////
+WriteMQFromSelectionCoProcessor::WriteMQFromSelectionCoProcessor(SPtr<Grid3D> grid, SPtr<UbScheduler> s,
+ SPtr<GbObject3D> gbObject,
+ const std::string& path, WbWriter* const writer,
+ SPtr<LBMUnitConverter> conv,
+ SPtr<Communicator> comm)
+ : CoProcessor(grid, s),
+ gbObject(gbObject),
+ path(path),
+ writer(writer),
+ conv(conv),
+ comm(comm)
+{
+ gridRank = comm->getProcessID();
+ minInitLevel = this->grid->getCoarsestInitializedLevel();
+ maxInitLevel = this->grid->getFinestInitializedLevel();
+
+ blockVector.resize(maxInitLevel+1);
+
+ for (int level = minInitLevel; level<=maxInitLevel; level++)
+ {
+ grid->getBlocks(level, gridRank, true, blockVector[level]);
+ }
+}
+//////////////////////////////////////////////////////////////////////////
+void WriteMQFromSelectionCoProcessor::init()
+{
+
+}
+//////////////////////////////////////////////////////////////////////////
+void WriteMQFromSelectionCoProcessor::process(double step)
+{
+ if(scheduler->isDue(step) )
+ collectData(step);
+
+ UBLOG(logDEBUG3, "WriteMQFromSelectionCoProcessor::update:" << step);
+}
+//////////////////////////////////////////////////////////////////////////
+void WriteMQFromSelectionCoProcessor::collectData(double step)
+{
+ int istep = static_cast<int>(step);
+
+ for(int level = minInitLevel; level<=maxInitLevel;level++)
+ {
+ for(SPtr<Block3D> block : blockVector[level])
+ {
+ if (block)
+ {
+ UbTupleDouble3 org = grid->getBlockWorldCoordinates(block);
+ UbTupleDouble3 blockLengths = grid->getBlockLengths(block);
+
+ double minX1 = val<1>(org);
+ double minX2 = val<2>(org);
+ double minX3 = val<3>(org);
+ double maxX1 = val<1>(org)+val<1>(blockLengths);
+ double maxX2 = val<2>(org)+val<2>(blockLengths);
+ double maxX3 = val<3>(org)+val<3>(blockLengths);
+
+ if (gbObject->isCellInsideOrCuttingGbObject3D(minX1, minX2, minX3, maxX1, maxX2, maxX3))
+ {
+ addDataMQ(block);
+ }
+ }
+ }
+ }
+
+ std::string pfilePath, partPath, subfolder, cfilePath;
+
+ subfolder = "mqSelect"+UbSystem::toString(istep);
+ pfilePath = path+"/mqSelect/"+subfolder;
+ cfilePath = path+"/mqSelect/mq_collection";
+ partPath = pfilePath+"/mqSelect"+UbSystem::toString(gridRank)+ "_" + UbSystem::toString(istep);
+
+
+ std::string partName = writer->writeNodesWithNodeData(partPath,nodes,datanames,data);
+ size_t found=partName.find_last_of("/");
+ std::string piece = partName.substr(found+1);
+ piece = subfolder + "/" + piece;
+
+ std::vector<std::string> cellDataNames;
+ SPtr<Communicator> comm = Communicator::getInstance();
+ std::vector<std::string> pieces = comm->gather(piece);
+ if (comm->getProcessID() == comm->getRoot())
+ {
+ std::string pname = WbWriterVtkXmlASCII::getInstance()->writeParallelFile(pfilePath,pieces,datanames,cellDataNames);
+ found=pname.find_last_of("/");
+ piece = pname.substr(found+1);
+
+ std::vector<std::string> filenames;
+ filenames.push_back(piece);
+ if (step == CoProcessor::scheduler->getMinBegin())
+ {
+ WbWriterVtkXmlASCII::getInstance()->writeCollection(cfilePath,filenames,istep,false);
+ }
+ else
+ {
+ WbWriterVtkXmlASCII::getInstance()->addFilesToCollection(cfilePath,filenames,istep,false);
+ }
+ UBLOG(logINFO,"WriteMQFromSelectionCoProcessor step: " << istep);
+ }
+
+ clearData();
+}
+//////////////////////////////////////////////////////////////////////////
+void WriteMQFromSelectionCoProcessor::clearData()
+{
+ nodes.clear();
+ datanames.clear();
+ data.clear();
+}
+//////////////////////////////////////////////////////////////////////////
+void WriteMQFromSelectionCoProcessor::addDataMQ(SPtr<Block3D> block)
+{
+ double level = (double)block->getLevel();
+ double blockID = (double)block->getGlobalID();
+
+ //Diese Daten werden geschrieben:
+ datanames.resize(0);
+ datanames.push_back("Rho");
+ datanames.push_back("Vx");
+ datanames.push_back("Vy");
+ datanames.push_back("Vz");
+ //datanames.push_back("Press");
+ datanames.push_back("Level");
+ //datanames.push_back("BlockID");
+
+
+
+ data.resize(datanames.size());
+
+ SPtr<ILBMKernel> kernel = block->getKernel();
+ SPtr<BCArray3D> bcArray = kernel->getBCProcessor()->getBCArray();
+ SPtr<DistributionArray3D> distributions = kernel->getDataSet()->getFdistributions();
+ LBMReal f[D3Q27System::ENDF+1];
+ LBMReal vx1,vx2,vx3,rho;
+
+ if(block->getKernel()->getCompressible())
+ {
+ calcMacros = &D3Q27System::calcCompMacroscopicValues;
+ }
+ else
+ {
+ calcMacros = &D3Q27System::calcIncompMacroscopicValues;
+ }
+
+ int minX1 = 0;
+ int minX2 = 0;
+ int minX3 = 0;
+
+ int maxX1 = (int)(distributions->getNX1());
+ int maxX2 = (int)(distributions->getNX2());
+ int maxX3 = (int)(distributions->getNX3());
+
+ //int minX1 = 1;
+ //int minX2 = 1;
+ //int minX3 = 1;
+
+ //int maxX1 = (int)(distributions->getNX1());
+ //int maxX2 = (int)(distributions->getNX2());
+ //int maxX3 = (int)(distributions->getNX3());
+
+ //nummern vergeben und node vector erstellen + daten sammeln
+ CbArray3D<int> nodeNumbers((int)maxX1, (int)maxX2, (int)maxX3,-1);
+ maxX1 -= 2;
+ maxX2 -= 2;
+ maxX3 -= 2;
+
+ //D3Q27BoundaryConditionPtr bcPtr;
+ int nr = (int)nodes.size();
+
+ for(int ix3=minX3; ix3<=maxX3; ix3++)
+ {
+ for(int ix2=minX2; ix2<=maxX2; ix2++)
+ {
+ for(int ix1=minX1; ix1<=maxX1; ix1++)
+ {
+ if(!bcArray->isUndefined(ix1,ix2,ix3) && !bcArray->isSolid(ix1,ix2,ix3))
+ {
+ Vector3D worldCoordinates = grid->getNodeCoordinates(block, ix1, ix2, ix3);
+ if (gbObject->isPointInGbObject3D(worldCoordinates[0], worldCoordinates[1], worldCoordinates[2]))
+ {
+ int index = 0;
+ nodeNumbers(ix1, ix2, ix3) = nr++;
+
+ nodes.push_back(UbTupleFloat3(float(worldCoordinates[0]),
+ float(worldCoordinates[1]),
+ float(worldCoordinates[2])));
+
+ distributions->getDistribution(f, ix1, ix2, ix3);
+ calcMacros(f, rho, vx1, vx2, vx3);
+
+ if (UbMath::isNaN(rho) || UbMath::isInfinity(rho))
+ UB_THROW(UbException(UB_EXARGS, "rho is not a number (nan or -1.#IND) or infinity number -1.#INF in block="+block->toString()+
+ ", node="+UbSystem::toString(ix1)+","+UbSystem::toString(ix2)+","+UbSystem::toString(ix3)));
+ if (UbMath::isNaN(vx1) || UbMath::isInfinity(vx1))
+ UB_THROW(UbException(UB_EXARGS, "vx1 is not a number (nan or -1.#IND) or infinity number -1.#INF in block="+block->toString()+
+ ", node="+UbSystem::toString(ix1)+","+UbSystem::toString(ix2)+","+UbSystem::toString(ix3)));
+ //vx1=999.0;
+ if (UbMath::isNaN(vx2) || UbMath::isInfinity(vx2))
+ UB_THROW(UbException(UB_EXARGS, "vx2 is not a number (nan or -1.#IND) or infinity number -1.#INF in block="+block->toString()+
+ ", node="+UbSystem::toString(ix1)+","+UbSystem::toString(ix2)+","+UbSystem::toString(ix3)));
+ //vx2=999.0;
+ if (UbMath::isNaN(vx3) || UbMath::isInfinity(vx3))
+ UB_THROW(UbException(UB_EXARGS, "vx3 is not a number (nan or -1.#IND) or infinity number -1.#INF in block="+block->toString()+
+ ", node="+UbSystem::toString(ix1)+","+UbSystem::toString(ix2)+","+UbSystem::toString(ix3)));
+
+ data[index++].push_back(rho);
+ data[index++].push_back(vx1);
+ data[index++].push_back(vx2);
+ data[index++].push_back(vx3);
+ data[index++].push_back(level);
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/source/VirtualFluidsCore/CoProcessors/WriteMQFromSelectionCoProcessor.h b/source/VirtualFluidsCore/CoProcessors/WriteMQFromSelectionCoProcessor.h
new file mode 100644
index 000000000..886fed0f8
--- /dev/null
+++ b/source/VirtualFluidsCore/CoProcessors/WriteMQFromSelectionCoProcessor.h
@@ -0,0 +1,57 @@
+#ifndef WriteMQFromSelectionCoProcessor_H
+#define WriteMQFromSelectionCoProcessor_H
+
+#include <PointerDefinitions.h>
+#include <string>
+#include <vector>
+
+#include "CoProcessor.h"
+
+#include "LBMSystem.h"
+
+class Communicator;
+class Grid3D;
+class UbScheduler;
+class LBMUnitConverter;
+class WbWriter;
+class Block3D;
+class GbObject3D;
+
+class WriteMQFromSelectionCoProcessor : public CoProcessor
+{
+public:
+ WriteMQFromSelectionCoProcessor();
+ WriteMQFromSelectionCoProcessor(SPtr<Grid3D> grid, SPtr<UbScheduler> s,
+ SPtr<GbObject3D> gbObject,
+ const std::string& path, WbWriter* const writer,
+ SPtr<LBMUnitConverter> conv, SPtr<Communicator> comm);
+ ~WriteMQFromSelectionCoProcessor(){}
+
+ void process(double step) override;
+
+protected:
+ void collectData(double step);
+ void addDataMQ(SPtr<Block3D> block);
+ void clearData();
+
+private:
+ void init();
+ std::vector<UbTupleFloat3> nodes;
+ std::vector<std::string> datanames;
+ std::vector<std::vector<double> > data;
+ std::string path;
+ WbWriter* writer;
+ SPtr<LBMUnitConverter> conv;
+ bool bcInformation;
+ std::vector<std::vector<SPtr<Block3D> > > blockVector;
+ int minInitLevel;
+ int maxInitLevel;
+ int gridRank;
+ SPtr<Communicator> comm;
+ SPtr<GbObject3D> gbObject;
+
+ typedef void(*CalcMacrosFct)(const LBMReal* const& /*feq[27]*/, LBMReal& /*(d)rho*/, LBMReal& /*vx1*/, LBMReal& /*vx2*/, LBMReal& /*vx3*/);
+ CalcMacrosFct calcMacros;
+};
+
+#endif
diff --git a/source/VirtualFluidsCore/CoProcessors/WriteMacroscopicQuantitiesCoProcessor.h b/source/VirtualFluidsCore/CoProcessors/WriteMacroscopicQuantitiesCoProcessor.h
index 6618468bf..ca6e2fcb4 100644
--- a/source/VirtualFluidsCore/CoProcessors/WriteMacroscopicQuantitiesCoProcessor.h
+++ b/source/VirtualFluidsCore/CoProcessors/WriteMacroscopicQuantitiesCoProcessor.h
@@ -1,5 +1,5 @@
-#ifndef D3Q27MACROSCOPICQUANTITIESCoProcessor_H
-#define D3Q27MACROSCOPICQUANTITIESCoProcessor_H
+#ifndef WriteMacroscopicQuantitiesCoProcessor_H
+#define WriteMacroscopicQuantitiesCoProcessor_H
#include <PointerDefinitions.h>
#include <string>
@@ -30,7 +30,6 @@ public:
protected:
void collectData(double step);
void addDataMQ(SPtr<Block3D> block);
- void addDataGeo(SPtr<Block3D> block);
void clearData();
private:
diff --git a/source/VirtualFluidsCore/Interactors/Interactor3D.cpp b/source/VirtualFluidsCore/Interactors/Interactor3D.cpp
index b8b17262c..d4380b076 100644
--- a/source/VirtualFluidsCore/Interactors/Interactor3D.cpp
+++ b/source/VirtualFluidsCore/Interactors/Interactor3D.cpp
@@ -218,12 +218,12 @@ void Interactor3D::setSolidBlock(SPtr<Block3D> block)
UbTupleDouble3 nodeOffset = grid.lock()->getNodeOffset(block);
//coordinates of block without ghost layer
- minX1 = val<1>(org) - val<1>(nodeOffset);
- minX2 = val<2>(org) - val<2>(nodeOffset);
- minX3 = val<3>(org) - val<3>(nodeOffset);
- maxX1 = val<1>(org) + val<1>(blockLengths) + val<1>(nodeOffset);
- maxX2 = val<2>(org) + val<2>(blockLengths) + val<2>(nodeOffset);
- maxX3 = val<3>(org) + val<3>(blockLengths) + val<3>(nodeOffset);
+ minX1 = val<1>(org) + val<1>(nodeOffset);
+ minX2 = val<2>(org) + val<2>(nodeOffset);
+ minX3 = val<3>(org) + val<3>(nodeOffset);
+ maxX1 = val<1>(org) + val<1>(blockLengths) - val<1>(nodeOffset);
+ maxX2 = val<2>(org) + val<2>(blockLengths) - val<2>(nodeOffset);
+ maxX3 = val<3>(org) + val<3>(blockLengths) - val<3>(nodeOffset);
if(this->isInverseSolid())
{
diff --git a/source/VirtualFluidsCore/Visitors/GenBlocksGridVisitor.cpp b/source/VirtualFluidsCore/Visitors/GenBlocksGridVisitor.cpp
index 2ad82c218..1d3f09c94 100644
--- a/source/VirtualFluidsCore/Visitors/GenBlocksGridVisitor.cpp
+++ b/source/VirtualFluidsCore/Visitors/GenBlocksGridVisitor.cpp
@@ -55,12 +55,20 @@ void GenBlocksGridVisitor::genBlocks(SPtr<Grid3D> grid)
double geoMaxX3 = boundingBox->getX3Maximum();
maxInd = grid->getBlockIndexes(geoMaxX1, geoMaxX2, geoMaxX3);
UbTupleDouble3 blockCoord = grid->getBlockWorldCoordinates(static_cast<int>(val<1>(maxInd)), static_cast<int>(val<2>(maxInd)), static_cast<int>(val<3>(maxInd)), 0);
- if (geoMaxX1 > val<1>(blockCoord))
- val<1>(maxInd) += 1;
- if (geoMaxX2 > val<2>(blockCoord))
- val<2>(maxInd) += 1;
- if (geoMaxX3 > val<3>(blockCoord))
- val<3>(maxInd) += 1;
+ //if (geoMaxX1 > val<1>(blockCoord))
+ // val<1>(maxInd) += 1;
+ //if (geoMaxX2 > val<2>(blockCoord))
+ // val<2>(maxInd) += 1;
+ //if (geoMaxX3 > val<3>(blockCoord))
+ // val<3>(maxInd) += 1;
+
+ double dx = grid->getDeltaX(0);
+ if (fabs(geoMaxX1-val<1>(blockCoord)) > dx)
+ val<1>(maxInd) += 1;
+ if (fabs(geoMaxX2-val<2>(blockCoord)) > dx)
+ val<2>(maxInd) += 1;
+ if (fabs(geoMaxX3-val<3>(blockCoord)) > dx)
+ val<3>(maxInd) += 1;
this->fillExtentWithBlocks(grid);
diff --git a/source/VirtualFluidsCore/Visitors/SpongeLayerBlockVisitor.cpp b/source/VirtualFluidsCore/Visitors/SpongeLayerBlockVisitor.cpp
index c9cd16c09..0db46d74d 100644
--- a/source/VirtualFluidsCore/Visitors/SpongeLayerBlockVisitor.cpp
+++ b/source/VirtualFluidsCore/Visitors/SpongeLayerBlockVisitor.cpp
@@ -4,16 +4,18 @@
#include "Grid3D.h"
#include "Block3D.h"
-#include "ILBMKernel.h"
+#include "LBMKernel.h"
#include "UbException.h"
-
-#include "CompressibleCumulant4thOrderViscosityLBMKernel.h"
-
+#include "D3Q27System.h"
#include <numerics/geometry3d/GbCuboid3D.h>
using namespace std;
-SpongeLayerBlockVisitor::SpongeLayerBlockVisitor() : Block3DVisitor(0, Grid3DSystem::MAXLEVEL)
+SpongeLayerBlockVisitor::SpongeLayerBlockVisitor(SPtr<GbCuboid3D> boundingBox, SPtr<LBMKernel> kernel, int dir) :
+ Block3DVisitor(0, Grid3DSystem::MAXLEVEL),
+ boundingBox(boundingBox),
+ kernel(kernel),
+ dir(dir)
{
}
@@ -71,15 +73,41 @@ void SpongeLayerBlockVisitor::visit(SPtr<Grid3D> grid, SPtr<Block3D> block)
double oldCollFactor = newKernel->getCollisionFactor();
- int ibX1 = block->getX1();
-
+
UbTupleInt3 ixMin = grid->getBlockIndexes(boundingBox->getX1Minimum(),boundingBox->getX2Minimum(),boundingBox->getX3Minimum());
UbTupleInt3 ixMax = grid->getBlockIndexes(boundingBox->getX1Maximum(),boundingBox->getX2Maximum(),boundingBox->getX3Maximum());
- int ibMax = val<1>(ixMax)-val<1>(ixMin)+1;
- double index = (double)(ibX1-val<1>(ixMin)+1);
+ double newCollFactor;
+
+ if (dir == D3Q27System::E)
+ {
+ int ibX1 = block->getX1();
+ int ibMax = val<1>(ixMax)-val<1>(ixMin)+1;
+ double index = (double)(ibX1-val<1>(ixMin)+1);
+ newCollFactor = oldCollFactor - (oldCollFactor-1.0)/(double)(ibMax)*index;
+ }
+ else if (dir == D3Q27System::W)
+ {
+ int ibX1 = block->getX1();
+ int ibMax = val<1>(ixMax)-val<1>(ixMin)+1;
+ double index = (double)(ibX1-val<1>(ixMin)+1);
+ newCollFactor = (oldCollFactor-1.0)/(double)(ibMax)*index;
+ }
+ else if (dir == D3Q27System::T)
+ {
+ int ibX3 = block->getX3();
+ int ibMax = val<3>(ixMax)-val<3>(ixMin)+1;
+ double index = (double)(ibX3-val<3>(ixMin)+1);
+ newCollFactor = oldCollFactor - (oldCollFactor-1.0)/(double)(ibMax)*index;
+ }
+ else if (dir == D3Q27System::B)
+ {
+ int ibX3 = block->getX3();
+ int ibMax = val<3>(ixMax)-val<3>(ixMin)+1;
+ double index = (double)(ibX3-val<3>(ixMin)+1);
+ newCollFactor = (oldCollFactor-1.0)/(double)(ibMax)*index;
+ }
- double newCollFactor = oldCollFactor - (oldCollFactor-1.0)/(double)(ibMax)*index;
newKernel->setCollisionFactor(newCollFactor);
block->setKernel(newKernel);
@@ -87,14 +115,6 @@ void SpongeLayerBlockVisitor::visit(SPtr<Grid3D> grid, SPtr<Block3D> block)
}
}
//////////////////////////////////////////////////////////////////////////
-void SpongeLayerBlockVisitor::setBoundingBox(SPtr<GbCuboid3D> bb)
-{
- boundingBox = bb;
-}
-//////////////////////////////////////////////////////////////////////////
-void SpongeLayerBlockVisitor::setKernel(SPtr<LBMKernel> k)
-{
- kernel = k;
-}
+
diff --git a/source/VirtualFluidsCore/Visitors/SpongeLayerBlockVisitor.h b/source/VirtualFluidsCore/Visitors/SpongeLayerBlockVisitor.h
index 9e4adc570..8518ab22b 100644
--- a/source/VirtualFluidsCore/Visitors/SpongeLayerBlockVisitor.h
+++ b/source/VirtualFluidsCore/Visitors/SpongeLayerBlockVisitor.h
@@ -15,18 +15,15 @@ class LBMKernel;
class SpongeLayerBlockVisitor : public Block3DVisitor
{
public:
- SpongeLayerBlockVisitor();
+ SpongeLayerBlockVisitor(SPtr<GbCuboid3D> boundingBox, SPtr<LBMKernel> kernel, int dir);
virtual ~SpongeLayerBlockVisitor();
void visit(SPtr<Grid3D> grid, SPtr<Block3D> block) override;
- void setBoundingBox(SPtr<GbCuboid3D> boundingBox);
- void setKernel(SPtr<LBMKernel> k);
-
private:
SPtr<GbCuboid3D> boundingBox;
SPtr<LBMKernel> kernel;
- LBMReal viscosity;
+ int dir;
};
#endif // SetSpongeLayerBlockVisitor_h__
--
GitLab