diff --git a/CMake/cmake_config_files/CSE01.config.cmake b/CMake/cmake_config_files/CSE01.config.cmake
index baa0f94981c2e9f9ac05b62468311f4bead32ff3..cad3f60ce31edac1069d1edce3fdd43b49a72b6e 100644
--- a/CMake/cmake_config_files/CSE01.config.cmake
+++ b/CMake/cmake_config_files/CSE01.config.cmake
@@ -2,22 +2,22 @@
#################################################################################
# BOOST
#################################################################################
-SET(BOOST_VERSION "1.60.0")
-SET(BOOST_ROOT "d:/boost/boost_1_60_0")
-SET(BOOST_DIR ${BOOST_ROOT})
-SET(BOOST_LIBRARYDIR ${BOOST_ROOT}"/stageMSVC64/lib")
+#SET(BOOST_VERSION "1.60.0")
+#SET(BOOST_ROOT "d:/boost/boost_1_60_0")
+#SET(BOOST_DIR ${BOOST_ROOT})
+#SET(BOOST_LIBRARYDIR ${BOOST_ROOT}"/stageMSVC64/lib")
#################################################################################
#################################################################################
# METIS
#################################################################################
-IF(${USE_METIS})
- SET(METIS_INCLUDEDIR "d:/metis-5.1.0/include")
- SET(METIS_DEBUG_LIBRARY "d:/metis-5.1.0/build/libmetis/Debug/metis.lib")
- SET(METIS_RELEASE_LIBRARY "d:/metis-5.1.0/build/libmetis/Release/metis.lib")
-ENDIF()
+#IF(${USE_METIS})
+# SET(METIS_INCLUDEDIR "d:/metis-5.1.0/include")
+# SET(METIS_DEBUG_LIBRARY "d:/metis-5.1.0/build/libmetis/Debug/metis.lib")
+# SET(METIS_RELEASE_LIBRARY "d:/metis-5.1.0/build/libmetis/Release/metis.lib")
+#ENDIF()
#################################################################################
# VTK
#################################################################################
-set(VTK_DIR "d:/tools/VTK/build/VTK-8.2.0")
+#set(VTK_DIR "d:/tools/VTK/build/VTK-8.2.0")
#################################################################################
\ No newline at end of file
diff --git a/apps/cpu/Multiphase/Multiphase.cpp b/apps/cpu/Multiphase/Multiphase.cpp
index af7b309adfe24bf5ece7446f83db6b198d3bfcf2..bb99f5efdb092fb69121b9025663918cd9807d07 100644
--- a/apps/cpu/Multiphase/Multiphase.cpp
+++ b/apps/cpu/Multiphase/Multiphase.cpp
@@ -9,9 +9,6 @@ using namespace std;
void run(string configname)
{
try {
-
- //Sleep(20000);
-
ConfigurationFile config;
config.load(configname);
@@ -73,7 +70,7 @@ void run(string configname)
}
}
- // Sleep(30000);
+ //Sleep(30000);
// LBMReal dLB = 0; // = length[1] / dx;
LBMReal rhoLB = 0.0;
@@ -88,7 +85,7 @@ void run(string configname)
kernel = SPtr<LBMKernel>(new MultiphaseScratchCumulantLBMKernel());
//kernel = SPtr<LBMKernel>(new MultiphaseCumulantLBMKernel());
//kernel = SPtr<LBMKernel>(new MultiphaseTwoPhaseFieldsCumulantLBMKernel());
-
+
kernel->setWithForcing(true);
kernel->setForcingX1(0.0);
kernel->setForcingX2(gr);
@@ -111,10 +108,15 @@ void run(string configname)
//////////////////////////////////////////////////////////////////////////
// restart
SPtr<UbScheduler> rSch(new UbScheduler(cpStep, cpStart));
- SPtr<MPIIOMigrationBECoProcessor> rcp = make_shared<MPIIOMigrationBECoProcessor>(grid, rSch, pathname, comm);
+ //SPtr<MPIIORestartCoProcessor> rcp(new MPIIORestartCoProcessor(grid, rSch, pathname, comm));
+ //SPtr<MPIIOMigrationCoProcessor> rcp(new MPIIOMigrationCoProcessor(grid, rSch, pathname, comm));
+ SPtr<MPIIOMigrationBECoProcessor> rcp(new MPIIOMigrationBECoProcessor(grid, rSch, pathname, comm));
+ rcp->setNu(nuLB);
+ rcp->setNuLG(nuL, nuG);
+ rcp->setDensityRatio(densityRatio);
+
rcp->setLBMKernel(kernel);
rcp->setBCProcessor(bcProc);
- rcp->setNu(nuLB);
//////////////////////////////////////////////////////////////////////////
mu::Parser fctF1;
@@ -125,19 +127,17 @@ void run(string configname)
fctF1.DefineConst("R", 8.0);
fctF1.DefineConst("x0", 0.0);
fctF1.DefineConst("z0", 0.0);
- //SPtr<BCAdapter> velBCAdapterF1(
- // new MultiphaseVelocityBCAdapter(false, true, false, fctF1, phiH, 0.0, BCFunction::INFCONST));
mu::Parser fctF2;
fctF2.SetExpr("vy1");
- fctF2.DefineConst("vy1", uLB);
+ fctF2.DefineConst("vy1", -uLB);
- double startTime = 500;
- SPtr<BCAdapter> velBCAdapterF1(new MultiphaseVelocityBCAdapter(true, false, false, fctF1, phiH, 0.0, startTime));
- SPtr<BCAdapter> velBCAdapterF2(new MultiphaseVelocityBCAdapter(true, false, false, fctF2, phiH, startTime, endTime));
+ double startTime = 1;
+ SPtr<BCAdapter> velBCAdapterF1(new MultiphaseVelocityBCAdapter(false, true, false, fctF1, phiH, 0.0, startTime));
+ SPtr<BCAdapter> velBCAdapterF2(new MultiphaseVelocityBCAdapter(false, true, false, fctF2, phiH, startTime, endTime));
SPtr<D3Q27Interactor> inflowF1Int;
- SPtr<D3Q27Interactor> cylInt;
+
if (newStart) {
// if (newStart) {
@@ -160,36 +160,52 @@ void run(string configname)
double g_maxX3 = boundingBox[5];
// geometry
+
+ // GbObject3DPtr innerCube(new GbCuboid3D(g_minX1+2, g_minX2+2, g_minX3+2, g_maxX1-2, g_maxX2-2,
+ // g_maxX3-2));
+
+ // GbObject3DPtr cylinder1(new GbCylinder3D(g_minX1 - 2.0*dx, g_maxX2/2, g_maxX3/2, g_minX1 + 12.0*dx,
+ // g_maxX2/2, g_maxX3/2, radius)); GbObject3DPtr cylinder2(new GbCylinder3D(g_minX1 + 12.0*dx, g_maxX2/2,
+ // g_maxX3/2, g_maxX1 + 2.0*dx, g_maxX2/2, g_maxX3/2, dLB / 2.0));
+
+ // GbObject3DPtr cylinder(new GbCylinder3D(g_minX1 - 2.0*dx, g_maxX2/2, g_maxX3/2, g_maxX1 + 2.0*dx,
+ // g_maxX2/2, g_maxX3/2, dLB / 2.0)); GbObject3DPtr cylinders(new GbObject3DManager()); GbObject3DPtr
+ // cylinders1(new GbObjectGroup3D());
+
SPtr<GbObject3D> gridCube(new GbCuboid3D(g_minX1, g_minX2, g_minX3, g_maxX1, g_maxX2, g_maxX3));
if (myid == 0)
GbSystem3D::writeGeoObject(gridCube.get(), pathname + "/geo/gridCube",
WbWriterVtkXmlBinary::getInstance());
+
if (myid == 0) UBLOG(logINFO, "Read geoFile:start");
SPtr<GbTriFaceMesh3D> cylinder = make_shared<GbTriFaceMesh3D>();
- cylinder->readMeshFromSTLFileBinary(pathGeo + "/" + geoFile, false);
+ cylinder->readMeshFromSTLFileASCII(pathGeo + "/" + geoFile, false);
GbSystem3D::writeGeoObject(cylinder.get(), pathname + "/geo/Stlgeo", WbWriterVtkXmlBinary::getInstance());
if (myid == 0) UBLOG(logINFO, "Read geoFile:stop");
// inflow
- //GbCuboid3DPtr geoInflowF1(new GbCuboid3D(g_minX1, g_minX2 - 0.5 * dx, g_minX3, g_maxX1, g_minX2 - 1.0 * dx, g_maxX3));
- GbCuboid3DPtr geoInflowF1(
- new GbCuboid3D(g_minX1*0.5 - dx, g_minX2 - dx, g_minX3*0.5 - dx, g_maxX1*0.5 + dx, g_minX2, g_maxX3*0.5 + dx));
- if (myid == 0) GbSystem3D::writeGeoObject(geoInflowF1.get(), pathname + "/geo/geoInflowF1", WbWriterVtkXmlASCII::getInstance());
+ // GbCuboid3DPtr geoInflowF1(new GbCuboid3D(40.0, 628.0, 40.0, 80, 631.0, 80.0)); // For JetBreakup
+ // (Original) GbCuboid3DPtr geoInflowF1(new GbCuboid3D(g_minX1-2.0*dx, g_minX2-2.0*dx, g_minX3-2.0*dx,
+ // g_maxX1+2.0*dx, g_minX2+2.0*dx, g_maxX3+2.0*dx)); if (myid == 0)
+ // GbSystem3D::writeGeoObject(geoInflowF1.get(), pathname + "/geo/geoInflowF1",
+ // WbWriterVtkXmlASCII::getInstance());
+
+ ////outflow
+ ////GbCuboid3DPtr geoOutflow(new GbCuboid3D(-1.0, -1, -1.0, 121.0, 1.0, 121.0)); // For JetBreakup
+ ///(Original)
+ // GbCuboid3DPtr geoOutflow(new GbCuboid3D(g_minX1-2.0*dx, g_maxX2, g_minX3-2.0*dx, g_maxX1+2.0*dx,
+ // g_maxX2+2.0*dx, g_maxX3+2.0*dx)); if (myid == 0) GbSystem3D::writeGeoObject(geoOutflow.get(), pathname +
+ // "/geo/geoOutflow", WbWriterVtkXmlASCII::getInstance());
- GbCylinder3DPtr cylinder1(new GbCylinder3D(g_minX1-dx, 0.0, 0.0, g_minX1+dx, 0.0, 0.0, 3e-3));
+ GbCuboid3DPtr geoInflowF1(
+ new GbCuboid3D(g_minX1, g_minX2 - 0.5 * dx, g_minX3, g_maxX1, g_minX2 - 1.0 * dx, g_maxX3));
if (myid == 0)
- GbSystem3D::writeGeoObject(cylinder1.get(), pathname + "/geo/cylinder1",
+ GbSystem3D::writeGeoObject(geoInflowF1.get(), pathname + "/geo/geoInflowF1",
WbWriterVtkXmlASCII::getInstance());
- //GbCylinder3DPtr cylinder2(
- // new GbCylinder3D(0.0, g_minX2 - 2.0 * dx / 2.0, 0.0, 0.0, g_minX2 + 4.0 * dx, 0.0, 8.0+2.0*dx));
- //if (myid == 0)
- // GbSystem3D::writeGeoObject(cylinder2.get(), pathname + "/geo/cylinder2",
- // WbWriterVtkXmlASCII::getInstance());
// outflow
// GbCuboid3DPtr geoOutflow(new GbCuboid3D(-1.0, -1, -1.0, 121.0, 1.0, 121.0)); // For JetBreakup (Original)
- //GbCuboid3DPtr geoOutflow(new GbCuboid3D(g_minX1, g_maxX2 - 40 * dx, g_minX3, g_maxX1, g_maxX2, g_maxX3));
- GbCuboid3DPtr geoOutflow(new GbCuboid3D(g_minX1, g_maxX2, g_minX3, g_maxX1, g_maxX2 + dx, g_maxX3));
+ GbCuboid3DPtr geoOutflow(new GbCuboid3D(g_minX1, g_maxX2 - 1 * dx, g_minX3, g_maxX1, g_maxX2, g_maxX3));
if (myid == 0)
GbSystem3D::writeGeoObject(geoOutflow.get(), pathname + "/geo/geoOutflow",
WbWriterVtkXmlASCII::getInstance());
@@ -223,6 +239,11 @@ void run(string configname)
SPtr<BCAdapter> denBCAdapter(new DensityBCAdapter(rhoLB));
denBCAdapter->setBcAlgorithm(SPtr<BCAlgorithm>(new MultiphaseNonReflectingOutflowBCAlgorithm()));
+ // double r = 5.0; //boost::dynamic_pointer_cast<GbCylinder3D>(cylinder)->getRadius();
+ // double cx1 = g_minX1;
+ // double cx2 = 0.0; //cylinder->getX2Centroid();
+ // double cx3 = 0.0; //cylinder->getX3Centroid();
+
mu::Parser fctPhi_F1;
fctPhi_F1.SetExpr("phiH");
fctPhi_F1.DefineConst("phiH", phiH);
@@ -235,70 +256,74 @@ void run(string configname)
fctvel_F2_init.SetExpr("U");
fctvel_F2_init.DefineConst("U", 0);
+ // fct.SetExpr("U");
+ // fct.DefineConst("U", uLB);
+ // BCAdapterPtr velBCAdapter(new VelocityBCAdapter(true, false, false, fct, 0, BCFunction::INFCONST));
+
+
+
+
+
+ // BCAdapterPtr velBCAdapterF2_1_init(new MultiphaseVelocityBCAdapter(false, false, true, fctF2_1, phiH,
+ // 0.0, endTime)); BCAdapterPtr velBCAdapterF2_2_init(new MultiphaseVelocityBCAdapter(false, false, true,
+ // fctF2_2, phiH, 0.0, endTime));
+
+ // BCAdapterPtr velBCAdapterF2_1_init(new MultiphaseVelocityBCAdapter(false, false, true, fctvel_F2_init,
+ // phiL, 0.0, endTime)); BCAdapterPtr velBCAdapterF2_2_init(new MultiphaseVelocityBCAdapter(false, false,
+ // true, fctvel_F2_init, phiL, 0.0, endTime));
+
velBCAdapterF1->setBcAlgorithm(SPtr<BCAlgorithm>(new MultiphaseVelocityBCAlgorithm()));
+ //velBCAdapterF1->setBcAlgorithm(SPtr<BCAlgorithm>(new VelocityBCAlgorithm()));
+ // velBCAdapterF2_1_init->setBcAlgorithm(BCAlgorithmPtr(new MultiphaseVelocityBCAlgorithm()));
+ // velBCAdapterF2_2_init->setBcAlgorithm(BCAlgorithmPtr(new MultiphaseVelocityBCAlgorithm()));
+
+ // velBCAdapter->setBcAlgorithm(BCAlgorithmPtr(new VelocityWithDensityBCAlgorithm()));
+ // mu::Parser fct;
+ // fct.SetExpr("U");
+ // fct.DefineConst("U", uLB);
+ // BCAdapterPtr velBCAdapter(new VelocityBCAdapter(true, false, false, fct, 0, BCFunction::INFCONST));
+ // velBCAdapter->setBcAlgorithm(BCAlgorithmPtr(new NonReflectingVelocityBCAlgorithm()));
+
//////////////////////////////////////////////////////////////////////////////////
// BC visitor
MultiphaseBoundaryConditionsBlockVisitor bcVisitor;
bcVisitor.addBC(noSlipBCAdapter);
- bcVisitor.addBC(denBCAdapter); //Ohne das BB?
+ // bcVisitor.addBC(denBCAdapter); //Ohne das BB?
bcVisitor.addBC(velBCAdapterF1);
+ // bcVisitor.addBC(velBCAdapterF2_1_init);
+ // bcVisitor.addBC(velBCAdapterF2_2_init);
SPtr<WriteBlocksCoProcessor> ppblocks(new WriteBlocksCoProcessor(
grid, SPtr<UbScheduler>(new UbScheduler(1)), pathname, WbWriterVtkXmlBinary::getInstance(), comm));
- SPtr<Interactor3D> tubes(new D3Q27TriFaceMeshInteractor(cylinder, grid, noSlipBCAdapter, Interactor3D::SOLID, Interactor3D::POINTS));
+ //ppblocks->process(0);
+
+ SPtr<Interactor3D> tubes(new D3Q27TriFaceMeshInteractor(cylinder, grid, noSlipBCAdapter, Interactor3D::SOLID));
- //inflowF1Int =
- // SPtr<D3Q27Interactor>(new D3Q27Interactor(cylinder1, grid, noSlipBCAdapter, Interactor3D::SOLID));
- //inflowF1Int->addBCAdapter(velBCAdapterF2);
+ inflowF1Int = SPtr<D3Q27Interactor> (new D3Q27Interactor(geoInflowF1, grid, velBCAdapterF1, Interactor3D::SOLID));
+ inflowF1Int->addBCAdapter(velBCAdapterF2);
+
+ // inflowF1Int->addBCAdapter(velBCAdapterFStart);
+
+ // D3Q27InteractorPtr inflowF2_1Int_init = D3Q27InteractorPtr(new D3Q27Interactor(geoInflowF2_1, grid,
+ // velBCAdapterF2_1_init, Interactor3D::SOLID));
+
+ // D3Q27InteractorPtr inflowF2_2Int_init = D3Q27InteractorPtr(new D3Q27Interactor(geoInflowF2_2, grid,
+ // velBCAdapterF2_2_init, Interactor3D::SOLID));
SPtr<D3Q27Interactor> outflowInt(new D3Q27Interactor(geoOutflow, grid, denBCAdapter, Interactor3D::SOLID));
- // Create boundary conditions geometry
- GbCuboid3DPtr wallXmin(new GbCuboid3D(g_minX1 - dx, g_minX2 - dx, g_minX3 - dx, g_minX1, g_maxX2 + dx, g_maxX3));
- GbSystem3D::writeGeoObject(wallXmin.get(), pathname + "/geo/wallXmin", WbWriterVtkXmlASCII::getInstance());
- GbCuboid3DPtr wallXmax(new GbCuboid3D(g_maxX1, g_minX2 - dx, g_minX3 - dx, g_maxX1 + dx, g_maxX2 + dx, g_maxX3));
- GbSystem3D::writeGeoObject(wallXmax.get(), pathname + "/geo/wallXmax", WbWriterVtkXmlASCII::getInstance());
- GbCuboid3DPtr wallZmin(new GbCuboid3D(g_minX1 - dx, g_minX2 - dx, g_minX3 - dx, g_maxX1 + dx, g_maxX2 + dx, g_minX3));
- GbSystem3D::writeGeoObject(wallZmin.get(), pathname + "/geo/wallZmin", WbWriterVtkXmlASCII::getInstance());
- GbCuboid3DPtr wallZmax(new GbCuboid3D(g_minX1 - dx, g_minX2 - dx, g_maxX3, g_maxX1 + dx, g_maxX2 + dx, g_maxX3 + dx));
- GbSystem3D::writeGeoObject(wallZmax.get(), pathname + "/geo/wallZmax", WbWriterVtkXmlASCII::getInstance());
- GbCuboid3DPtr wallYmin(new GbCuboid3D(g_minX1 - dx, g_minX2 - dx, g_minX3 - dx, g_maxX1 + dx, g_minX2, g_maxX3));
- GbSystem3D::writeGeoObject(wallYmin.get(), pathname + "/geo/wallYmin", WbWriterVtkXmlASCII::getInstance());
- GbCuboid3DPtr wallYmax(new GbCuboid3D(g_minX1 - dx, g_maxX2, g_minX3 - dx, g_maxX1 + dx, g_maxX2 + dx, g_maxX3));
- GbSystem3D::writeGeoObject(wallYmax.get(), pathname + "/geo/wallYmax", WbWriterVtkXmlASCII::getInstance());
-
- // Add boundary conditions to grid generator
- SPtr<D3Q27Interactor> wallXminInt(new D3Q27Interactor(wallXmin, grid, noSlipBCAdapter, Interactor3D::SOLID));
- SPtr<D3Q27Interactor> wallXmaxInt(new D3Q27Interactor(wallXmax, grid, noSlipBCAdapter, Interactor3D::SOLID));
- SPtr<D3Q27Interactor> wallZminInt(new D3Q27Interactor(wallZmin, grid, noSlipBCAdapter, Interactor3D::SOLID));
- SPtr<D3Q27Interactor> wallZmaxInt(new D3Q27Interactor(wallZmax, grid, noSlipBCAdapter, Interactor3D::SOLID));
- SPtr<D3Q27Interactor> wallYminInt(new D3Q27Interactor(wallYmin, grid, noSlipBCAdapter, Interactor3D::SOLID));
- SPtr<D3Q27Interactor> wallYmaxInt(new D3Q27Interactor(wallYmax, grid, noSlipBCAdapter, Interactor3D::SOLID));
-
-
- cylInt = SPtr<D3Q27Interactor>(new D3Q27Interactor(cylinder1, grid, velBCAdapterF1, Interactor3D::SOLID));
- cylInt->addBCAdapter(velBCAdapterF2);
- //SPtr<D3Q27Interactor> cyl2Int(new D3Q27Interactor(cylinder2, grid, noSlipBCAdapter, Interactor3D::SOLID));
+ // SetSolidBlockVisitor visitor1(inflowF2_1Int, SetSolidBlockVisitor::BC);
+ // grid->accept(visitor1);
+ // SetSolidBlockVisitor visitor2(inflowF2_2Int, SetSolidBlockVisitor::BC);
+ // grid->accept(visitor2);
SPtr<Grid3DVisitor> metisVisitor(
new MetisPartitioningGridVisitor(comm, MetisPartitioningGridVisitor::LevelBased, D3Q27System::BSW));
- InteractorsHelper intHelper(grid, metisVisitor, true);
- intHelper.addInteractor(cylInt);
+ InteractorsHelper intHelper(grid, metisVisitor);
intHelper.addInteractor(tubes);
- //intHelper.addInteractor(outflowInt);
- //intHelper.addInteractor(cyl2Int);
-
-
- intHelper.addInteractor(wallXminInt);
- intHelper.addInteractor(wallXmaxInt);
- intHelper.addInteractor(wallZminInt);
- intHelper.addInteractor(wallZmaxInt);
- intHelper.addInteractor(wallYminInt);
- intHelper.addInteractor(wallYmaxInt);
- //intHelper.addInteractor(inflowF1Int);
-
-
+ intHelper.addInteractor(inflowF1Int);
+ intHelper.addInteractor(outflowInt);
intHelper.selectBlocks();
ppblocks->process(0);
@@ -330,6 +355,26 @@ void run(string configname)
UBLOG(logINFO, "Available memory per process = " << availMem << " bytes");
}
+ // LBMKernelPtr kernel;
+
+ // kernel = LBMKernelPtr(new MultiphaseCumulantLBMKernel(blocknx[0], blocknx[1], blocknx[2],
+ // MultiphaseCumulantLBMKernel::NORMAL));
+
+ // kernel->setWithForcing(true);
+ // kernel->setForcingX1(0.0);
+ // kernel->setForcingX2(gr);
+ // kernel->setForcingX3(0.0);
+
+ // kernel->setPhiL(phiL);
+ // kernel->setPhiH(phiH);
+ // kernel->setPhaseFieldRelaxation(tauH);
+ // kernel->setMobility(mob);
+
+ // BCProcessorPtr bcProc(new BCProcessor());
+ // //BCProcessorPtr bcProc(new ThinWallBCProcessor());
+
+ // kernel->setBCProcessor(bcProc);
+
MultiphaseSetKernelBlockVisitor kernelVisitor(kernel, nuL, nuG, densityRatio, beta, kappa, theta, availMem,
needMem);
@@ -340,18 +385,68 @@ void run(string configname)
grid->accept(undefNodesVisitor);
}
+ // inflowF2_1Int->initInteractor();
+ // inflowF2_2Int->initInteractor();
+
intHelper.setBC();
grid->accept(bcVisitor);
// initialization of distributions
+ LBMReal x1c = radius; // g_minX1; //radius; //19; //(g_maxX1+g_minX1)/2;
+ LBMReal x2c = (g_maxX2 + g_minX2) / 2; // g_minX2 + 2;
+ LBMReal x3c = (g_maxX3 + g_minX3) / 2;
mu::Parser fct1;
+
+ // fct1.SetExpr("0.5-0.5*tanh(2*(sqrt((x1-x1c)^2+(x2-x2c)^2+(x3-x3c)^2)-radius)/interfaceThickness)");
+ // fct1.SetExpr("phiM-phiM*tanh((sqrt((x1-x1c)^2+(x2-x2c)^2+(x3-x3c)^2)-radius)/(interfaceThickness*phiM))");
+
+ // fct1.SetExpr("0.5*(phiH + phiL)-0.5*(phiH -
+ // phiL)*tanh(2*(sqrt((x1-x1c)^2+(x2-x2c)^2+(x3-x3c)^2)-radius)/interfaceThickness)");
+
+ // fct1.SetExpr("0.5*(phiH + phiL) + 0.5*(phiH - phiL)*tanh(2*((x2-radius))/interfaceThickness)");
fct1.SetExpr("phiL");
+ fct1.DefineConst("x1c", x1c);
+ fct1.DefineConst("x2c", x2c);
+ fct1.DefineConst("x3c", x3c);
fct1.DefineConst("phiL", phiL);
+ fct1.DefineConst("phiH", phiH);
+ fct1.DefineConst("radius", radius);
+ fct1.DefineConst("interfaceThickness", interfaceThickness);
+
+ mu::Parser fct2;
+ // fct2.SetExpr("vx1*(1-((x2-y0)^2+(x3-z0)^2)/(R^2))");
+ fct2.SetExpr("vx1");
+ fct2.DefineConst("R", 10.0);
+ fct2.DefineConst("vx1", uLB);
+ fct2.DefineConst("y0", 1.0);
+ fct2.DefineConst("z0", 31.0);
+ /*fct2.SetExpr("0.5*uLB-uLB*0.5*tanh(2*(sqrt((x1-x1c)^2+(x2-x2c)^2+(x3-x3c)^2)-radius)/interfaceThickness)");
+ fct2.DefineConst("uLB", uLB);
+ fct2.DefineConst("x1c", x1c);
+ fct2.DefineConst("x2c", x2c);
+ fct2.DefineConst("x3c", x3c);
+ fct2.DefineConst("radius", radius);
+ fct2.DefineConst("interfaceThickness", interfaceThickness);*/
+
MultiphaseInitDistributionsBlockVisitor initVisitor(densityRatio, interfaceThickness, radius);
initVisitor.setPhi(fct1);
+ // initVisitor.setVx1(fct2);
grid->accept(initVisitor);
+ // set connectors
+ //InterpolationProcessorPtr iProcessor(new IncompressibleOffsetInterpolationProcessor());
+ // InterpolationProcessorPtr iProcessor(new CompressibleOffsetInterpolationProcessor());
+ //SetConnectorsBlockVisitor setConnsVisitor(comm, true, D3Q27System::ENDDIR, nuLB, iProcessor);
+ // ConnectorFactoryPtr factory(new Block3DConnectorFactory());
+ // ConnectorBlockVisitor setConnsVisitor(comm, nuLB, iProcessor, factory);
+ //ThreeDistributionsSetConnectorsBlockVisitor setConnsVisitor(comm);
+ //grid->accept(setConnsVisitor);
+
+ // domain decomposition for threads
+ // PQueuePartitioningGridVisitor pqPartVisitor(numOfThreads);
+ // grid->accept(pqPartVisitor);
+
// boundary conditions grid
{
SPtr<UbScheduler> geoSch(new UbScheduler(1));
@@ -379,6 +474,18 @@ void run(string configname)
rcp->restart((int)restartStep);
grid->setTimeStep(restartStep);
+ // BCAdapterPtr velBCAdapter(new VelocityBCAdapter());
+ // velBCAdapter->setBcAlgorithm(BCAlgorithmPtr(new VelocityBCAlgorithm()));
+ // velBCAdapter->setBcAlgorithm(BCAlgorithmPtr(new VelocityWithDensityBCAlgorithm()));
+ // bcVisitor.addBC(velBCAdapter);
+ // grid->accept(bcVisitor);
+
+ // set connectors
+ // InterpolationProcessorPtr iProcessor(new IncompressibleOffsetInterpolationProcessor());
+ //InterpolationProcessorPtr iProcessor(new CompressibleOffsetInterpolationProcessor());
+ //SetConnectorsBlockVisitor setConnsVisitor(comm, true, D3Q27System::ENDDIR, nuLB, iProcessor);
+ //grid->accept(setConnsVisitor);
+
if (myid == 0)
UBLOG(logINFO, "Restart - end");
}
@@ -391,20 +498,16 @@ void run(string configname)
SPtr<UbScheduler> visSch(new UbScheduler(outTime));
SPtr<WriteMultiphaseQuantitiesCoProcessor> pp(new WriteMultiphaseQuantitiesCoProcessor(
- //SPtr<WriteMacroscopicQuantitiesCoProcessor> pp(new WriteMacroscopicQuantitiesCoProcessor(
grid, visSch, pathname, WbWriterVtkXmlBinary::getInstance(), conv, comm));
- pp->process(0);
+ //SPtr<WriteMacroscopicQuantitiesCoProcessor> pp(new WriteMacroscopicQuantitiesCoProcessor(
+ // grid, visSch, pathname, WbWriterVtkXmlBinary::getInstance(), conv, comm));
SPtr<UbScheduler> nupsSch(new UbScheduler(10, 30, 100));
SPtr<NUPSCounterCoProcessor> npr(new NUPSCounterCoProcessor(grid, nupsSch, numOfThreads, comm));
SPtr<UbScheduler> timeBCSch(new UbScheduler(1, startTime, startTime));
auto timeDepBC = make_shared<TimeDependentBCCoProcessor>(TimeDependentBCCoProcessor(grid, timeBCSch));
- timeDepBC->addInteractor(cylInt);
-
-#ifdef _OPENMP
- omp_set_num_threads(numOfThreads);
-#endif
+ timeDepBC->addInteractor(inflowF1Int);
SPtr<UbScheduler> stepGhostLayer(new UbScheduler(1));
SPtr<Calculator> calculator(new BasicCalculator(grid, stepGhostLayer, endTime));
@@ -413,7 +516,6 @@ void run(string configname)
calculator->addCoProcessor(timeDepBC);
calculator->addCoProcessor(rcp);
-
if (myid == 0)
diff --git a/src/cpu/VirtualFluidsCore/CoProcessors/MPIIOCoProcessor.cpp b/src/cpu/VirtualFluidsCore/CoProcessors/MPIIOCoProcessor.cpp
index 028cb68ca562771b263b801c48c5f3b3791c723f..a3572c8c40ed63144080c1803d728393eaf30547 100644
--- a/src/cpu/VirtualFluidsCore/CoProcessors/MPIIOCoProcessor.cpp
+++ b/src/cpu/VirtualFluidsCore/CoProcessors/MPIIOCoProcessor.cpp
@@ -70,7 +70,7 @@ MPIIOCoProcessor::MPIIOCoProcessor(SPtr<Grid3D> grid, SPtr<UbScheduler> s, const
//---------------------------------------
- MPI_Type_contiguous(7, MPI_CHAR, &arrayPresenceType);
+ MPI_Type_contiguous(8, MPI_CHAR, &arrayPresenceType);
MPI_Type_commit(&arrayPresenceType);
}
@@ -378,10 +378,24 @@ void MPIIOCoProcessor::clearAllFiles(int step)
MPI_File_set_size(file_handler, new_size);
MPI_File_close(&file_handler);
- std::string filename2 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSet.bin";
- int rc2 = MPI_File_open(MPI_COMM_WORLD, filename2.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
- if (rc2 != MPI_SUCCESS)
- throw UbException(UB_EXARGS, "couldn't open file " + filename2);
+ std::string filename21 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSetF.bin";
+ int rc21 = MPI_File_open(MPI_COMM_WORLD, filename21.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc21 != MPI_SUCCESS)
+ throw UbException(UB_EXARGS, "couldn't open file " + filename21);
+ MPI_File_set_size(file_handler, new_size);
+ MPI_File_close(&file_handler);
+
+ std::string filename22 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSetH1.bin";
+ int rc22 = MPI_File_open(MPI_COMM_WORLD, filename22.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc22 != MPI_SUCCESS)
+ throw UbException(UB_EXARGS, "couldn't open file " + filename22);
+ MPI_File_set_size(file_handler, new_size);
+ MPI_File_close(&file_handler);
+
+ std::string filename23 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSetH2.bin";
+ int rc23 = MPI_File_open(MPI_COMM_WORLD, filename23.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc23 != MPI_SUCCESS)
+ throw UbException(UB_EXARGS, "couldn't open file " + filename23);
MPI_File_set_size(file_handler, new_size);
MPI_File_close(&file_handler);
@@ -441,25 +455,20 @@ void MPIIOCoProcessor::clearAllFiles(int step)
MPI_File_set_size(file_handler, new_size);
MPI_File_close(&file_handler);
- std::string filename10 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpPhaseField.bin";
+ std::string filename10 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpPhaseField1.bin";
int rc10 = MPI_File_open(MPI_COMM_WORLD, filename10.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
if (rc10 != MPI_SUCCESS)
throw UbException(UB_EXARGS, "couldn't open file " + filename10);
MPI_File_set_size(file_handler, new_size);
MPI_File_close(&file_handler);
-
- /*std::string filename10 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpBC1.bin";
- int rc10 = MPI_File_open(MPI_COMM_WORLD, filename10.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info,
- &file_handler); if (rc10 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename10);
+ std::string filename11 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpPhaseField2.bin";
+ int rc11 = MPI_File_open(MPI_COMM_WORLD, filename11.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc11 != MPI_SUCCESS)
+ throw UbException(UB_EXARGS, "couldn't open file " + filename11);
MPI_File_set_size(file_handler, new_size);
MPI_File_close(&file_handler);
- std::string filename11 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpBC2.bin";
- int rc11 = MPI_File_open(MPI_COMM_WORLD, filename11.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info,
- &file_handler); if (rc11 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename11);
- MPI_File_set_size(file_handler, new_size);
- MPI_File_close(&file_handler);*/
}
void MPIIOCoProcessor::writeCpTimeStep(int step)
diff --git a/src/cpu/VirtualFluidsCore/CoProcessors/MPIIOMigrationBECoProcessor.cpp b/src/cpu/VirtualFluidsCore/CoProcessors/MPIIOMigrationBECoProcessor.cpp
index 39b32b6aac7110eb873a52351d46560f8843820e..0e27f03a9ff1f7e1eb905cdb87e6e5cfcb97454c 100644
--- a/src/cpu/VirtualFluidsCore/CoProcessors/MPIIOMigrationBECoProcessor.cpp
+++ b/src/cpu/VirtualFluidsCore/CoProcessors/MPIIOMigrationBECoProcessor.cpp
@@ -25,9 +25,8 @@ using namespace MPIIODataStructures;
#define MESSAGE_TAG 80
#define SEND_BLOCK_SIZE 100000
-MPIIOMigrationBECoProcessor::MPIIOMigrationBECoProcessor(SPtr<Grid3D> grid, SPtr<UbScheduler> s,
- const std::string &path, SPtr<Communicator> comm)
- : MPIIOCoProcessor(grid, s, path, comm), nue(-999.999)
+MPIIOMigrationBECoProcessor::MPIIOMigrationBECoProcessor(SPtr<Grid3D> grid, SPtr<UbScheduler> s, const std::string &path, SPtr<Communicator> comm)
+ : MPIIOCoProcessor(grid, s, path, comm), nue(-999.999), nuL(-999.999), nuG(-999.999), densityRatio(-999.999)
{
memset(&boundCondParamStr, 0, sizeof(boundCondParamStr));
@@ -107,44 +106,60 @@ void MPIIOMigrationBECoProcessor::writeDataSet(int step)
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
int maxInitLevel = this->grid->getFinestInitializedLevel();
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
grid->getBlocks(level, rank, blocksVector[level]);
blocksCount += static_cast<int>(blocksVector[level].size());
}
dataSetParam dataSetParamStr1, dataSetParamStr2, dataSetParamStr3;
int firstGlobalID;
- std::vector<double> doubleValuesArray; // double-values (arrays of f's) in all blocks
+ std::vector<double> doubleValuesArrayF; // double-values (arrays of f's) in all blocks Fdistribution
+ std::vector<double> doubleValuesArrayH1; // double-values (arrays of f's) in all blocks H1distribution
+ // std::vector<double> doubleValuesArrayH2; // double-values (arrays of f's) in all blocks H2distribution
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::writeDataSet start collect data rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
+ bool multiPhase = false;
DSArraysPresence arrPresence;
bool firstBlock = true;
int doubleCountInBlock = 0;
int ic = 0;
- SPtr<D3Q27EsoTwist3DSplittedVector> D3Q27EsoTwist3DSplittedVectorPtrF, D3Q27EsoTwist3DSplittedVectorPtrH;
- CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr localDistributionsF, localDistributionsH;
- CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr nonLocalDistributionsF, nonLocalDistributionsH;
- CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr zeroDistributionsF, zeroDistributionsH;
+ SPtr<D3Q27EsoTwist3DSplittedVector> D3Q27EsoTwist3DSplittedVectorPtrF = 0, D3Q27EsoTwist3DSplittedVectorPtrH1 = 0, D3Q27EsoTwist3DSplittedVectorPtrH2 = 0;
+ CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr localDistributionsF = 0, localDistributionsH1 = 0, localDistributionsH2 = 0;
+ CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr nonLocalDistributionsF = 0, nonLocalDistributionsH1 = 0, nonLocalDistributionsH2 = 0;
+ CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr zeroDistributionsF = 0, zeroDistributionsH1 = 0, zeroDistributionsH2 = 0;
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
{
- D3Q27EsoTwist3DSplittedVectorPtrF = dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(
- block->getKernel()->getDataSet()->getFdistributions());
+ D3Q27EsoTwist3DSplittedVectorPtrF = dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(block->getKernel()->getDataSet()->getFdistributions());
localDistributionsF = D3Q27EsoTwist3DSplittedVectorPtrF->getLocalDistributions();
nonLocalDistributionsF = D3Q27EsoTwist3DSplittedVectorPtrF->getNonLocalDistributions();
zeroDistributionsF = D3Q27EsoTwist3DSplittedVectorPtrF->getZeroDistributions();
- D3Q27EsoTwist3DSplittedVectorPtrH = dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(
- block->getKernel()->getDataSet()->getHdistributions());
- localDistributionsH = D3Q27EsoTwist3DSplittedVectorPtrH->getLocalDistributions();
- nonLocalDistributionsH = D3Q27EsoTwist3DSplittedVectorPtrH->getNonLocalDistributions();
- zeroDistributionsH = D3Q27EsoTwist3DSplittedVectorPtrH->getZeroDistributions();
+ D3Q27EsoTwist3DSplittedVectorPtrH1 = dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(block->getKernel()->getDataSet()->getHdistributions());
+ if (D3Q27EsoTwist3DSplittedVectorPtrH1 != 0)
+ {
+ multiPhase = true;
+ localDistributionsH1 = D3Q27EsoTwist3DSplittedVectorPtrH1->getLocalDistributions();
+ nonLocalDistributionsH1 = D3Q27EsoTwist3DSplittedVectorPtrH1->getNonLocalDistributions();
+ zeroDistributionsH1 = D3Q27EsoTwist3DSplittedVectorPtrH1->getZeroDistributions();
+ }
+
+ /*D3Q27EsoTwist3DSplittedVectorPtrH2 = dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(block->getKernel()->getDataSet()->getH2distributions());
+ if (D3Q27EsoTwist3DSplittedVectorPtrH2 != 0)
+ {
+ localDistributionsH2 = D3Q27EsoTwist3DSplittedVectorPtrH2->getLocalDistributions();
+ nonLocalDistributionsH2 = D3Q27EsoTwist3DSplittedVectorPtrH2->getNonLocalDistributions();
+ zeroDistributionsH2 = D3Q27EsoTwist3DSplittedVectorPtrH2->getZeroDistributions();
+ }*/
+
if (firstBlock) // && block->getKernel()) // when first (any) valid block...
{
@@ -171,94 +186,92 @@ void MPIIOMigrationBECoProcessor::writeDataSet(int step)
}
// ... than save some parameters that are equal in all blocks
- dataSetParamStr1.nx1 = dataSetParamStr2.nx1 = dataSetParamStr3.nx1 =
- static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX1());
- dataSetParamStr1.nx2 = dataSetParamStr2.nx2 = dataSetParamStr3.nx2 =
- static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX2());
- dataSetParamStr1.nx3 = dataSetParamStr2.nx3 = dataSetParamStr3.nx3 =
- static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX3());
-
- // Fdistributions + Hdistributions
- doubleCountInBlock =
- (dataSetParamStr1.nx[0] * dataSetParamStr1.nx[1] * dataSetParamStr1.nx[2] * dataSetParamStr1.nx[3] +
+ dataSetParamStr1.nx1 = dataSetParamStr2.nx1 = dataSetParamStr3.nx1 = static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX1());
+ dataSetParamStr1.nx2 = dataSetParamStr2.nx2 = dataSetParamStr3.nx2 = static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX2());
+ dataSetParamStr1.nx3 = dataSetParamStr2.nx3 = dataSetParamStr3.nx3 = static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX3());
+
+ doubleCountInBlock = dataSetParamStr1.nx[0] * dataSetParamStr1.nx[1] * dataSetParamStr1.nx[2] * dataSetParamStr1.nx[3] +
dataSetParamStr2.nx[0] * dataSetParamStr2.nx[1] * dataSetParamStr2.nx[2] * dataSetParamStr2.nx[3] +
- dataSetParamStr3.nx[0] * dataSetParamStr3.nx[1] * dataSetParamStr3.nx[2] * dataSetParamStr3.nx[3]) * 2;
+ dataSetParamStr3.nx[0] * dataSetParamStr3.nx[1] * dataSetParamStr3.nx[2] * dataSetParamStr3.nx[3];
- SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> averageDensityArray =
- block->getKernel()->getDataSet()->getAverageDensity();
+ SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> averageDensityArray = block->getKernel()->getDataSet()->getAverageDensity();
if (averageDensityArray)
arrPresence.isAverageDensityArrayPresent = true;
else
arrPresence.isAverageDensityArrayPresent = false;
- SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> AverageVelocityArray3DPtr =
- block->getKernel()->getDataSet()->getAverageVelocity();
+ SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> AverageVelocityArray3DPtr = block->getKernel()->getDataSet()->getAverageVelocity();
if (AverageVelocityArray3DPtr)
arrPresence.isAverageVelocityArrayPresent = true;
else
arrPresence.isAverageVelocityArrayPresent = false;
- SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> AverageFluctArray3DPtr =
- block->getKernel()->getDataSet()->getAverageFluctuations();
+ SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> AverageFluctArray3DPtr = block->getKernel()->getDataSet()->getAverageFluctuations();
if (AverageFluctArray3DPtr)
arrPresence.isAverageFluktuationsArrayPresent = true;
else
arrPresence.isAverageFluktuationsArrayPresent = false;
- SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> AverageTripleArray3DPtr =
- block->getKernel()->getDataSet()->getAverageTriplecorrelations();
+ SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> AverageTripleArray3DPtr = block->getKernel()->getDataSet()->getAverageTriplecorrelations();
if (AverageTripleArray3DPtr)
arrPresence.isAverageTripleArrayPresent = true;
else
arrPresence.isAverageTripleArrayPresent = false;
- SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> ShearStressValArray3DPtr =
- block->getKernel()->getDataSet()->getShearStressValues();
+ SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> ShearStressValArray3DPtr = block->getKernel()->getDataSet()->getShearStressValues();
if (ShearStressValArray3DPtr)
arrPresence.isShearStressValArrayPresent = true;
else
arrPresence.isShearStressValArrayPresent = false;
- SPtr<CbArray3D<LBMReal, IndexerX3X2X1>> relaxationFactor3DPtr =
- block->getKernel()->getDataSet()->getRelaxationFactor();
+ SPtr<CbArray3D<LBMReal, IndexerX3X2X1>> relaxationFactor3DPtr = block->getKernel()->getDataSet()->getRelaxationFactor();
if (relaxationFactor3DPtr)
arrPresence.isRelaxationFactorPresent = true;
else
arrPresence.isRelaxationFactorPresent = false;
- SPtr<CbArray3D<LBMReal, IndexerX3X2X1>> phaseField3DPtr =
- block->getKernel()->getDataSet()->getPhaseField();
- if (phaseField3DPtr)
- arrPresence.isPhaseFieldPresent = true;
+ SPtr<CbArray3D<LBMReal, IndexerX3X2X1>> phaseField3DPtr1 = block->getKernel()->getDataSet()->getPhaseField();
+ if (phaseField3DPtr1)
+ arrPresence.isPhaseField1Present = true;
else
- arrPresence.isPhaseFieldPresent = false;
+ arrPresence.isPhaseField1Present = false;
+
+ SPtr<CbArray3D<LBMReal, IndexerX3X2X1>> phaseField3DPtr2 = block->getKernel()->getDataSet()->getPhaseField2();
+ if (phaseField3DPtr2)
+ arrPresence.isPhaseField2Present = true;
+ else
+ arrPresence.isPhaseField2Present = false;
+
firstBlock = false;
}
- if (localDistributionsF && (dataSetParamStr1.nx[0] > 0) && (dataSetParamStr1.nx[1] > 0) &&
- (dataSetParamStr1.nx[2] > 0) && (dataSetParamStr1.nx[3] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), localDistributionsF->getDataVector().begin(),
- localDistributionsF->getDataVector().end());
- if (nonLocalDistributionsF && (dataSetParamStr2.nx[0] > 0) && (dataSetParamStr2.nx[1] > 0) &&
- (dataSetParamStr2.nx[2] > 0) && (dataSetParamStr2.nx[3] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), nonLocalDistributionsF->getDataVector().begin(),
- nonLocalDistributionsF->getDataVector().end());
+ if (localDistributionsF && (dataSetParamStr1.nx[0] > 0) && (dataSetParamStr1.nx[1] > 0) && (dataSetParamStr1.nx[2] > 0) && (dataSetParamStr1.nx[3] > 0))
+ doubleValuesArrayF.insert(doubleValuesArrayF.end(), localDistributionsF->getDataVector().begin(), localDistributionsF->getDataVector().end());
+ if (nonLocalDistributionsF && (dataSetParamStr2.nx[0] > 0) && (dataSetParamStr2.nx[1] > 0) && (dataSetParamStr2.nx[2] > 0) && (dataSetParamStr2.nx[3] > 0))
+ doubleValuesArrayF.insert(doubleValuesArrayF.end(), nonLocalDistributionsF->getDataVector().begin(), nonLocalDistributionsF->getDataVector().end());
if (zeroDistributionsF && (dataSetParamStr3.nx[0] > 0) && (dataSetParamStr3.nx[1] > 0) && (dataSetParamStr3.nx[2] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), zeroDistributionsF->getDataVector().begin(),
- zeroDistributionsF->getDataVector().end());
-
- if (localDistributionsH && (dataSetParamStr1.nx[0] > 0) && (dataSetParamStr1.nx[1] > 0) &&
- (dataSetParamStr1.nx[2] > 0) && (dataSetParamStr1.nx[3] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), localDistributionsH->getDataVector().begin(),
- localDistributionsH->getDataVector().end());
- if (nonLocalDistributionsH && (dataSetParamStr2.nx[0] > 0) && (dataSetParamStr2.nx[1] > 0) &&
- (dataSetParamStr2.nx[2] > 0) && (dataSetParamStr2.nx[3] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), nonLocalDistributionsH->getDataVector().begin(),
- nonLocalDistributionsH->getDataVector().end());
- if (zeroDistributionsH && (dataSetParamStr3.nx[0] > 0) && (dataSetParamStr3.nx[1] > 0) && (dataSetParamStr3.nx[2] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), zeroDistributionsH->getDataVector().begin(),
- zeroDistributionsH->getDataVector().end());
+ doubleValuesArrayF.insert(doubleValuesArrayF.end(), zeroDistributionsF->getDataVector().begin(), zeroDistributionsF->getDataVector().end());
+
+ if (multiPhase)
+ {
+ if (localDistributionsH1 && (dataSetParamStr1.nx[0] > 0) && (dataSetParamStr1.nx[1] > 0) && (dataSetParamStr1.nx[2] > 0) && (dataSetParamStr1.nx[3] > 0))
+ doubleValuesArrayH1.insert(doubleValuesArrayH1.end(), localDistributionsH1->getDataVector().begin(), localDistributionsH1->getDataVector().end());
+ if (nonLocalDistributionsH1 && (dataSetParamStr2.nx[0] > 0) && (dataSetParamStr2.nx[1] > 0) && (dataSetParamStr2.nx[2] > 0) && (dataSetParamStr2.nx[3] > 0))
+ doubleValuesArrayH1.insert(doubleValuesArrayH1.end(), nonLocalDistributionsH1->getDataVector().begin(), nonLocalDistributionsH1->getDataVector().end());
+ if (zeroDistributionsH1 && (dataSetParamStr3.nx[0] > 0) && (dataSetParamStr3.nx[1] > 0) && (dataSetParamStr3.nx[2] > 0))
+ doubleValuesArrayH1.insert(doubleValuesArrayH1.end(), zeroDistributionsH1->getDataVector().begin(), zeroDistributionsH1->getDataVector().end());
+ }
+
+ /*if (D3Q27EsoTwist3DSplittedVectorPtrH2 != 0)
+ {
+ if (localDistributionsH2 && (dataSetParamStr1.nx[0] > 0) && (dataSetParamStr1.nx[1] > 0) && (dataSetParamStr1.nx[2] > 0) && (dataSetParamStr1.nx[3] > 0))
+ doubleValuesArrayH2.insert(doubleValuesArrayH2.end(), localDistributionsH2->getDataVector().begin(), localDistributionsH2->getDataVector().end());
+ if (nonLocalDistributionsH2 && (dataSetParamStr2.nx[0] > 0) && (dataSetParamStr2.nx[1] > 0) && (dataSetParamStr2.nx[2] > 0) && (dataSetParamStr2.nx[3] > 0))
+ doubleValuesArrayH2.insert(doubleValuesArrayH2.end(), nonLocalDistributionsH2->getDataVector().begin(), nonLocalDistributionsH2->getDataVector().end());
+ if (zeroDistributionsH2 && (dataSetParamStr3.nx[0] > 0) && (dataSetParamStr3.nx[1] > 0) && (dataSetParamStr3.nx[2] > 0))
+ doubleValuesArrayH2.insert(doubleValuesArrayH2.end(), zeroDistributionsH2->getDataVector().begin(), zeroDistributionsH2->getDataVector().end());
+ }*/
ic++;
}
@@ -267,10 +280,10 @@ void MPIIOMigrationBECoProcessor::writeDataSet(int step)
MPI_Type_contiguous(doubleCountInBlock , MPI_DOUBLE, &dataSetDoubleType);
MPI_Type_commit(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::writeDataSet start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
@@ -287,28 +300,57 @@ void MPIIOMigrationBECoProcessor::writeDataSet(int step)
// write to the file
MPI_File file_handler;
- std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSet.bin";
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSetF.bin";
int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
if (rc != MPI_SUCCESS)
throw UbException(UB_EXARGS, "couldn't open file " + filename);
- MPI_Offset write_offset = (MPI_Offset)(3 * sizeof(dataSetParam)) + (MPI_Offset)(firstGlobalID) *
- (MPI_Offset)(doubleCountInBlock) *
- (MPI_Offset)(sizeof(double));
+ MPI_Offset write_offset = (MPI_Offset)(3 * sizeof(dataSetParam)) + (MPI_Offset)(firstGlobalID) * (MPI_Offset)(doubleCountInBlock) * (MPI_Offset)(sizeof(double));
MPI_File_write_at(file_handler, (MPI_Offset)0, &dataSetParamStr1, 1, dataSetParamType, MPI_STATUS_IGNORE);
- MPI_File_write_at(file_handler, (MPI_Offset)(sizeof(dataSetParam)), &dataSetParamStr2, 1, dataSetParamType,
- MPI_STATUS_IGNORE);
- MPI_File_write_at(file_handler, (MPI_Offset)(2 * sizeof(dataSetParam)), &dataSetParamStr3, 1, dataSetParamType,
- MPI_STATUS_IGNORE);
- MPI_File_write_at(file_handler, write_offset, &doubleValuesArray[0], blocksCount, dataSetDoubleType,
- MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(sizeof(dataSetParam)), &dataSetParamStr2, 1, dataSetParamType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(2 * sizeof(dataSetParam)), &dataSetParamStr3, 1, dataSetParamType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, write_offset, &doubleValuesArrayF[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
MPI_File_sync(file_handler);
MPI_File_close(&file_handler);
+
+ //-------------------------------- H1 ------------------------------------------------
+ if (multiPhase)
+ {
+ filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSetH1.bin";
+ rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc != MPI_SUCCESS)
+ throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ write_offset = (MPI_Offset)(firstGlobalID) * (MPI_Offset)(doubleCountInBlock) * (MPI_Offset)(sizeof(double));
+ MPI_File_write_at(file_handler, write_offset, &doubleValuesArrayH1[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
+
+ MPI_File_sync(file_handler);
+ MPI_File_close(&file_handler);
+ }
+
+ //-------------------------------- H2 --------------------------------------------------
+ /*if (D3Q27EsoTwist3DSplittedVectorPtr2 != 0)
+ {
+ filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSetH2.bin";
+ rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc != MPI_SUCCESS)
+ throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ write_offset = (MPI_Offset)(firstGlobalID) * (MPI_Offset)(doubleCountInBlock) * (MPI_Offset)(sizeof(double));
+ MPI_File_write_at(file_handler, write_offset, &doubleValuesArrayH2[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
+
+ MPI_File_sync(file_handler);
+ MPI_File_close(&file_handler);
+ } */
+
+ //--------------------------------
+
MPI_Type_free(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::writeDataSet time: " << finish - start << " s");
}
@@ -346,9 +388,12 @@ void MPIIOMigrationBECoProcessor::writeDataSet(int step)
write3DArray(step, RelaxationFactor, std::string("/cpRelaxationFactor.bin"));
// writeRelaxationFactor(step);
- if (arrPresence.isPhaseFieldPresent)
- write3DArray(step, PhaseField, std::string("/cpPhaseField.bin"));
-}
+ if (arrPresence.isPhaseField1Present)
+ write3DArray(step, PhaseField1, std::string("/cpPhaseField1.bin"));
+
+ if (arrPresence.isPhaseField2Present)
+ write3DArray(step, PhaseField2, std::string("/cpPhaseField2.bin"));
+ }
void MPIIOMigrationBECoProcessor::write4DArray(int step, Arrays arrayType, std::string fname)
{
@@ -361,7 +406,8 @@ void MPIIOMigrationBECoProcessor::write4DArray(int step, Arrays arrayType, std::
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
int maxInitLevel = this->grid->getFinestInitializedLevel();
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
grid->getBlocks(level, rank, blocksVector[level]);
blocksCount += static_cast<int>(blocksVector[level].size());
}
@@ -374,16 +420,18 @@ void MPIIOMigrationBECoProcessor::write4DArray(int step, Arrays arrayType, std::
int ic = 0;
SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> ___Array;
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::write4DArray start collect data rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
{
- switch (arrayType) {
+ switch (arrayType)
+ {
case AverageDensity:
___Array = block->getKernel()->getDataSet()->getAverageDensity();
break;
@@ -400,8 +448,7 @@ void MPIIOMigrationBECoProcessor::write4DArray(int step, Arrays arrayType, std::
___Array = block->getKernel()->getDataSet()->getShearStressValues();
break;
default:
- UB_THROW(UbException(UB_EXARGS,
- "MPIIOMigrationBECoProcessor::write4DArray : 4D array type does not exist!"));
+ UB_THROW(UbException(UB_EXARGS, "MPIIOMigrationBECoProcessor::write4DArray : 4D array type does not exist!"));
break;
}
@@ -410,29 +457,26 @@ void MPIIOMigrationBECoProcessor::write4DArray(int step, Arrays arrayType, std::
firstGlobalID = block->getGlobalID();
dataSetParamStr.nx1 = dataSetParamStr.nx2 = dataSetParamStr.nx3 = 0;
- dataSetParamStr.nx[0] = static_cast<int>(___Array->getNX1());
- dataSetParamStr.nx[1] = static_cast<int>(___Array->getNX2());
- dataSetParamStr.nx[2] = static_cast<int>(___Array->getNX3());
- dataSetParamStr.nx[3] = static_cast<int>(___Array->getNX4());
- doubleCountInBlock =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ dataSetParamStr.nx[0] = static_cast<int>(___Array->getNX1());
+ dataSetParamStr.nx[1] = static_cast<int>(___Array->getNX2());
+ dataSetParamStr.nx[2] = static_cast<int>(___Array->getNX3());
+ dataSetParamStr.nx[3] = static_cast<int>(___Array->getNX4());
+ doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
firstBlock = false;
}
- if ((dataSetParamStr.nx[0] > 0) && (dataSetParamStr.nx[1] > 0) && (dataSetParamStr.nx[2] > 0) &&
- (dataSetParamStr.nx[3] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), ___Array->getDataVector().begin(),
- ___Array->getDataVector().end());
+ if ((dataSetParamStr.nx[0] > 0) && (dataSetParamStr.nx[1] > 0) && (dataSetParamStr.nx[2] > 0) && (dataSetParamStr.nx[3] > 0))
+ doubleValuesArray.insert(doubleValuesArray.end(), ___Array->getDataVector().begin(), ___Array->getDataVector().end());
ic++;
}
}
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::write4DArray start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
// register new MPI-type depending on the block-specific information
@@ -451,20 +495,18 @@ void MPIIOMigrationBECoProcessor::write4DArray(int step, Arrays arrayType, std::
if (rc != MPI_SUCCESS)
throw UbException(UB_EXARGS, "couldn't open file " + filename);
- MPI_Offset write_offset = (MPI_Offset)(sizeof(dataSetParam)) + (MPI_Offset)(firstGlobalID) *
- (MPI_Offset)(doubleCountInBlock) *
- (MPI_Offset)(sizeof(double));
+ MPI_Offset write_offset = (MPI_Offset)(sizeof(dataSetParam)) + (MPI_Offset)(firstGlobalID) * (MPI_Offset)(doubleCountInBlock) * (MPI_Offset)(sizeof(double));
// each process writes common parameters of a dataSet
MPI_File_write_at(file_handler, 0, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
- MPI_File_write_at(file_handler, write_offset, &doubleValuesArray[0], blocksCount, dataSetDoubleType,
- MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, write_offset, &doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
MPI_File_sync(file_handler);
MPI_File_close(&file_handler);
MPI_Type_free(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::write4DArray time: " << finish - start << " s");
}
@@ -481,7 +523,8 @@ void MPIIOMigrationBECoProcessor::write3DArray(int step, Arrays arrayType, std::
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
int maxInitLevel = this->grid->getFinestInitializedLevel();
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
grid->getBlocks(level, rank, blocksVector[level]);
blocksCount += static_cast<int>(blocksVector[level].size());
}
@@ -494,25 +537,30 @@ void MPIIOMigrationBECoProcessor::write3DArray(int step, Arrays arrayType, std::
int ic = 0;
SPtr<CbArray3D<LBMReal, IndexerX3X2X1>> ___Array;
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::write3DArray start collect data rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
{
- switch (arrayType) {
+ switch (arrayType)
+ {
case RelaxationFactor:
___Array = block->getKernel()->getDataSet()->getRelaxationFactor();
break;
- case PhaseField:
+ case PhaseField1:
___Array = block->getKernel()->getDataSet()->getPhaseField();
break;
+ case PhaseField2:
+ ___Array = block->getKernel()->getDataSet()->getPhaseField2();
+ break;
default:
UB_THROW(UbException(UB_EXARGS,
- "MPIIOMigrationBECoProcessor::write3DArray : 3D array type does not exist!"));
+ "MPIIOMigrationBECoProcessor::write3DArray : 3D array type does not exist!"));
break;
}
@@ -531,17 +579,16 @@ void MPIIOMigrationBECoProcessor::write3DArray(int step, Arrays arrayType, std::
}
if ((dataSetParamStr.nx[0] > 0) && (dataSetParamStr.nx[1] > 0) && (dataSetParamStr.nx[2] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), ___Array->getDataVector().begin(),
- ___Array->getDataVector().end());
+ doubleValuesArray.insert(doubleValuesArray.end(), ___Array->getDataVector().begin(), ___Array->getDataVector().end());
ic++;
}
}
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::write3DArray start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
// register new MPI-type depending on the block-specific information
@@ -560,37 +607,33 @@ void MPIIOMigrationBECoProcessor::write3DArray(int step, Arrays arrayType, std::
if (rc != MPI_SUCCESS)
throw UbException(UB_EXARGS, "couldn't open file " + filename);
- MPI_Offset write_offset = (MPI_Offset)(sizeof(dataSetParam)) + (MPI_Offset)(firstGlobalID) *
- (MPI_Offset)(doubleCountInBlock) *
- (MPI_Offset)(sizeof(double));
+ MPI_Offset write_offset = (MPI_Offset)(sizeof(dataSetParam)) + (MPI_Offset)(firstGlobalID) * (MPI_Offset)(doubleCountInBlock) * (MPI_Offset)(sizeof(double));
// each process writes common parameters of a dataSet
MPI_File_write_at(file_handler, 0, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
- MPI_File_write_at(file_handler, write_offset, &doubleValuesArray[0], blocksCount, dataSetDoubleType,
- MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, write_offset, &doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
MPI_File_sync(file_handler);
MPI_File_close(&file_handler);
MPI_Type_free(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::write3DArray time: " << finish - start << " s");
}
}
-//---------------------------------------------------------------------------------
-
void MPIIOMigrationBECoProcessor::writeBoundaryConds(int step)
{
int rank, size;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::writeBoundaryConds start collect data rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
int blocksCount = 0; // quantity of blocks, that belong to this process
@@ -601,7 +644,8 @@ void MPIIOMigrationBECoProcessor::writeBoundaryConds(int step)
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
int maxInitLevel = this->grid->getFinestInitializedLevel();
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
grid->getBlocks(level, rank, blocksVector[level]);
blocksCount += static_cast<int>(blocksVector[level].size());
}
@@ -616,24 +660,26 @@ void MPIIOMigrationBECoProcessor::writeBoundaryConds(int step)
int ic = 0;
SPtr<BCArray3D> bcArr;
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
for (SPtr<Block3D> block : blocksVector[level]) // all the blocks of the current level
{
bcArr = block->getKernel()->getBCProcessor()->getBCArray();
- bcAddArray[ic].globalID =
- block->getGlobalID(); // id of the block needed to find it while regenerating the grid
+ bcAddArray[ic].globalID = block->getGlobalID(); // id of the block needed to find it while regenerating the grid
bcAddArray[ic].boundCond_count = 0; // how many BoundaryConditions in this block
bcAddArray[ic].indexContainer_count = 0; // how many indexContainer-values in this block
bytesCount[ic] = sizeof(BCAddMigration);
bcVector[ic].resize(0);
indexContainerVector[ic].resize(0);
- for (std::size_t bc = 0; bc < bcArr->getBCVectorSize(); bc++) {
+ for (std::size_t bc = 0; bc < bcArr->getBCVectorSize(); bc++)
+ {
BoundaryCondition *bouCond = new BoundaryCondition();
if (bcArr->bcvector[bc] == NULL)
memset(bouCond, 0, sizeof(BoundaryCondition));
- else {
+ else
+ {
bouCond->noslipBoundaryFlags = bcArr->bcvector[bc]->getNoSlipBoundary();
bouCond->slipBoundaryFlags = bcArr->bcvector[bc]->getSlipBoundary();
bouCond->velocityBoundaryFlags = bcArr->bcvector[bc]->getVelocityBoundary();
@@ -662,7 +708,8 @@ void MPIIOMigrationBECoProcessor::writeBoundaryConds(int step)
bytesCount[ic] += sizeof(BoundaryCondition);
}
- if (bcindexmatrixCountNotInit) {
+ if (bcindexmatrixCountNotInit)
+ {
boundCondParamStr.nx1 = static_cast<int>(bcArr->bcindexmatrix.getNX1());
boundCondParamStr.nx2 = static_cast<int>(bcArr->bcindexmatrix.getNX2());
boundCondParamStr.nx3 = static_cast<int>(bcArr->bcindexmatrix.getNX3());
@@ -670,11 +717,9 @@ void MPIIOMigrationBECoProcessor::writeBoundaryConds(int step)
bcindexmatrixCountNotInit = false;
}
- bcindexmatrixVector.insert(bcindexmatrixVector.end(), bcArr->bcindexmatrix.getDataVector().begin(),
- bcArr->bcindexmatrix.getDataVector().end());
+ bcindexmatrixVector.insert(bcindexmatrixVector.end(), bcArr->bcindexmatrix.getDataVector().begin(), bcArr->bcindexmatrix.getDataVector().end());
- indexContainerVector[ic].insert(indexContainerVector[ic].begin(), bcArr->indexContainer.begin(),
- bcArr->indexContainer.end());
+ indexContainerVector[ic].insert(indexContainerVector[ic].begin(), bcArr->indexContainer.begin(), bcArr->indexContainer.end());
bcAddArray[ic].indexContainer_count = static_cast<int>(bcArr->indexContainer.size());
count_indexContainer += bcAddArray[ic].indexContainer_count;
bytesCount[ic] += bcAddArray[ic].indexContainer_count * sizeof(int);
@@ -688,10 +733,10 @@ void MPIIOMigrationBECoProcessor::writeBoundaryConds(int step)
MPI_Type_contiguous(boundCondParamStr.bcindexmatrixCount, MPI_INT, &bcindexmatrixType);
MPI_Type_commit(&bcindexmatrixType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::writeBoundaryConds start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
@@ -712,13 +757,10 @@ void MPIIOMigrationBECoProcessor::writeBoundaryConds(int step)
if (rc != MPI_SUCCESS)
throw UbException(UB_EXARGS, "couldn't open file " + filename);
- MPI_Offset write_offset = (MPI_Offset)(sizeof(int)) + (MPI_Offset)(bcAddArray[0].globalID) *
- (MPI_Offset)(boundCondParamStr.bcindexmatrixCount) *
- (MPI_Offset)(sizeof(int));
+ MPI_Offset write_offset = (MPI_Offset)(sizeof(int)) + (MPI_Offset)(bcAddArray[0].globalID) * (MPI_Offset)(boundCondParamStr.bcindexmatrixCount) * (MPI_Offset)(sizeof(int));
MPI_File_write_at(file_handler, 0, &boundCondParamStr.bcindexmatrixCount, 1, MPI_INT, MPI_STATUS_IGNORE);
- MPI_File_write_at(file_handler, write_offset, &bcindexmatrixVector[0], blocksCount, bcindexmatrixType,
- MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, write_offset, &bcindexmatrixVector[0], blocksCount, bcindexmatrixType, MPI_STATUS_IGNORE);
MPI_File_sync(file_handler);
MPI_File_close(&file_handler);
@@ -732,14 +774,17 @@ void MPIIOMigrationBECoProcessor::writeBoundaryConds(int step)
MPI_File_write_at(file_handler, 0, &boundCondParamStr, 4, MPI_INT, MPI_STATUS_IGNORE);
- write_offset =
- (MPI_Offset)(sizeof(boundCondParam)) + (MPI_Offset)(grid->getNumberOfBlocks()) * (MPI_Offset)(sizeof(size_t));
+ write_offset = (MPI_Offset)(sizeof(boundCondParam)) + (MPI_Offset)(grid->getNumberOfBlocks()) * (MPI_Offset)(sizeof(size_t));
size_t next_file_offset = 0;
- if (size > 1) {
- if (rank == 0) {
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
next_file_offset = write_offset + allBytesCount;
MPI_Send(&next_file_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
- } else {
+ }
+ else
+ {
MPI_Recv(&write_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
next_file_offset = write_offset + allBytesCount;
if (rank < size - 1)
@@ -749,21 +794,17 @@ void MPIIOMigrationBECoProcessor::writeBoundaryConds(int step)
MPI_Offset write_offsetIndex;
- for (int nb = 0; nb < blocksCount; nb++) {
- write_offsetIndex =
- (MPI_Offset)(sizeof(boundCondParam)) + (MPI_Offset)(bcAddArray[nb].globalID) * (MPI_Offset)(sizeof(size_t));
+ for (int nb = 0; nb < blocksCount; nb++)
+ {
+ write_offsetIndex = (MPI_Offset)(sizeof(boundCondParam)) + (MPI_Offset)(bcAddArray[nb].globalID) * (MPI_Offset)(sizeof(size_t));
MPI_File_write_at(file_handler, write_offsetIndex, &write_offset, 1, MPI_LONG_LONG_INT, MPI_STATUS_IGNORE);
MPI_File_write_at(file_handler, write_offset, &bcAddArray[nb], 3, MPI_INT, MPI_STATUS_IGNORE);
if (bcVector[nb].size() > 0)
- MPI_File_write_at(file_handler, write_offset + (MPI_Offset)(sizeof(BCAddMigration)), &bcVector[nb][0],
- bcAddArray[nb].boundCond_count, boundCondType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, write_offset + (MPI_Offset)(sizeof(BCAddMigration)), &bcVector[nb][0], bcAddArray[nb].boundCond_count, boundCondType, MPI_STATUS_IGNORE);
if (indexContainerVector[nb].size() > 0)
- MPI_File_write_at(
- file_handler,
- write_offset + (MPI_Offset)(sizeof(BCAddMigration)) +
- (MPI_Offset)(bcAddArray[nb].boundCond_count) * (MPI_Offset)(sizeof(BoundaryCondition)),
+ MPI_File_write_at(file_handler, write_offset + (MPI_Offset)(sizeof(BCAddMigration)) + (MPI_Offset)(bcAddArray[nb].boundCond_count) * (MPI_Offset)(sizeof(BoundaryCondition)),
&indexContainerVector[nb][0], bcAddArray[nb].indexContainer_count, MPI_INT, MPI_STATUS_IGNORE);
write_offset += bytesCount[nb];
@@ -772,7 +813,8 @@ void MPIIOMigrationBECoProcessor::writeBoundaryConds(int step)
MPI_File_sync(file_handler);
MPI_File_close(&file_handler);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::writeBoundaryConds time: " << finish - start << " s");
}
@@ -792,8 +834,7 @@ void MPIIOMigrationBECoProcessor::restart(int step)
UBLOG(logINFO, "Load check point - start");
readBlocks(step);
- SPtr<Grid3DVisitor> newMetisVisitor(new MetisPartitioningGridVisitor(comm, MetisPartitioningGridVisitor::LevelBased,
- D3Q27System::BSW, MetisPartitioner::KWAY));
+ SPtr<Grid3DVisitor> newMetisVisitor(new MetisPartitioningGridVisitor(comm, MetisPartitioningGridVisitor::LevelBased, D3Q27System::BSW, MetisPartitioner::KWAY));
grid->accept(newMetisVisitor);
readDataSet(step);
@@ -806,8 +847,7 @@ void MPIIOMigrationBECoProcessor::restart(int step)
void MPIIOMigrationBECoProcessor::readBlocks(int step) { MPIIOCoProcessor::readBlocks(step); }
-void MPIIOMigrationBECoProcessor::blocksExchange(int tagN, int ind1, int ind2, int doubleCountInBlock,
- std::vector<double> &pV, std::vector<double> *rawDataReceive)
+void MPIIOMigrationBECoProcessor::blocksExchange(int tagN, int ind1, int ind2, int doubleCountInBlock, std::vector<double> &pV, std::vector<double> *rawDataReceive)
{
int rank, size;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
@@ -821,7 +861,8 @@ void MPIIOMigrationBECoProcessor::blocksExchange(int tagN, int ind1, int ind2, i
int *blocksCounterRec = new int[size];
std::vector<double> *rawDataSend = new std::vector<double>[size];
- for (int r = 0; r < size; r++) {
+ for (int r = 0; r < size; r++)
+ {
rawDataSend[r].resize(0);
blocksCounterSend[r] = 0;
blocksCounterRec[r] = 0;
@@ -842,8 +883,7 @@ void MPIIOMigrationBECoProcessor::blocksExchange(int tagN, int ind1, int ind2, i
{
blocksCounterRec[tempRank]++;
rawDataReceive[tempRank].push_back(double(indexB + ind));
- rawDataReceive[tempRank].insert(rawDataReceive[tempRank].end(),
- pV.begin() + ind * size_t(doubleCountInBlock),
+ rawDataReceive[tempRank].insert(rawDataReceive[tempRank].end(), pV.begin() + ind * size_t(doubleCountInBlock),
pV.begin() + ind * size_t(doubleCountInBlock) + size_t(doubleCountInBlock));
} else // we must send data to other processes
{
@@ -856,17 +896,20 @@ void MPIIOMigrationBECoProcessor::blocksExchange(int tagN, int ind1, int ind2, i
MPI_Request *requests = new MPI_Request[size * 2]; // send + receive
int requestCount = 0;
- // MPI_Status status;
- for (int r = 0; r < size; r++) {
- if (r != rank) {
+ for (int r = 0; r < size; r++)
+ {
+ if (r != rank)
+ {
MPI_Irecv(&blocksCounterRec[r], 1, MPI_INT, r, tagN, MPI_COMM_WORLD, &requests[requestCount]);
requestCount++;
}
}
- for (int r = 0; r < size; r++) {
- if (r != rank) {
+ for (int r = 0; r < size; r++)
+ {
+ if (r != rank)
+ {
MPI_Isend(&blocksCounterSend[r], 1, MPI_INT, r, tagN, MPI_COMM_WORLD, &requests[requestCount]);
requestCount++;
}
@@ -877,7 +920,8 @@ void MPIIOMigrationBECoProcessor::blocksExchange(int tagN, int ind1, int ind2, i
MPI_Type_contiguous(doubleCountInBlock + 1, MPI_DOUBLE, &sendBlockDoubleType);
MPI_Type_commit(&sendBlockDoubleType);
- for (int r = 0; r < size; r++) {
+ for (int r = 0; r < size; r++)
+ {
if (r != rank)
rawDataReceive[r].resize(size_t(blocksCounterRec[r]) * size_t(doubleCountInBlock + 1));
}
@@ -888,35 +932,39 @@ void MPIIOMigrationBECoProcessor::blocksExchange(int tagN, int ind1, int ind2, i
const int maxQuant = 400;
int restQuant;
- for (int r = 0; r < size; r++) {
- if (r != rank) {
+ for (int r = 0; r < size; r++)
+ {
+ if (r != rank)
+ {
sendRecCount = int(blocksCounterRec[r] / maxQuant);
if (sendRecCount * maxQuant < blocksCounterRec[r])
sendRecCount++;
requests = (MPI_Request *)realloc(requests, (requestCount + sendRecCount) * sizeof(MPI_Request));
- for (int sc = 0; sc < sendRecCount; sc++) {
+ for (int sc = 0; sc < sendRecCount; sc++)
+ {
restQuant = (sc < sendRecCount - 1) ? maxQuant : blocksCounterRec[r] - sc * maxQuant;
sendRecOffset = size_t(sc) * size_t(maxQuant) * size_t((doubleCountInBlock + 1));
- MPI_Irecv(&rawDataReceive[r][sendRecOffset], restQuant, sendBlockDoubleType, r, tagN, MPI_COMM_WORLD,
- &requests[requestCount]);
+ MPI_Irecv(&rawDataReceive[r][sendRecOffset], restQuant, sendBlockDoubleType, r, tagN, MPI_COMM_WORLD, &requests[requestCount]);
requestCount++;
}
}
}
- for (int r = 0; r < size; r++) {
- if (r != rank) {
+ for (int r = 0; r < size; r++)
+ {
+ if (r != rank)
+ {
sendRecCount = int(blocksCounterSend[r] / maxQuant);
if (sendRecCount * maxQuant < blocksCounterSend[r])
sendRecCount++;
requests = (MPI_Request *)realloc(requests, (requestCount + sendRecCount) * sizeof(MPI_Request));
- for (int sc = 0; sc < sendRecCount; sc++) {
+ for (int sc = 0; sc < sendRecCount; sc++)
+ {
restQuant = (sc < sendRecCount - 1) ? maxQuant : blocksCounterSend[r] - sc * maxQuant;
sendRecOffset = size_t(sc) * size_t(maxQuant) * size_t((doubleCountInBlock + 1));
- MPI_Isend(&rawDataSend[r][sendRecOffset], restQuant, sendBlockDoubleType, r, tagN, MPI_COMM_WORLD,
- &requests[requestCount]);
+ MPI_Isend(&rawDataSend[r][sendRecOffset], restQuant, sendBlockDoubleType, r, tagN, MPI_COMM_WORLD, &requests[requestCount]);
requestCount++;
}
}
@@ -942,13 +990,20 @@ void MPIIOMigrationBECoProcessor::readDataSet(int step)
UB_THROW(UbException(UB_EXARGS, "bcProcessor does not exist!"));
if (nue == -999.999)
UB_THROW(UbException(UB_EXARGS, "nue is not initialised!"));
+ if (nuL == -999.999 )
+ UB_THROW(UbException(UB_EXARGS, "nuL is not initialised!"));
+ if (nuG == -999.999)
+ UB_THROW(UbException(UB_EXARGS, "nuG is not initialised!"));
+ if (densityRatio == -999.999)
+ UB_THROW(UbException(UB_EXARGS, "densityRatio is not initialised!"));
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::readDataSet start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
+ bool multiPhase = false;
dataSetParam dataSetParamStr1, dataSetParamStr2, dataSetParamStr3;
int blocksCountAll = grid->getNumberOfBlocks(); // quantity of all blocks in the grid
@@ -968,150 +1023,183 @@ void MPIIOMigrationBECoProcessor::readDataSet(int step)
start = MPI_Wtime();
MPI_File file_handler;
- std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSet.bin";
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSetF.bin";
int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
if (rc != MPI_SUCCESS)
throw UbException(UB_EXARGS, "couldn't open file " + filename);
MPI_File_read_at(file_handler, (MPI_Offset)0, &dataSetParamStr1, 1, dataSetParamType, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, (MPI_Offset)(sizeof(dataSetParam)), &dataSetParamStr2, 1, dataSetParamType,
- MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, (MPI_Offset)(2 * sizeof(dataSetParam)), &dataSetParamStr3, 1, dataSetParamType,
- MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(sizeof(dataSetParam)), &dataSetParamStr2, 1, dataSetParamType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(2 * sizeof(dataSetParam)), &dataSetParamStr3, 1, dataSetParamType, MPI_STATUS_IGNORE);
- size_t doubleCountInBlock =
- (dataSetParamStr1.nx[0] * dataSetParamStr1.nx[1] * dataSetParamStr1.nx[2] * dataSetParamStr1.nx[3] +
+ size_t doubleCountInBlock = dataSetParamStr1.nx[0] * dataSetParamStr1.nx[1] * dataSetParamStr1.nx[2] * dataSetParamStr1.nx[3] +
dataSetParamStr2.nx[0] * dataSetParamStr2.nx[1] * dataSetParamStr2.nx[2] * dataSetParamStr2.nx[3] +
- dataSetParamStr3.nx[0] * dataSetParamStr3.nx[1] * dataSetParamStr3.nx[2] * dataSetParamStr3.nx[3]) * 2;
- std::vector<double> doubleValuesArray(myBlocksCount * doubleCountInBlock); // double-values in all blocks
+ dataSetParamStr3.nx[0] * dataSetParamStr3.nx[1] * dataSetParamStr3.nx[2] * dataSetParamStr3.nx[3];
+ std::vector<double> doubleValuesArrayF(size_t(myBlocksCount * doubleCountInBlock)); // double-values in all blocks Fdistributions
+ std::vector<double> doubleValuesArrayH1; // double-values in all blocks H1distributions
+ //std::vector<double> doubleValuesArrayH2; // double-values in all blocks H2distributions
MPI_Type_contiguous(int(doubleCountInBlock), MPI_DOUBLE, &dataSetDoubleType);
MPI_Type_commit(&dataSetDoubleType);
- MPI_Offset read_offset =
- (MPI_Offset)(3 * sizeof(dataSetParam)) + (MPI_Offset)(indexB * doubleCountInBlock * sizeof(double));
- MPI_File_read_at(file_handler, read_offset, &doubleValuesArray[0], int(myBlocksCount), dataSetDoubleType,
- MPI_STATUS_IGNORE);
+ //--------------------------------- F ---------------------------------------------------------
+ MPI_Offset read_offset = (MPI_Offset)(3 * sizeof(dataSetParam)) + (MPI_Offset)(indexB * doubleCountInBlock * sizeof(double));
+ MPI_File_read_at(file_handler, read_offset, &doubleValuesArrayF[0], int(myBlocksCount), dataSetDoubleType, MPI_STATUS_IGNORE);
MPI_File_close(&file_handler);
+
+ //--------------------------------- H1 ---------------------------------------------------------
+ MPI_Offset fsize;
+ filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSetH1.bin";
+ rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
+ if (rc != MPI_SUCCESS)
+ throw UbException(UB_EXARGS, "couldn't open file " + filename);
+ int fs = MPI_File_get_size(file_handler, &fsize);
+ if (fsize > 0)
+ {
+ multiPhase = true;
+ doubleValuesArrayH1.resize(myBlocksCount * doubleCountInBlock);
+
+ read_offset = (MPI_Offset)(indexB * doubleCountInBlock * sizeof(double)) ;
+ MPI_File_read_at(file_handler, read_offset, &doubleValuesArrayH1[0], int(myBlocksCount), dataSetDoubleType, MPI_STATUS_IGNORE);
+ }
+ MPI_File_close(&file_handler);
+
MPI_Type_free(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::readDataSet time: " << finish - start << " s");
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::readDataSet start of exchange of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
- std::vector<double> *rawDataReceive = new std::vector<double>[size];
+ std::vector<double>* rawDataReceiveF = new std::vector<double>[size];
for (int r = 0; r < size; r++)
- rawDataReceive[r].resize(0);
+ rawDataReceiveF[r].resize(0);
+ blocksExchange(MESSAGE_TAG, indexB, indexE, int(doubleCountInBlock), doubleValuesArrayF, rawDataReceiveF);
- blocksExchange(MESSAGE_TAG, indexB, indexE, int(doubleCountInBlock), doubleValuesArray, rawDataReceive);
+ std::vector<double>* rawDataReceiveH1 = new std::vector<double>[size];
+ for (int r = 0; r < size; r++)
+ rawDataReceiveH1[r].resize(0);
+ blocksExchange(MESSAGE_TAG, indexB, indexE, int(doubleCountInBlock), doubleValuesArrayH1, rawDataReceiveH1);
- if (comm->isRoot()) {
+ /* std::vector<double>* rawDataReceiveH2 = new std::vector<double>[size];
+ for (int r = 0; r < size; r++)
+ rawDataReceiveH2[r].resize(0);
+ blocksExchange(MESSAGE_TAG, indexB, indexE, int(doubleCountInBlock), doubleValuesArrayH2, rawDataReceiveH2);*/
+
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::readDataSet time: " << finish - start << " s");
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::readDataSet start of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
-
+
//-------------------------------------- restore blocks ---------------------------------
int blockID;
std::vector<double> vectorsOfValuesF1, vectorsOfValuesF2, vectorsOfValuesF3;
- std::vector<double> vectorsOfValuesH1, vectorsOfValuesH2, vectorsOfValuesH3;
+ std::vector<double> vectorsOfValuesH11, vectorsOfValuesH12, vectorsOfValuesH13;
+ //std::vector<double> vectorsOfValuesH21, vectorsOfValuesH22, vectorsOfValuesH23;
- size_t vectorSize1 =
- dataSetParamStr1.nx[0] * dataSetParamStr1.nx[1] * dataSetParamStr1.nx[2] * dataSetParamStr1.nx[3];
- size_t vectorSize2 =
- dataSetParamStr2.nx[0] * dataSetParamStr2.nx[1] * dataSetParamStr2.nx[2] * dataSetParamStr2.nx[3];
- size_t vectorSize3 =
- dataSetParamStr3.nx[0] * dataSetParamStr3.nx[1] * dataSetParamStr3.nx[2] * dataSetParamStr3.nx[3];
+ size_t vectorSize1 = dataSetParamStr1.nx[0] * dataSetParamStr1.nx[1] * dataSetParamStr1.nx[2] * dataSetParamStr1.nx[3];
+ size_t vectorSize2 = dataSetParamStr2.nx[0] * dataSetParamStr2.nx[1] * dataSetParamStr2.nx[2] * dataSetParamStr2.nx[3];
+ size_t vectorSize3 = dataSetParamStr3.nx[0] * dataSetParamStr3.nx[1] * dataSetParamStr3.nx[2] * dataSetParamStr3.nx[3];
size_t index;
- for (int r = 0; r < size; r++) {
+ for (int r = 0; r < size; r++)
+ {
index = 0;
- for (int ii = 0; ii < int(rawDataReceive[r].size() / doubleCountInBlock); ii++) {
- blockID = (int)(rawDataReceive[r][index]);
+ for (int ii = 0; ii < int(rawDataReceiveF[r].size() / doubleCountInBlock); ii++)
+ {
+ blockID = (int)(rawDataReceiveF[r][index]);
index += 1;
- vectorsOfValuesF1.assign(rawDataReceive[r].data() + index, rawDataReceive[r].data() + index + vectorSize1);
- index += vectorSize1;
-
- vectorsOfValuesF2.assign(rawDataReceive[r].data() + index, rawDataReceive[r].data() + index + vectorSize2);
- index += vectorSize2;
-
- vectorsOfValuesF3.assign(rawDataReceive[r].data() + index, rawDataReceive[r].data() + index + vectorSize3);
- index += vectorSize3;
-
- vectorsOfValuesH1.assign(rawDataReceive[r].data() + index, rawDataReceive[r].data() + index + vectorSize1);
+ vectorsOfValuesF1.assign(rawDataReceiveF[r].data() + index, rawDataReceiveF[r].data() + index + vectorSize1);
+ if(multiPhase)
+ vectorsOfValuesH11.assign(rawDataReceiveH1[r].data() + index, rawDataReceiveH1[r].data() + index + vectorSize1);
+ //vectorsOfValuesH21.assign(rawDataReceiveH2[r].data() + index, rawDataReceiveH2[r].data() + index + vectorSize1);
index += vectorSize1;
- vectorsOfValuesH2.assign(rawDataReceive[r].data() + index, rawDataReceive[r].data() + index + vectorSize2);
+ vectorsOfValuesF2.assign(rawDataReceiveF[r].data() + index, rawDataReceiveF[r].data() + index + vectorSize2);
+ if (multiPhase)
+ vectorsOfValuesH12.assign(rawDataReceiveH1[r].data() + index, rawDataReceiveH1[r].data() + index + vectorSize1);
+ //vectorsOfValuesH22.assign(rawDataReceiveH2[r].data() + index, rawDataReceiveH2[r].data() + index + vectorSize1);
index += vectorSize2;
- vectorsOfValuesH3.assign(rawDataReceive[r].data() + index, rawDataReceive[r].data() + index + vectorSize3);
+ vectorsOfValuesF3.assign(rawDataReceiveF[r].data() + index, rawDataReceiveF[r].data() + index + vectorSize3);
+ if (multiPhase)
+ vectorsOfValuesH13.assign(rawDataReceiveH1[r].data() + index, rawDataReceiveH1[r].data() + index + vectorSize1);
+ //vectorsOfValuesH23.assign(rawDataReceiveH2[r].data() + index, rawDataReceiveH2[r].data() + index + vectorSize1);
index += vectorSize3;
SPtr<DistributionArray3D> mFdistributions(new D3Q27EsoTwist3DSplittedVector());
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)
- ->setLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
- new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesF1, dataSetParamStr1.nx[0],
- dataSetParamStr1.nx[1], dataSetParamStr1.nx[2],
- dataSetParamStr1.nx[3])));
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)
- ->setNonLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
- new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesF2, dataSetParamStr2.nx[0],
- dataSetParamStr2.nx[1], dataSetParamStr2.nx[2],
- dataSetParamStr2.nx[3])));
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)
- ->setZeroDistributions(CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
+ new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesF1, dataSetParamStr1.nx[0], dataSetParamStr1.nx[1], dataSetParamStr1.nx[2], dataSetParamStr1.nx[3])));
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNonLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
+ new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesF2, dataSetParamStr2.nx[0], dataSetParamStr2.nx[1], dataSetParamStr2.nx[2], dataSetParamStr2.nx[3])));
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setZeroDistributions(CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(
vectorsOfValuesF3, dataSetParamStr3.nx[0], dataSetParamStr3.nx[1], dataSetParamStr3.nx[2])));
dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNX1(dataSetParamStr1.nx1);
dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNX2(dataSetParamStr1.nx2);
dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNX3(dataSetParamStr1.nx3);
- SPtr<DistributionArray3D> mHdistributions(new D3Q27EsoTwist3DSplittedVector());
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mHdistributions)
- ->setLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
- new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesH1, dataSetParamStr1.nx[0],
- dataSetParamStr1.nx[1], dataSetParamStr1.nx[2],
- dataSetParamStr1.nx[3])));
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mHdistributions)
- ->setNonLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
- new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesH2, dataSetParamStr2.nx[0],
- dataSetParamStr2.nx[1], dataSetParamStr2.nx[2],
- dataSetParamStr2.nx[3])));
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mHdistributions)
- ->setZeroDistributions( CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(
- vectorsOfValuesH3, dataSetParamStr3.nx[0], dataSetParamStr3.nx[1], dataSetParamStr3.nx[2])));
-
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mHdistributions)->setNX1(dataSetParamStr1.nx1);
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mHdistributions)->setNX2(dataSetParamStr1.nx2);
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mHdistributions)->setNX3(dataSetParamStr1.nx3);
+ SPtr<DistributionArray3D> mH1distributions(new D3Q27EsoTwist3DSplittedVector());
+ if (multiPhase)
+ {
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH1distributions)->setLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
+ new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesH11, dataSetParamStr1.nx[0], dataSetParamStr1.nx[1], dataSetParamStr1.nx[2], dataSetParamStr1.nx[3])));
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH1distributions)->setNonLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
+ new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesH12, dataSetParamStr2.nx[0], dataSetParamStr2.nx[1], dataSetParamStr2.nx[2], dataSetParamStr2.nx[3])));
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH1distributions)->setZeroDistributions(CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(
+ vectorsOfValuesH13, dataSetParamStr3.nx[0], dataSetParamStr3.nx[1], dataSetParamStr3.nx[2])));
+
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH1distributions)->setNX1(dataSetParamStr1.nx1);
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH1distributions)->setNX2(dataSetParamStr1.nx2);
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH1distributions)->setNX3(dataSetParamStr1.nx3);
+ }
+
+ /*SPtr<DistributionArray3D> mH2distributions(new D3Q27EsoTwist3DSplittedVector());
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH2distributions)->setLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
+ new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesH21, dataSetParamStr1.nx[0], dataSetParamStr1.nx[1], dataSetParamStr1.nx[2], dataSetParamStr1.nx[3])));
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH2distributions)->setNonLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
+ new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesH22, dataSetParamStr2.nx[0], dataSetParamStr2.nx[1], dataSetParamStr2.nx[2], dataSetParamStr2.nx[3])));
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH2distributions)->setZeroDistributions(CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(
+ vectorsOfValuesH23, dataSetParamStr3.nx[0], dataSetParamStr3.nx[1], dataSetParamStr3.nx[2])));
+
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH2distributions)->setNX1(dataSetParamStr1.nx1);
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH2distributions)->setNX2(dataSetParamStr1.nx2);
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH2distributions)->setNX3(dataSetParamStr1.nx3);*/
+
// find the nesessary block and fill it
SPtr<Block3D> block = grid->getBlock(blockID);
this->lbmKernel->setBlock(block);
SPtr<LBMKernel> kernel = this->lbmKernel->clone();
- LBMReal collFactor = LBMSystem::calcCollisionFactor(this->nue, block->getLevel());
+ LBMReal collFactor = LBMSystem::calcCollisionFactor(this->nue, block->getLevel());
+ LBMReal collFactorL = LBMSystem::calcCollisionFactor(this->nuL, block->getLevel());
+ LBMReal collFactorG = LBMSystem::calcCollisionFactor(this->nuG, block->getLevel());
kernel->setCollisionFactor(collFactor);
kernel->setIndex(block->getX1(), block->getX2(), block->getX3());
kernel->setDeltaT(LBMSystem::getDeltaT(block->getLevel()));
+ kernel->setCollisionFactorMultiphase(collFactorL, collFactorG);
+ kernel->setDensityRatio(this->densityRatio);
SPtr<DataSet3D> dataSetPtr = SPtr<DataSet3D>(new DataSet3D());
dataSetPtr->setFdistributions(mFdistributions);
- dataSetPtr->setHdistributions(mHdistributions);
+ if (multiPhase)
+ dataSetPtr->setHdistributions(mH1distributions);
+// dataSetPtr->setHdistributions(mH2distributions);
kernel->setDataSet(dataSetPtr);
block->setKernel(kernel);
}
}
- if (comm->isRoot()) {
+ //if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::readDataSet end of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
//-------------------------------------------------------------
@@ -1149,11 +1237,16 @@ void MPIIOMigrationBECoProcessor::readDataSet(int step)
if (arrPresence.isRelaxationFactorPresent)
readArray(step, RelaxationFactor, std::string("/cpRelaxationFactor.bin"));
// readRelaxationFactor(step);
+
+ if (arrPresence.isPhaseField1Present)
+ readArray(step, PhaseField1, std::string("/cpPhaseField1.bin"));
- if (arrPresence.isPhaseFieldPresent)
- readArray(step, PhaseField, std::string("/cpPhaseField.bin"));
+ if (arrPresence.isPhaseField2Present)
+ readArray(step, PhaseField2, std::string("/cpPhaseField2.bin"));
- delete[] rawDataReceive;
+ delete[] rawDataReceiveF;
+// delete[] rawDataReceiveH1;
+// delete[] rawDataReceiveH2;
}
void MPIIOMigrationBECoProcessor::readArray(int step, Arrays arrType, std::string fname)
@@ -1162,11 +1255,12 @@ void MPIIOMigrationBECoProcessor::readArray(int step, Arrays arrType, std::strin
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::readArray start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
+
double start, finish;
if (comm->isRoot())
start = MPI_Wtime();
@@ -1194,56 +1288,53 @@ void MPIIOMigrationBECoProcessor::readArray(int step, Arrays arrType, std::strin
MPI_File_read_at(file_handler, (MPI_Offset)0, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
- size_t doubleCountInBlock =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ size_t doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
std::vector<double> doubleValuesArray(myBlocksCount * doubleCountInBlock); // double-values in all blocks
MPI_Type_contiguous(int(doubleCountInBlock), MPI_DOUBLE, &dataSetDoubleType);
MPI_Type_commit(&dataSetDoubleType);
- MPI_Offset read_offset = (MPI_Offset)(sizeof(dataSetParam)) +
- (MPI_Offset)(indexB) * (MPI_Offset)(doubleCountInBlock) * (MPI_Offset)(sizeof(double));
- MPI_File_read_at(file_handler, read_offset, &doubleValuesArray[0], int(myBlocksCount), dataSetDoubleType,
- MPI_STATUS_IGNORE);
+ MPI_Offset read_offset = (MPI_Offset)(sizeof(dataSetParam)) + (MPI_Offset)(indexB) * (MPI_Offset)(doubleCountInBlock) * (MPI_Offset)(sizeof(double));
+ MPI_File_read_at(file_handler, read_offset, &doubleValuesArray[0], int(myBlocksCount), dataSetDoubleType, MPI_STATUS_IGNORE);
MPI_File_close(&file_handler);
MPI_Type_free(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::readArray time: " << finish - start << " s");
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::readArray start of exchange of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
std::vector<double> *rawDataReceive = new std::vector<double>[size];
for (int r = 0; r < size; r++)
rawDataReceive[r].resize(0);
- blocksExchange(MESSAGE_TAG + int(arrType), indexB, indexE, int(doubleCountInBlock), doubleValuesArray,
- rawDataReceive);
+ blocksExchange(MESSAGE_TAG + int(arrType), indexB, indexE, int(doubleCountInBlock), doubleValuesArray, rawDataReceive);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::readArray end of exchange of data, rank = " << rank);
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::readArray time: " << finish - start << " s");
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::readArray start of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
//----------------------------- restore data ---------------------------------
int blockID;
std::vector<double> vectorsOfValues;
size_t index;
- size_t nextVectorSize =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ size_t nextVectorSize = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
- for (int r = 0; r < size; r++) {
+ for (int r = 0; r < size; r++)
+ {
index = 0;
- for (int ii = 0; ii < int(rawDataReceive[r].size() / doubleCountInBlock); ii++) {
- blockID = (int)(rawDataReceive[r][index]);
+ for (int ii = 0; ii < int(rawDataReceive[r].size() / doubleCountInBlock); ii++)
+ {
+ blockID = (int)(rawDataReceive[r][index]);
SPtr<Block3D> block = grid->getBlock(blockID);
index += 1;
@@ -1254,40 +1345,31 @@ void MPIIOMigrationBECoProcessor::readArray(int step, Arrays arrType, std::strin
SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> ___4DArray;
SPtr<CbArray3D<LBMReal, IndexerX3X2X1>> ___3DArray;
- switch (arrType) {
+ switch (arrType)
+ {
case AverageDensity:
- ___4DArray =
- CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(
- vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2],
- dataSetParamStr.nx[3]));
+ ___4DArray = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(
+ vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
block->getKernel()->getDataSet()->setAverageDensity(___4DArray);
break;
case AverageVelocity:
- ___4DArray =
- CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(
- vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2],
- dataSetParamStr.nx[3]));
+ ___4DArray = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(
+ vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
block->getKernel()->getDataSet()->setAverageVelocity(___4DArray);
break;
case AverageFluktuations:
- ___4DArray =
- CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(
- vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2],
- dataSetParamStr.nx[3]));
+ ___4DArray = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(
+ vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
block->getKernel()->getDataSet()->setAverageFluctuations(___4DArray);
break;
case AverageTriple:
- ___4DArray =
- CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(
- vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2],
- dataSetParamStr.nx[3]));
+ ___4DArray = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(
+ vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
block->getKernel()->getDataSet()->setAverageTriplecorrelations(___4DArray);
break;
case ShearStressVal:
- ___4DArray =
- CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(
- vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2],
- dataSetParamStr.nx[3]));
+ ___4DArray = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(
+ vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
block->getKernel()->getDataSet()->setShearStressValues(___4DArray);
break;
case RelaxationFactor:
@@ -1295,14 +1377,18 @@ void MPIIOMigrationBECoProcessor::readArray(int step, Arrays arrType, std::strin
vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2]));
block->getKernel()->getDataSet()->setRelaxationFactor(___3DArray);
break;
- case PhaseField:
+ case PhaseField1:
___3DArray = CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(
vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2]));
block->getKernel()->getDataSet()->setPhaseField(___3DArray);
break;
+ case PhaseField2:
+ ___3DArray = CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(
+ vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2]));
+ block->getKernel()->getDataSet()->setPhaseField2(___3DArray);
+ break;
default:
- UB_THROW(
- UbException(UB_EXARGS, "MPIIOMigrationBECoProcessor::readArray : array type does not exist!"));
+ UB_THROW(UbException(UB_EXARGS, "MPIIOMigrationBECoProcessor::readArray : array type does not exist!"));
break;
}
}
@@ -1310,10 +1396,10 @@ void MPIIOMigrationBECoProcessor::readArray(int step, Arrays arrType, std::strin
delete[] rawDataReceive;
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::readArray end of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
}
@@ -1323,10 +1409,10 @@ void MPIIOMigrationBECoProcessor::readBoundaryConds(int step)
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::readBoundaryConds start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
@@ -1360,25 +1446,24 @@ void MPIIOMigrationBECoProcessor::readBoundaryConds(int step)
MPI_Type_contiguous(sizeOfBIM, MPI_INT, &bcindexmatrixType);
MPI_Type_commit(&bcindexmatrixType);
- MPI_Offset read_offset =
- (MPI_Offset)(sizeof(int)) + (MPI_Offset)(indexB) * (MPI_Offset)(sizeOfBIM) * (MPI_Offset)(sizeof(int));
- MPI_File_read_at(file_handler, read_offset, &bcindexmatrixVAll[0], int(myBlocksCount), bcindexmatrixType,
- MPI_STATUS_IGNORE);
+ MPI_Offset read_offset = (MPI_Offset)(sizeof(int)) + (MPI_Offset)(indexB) * (MPI_Offset)(sizeOfBIM) * (MPI_Offset)(sizeof(int));
+ MPI_File_read_at(file_handler, read_offset, &bcindexmatrixVAll[0], int(myBlocksCount), bcindexmatrixType, MPI_STATUS_IGNORE);
MPI_File_close(&file_handler);
MPI_Type_free(&bcindexmatrixType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::readBoundaryConds time: " << finish - start << " s");
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::readBoundaryConds start of exchange of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
std::vector<int> *rawDataReceive = new std::vector<int>[size];
std::vector<int> *rawDataSend = new std::vector<int>[size];
- for (int r = 0; r < size; r++) {
+ for (int r = 0; r < size; r++)
+ {
rawDataReceive[r].resize(0);
rawDataSend[r].resize(0);
rawDataReceive[r].push_back(0);
@@ -1396,14 +1481,12 @@ void MPIIOMigrationBECoProcessor::readBoundaryConds(int step)
{
rawDataReceive[tempRank][0]++;
rawDataReceive[tempRank].push_back(indexB + ind);
- rawDataReceive[tempRank].insert(rawDataReceive[tempRank].end(), bcindexmatrixVAll.begin() + ind * sizeOfBIM,
- bcindexmatrixVAll.begin() + ind * sizeOfBIM + sizeOfBIM);
+ rawDataReceive[tempRank].insert(rawDataReceive[tempRank].end(), bcindexmatrixVAll.begin() + ind * sizeOfBIM, bcindexmatrixVAll.begin() + ind * sizeOfBIM + sizeOfBIM);
} else // we must send data to other processes
{
rawDataSend[tempRank][0]++;
rawDataSend[tempRank].push_back(indexB + ind);
- rawDataSend[tempRank].insert(rawDataSend[tempRank].end(), bcindexmatrixVAll.begin() + ind * sizeOfBIM,
- bcindexmatrixVAll.begin() + ind * sizeOfBIM + sizeOfBIM);
+ rawDataSend[tempRank].insert(rawDataSend[tempRank].end(), bcindexmatrixVAll.begin() + ind * sizeOfBIM, bcindexmatrixVAll.begin() + ind * sizeOfBIM + sizeOfBIM);
}
}
@@ -1414,9 +1497,11 @@ void MPIIOMigrationBECoProcessor::readBoundaryConds(int step)
int intBlockCount;
int rds;
- for (int r = 0; r < size; r++) {
- if (r != rank) {
- rds = int(rawDataSend[r].size());
+ for (int r = 0; r < size; r++)
+ {
+ if (r != rank)
+ {
+ rds = int(rawDataSend[r].size());
intBlockCount = (int)(rds / SEND_BLOCK_SIZE);
if (intBlockCount * SEND_BLOCK_SIZE < rds)
intBlockCount += 1;
@@ -1424,21 +1509,21 @@ void MPIIOMigrationBECoProcessor::readBoundaryConds(int step)
for (int i = rds; i < intBlockCount * SEND_BLOCK_SIZE; i++)
rawDataSend[r].push_back(0);
- MPI_Isend(&rawDataSend[r][0], intBlockCount, sendBlockIntType, r, MESSAGE_TAG + 7, MPI_COMM_WORLD,
- &requests[requestCount]);
+ MPI_Isend(&rawDataSend[r][0], intBlockCount, sendBlockIntType, r, MESSAGE_TAG + 7, MPI_COMM_WORLD, &requests[requestCount]);
// MPI_Isend(&rawDataSend[r][0], rawDataSend[r].size(), MPI_INT, r, MESSAGE_TAG + 7, MPI_COMM_WORLD,
// &requests[requestCount]);
requestCount++;
}
}
- for (int r = 0; r < size; r++) {
- if (r != rank) {
+ for (int r = 0; r < size; r++)
+ {
+ if (r != rank)
+ {
MPI_Probe(r, MESSAGE_TAG + 7, MPI_COMM_WORLD, &status);
MPI_Get_count(&status, sendBlockIntType, &quant);
rawDataReceive[r].resize(quant * SEND_BLOCK_SIZE);
- MPI_Irecv(&rawDataReceive[r][0], quant, sendBlockIntType, r, MESSAGE_TAG + 7, MPI_COMM_WORLD,
- &requests[requestCount]);
+ MPI_Irecv(&rawDataReceive[r][0], quant, sendBlockIntType, r, MESSAGE_TAG + 7, MPI_COMM_WORLD, &requests[requestCount]);
requestCount++;
}
}
@@ -1447,17 +1532,17 @@ void MPIIOMigrationBECoProcessor::readBoundaryConds(int step)
//- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::readBoundaryConds end of exchange of data, rank = " << rank);
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::readBoundaryConds time: " << finish - start << " s");
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::readBoundaryConds start of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpBC2.bin";
- rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
+ rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
if (rc != MPI_SUCCESS)
throw UbException(UB_EXARGS, "couldn't open file " + filename);
@@ -1475,10 +1560,12 @@ void MPIIOMigrationBECoProcessor::readBoundaryConds(int step)
std::vector<int> indexContainerV;
std::vector<int> bcindexmatrixV;
- for (int r = 0; r < size; r++) {
+ for (int r = 0; r < size; r++)
+ {
index = 1;
- for (int ii = 0; ii < rawDataReceive[r][0]; ii++) {
+ for (int ii = 0; ii < rawDataReceive[r][0]; ii++)
+ {
blockID = (int)(rawDataReceive[r][index]);
index += 1;
@@ -1498,18 +1585,19 @@ void MPIIOMigrationBECoProcessor::readBoundaryConds(int step)
bcAddArray.boundCond_count, boundCondType, MPI_STATUS_IGNORE);
if (bcAddArray.indexContainer_count > 0)
- MPI_File_read_at(file_handler,
- read_offset2 + (MPI_Offset)(sizeof(BCAddMigration)) +
+ MPI_File_read_at(file_handler, read_offset2 + (MPI_Offset)(sizeof(BCAddMigration)) +
(MPI_Offset)(bcAddArray.boundCond_count) * (MPI_Offset)(sizeof(BoundaryCondition)),
&indexContainerV[0], bcAddArray.indexContainer_count, MPI_INT, MPI_STATUS_IGNORE);
bcVector.resize(0);
- for (int ibc = 0; ibc < bcAddArray.boundCond_count; ibc++) {
+ for (int ibc = 0; ibc < bcAddArray.boundCond_count; ibc++)
+ {
SPtr<BoundaryConditions> bc;
if (memcmp(&bcArray[ibc], nullBouCond, sizeof(BoundaryCondition)) == 0)
bc = SPtr<BoundaryConditions>();
- else {
+ else
+ {
bc = SPtr<BoundaryConditions>(new BoundaryConditions);
bc->noslipBoundaryFlags = bcArray[ibc].noslipBoundaryFlags;
bc->slipBoundaryFlags = bcArray[ibc].slipBoundaryFlags;
@@ -1537,8 +1625,7 @@ void MPIIOMigrationBECoProcessor::readBoundaryConds(int step)
bcVector.push_back(bc);
}
- CbArray3D<int, IndexerX3X2X1> bcim(bcindexmatrixV, boundCondParamStr.nx1, boundCondParamStr.nx2,
- boundCondParamStr.nx3);
+ CbArray3D<int, IndexerX3X2X1> bcim(bcindexmatrixV, boundCondParamStr.nx1, boundCondParamStr.nx2, boundCondParamStr.nx3);
SPtr<Block3D> block1 = grid->getBlock(blockID);
SPtr<BCProcessor> bcProc = bcProcessor->clone(block1->getKernel());
@@ -1562,12 +1649,12 @@ void MPIIOMigrationBECoProcessor::readBoundaryConds(int step)
delete[] rawDataSend;
delete[] requests;
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::readBoundaryConds end of restore of data, rank = " << rank);
UBLOG(logINFO, "MPIIOMigrationBECoProcessor::readBoundaryConds time: " << finish - start << " s");
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
}
@@ -1577,3 +1664,8 @@ void MPIIOMigrationBECoProcessor::setLBMKernel(SPtr<LBMKernel> kernel) { this->l
void MPIIOMigrationBECoProcessor::setBCProcessor(SPtr<BCProcessor> bcProcessor) { this->bcProcessor = bcProcessor; }
//////////////////////////////////////////////////////////////////////////
void MPIIOMigrationBECoProcessor::setNu(double nu) { this->nue = nu; }
+
+void MPIIOMigrationBECoProcessor::setNuLG(double cfL, double cfG) { this->nuL = cfL; this->nuG = cfG; }
+
+void MPIIOMigrationBECoProcessor::setDensityRatio(double dr) { this->densityRatio = dr; }
+
diff --git a/src/cpu/VirtualFluidsCore/CoProcessors/MPIIOMigrationBECoProcessor.h b/src/cpu/VirtualFluidsCore/CoProcessors/MPIIOMigrationBECoProcessor.h
index 4d825fde3956dcbe711f49b18b57cd929ba986d9..9a89ada1ae039d10cd53b06b189e5709398911c8 100644
--- a/src/cpu/VirtualFluidsCore/CoProcessors/MPIIOMigrationBECoProcessor.h
+++ b/src/cpu/VirtualFluidsCore/CoProcessors/MPIIOMigrationBECoProcessor.h
@@ -25,7 +25,8 @@ class MPIIOMigrationBECoProcessor : public MPIIOCoProcessor
AverageTriple = 4,
ShearStressVal = 5,
RelaxationFactor = 6,
- PhaseField = 7
+ PhaseField1 = 7,
+ PhaseField2 = 8
};
public:
@@ -71,6 +72,8 @@ public:
//! The function truncates the data files
void clearAllFiles(int step);
void setNu(double nu);
+ void setNuLG(double cfL, double cfG);
+ void setDensityRatio(double dr);
void blocksExchange(int tagN, int ind1, int ind2, int doubleCountInBlock, std::vector<double> &pV,
std::vector<double> *rawDataReceive);
@@ -91,6 +94,10 @@ private:
SPtr<LBMKernel> lbmKernel;
SPtr<BCProcessor> bcProcessor;
double nue;
+ double nuL;
+ double nuG;
+ double densityRatio;
+
};
#endif
diff --git a/src/cpu/VirtualFluidsCore/CoProcessors/MPIIOMigrationCoProcessor.cpp b/src/cpu/VirtualFluidsCore/CoProcessors/MPIIOMigrationCoProcessor.cpp
index 69cc8795bb7fdb4791420473a242a0f1ff96a06d..d0ce025fda5428a74ec27901202b06acef4781e2 100644
--- a/src/cpu/VirtualFluidsCore/CoProcessors/MPIIOMigrationCoProcessor.cpp
+++ b/src/cpu/VirtualFluidsCore/CoProcessors/MPIIOMigrationCoProcessor.cpp
@@ -22,8 +22,7 @@
using namespace MPIIODataStructures;
-MPIIOMigrationCoProcessor::MPIIOMigrationCoProcessor(SPtr<Grid3D> grid, SPtr<UbScheduler> s, const std::string &path,
- SPtr<Communicator> comm)
+MPIIOMigrationCoProcessor::MPIIOMigrationCoProcessor(SPtr<Grid3D> grid, SPtr<UbScheduler> s, const std::string &path, SPtr<Communicator> comm)
: MPIIOCoProcessor(grid, s, path, comm)
{
memset(&boundCondParamStr, 0, sizeof(boundCondParamStr));
@@ -31,7 +30,7 @@ MPIIOMigrationCoProcessor::MPIIOMigrationCoProcessor(SPtr<Grid3D> grid, SPtr<UbS
//------------------------- define MPI types ---------------------------------
MPI_Datatype typesDataSet[3] = { MPI_DOUBLE, MPI_INT, MPI_CHAR };
- int blocksDataSet[3] = { 2, 2, 2 };
+ int blocksDataSet[3] = { 5, 2, 2 };
MPI_Aint offsetsDatatSet[3], lbDataSet, extentDataSet;
offsetsDatatSet[0] = 0;
@@ -71,7 +70,8 @@ MPIIOMigrationCoProcessor::~MPIIOMigrationCoProcessor()
//////////////////////////////////////////////////////////////////////////
void MPIIOMigrationCoProcessor::process(double step)
{
- if (scheduler->isDue(step)) {
+ if (scheduler->isDue(step))
+ {
if (comm->isRoot())
UBLOG(logINFO, "MPIIOMigrationCoProcessor save step: " << step);
if (comm->isRoot())
@@ -100,8 +100,7 @@ void MPIIOMigrationCoProcessor::clearAllFiles(int step)
UbSystem::makeDirectory(path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step));
std::string filename10 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpBC.bin";
- int rc10 =
- MPI_File_open(MPI_COMM_WORLD, filename10.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ int rc10 = MPI_File_open(MPI_COMM_WORLD, filename10.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
if (rc10 != MPI_SUCCESS)
throw UbException(UB_EXARGS, "couldn't open file " + filename10);
MPI_File_set_size(file_handler, new_size);
@@ -125,69 +124,93 @@ void MPIIOMigrationCoProcessor::writeDataSet(int step)
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
int maxInitLevel = this->grid->getFinestInitializedLevel();
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
grid->getBlocks(level, rank, blocksVector[level]);
blocksCount += static_cast<int>(blocksVector[level].size());
}
dataSetParam dataSetParamStr1, dataSetParamStr2, dataSetParamStr3;
DataSetMigration *dataSetArray = new DataSetMigration[blocksCount];
- std::vector<double> doubleValuesArray; // double-values (arrays of f's) in all blocks
+ std::vector<double> doubleValuesArrayF; // double-values (arrays of f's) in all blocks Fdistribution
+ std::vector<double> doubleValuesArrayH1; // double-values (arrays of f's) in all blocks H1distribution
+ // std::vector<double> doubleValuesArrayH2; // double-values (arrays of f's) in all blocks H2distribution
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeDataSet start collect data rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
+ bool multiPhase = false;
DSArraysPresence arrPresence;
bool firstBlock = true;
size_t doubleCountInBlock = 0;
int ic = 0;
- SPtr<D3Q27EsoTwist3DSplittedVector> D3Q27EsoTwist3DSplittedVectorPtrF, D3Q27EsoTwist3DSplittedVectorPtrH;
- CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr localDistributionsF, localDistributionsH;
- CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr nonLocalDistributionsF, nonLocalDistributionsH;
- CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr zeroDistributionsF, zeroDistributionsH;
+ SPtr<D3Q27EsoTwist3DSplittedVector> D3Q27EsoTwist3DSplittedVectorPtrF = 0, D3Q27EsoTwist3DSplittedVectorPtrH1 = 0, D3Q27EsoTwist3DSplittedVectorPtrH2 = 0;
+ CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr localDistributionsF = 0, localDistributionsH1 = 0, localDistributionsH2 = 0;
+ CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr nonLocalDistributionsF = 0, nonLocalDistributionsH1 = 0, nonLocalDistributionsH2 = 0;
+ CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr zeroDistributionsF = 0, zeroDistributionsH1 = 0, zeroDistributionsH2 = 0;
+
+ SPtr<LBMKernel> kernel;
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
{
- dataSetArray[ic].globalID =
- block->getGlobalID(); // id of the block needed to find it while regenerating the grid
- dataSetArray[ic].ghostLayerWidth = block->getKernel()->getGhostLayerWidth();
- dataSetArray[ic].collFactor = block->getKernel()->getCollisionFactor();
- dataSetArray[ic].deltaT = block->getKernel()->getDeltaT();
- dataSetArray[ic].compressible = block->getKernel()->getCompressible();
- dataSetArray[ic].withForcing = block->getKernel()->getWithForcing();
-
- D3Q27EsoTwist3DSplittedVectorPtrF = dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(
- block->getKernel()->getDataSet()->getFdistributions());
+ kernel = dynamicPointerCast<LBMKernel>(block->getKernel());
+
+ dataSetArray[ic].globalID = block->getGlobalID(); // id of the block needed to find it while regenerating the grid
+ dataSetArray[ic].ghostLayerWidth = kernel->getGhostLayerWidth();
+ dataSetArray[ic].collFactor = kernel->getCollisionFactor();
+ dataSetArray[ic].deltaT = kernel->getDeltaT();
+ dataSetArray[ic].compressible = kernel->getCompressible();
+ dataSetArray[ic].withForcing = kernel->getWithForcing();
+ dataSetArray[ic].collFactorL = kernel->getCollisionFactorL();
+ dataSetArray[ic].collFactorG = kernel->getCollisionFactorG();
+ dataSetArray[ic].densityRatio = kernel->getDensityRatio();
+
+ D3Q27EsoTwist3DSplittedVectorPtrF = dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(block->getKernel()->getDataSet()->getFdistributions());
localDistributionsF = D3Q27EsoTwist3DSplittedVectorPtrF->getLocalDistributions();
nonLocalDistributionsF = D3Q27EsoTwist3DSplittedVectorPtrF->getNonLocalDistributions();
zeroDistributionsF = D3Q27EsoTwist3DSplittedVectorPtrF->getZeroDistributions();
- D3Q27EsoTwist3DSplittedVectorPtrH = dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(
- block->getKernel()->getDataSet()->getHdistributions());
- localDistributionsH = D3Q27EsoTwist3DSplittedVectorPtrH->getLocalDistributions();
- nonLocalDistributionsH = D3Q27EsoTwist3DSplittedVectorPtrH->getNonLocalDistributions();
- zeroDistributionsH = D3Q27EsoTwist3DSplittedVectorPtrH->getZeroDistributions();
+ D3Q27EsoTwist3DSplittedVectorPtrH1 = dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(block->getKernel()->getDataSet()->getHdistributions());
+ if (D3Q27EsoTwist3DSplittedVectorPtrH1 != 0)
+ {
+ multiPhase = true;
+ localDistributionsH1 = D3Q27EsoTwist3DSplittedVectorPtrH1->getLocalDistributions();
+ nonLocalDistributionsH1 = D3Q27EsoTwist3DSplittedVectorPtrH1->getNonLocalDistributions();
+ zeroDistributionsH1 = D3Q27EsoTwist3DSplittedVectorPtrH1->getZeroDistributions();
+ }
+
+ /*D3Q27EsoTwist3DSplittedVectorPtrH2 = dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(block->getKernel()->getDataSet()->getH2distributions());
+ if (D3Q27EsoTwist3DSplittedVectorPtrH2 != 0)
+ {
+ localDistributionsH2 = D3Q27EsoTwist3DSplittedVectorPtrH2->getLocalDistributions();
+ nonLocalDistributionsH2 = D3Q27EsoTwist3DSplittedVectorPtrH2->getNonLocalDistributions();
+ zeroDistributionsH2 = D3Q27EsoTwist3DSplittedVectorPtrH2->getZeroDistributions();
+ }*/
if (firstBlock) // && block->getKernel()) // when first (any) valid block...
{
- if (localDistributionsF) {
+ if (localDistributionsF)
+ {
dataSetParamStr1.nx[0] = static_cast<int>(localDistributionsF->getNX1());
dataSetParamStr1.nx[1] = static_cast<int>(localDistributionsF->getNX2());
dataSetParamStr1.nx[2] = static_cast<int>(localDistributionsF->getNX3());
dataSetParamStr1.nx[3] = static_cast<int>(localDistributionsF->getNX4());
}
- if (nonLocalDistributionsF) {
+ if (nonLocalDistributionsF)
+ {
dataSetParamStr2.nx[0] = static_cast<int>(nonLocalDistributionsF->getNX1());
dataSetParamStr2.nx[1] = static_cast<int>(nonLocalDistributionsF->getNX2());
dataSetParamStr2.nx[2] = static_cast<int>(nonLocalDistributionsF->getNX3());
dataSetParamStr2.nx[3] = static_cast<int>(nonLocalDistributionsF->getNX4());
}
- if (zeroDistributionsF) {
+ if (zeroDistributionsF)
+ {
dataSetParamStr3.nx[0] = static_cast<int>(zeroDistributionsF->getNX1());
dataSetParamStr3.nx[1] = static_cast<int>(zeroDistributionsF->getNX2());
dataSetParamStr3.nx[2] = static_cast<int>(zeroDistributionsF->getNX3());
@@ -195,94 +218,91 @@ void MPIIOMigrationCoProcessor::writeDataSet(int step)
}
// ... than save some parameters that are equal in all blocks
- dataSetParamStr1.nx1 = dataSetParamStr2.nx1 = dataSetParamStr3.nx1 =
- static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX1());
- dataSetParamStr1.nx2 = dataSetParamStr2.nx2 = dataSetParamStr3.nx2 =
- static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX2());
- dataSetParamStr1.nx3 = dataSetParamStr2.nx3 = dataSetParamStr3.nx3 =
- static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX3());
-
- // Fdistributions + Hdistributions
- doubleCountInBlock =
- (dataSetParamStr1.nx[0] * dataSetParamStr1.nx[1] * dataSetParamStr1.nx[2] * dataSetParamStr1.nx[3] +
- dataSetParamStr2.nx[0] * dataSetParamStr2.nx[1] * dataSetParamStr2.nx[2] * dataSetParamStr2.nx[3] +
- dataSetParamStr3.nx[0] * dataSetParamStr3.nx[1] * dataSetParamStr3.nx[2] * dataSetParamStr3.nx[3]) * 2;
-
- SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> averageDensityArray =
- block->getKernel()->getDataSet()->getAverageDensity();
+ dataSetParamStr1.nx1 = dataSetParamStr2.nx1 = dataSetParamStr3.nx1 = static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX1());
+ dataSetParamStr1.nx2 = dataSetParamStr2.nx2 = dataSetParamStr3.nx2 = static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX2());
+ dataSetParamStr1.nx3 = dataSetParamStr2.nx3 = dataSetParamStr3.nx3 = static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX3());
+
+ doubleCountInBlock = dataSetParamStr1.nx[0] * dataSetParamStr1.nx[1] * dataSetParamStr1.nx[2] * dataSetParamStr1.nx[3] +
+ dataSetParamStr2.nx[0] * dataSetParamStr2.nx[1] * dataSetParamStr2.nx[2] * dataSetParamStr2.nx[3] +
+ dataSetParamStr3.nx[0] * dataSetParamStr3.nx[1] * dataSetParamStr3.nx[2] * dataSetParamStr3.nx[3];
+
+ SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> averageDensityArray = block->getKernel()->getDataSet()->getAverageDensity();
if (averageDensityArray)
arrPresence.isAverageDensityArrayPresent = true;
else
arrPresence.isAverageDensityArrayPresent = false;
- SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> AverageVelocityArray3DPtr =
- block->getKernel()->getDataSet()->getAverageVelocity();
+ SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> AverageVelocityArray3DPtr = block->getKernel()->getDataSet()->getAverageVelocity();
if (AverageVelocityArray3DPtr)
arrPresence.isAverageVelocityArrayPresent = true;
else
arrPresence.isAverageVelocityArrayPresent = false;
- SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> AverageFluctArray3DPtr =
- block->getKernel()->getDataSet()->getAverageFluctuations();
+ SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> AverageFluctArray3DPtr = block->getKernel()->getDataSet()->getAverageFluctuations();
if (AverageFluctArray3DPtr)
arrPresence.isAverageFluktuationsArrayPresent = true;
else
arrPresence.isAverageFluktuationsArrayPresent = false;
- SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> AverageTripleArray3DPtr =
- block->getKernel()->getDataSet()->getAverageTriplecorrelations();
+ SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> AverageTripleArray3DPtr = block->getKernel()->getDataSet()->getAverageTriplecorrelations();
if (AverageTripleArray3DPtr)
arrPresence.isAverageTripleArrayPresent = true;
else
arrPresence.isAverageTripleArrayPresent = false;
- SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> ShearStressValArray3DPtr =
- block->getKernel()->getDataSet()->getShearStressValues();
+ SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> ShearStressValArray3DPtr = block->getKernel()->getDataSet()->getShearStressValues();
if (ShearStressValArray3DPtr)
arrPresence.isShearStressValArrayPresent = true;
else
arrPresence.isShearStressValArrayPresent = false;
- SPtr<CbArray3D<LBMReal, IndexerX3X2X1>> relaxationFactor3DPtr =
- block->getKernel()->getDataSet()->getRelaxationFactor();
+ SPtr<CbArray3D<LBMReal, IndexerX3X2X1>> relaxationFactor3DPtr = block->getKernel()->getDataSet()->getRelaxationFactor();
if (relaxationFactor3DPtr)
arrPresence.isRelaxationFactorPresent = true;
else
arrPresence.isRelaxationFactorPresent = false;
- SPtr<CbArray3D<LBMReal, IndexerX3X2X1>> phaseField3DPtr =
- block->getKernel()->getDataSet()->getPhaseField();
- if (phaseField3DPtr)
- arrPresence.isPhaseFieldPresent = true;
+ SPtr<CbArray3D<LBMReal, IndexerX3X2X1>> phaseField3DPtr1 = block->getKernel()->getDataSet()->getPhaseField();
+ if (phaseField3DPtr1)
+ arrPresence.isPhaseField1Present = true;
+ else
+ arrPresence.isPhaseField1Present = false;
+
+ SPtr<CbArray3D<LBMReal, IndexerX3X2X1>> phaseField3DPtr2 = block->getKernel()->getDataSet()->getPhaseField2();
+ if (phaseField3DPtr2)
+ arrPresence.isPhaseField2Present = true;
else
- arrPresence.isPhaseFieldPresent = false;
+ arrPresence.isPhaseField2Present = false;
firstBlock = false;
}
- if (localDistributionsF && (dataSetParamStr1.nx[0] > 0) && (dataSetParamStr1.nx[1] > 0) &&
- (dataSetParamStr1.nx[2] > 0) && (dataSetParamStr1.nx[3] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), localDistributionsF->getDataVector().begin(),
- localDistributionsF->getDataVector().end());
- if (nonLocalDistributionsF && (dataSetParamStr2.nx[0] > 0) && (dataSetParamStr2.nx[1] > 0) &&
- (dataSetParamStr2.nx[2] > 0) && (dataSetParamStr2.nx[3] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), nonLocalDistributionsF->getDataVector().begin(),
- nonLocalDistributionsF->getDataVector().end());
+ if (localDistributionsF && (dataSetParamStr1.nx[0] > 0) && (dataSetParamStr1.nx[1] > 0) && (dataSetParamStr1.nx[2] > 0) && (dataSetParamStr1.nx[3] > 0))
+ doubleValuesArrayF.insert(doubleValuesArrayF.end(), localDistributionsF->getDataVector().begin(), localDistributionsF->getDataVector().end());
+ if (nonLocalDistributionsF && (dataSetParamStr2.nx[0] > 0) && (dataSetParamStr2.nx[1] > 0) && (dataSetParamStr2.nx[2] > 0) && (dataSetParamStr2.nx[3] > 0))
+ doubleValuesArrayF.insert(doubleValuesArrayF.end(), nonLocalDistributionsF->getDataVector().begin(), nonLocalDistributionsF->getDataVector().end());
if (zeroDistributionsF && (dataSetParamStr3.nx[0] > 0) && (dataSetParamStr3.nx[1] > 0) && (dataSetParamStr3.nx[2] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), zeroDistributionsF->getDataVector().begin(),
- zeroDistributionsF->getDataVector().end());
-
- if (localDistributionsH && (dataSetParamStr1.nx[0] > 0) && (dataSetParamStr1.nx[1] > 0) &&
- (dataSetParamStr1.nx[2] > 0) && (dataSetParamStr1.nx[3] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), localDistributionsH->getDataVector().begin(),
- localDistributionsH->getDataVector().end());
- if (nonLocalDistributionsH && (dataSetParamStr2.nx[0] > 0) && (dataSetParamStr2.nx[1] > 0) &&
- (dataSetParamStr2.nx[2] > 0) && (dataSetParamStr2.nx[3] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), nonLocalDistributionsH->getDataVector().begin(),
- nonLocalDistributionsH->getDataVector().end());
- if (zeroDistributionsH && (dataSetParamStr3.nx[0] > 0) && (dataSetParamStr3.nx[1] > 0) && (dataSetParamStr3.nx[2] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), zeroDistributionsH->getDataVector().begin(),
- zeroDistributionsH->getDataVector().end());
+ doubleValuesArrayF.insert(doubleValuesArrayF.end(), zeroDistributionsF->getDataVector().begin(), zeroDistributionsF->getDataVector().end());
+
+ if (multiPhase)
+ {
+ if (localDistributionsH1 && (dataSetParamStr1.nx[0] > 0) && (dataSetParamStr1.nx[1] > 0) && (dataSetParamStr1.nx[2] > 0) && (dataSetParamStr1.nx[3] > 0))
+ doubleValuesArrayH1.insert(doubleValuesArrayH1.end(), localDistributionsH1->getDataVector().begin(), localDistributionsH1->getDataVector().end());
+ if (nonLocalDistributionsH1 && (dataSetParamStr2.nx[0] > 0) && (dataSetParamStr2.nx[1] > 0) && (dataSetParamStr2.nx[2] > 0) && (dataSetParamStr2.nx[3] > 0))
+ doubleValuesArrayH1.insert(doubleValuesArrayH1.end(), nonLocalDistributionsH1->getDataVector().begin(), nonLocalDistributionsH1->getDataVector().end());
+ if (zeroDistributionsH1 && (dataSetParamStr3.nx[0] > 0) && (dataSetParamStr3.nx[1] > 0) && (dataSetParamStr3.nx[2] > 0))
+ doubleValuesArrayH1.insert(doubleValuesArrayH1.end(), zeroDistributionsH1->getDataVector().begin(), zeroDistributionsH1->getDataVector().end());
+ }
+
+ /*if (D3Q27EsoTwist3DSplittedVectorPtrH2 != 0)
+ {
+ if (localDistributionsH2 && (dataSetParamStr1.nx[0] > 0) && (dataSetParamStr1.nx[1] > 0) && (dataSetParamStr1.nx[2] > 0) && (dataSetParamStr1.nx[3] > 0))
+ doubleValuesArrayH2.insert(doubleValuesArrayH2.end(), localDistributionsH2->getDataVector().begin(), localDistributionsH2->getDataVector().end());
+ if (nonLocalDistributionsH2 && (dataSetParamStr2.nx[0] > 0) && (dataSetParamStr2.nx[1] > 0) && (dataSetParamStr2.nx[2] > 0) && (dataSetParamStr2.nx[3] > 0))
+ doubleValuesArrayH2.insert(doubleValuesArrayH2.end(), nonLocalDistributionsH2->getDataVector().begin(), nonLocalDistributionsH2->getDataVector().end());
+ if (zeroDistributionsH2 && (dataSetParamStr3.nx[0] > 0) && (dataSetParamStr3.nx[1] > 0) && (dataSetParamStr3.nx[2] > 0))
+ doubleValuesArrayH2.insert(doubleValuesArrayH2.end(), zeroDistributionsH2->getDataVector().begin(), zeroDistributionsH2->getDataVector().end());
+ }*/
ic++;
}
@@ -292,10 +312,10 @@ void MPIIOMigrationCoProcessor::writeDataSet(int step)
MPI_Type_contiguous(int(doubleCountInBlock), MPI_DOUBLE, &dataSetDoubleType);
MPI_Type_commit(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeDataSet start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
@@ -311,32 +331,73 @@ void MPIIOMigrationCoProcessor::writeDataSet(int step)
// write to the file
MPI_File file_handler;
- std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSet.bin";
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSetF.bin";
int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
if (rc != MPI_SUCCESS)
throw UbException(UB_EXARGS, "couldn't open file " + filename);
MPI_File_write_at(file_handler, (MPI_Offset)0, &dataSetParamStr1, 1, dataSetParamType, MPI_STATUS_IGNORE);
- MPI_File_write_at(file_handler, (MPI_Offset)(sizeof(dataSetParam)), &dataSetParamStr2, 1, dataSetParamType,
- MPI_STATUS_IGNORE);
- MPI_File_write_at(file_handler, (MPI_Offset)(2 * sizeof(dataSetParam)), &dataSetParamStr3, 1, dataSetParamType,
- MPI_STATUS_IGNORE);
-
+ MPI_File_write_at(file_handler, (MPI_Offset)(sizeof(dataSetParam)), &dataSetParamStr2, 1, dataSetParamType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(2 * sizeof(dataSetParam)), &dataSetParamStr3, 1, dataSetParamType, MPI_STATUS_IGNORE);
+
MPI_Offset write_offset;
size_t sizeofOneDataSet = sizeof(DataSetMigration) + doubleCountInBlock * sizeof(double);
- for (int nb = 0; nb < blocksCount; nb++) {
+ for (int nb = 0; nb < blocksCount; nb++)
+ {
write_offset = (MPI_Offset)(3 * sizeof(dataSetParam) + dataSetArray[nb].globalID * sizeofOneDataSet);
MPI_File_write_at(file_handler, write_offset, &dataSetArray[nb], 1, dataSetType, MPI_STATUS_IGNORE);
- MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(DataSetMigration)),
- &doubleValuesArray[nb * doubleCountInBlock], 1, dataSetDoubleType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(DataSetMigration)), &doubleValuesArrayF[nb * doubleCountInBlock], 1, dataSetDoubleType, MPI_STATUS_IGNORE);
}
MPI_File_sync(file_handler);
MPI_File_close(&file_handler);
+
+ //-------------------------------- H1 ----------------------------------------------------
+ if (multiPhase)
+ {
+ filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSetH1.bin";
+ rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc != MPI_SUCCESS)
+ throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ sizeofOneDataSet = doubleCountInBlock * sizeof(double);
+
+ for (int nb = 0; nb < blocksCount; nb++)
+ {
+ write_offset = (MPI_Offset)(dataSetArray[nb].globalID * sizeofOneDataSet);
+ MPI_File_write_at(file_handler, write_offset, &doubleValuesArrayH1[nb * doubleCountInBlock], 1, dataSetDoubleType, MPI_STATUS_IGNORE);
+ }
+
+ MPI_File_sync(file_handler);
+ MPI_File_close(&file_handler);
+ }
+
+ //-------------------------------- H2 ----------------------------------------------------
+ /*if (D3Q27EsoTwist3DSplittedVectorPtrH2 != 0)
+ {
+ filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSetH2.bin";
+ rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc != MPI_SUCCESS)
+ throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ sizeofOneDataSet = doubleCountInBlock * sizeof(double);
+
+ for (int nb = 0; nb < blocksCount; nb++)
+ {
+ write_offset = (MPI_Offset)(dataSetArray[nb].globalID * sizeofOneDataSet);
+ MPI_File_write_at(file_handler, write_offset, &doubleValuesArrayH2[nb * doubleCountInBlock], 1, dataSetDoubleType, MPI_STATUS_IGNORE);
+ }
+
+ MPI_File_sync(file_handler);
+ MPI_File_close(&file_handler);
+ }*/
+ //--------------------------------
+
MPI_Type_free(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeDataSet time: " << finish - start << " s");
}
@@ -375,8 +436,11 @@ void MPIIOMigrationCoProcessor::writeDataSet(int step)
if (arrPresence.isRelaxationFactorPresent)
write3DArray(step, RelaxationFactor, std::string("/cpRelaxationFactor.bin"));
- if (arrPresence.isPhaseFieldPresent)
- write3DArray(step, PhaseField, std::string("/cpPhaseField.bin"));
+ if (arrPresence.isPhaseField1Present)
+ write3DArray(step, PhaseField1, std::string("/cpPhaseField1.bin"));
+
+ if (arrPresence.isPhaseField2Present)
+ write3DArray(step, PhaseField2, std::string("/cpPhaseField2.bin"));
}
@@ -391,7 +455,8 @@ void MPIIOMigrationCoProcessor::write4DArray(int step, Arrays arrayType, std::st
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
int maxInitLevel = this->grid->getFinestInitializedLevel();
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
grid->getBlocks(level, rank, blocksVector[level]);
blocksCount += static_cast<int>(blocksVector[level].size());
}
@@ -400,10 +465,10 @@ void MPIIOMigrationCoProcessor::write4DArray(int step, Arrays arrayType, std::st
std::vector<double> doubleValuesArray; // double-values of the AverageDensityArray in all blocks
dataSetParam dataSetParamStr;
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeAverageDensityArray start collect data rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
bool firstBlock = true;
@@ -411,13 +476,14 @@ void MPIIOMigrationCoProcessor::write4DArray(int step, Arrays arrayType, std::st
int ic = 0;
SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> ___Array;
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
{
- dataSetSmallArray[ic].globalID =
- block->getGlobalID(); // id of the block needed to find it while regenerating the grid
+ dataSetSmallArray[ic].globalID = block->getGlobalID(); // id of the block needed to find it while regenerating the grid
- switch (arrayType) {
+ switch (arrayType)
+ {
case AverageDensity:
___Array = block->getKernel()->getDataSet()->getAverageDensity();
break;
@@ -434,28 +500,24 @@ void MPIIOMigrationCoProcessor::write4DArray(int step, Arrays arrayType, std::st
___Array = block->getKernel()->getDataSet()->getShearStressValues();
break;
default:
- UB_THROW(UbException(UB_EXARGS,
- "MPIIOMigrationCoProcessor::write4DArray : 4D array type does not exist!"));
+ UB_THROW(UbException(UB_EXARGS, "MPIIOMigrationCoProcessor::write4DArray : 4D array type does not exist!"));
break;
}
if (firstBlock) // when first (any) valid block...
{
dataSetParamStr.nx1 = dataSetParamStr.nx2 = dataSetParamStr.nx3 = 0;
- dataSetParamStr.nx[0] = static_cast<int>(___Array->getNX1());
- dataSetParamStr.nx[1] = static_cast<int>(___Array->getNX2());
- dataSetParamStr.nx[2] = static_cast<int>(___Array->getNX3());
- dataSetParamStr.nx[3] = static_cast<int>(___Array->getNX4());
- doubleCountInBlock =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ dataSetParamStr.nx[0] = static_cast<int>(___Array->getNX1());
+ dataSetParamStr.nx[1] = static_cast<int>(___Array->getNX2());
+ dataSetParamStr.nx[2] = static_cast<int>(___Array->getNX3());
+ dataSetParamStr.nx[3] = static_cast<int>(___Array->getNX4());
+ doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
firstBlock = false;
}
- if (___Array && (dataSetParamStr.nx[0] > 0) && (dataSetParamStr.nx[1] > 0) && (dataSetParamStr.nx[2] > 0) &&
- (dataSetParamStr.nx[3] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), ___Array->getDataVector().begin(),
- ___Array->getDataVector().end());
+ if (___Array && (dataSetParamStr.nx[0] > 0) && (dataSetParamStr.nx[1] > 0) && (dataSetParamStr.nx[2] > 0) && (dataSetParamStr.nx[3] > 0))
+ doubleValuesArray.insert(doubleValuesArray.end(), ___Array->getDataVector().begin(), ___Array->getDataVector().end());
ic++;
}
@@ -465,10 +527,10 @@ void MPIIOMigrationCoProcessor::write4DArray(int step, Arrays arrayType, std::st
MPI_Type_contiguous(int(doubleCountInBlock), MPI_DOUBLE, &dataSetDoubleType);
MPI_Type_commit(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationCoProcessor::write4DArray start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
@@ -489,7 +551,8 @@ void MPIIOMigrationCoProcessor::write4DArray(int step, Arrays arrayType, std::st
MPI_Offset write_offset;
size_t sizeofOneDataSet = sizeof(DataSetSmallMigration) + doubleCountInBlock * sizeof(double);
- for (int nb = 0; nb < blocksCount; nb++) {
+ for (int nb = 0; nb < blocksCount; nb++)
+ {
write_offset = (MPI_Offset)(sizeof(dataSetParam) + dataSetSmallArray[nb].globalID * sizeofOneDataSet);
MPI_File_write_at(file_handler, write_offset, &dataSetSmallArray[nb], 1, dataSetSmallType, MPI_STATUS_IGNORE);
MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(DataSetSmallMigration)),
@@ -500,7 +563,8 @@ void MPIIOMigrationCoProcessor::write4DArray(int step, Arrays arrayType, std::st
MPI_File_close(&file_handler);
MPI_Type_free(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIOMigrationCoProcessor::write4DArray time: " << finish - start << " s");
}
@@ -519,7 +583,8 @@ void MPIIOMigrationCoProcessor::write3DArray(int step, Arrays arrayType, std::st
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
int maxInitLevel = this->grid->getFinestInitializedLevel();
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
grid->getBlocks(level, rank, blocksVector[level]);
blocksCount += static_cast<int>(blocksVector[level].size());
}
@@ -528,10 +593,10 @@ void MPIIOMigrationCoProcessor::write3DArray(int step, Arrays arrayType, std::st
std::vector<double> doubleValuesArray; // double-values (arrays of f's) in all blocks
dataSetParam dataSetParamStr;
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationCoProcessor::write3DArray start collect data rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
bool firstBlock = true;
@@ -539,41 +604,42 @@ void MPIIOMigrationCoProcessor::write3DArray(int step, Arrays arrayType, std::st
int ic = 0;
SPtr<CbArray3D<LBMReal, IndexerX3X2X1>> ___Array;
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
{
- dataSetSmallArray[ic].globalID =
- block->getGlobalID(); // id of the block needed to find it while regenerating the grid
+ dataSetSmallArray[ic].globalID = block->getGlobalID(); // id of the block needed to find it while regenerating the grid
- switch (arrayType) {
+ switch (arrayType)
+ {
case RelaxationFactor:
___Array = block->getKernel()->getDataSet()->getRelaxationFactor();
break;
- case PhaseField:
+ case PhaseField1:
___Array = block->getKernel()->getDataSet()->getPhaseField();
break;
+ case PhaseField2:
+ ___Array = block->getKernel()->getDataSet()->getPhaseField2();
+ break;
default:
- UB_THROW(UbException(UB_EXARGS,
- "MPIIOMigrationCoProcessor::write3DArray : 3D array type does not exist!"));
+ UB_THROW(UbException(UB_EXARGS, "MPIIOMigrationCoProcessor::write3DArray : 3D array type does not exist!"));
break;
}
if (firstBlock) // when first (any) valid block...
{
dataSetParamStr.nx1 = dataSetParamStr.nx2 = dataSetParamStr.nx3 = 0;
- dataSetParamStr.nx[0] = static_cast<int>(___Array->getNX1());
- dataSetParamStr.nx[1] = static_cast<int>(___Array->getNX2());
- dataSetParamStr.nx[2] = static_cast<int>(___Array->getNX3());
- dataSetParamStr.nx[3] = 1;
- doubleCountInBlock =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ dataSetParamStr.nx[0] = static_cast<int>(___Array->getNX1());
+ dataSetParamStr.nx[1] = static_cast<int>(___Array->getNX2());
+ dataSetParamStr.nx[2] = static_cast<int>(___Array->getNX3());
+ dataSetParamStr.nx[3] = 1;
+ doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
firstBlock = false;
}
if (___Array && (dataSetParamStr.nx[0] > 0) && (dataSetParamStr.nx[1] > 0) && (dataSetParamStr.nx[2] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), ___Array->getDataVector().begin(),
- ___Array->getDataVector().end());
+ doubleValuesArray.insert(doubleValuesArray.end(), ___Array->getDataVector().begin(), ___Array->getDataVector().end());
ic++;
}
@@ -583,10 +649,10 @@ void MPIIOMigrationCoProcessor::write3DArray(int step, Arrays arrayType, std::st
MPI_Type_contiguous(int(doubleCountInBlock), MPI_DOUBLE, &dataSetDoubleType);
MPI_Type_commit(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationCoProcessor::write3DArray start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
@@ -613,7 +679,8 @@ void MPIIOMigrationCoProcessor::write3DArray(int step, Arrays arrayType, std::st
size_t sizeofOneDataSet = sizeof(DataSetSmallMigration) + doubleCountInBlock * sizeof(double);
MPI_Offset write_offset;
- for (int nb = 0; nb < blocksCount; nb++) {
+ for (int nb = 0; nb < blocksCount; nb++)
+ {
write_offset = (MPI_Offset)(sizeof(dataSetParam) + dataSetSmallArray[nb].globalID * sizeofOneDataSet);
MPI_File_write_at(file_handler, write_offset, &dataSetSmallArray[nb], 1, dataSetSmallType, MPI_STATUS_IGNORE);
MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(DataSetSmallMigration)),
@@ -624,7 +691,8 @@ void MPIIOMigrationCoProcessor::write3DArray(int step, Arrays arrayType, std::st
MPI_File_close(&file_handler);
MPI_Type_free(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIOMigrationCoProcessor::write3DArray time: " << finish - start << " s");
}
@@ -1352,10 +1420,10 @@ void MPIIOMigrationCoProcessor::writeBoundaryConds(int step)
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeBoundaryConds start collect data rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
int blocksCount = 0; // quantity of blocks, that belong to this process
@@ -1366,7 +1434,8 @@ void MPIIOMigrationCoProcessor::writeBoundaryConds(int step)
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
int maxInitLevel = this->grid->getFinestInitializedLevel();
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
grid->getBlocks(level, rank, blocksVector[level]);
blocksCount += static_cast<int>(blocksVector[level].size());
}
@@ -1381,25 +1450,27 @@ void MPIIOMigrationCoProcessor::writeBoundaryConds(int step)
int ic = 0;
SPtr<BCArray3D> bcArr;
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
for (SPtr<Block3D> block : blocksVector[level]) // all the blocks of the current level
{
bcArr = block->getKernel()->getBCProcessor()->getBCArray();
- bcAddArray[ic].globalID =
- block->getGlobalID(); // id of the block needed to find it while regenerating the grid
+ bcAddArray[ic].globalID = block->getGlobalID(); // id of the block needed to find it while regenerating the grid
bcAddArray[ic].boundCond_count = 0; // how many BoundaryConditions in this block
bcAddArray[ic].indexContainer_count = 0; // how many indexContainer-values in this block
- bytesCount[ic] = sizeof(BCAddMigration);
+ bytesCount[ic] = sizeof(BCAddMigration);
bcVector[ic].resize(0);
bcindexmatrixVector[ic].resize(0);
indexContainerVector[ic].resize(0);
- for (std::size_t bc = 0; bc < bcArr->getBCVectorSize(); bc++) {
+ for (std::size_t bc = 0; bc < bcArr->getBCVectorSize(); bc++)
+ {
BoundaryCondition *bouCond = new BoundaryCondition();
- if (bcArr->bcvector[bc] == NULL) {
+ if (bcArr->bcvector[bc] == NULL)
memset(bouCond, 0, sizeof(BoundaryCondition));
- } else {
+ else
+ {
bouCond->noslipBoundaryFlags = bcArr->bcvector[bc]->getNoSlipBoundary();
bouCond->slipBoundaryFlags = bcArr->bcvector[bc]->getSlipBoundary();
bouCond->velocityBoundaryFlags = bcArr->bcvector[bc]->getVelocityBoundary();
@@ -1428,20 +1499,18 @@ void MPIIOMigrationCoProcessor::writeBoundaryConds(int step)
bytesCount[ic] += sizeof(BoundaryCondition);
}
- if (bcindexmatrixCountNotInit) {
+ if (bcindexmatrixCountNotInit)
+ {
boundCondParamStr.nx1 = static_cast<int>(bcArr->bcindexmatrix.getNX1());
boundCondParamStr.nx2 = static_cast<int>(bcArr->bcindexmatrix.getNX2());
boundCondParamStr.nx3 = static_cast<int>(bcArr->bcindexmatrix.getNX3());
boundCondParamStr.bcindexmatrixCount = static_cast<int>(bcArr->bcindexmatrix.getDataVector().size());
bcindexmatrixCountNotInit = false;
}
- bcindexmatrixVector[ic].insert(bcindexmatrixVector[ic].begin(),
- bcArr->bcindexmatrix.getDataVector().begin(),
- bcArr->bcindexmatrix.getDataVector().end());
+ bcindexmatrixVector[ic].insert(bcindexmatrixVector[ic].begin(), bcArr->bcindexmatrix.getDataVector().begin(), bcArr->bcindexmatrix.getDataVector().end());
bytesCount[ic] += boundCondParamStr.bcindexmatrixCount * sizeof(int);
- indexContainerVector[ic].insert(indexContainerVector[ic].begin(), bcArr->indexContainer.begin(),
- bcArr->indexContainer.end());
+ indexContainerVector[ic].insert(indexContainerVector[ic].begin(), bcArr->indexContainer.begin(), bcArr->indexContainer.end());
bcAddArray[ic].indexContainer_count = static_cast<int>(bcArr->indexContainer.size());
count_indexContainer += bcAddArray[ic].indexContainer_count;
bytesCount[ic] += bcAddArray[ic].indexContainer_count * sizeof(int);
@@ -1455,10 +1524,10 @@ void MPIIOMigrationCoProcessor::writeBoundaryConds(int step)
MPI_Type_contiguous(boundCondParamStr.bcindexmatrixCount, MPI_INT, &bcindexmatrixType);
MPI_Type_commit(&bcindexmatrixType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeBoundaryConds start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: "<< Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
@@ -1479,11 +1548,15 @@ void MPIIOMigrationCoProcessor::writeBoundaryConds(int step)
MPI_Offset write_offset = (MPI_Offset)(sizeof(boundCondParam) + grid->getNumberOfBlocks() * sizeof(size_t));
size_t next_file_offset = 0;
- if (size > 1) {
- if (rank == 0) {
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
next_file_offset = write_offset + allBytesCount;
MPI_Send(&next_file_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
- } else {
+ }
+ else
+ {
MPI_Recv(&write_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
next_file_offset = write_offset + allBytesCount;
if (rank < size - 1)
@@ -1495,7 +1568,8 @@ void MPIIOMigrationCoProcessor::writeBoundaryConds(int step)
MPI_Offset write_offsetIndex;
- for (int nb = 0; nb < blocksCount; nb++) {
+ for (int nb = 0; nb < blocksCount; nb++)
+ {
write_offsetIndex = (MPI_Offset)(sizeof(boundCondParam) + bcAddArray[nb].globalID * sizeof(size_t));
MPI_File_write_at(file_handler, write_offsetIndex, &write_offset, 1, MPI_LONG_LONG_INT, MPI_STATUS_IGNORE);
@@ -1505,17 +1579,12 @@ void MPIIOMigrationCoProcessor::writeBoundaryConds(int step)
bcAddArray[nb].boundCond_count, boundCondType, MPI_STATUS_IGNORE);
if (bcindexmatrixVector[nb].size() > 0)
- MPI_File_write_at(file_handler,
- (MPI_Offset)(write_offset + sizeof(BCAddMigration) +
- bcAddArray[nb].boundCond_count * sizeof(BoundaryCondition)),
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(BCAddMigration) + bcAddArray[nb].boundCond_count * sizeof(BoundaryCondition)),
&bcindexmatrixVector[nb][0], 1, bcindexmatrixType, MPI_STATUS_IGNORE);
if (indexContainerVector[nb].size() > 0)
- MPI_File_write_at(file_handler,
- (MPI_Offset)(write_offset + sizeof(BCAddMigration) +
- bcAddArray[nb].boundCond_count * sizeof(BoundaryCondition) +
- boundCondParamStr.bcindexmatrixCount * sizeof(int)),
- &indexContainerVector[nb][0], bcAddArray[nb].indexContainer_count, MPI_INT,
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(BCAddMigration) + bcAddArray[nb].boundCond_count * sizeof(BoundaryCondition) +
+ boundCondParamStr.bcindexmatrixCount * sizeof(int)), &indexContainerVector[nb][0], bcAddArray[nb].indexContainer_count, MPI_INT,
MPI_STATUS_IGNORE);
write_offset += bytesCount[nb];
@@ -1525,7 +1594,8 @@ void MPIIOMigrationCoProcessor::writeBoundaryConds(int step)
MPI_File_close(&file_handler);
MPI_Type_free(&bcindexmatrixType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeBoundaryConds time: " << finish - start << " s");
}
@@ -1547,8 +1617,7 @@ void MPIIOMigrationCoProcessor::restart(int step)
readBlocks(step);
- SPtr<Grid3DVisitor> metisVisitor(new MetisPartitioningGridVisitor(comm, MetisPartitioningGridVisitor::LevelBased,
- D3Q27System::BSW, MetisPartitioner::KWAY));
+ SPtr<Grid3DVisitor> metisVisitor(new MetisPartitioningGridVisitor(comm, MetisPartitioningGridVisitor::LevelBased, D3Q27System::BSW, MetisPartitioner::KWAY));
grid->accept(metisVisitor);
readDataSet(step);
@@ -1568,15 +1637,16 @@ void MPIIOMigrationCoProcessor::readDataSet(int step)
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationCoProcessor::readDataSet start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
if (comm->isRoot())
start = MPI_Wtime();
+ bool multiPhase = false;
size_t blocksCount = 0; // quantity of the blocks, that belong to this process
dataSetParam dataSetParamStr1, dataSetParamStr2, dataSetParamStr3;
@@ -1584,7 +1654,8 @@ void MPIIOMigrationCoProcessor::readDataSet(int step)
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
int maxInitLevel = this->grid->getFinestInitializedLevel();
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
grid->getBlocks(level, rank, blocksVector[level]);
blocksCount += static_cast<int>(blocksVector[level].size());
}
@@ -1592,22 +1663,21 @@ void MPIIOMigrationCoProcessor::readDataSet(int step)
DataSetMigration *dataSetArray = new DataSetMigration[blocksCount];
MPI_File file_handler;
- std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSet.bin";
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSetF.bin";
int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
if (rc != MPI_SUCCESS)
throw UbException(UB_EXARGS, "couldn't open file " + filename);
MPI_File_read_at(file_handler, (MPI_Offset)0, &dataSetParamStr1, 1, dataSetParamType, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, (MPI_Offset)(sizeof(dataSetParam)), &dataSetParamStr2, 1, dataSetParamType,
- MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, (MPI_Offset)(2 * sizeof(dataSetParam)), &dataSetParamStr3, 1, dataSetParamType,
- MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(sizeof(dataSetParam)), &dataSetParamStr2, 1, dataSetParamType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(2 * sizeof(dataSetParam)), &dataSetParamStr3, 1, dataSetParamType, MPI_STATUS_IGNORE);
- size_t doubleCountInBlock =
- (dataSetParamStr1.nx[0] * dataSetParamStr1.nx[1] * dataSetParamStr1.nx[2] * dataSetParamStr1.nx[3] +
+ size_t doubleCountInBlock = dataSetParamStr1.nx[0] * dataSetParamStr1.nx[1] * dataSetParamStr1.nx[2] * dataSetParamStr1.nx[3] +
dataSetParamStr2.nx[0] * dataSetParamStr2.nx[1] * dataSetParamStr2.nx[2] * dataSetParamStr2.nx[3] +
- dataSetParamStr3.nx[0] * dataSetParamStr3.nx[1] * dataSetParamStr3.nx[2] * dataSetParamStr3.nx[3] * 2);
- std::vector<double> doubleValuesArray(blocksCount * doubleCountInBlock); // double-values in all blocks
+ dataSetParamStr3.nx[0] * dataSetParamStr3.nx[1] * dataSetParamStr3.nx[2] * dataSetParamStr3.nx[3];
+ std::vector<double> doubleValuesArrayF(size_t(blocksCount * doubleCountInBlock)); // double-values in all blocks Fdistributions
+ std::vector<double> doubleValuesArrayH1; // double-values in all blocks H1distributions
+ //std::vector<double> doubleValuesArrayH2; // double-values in all blocks H2distributions
// define MPI_types depending on the block-specific information
MPI_Type_contiguous(int(doubleCountInBlock), MPI_DOUBLE, &dataSetDoubleType);
@@ -1617,95 +1687,145 @@ void MPIIOMigrationCoProcessor::readDataSet(int step)
MPI_Offset read_offset;
size_t sizeofOneDataSet = size_t(sizeof(DataSetMigration) + doubleCountInBlock * sizeof(double));
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
{
read_offset = (MPI_Offset)(3 * sizeof(dataSetParam) + block->getGlobalID() * sizeofOneDataSet);
MPI_File_read_at(file_handler, read_offset, &dataSetArray[ic], 1, dataSetType, MPI_STATUS_IGNORE);
MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(DataSetMigration)),
- &doubleValuesArray[ic * doubleCountInBlock], 1, dataSetDoubleType, MPI_STATUS_IGNORE);
+ &doubleValuesArrayF[ic * doubleCountInBlock], 1, dataSetDoubleType, MPI_STATUS_IGNORE);
ic++;
}
}
MPI_File_close(&file_handler);
+
+ //----------------------------------------- H1 ----------------------------------------------------
+ MPI_Offset fsize;
+ filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSetH1.bin";
+ rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
+ if (rc != MPI_SUCCESS)
+ throw UbException(UB_EXARGS, "couldn't open file " + filename);
+ int fs = MPI_File_get_size(file_handler, &fsize);
+ if (fsize > 0)
+ {
+ multiPhase = true;
+ doubleValuesArrayH1.resize(blocksCount * doubleCountInBlock);
+
+ sizeofOneDataSet = size_t(doubleCountInBlock * sizeof(double));
+
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
+ {
+ read_offset = (MPI_Offset)(block->getGlobalID() * sizeofOneDataSet);
+ MPI_File_read_at(file_handler, read_offset, &doubleValuesArrayH1[ic * doubleCountInBlock], 1, dataSetDoubleType, MPI_STATUS_IGNORE);
+ ic++;
+ }
+ }
+
+ }
+ MPI_File_close(&file_handler);
+
+ //----------------------------------------- H2 ----------------------------------------------------
+ /*filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSetH2.bin";
+ rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
+ if (rc != MPI_SUCCESS)
+ throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ sizeofOneDataSet = size_t(doubleCountInBlock * sizeof(double));
+ doubleValuesArrayH2.resize(blocksCount * doubleCountInBlock);
+
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
+ {
+ read_offset = (MPI_Offset)(block->getGlobalID() * sizeofOneDataSet);
+ MPI_File_read_at(file_handler, read_offset, &doubleValuesArrayH2[ic * doubleCountInBlock], 1, dataSetDoubleType, MPI_STATUS_IGNORE);
+ ic++;
+ }
+ }
+
+ MPI_File_close(&file_handler);*/
+
MPI_Type_free(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIOMigrationCoProcessor::readDataSet time: " << finish - start << " s");
UBLOG(logINFO, "MPIIOMigrationCoProcessor::readDataSet start of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
size_t index = 0;
std::vector<double> vectorsOfValuesF1, vectorsOfValuesF2, vectorsOfValuesF3;
- std::vector<double> vectorsOfValuesH1, vectorsOfValuesH2, vectorsOfValuesH3;
-
- size_t vectorSize1 =
- dataSetParamStr1.nx[0] * dataSetParamStr1.nx[1] * dataSetParamStr1.nx[2] * dataSetParamStr1.nx[3];
- size_t vectorSize2 =
- dataSetParamStr2.nx[0] * dataSetParamStr2.nx[1] * dataSetParamStr2.nx[2] * dataSetParamStr2.nx[3];
- size_t vectorSize3 =
- dataSetParamStr3.nx[0] * dataSetParamStr3.nx[1] * dataSetParamStr3.nx[2] * dataSetParamStr3.nx[3];
-
- for (std::size_t n = 0; n < blocksCount; n++) {
- vectorsOfValuesF1.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + vectorSize1);
+ std::vector<double> vectorsOfValuesH11, vectorsOfValuesH12, vectorsOfValuesH13;
+ //std::vector<double> vectorsOfValuesH21, vectorsOfValuesH22, vectorsOfValuesH23;
+
+ size_t vectorSize1 = dataSetParamStr1.nx[0] * dataSetParamStr1.nx[1] * dataSetParamStr1.nx[2] * dataSetParamStr1.nx[3];
+ size_t vectorSize2 = dataSetParamStr2.nx[0] * dataSetParamStr2.nx[1] * dataSetParamStr2.nx[2] * dataSetParamStr2.nx[3];
+ size_t vectorSize3 = dataSetParamStr3.nx[0] * dataSetParamStr3.nx[1] * dataSetParamStr3.nx[2] * dataSetParamStr3.nx[3];
+
+ for (std::size_t n = 0; n < blocksCount; n++)
+ {
+ vectorsOfValuesF1.assign(doubleValuesArrayF.data() + index, doubleValuesArrayF.data() + index + vectorSize1);
+ if(multiPhase)
+ vectorsOfValuesH11.assign(doubleValuesArrayH1.data() + index, doubleValuesArrayH1.data() + index + vectorSize1);
+ //vectorsOfValuesH21.assign(doubleValuesArrayH2.data() + index, doubleValuesArrayH2.data() + index + vectorSize1);
index += vectorSize1;
- vectorsOfValuesF2.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + vectorSize2);
+ vectorsOfValuesF2.assign(doubleValuesArrayF.data() + index, doubleValuesArrayF.data() + index + vectorSize2);
+ if (multiPhase)
+ vectorsOfValuesH12.assign(doubleValuesArrayH1.data() + index, doubleValuesArrayH1.data() + index + vectorSize2);
+ //vectorsOfValuesH22.assign(doubleValuesArrayH2.data() + index, doubleValuesArrayH2.data() + index + vectorSize2);
index += vectorSize2;
- vectorsOfValuesF3.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + vectorSize3);
- index += vectorSize3;
-
- vectorsOfValuesH1.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + vectorSize1);
- index += vectorSize1;
-
- vectorsOfValuesH2.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + vectorSize2);
- index += vectorSize2;
-
- vectorsOfValuesH3.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + vectorSize3);
+ vectorsOfValuesF3.assign(doubleValuesArrayF.data() + index, doubleValuesArrayF.data() + index + vectorSize3);
+ if (multiPhase)
+ vectorsOfValuesH13.assign(doubleValuesArrayH1.data() + index, doubleValuesArrayH1.data() + index + vectorSize3);
+ //vectorsOfValuesH23.assign(doubleValuesArrayH2.data() + index, doubleValuesArrayH2.data() + index + vectorSize3);
index += vectorSize3;
SPtr<DistributionArray3D> mFdistributions(new D3Q27EsoTwist3DSplittedVector());
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)
- ->setLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
- new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesF1, dataSetParamStr1.nx[0],
- dataSetParamStr1.nx[1], dataSetParamStr1.nx[2],
- dataSetParamStr1.nx[3])));
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)
- ->setNonLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
- new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesF2, dataSetParamStr2.nx[0],
- dataSetParamStr2.nx[1], dataSetParamStr2.nx[2],
- dataSetParamStr2.nx[3])));
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)
- ->setZeroDistributions(CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
+ new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesF1, dataSetParamStr1.nx[0], dataSetParamStr1.nx[1], dataSetParamStr1.nx[2], dataSetParamStr1.nx[3])));
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNonLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
+ new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesF2, dataSetParamStr2.nx[0], dataSetParamStr2.nx[1], dataSetParamStr2.nx[2], dataSetParamStr2.nx[3])));
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setZeroDistributions(CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(
vectorsOfValuesF3, dataSetParamStr3.nx[0], dataSetParamStr3.nx[1], dataSetParamStr3.nx[2])));
dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNX1(dataSetParamStr1.nx1);
dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNX2(dataSetParamStr1.nx2);
dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNX3(dataSetParamStr1.nx3);
- SPtr<DistributionArray3D> mHdistributions(new D3Q27EsoTwist3DSplittedVector());
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mHdistributions)
- ->setLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
- new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesH1, dataSetParamStr1.nx[0],
- dataSetParamStr1.nx[1], dataSetParamStr1.nx[2],
- dataSetParamStr1.nx[3])));
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mHdistributions)
- ->setNonLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
- new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesH2, dataSetParamStr2.nx[0],
- dataSetParamStr2.nx[1], dataSetParamStr2.nx[2],
- dataSetParamStr2.nx[3])));
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mHdistributions)
- ->setZeroDistributions(CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(
- vectorsOfValuesH3, dataSetParamStr3.nx[0], dataSetParamStr3.nx[1], dataSetParamStr3.nx[2])));
-
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mHdistributions)->setNX1(dataSetParamStr1.nx1);
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mHdistributions)->setNX2(dataSetParamStr1.nx2);
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mHdistributions)->setNX3(dataSetParamStr1.nx3);
+ SPtr<DistributionArray3D> mH1distributions(new D3Q27EsoTwist3DSplittedVector());
+ if (multiPhase)
+ {
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH1distributions)->setLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
+ new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesH11, dataSetParamStr1.nx[0], dataSetParamStr1.nx[1], dataSetParamStr1.nx[2], dataSetParamStr1.nx[3])));
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH1distributions)->setNonLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
+ new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesH12, dataSetParamStr2.nx[0], dataSetParamStr2.nx[1], dataSetParamStr2.nx[2], dataSetParamStr2.nx[3])));
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH1distributions)->setZeroDistributions(CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(
+ vectorsOfValuesH13, dataSetParamStr3.nx[0], dataSetParamStr3.nx[1], dataSetParamStr3.nx[2])));
+
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH1distributions)->setNX1(dataSetParamStr1.nx1);
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH1distributions)->setNX2(dataSetParamStr1.nx2);
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH1distributions)->setNX3(dataSetParamStr1.nx3);
+ }
+
+ /*SPtr<DistributionArray3D> mH2distributions(new D3Q27EsoTwist3DSplittedVector());
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH2distributions)->setLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
+ new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesH21, dataSetParamStr1.nx[0], dataSetParamStr1.nx[1], dataSetParamStr1.nx[2], dataSetParamStr1.nx[3])));
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH2distributions)->setNonLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
+ new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesH22, dataSetParamStr2.nx[0], dataSetParamStr2.nx[1], dataSetParamStr2.nx[2], dataSetParamStr2.nx[3])));
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH2distributions)->setZeroDistributions(CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(
+ vectorsOfValuesH23, dataSetParamStr3.nx[0], dataSetParamStr3.nx[1], dataSetParamStr3.nx[2])));
+
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH2distributions)->setNX1(dataSetParamStr1.nx1);
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH2distributions)->setNX2(dataSetParamStr1.nx2);
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH2distributions)->setNX3(dataSetParamStr1.nx3);*/
// find the nesessary block and fill it
SPtr<Block3D> block = grid->getBlock(dataSetArray[n].globalID);
@@ -1716,17 +1836,22 @@ void MPIIOMigrationCoProcessor::readDataSet(int step)
kernel->setDeltaT(dataSetArray[n].deltaT);
kernel->setCompressible(dataSetArray[n].compressible);
kernel->setWithForcing(dataSetArray[n].withForcing);
+ kernel->setCollisionFactorMultiphase(dataSetArray[n].collFactorL, dataSetArray[n].collFactorG);
+ kernel->setDensityRatio(dataSetArray[n].densityRatio);
+
SPtr<DataSet3D> dataSetPtr = SPtr<DataSet3D>(new DataSet3D());
dataSetPtr->setFdistributions(mFdistributions);
- dataSetPtr->setHdistributions(mHdistributions);
+ if (multiPhase)
+ dataSetPtr->setHdistributions(mH1distributions);
+ //dataSetPtr->setH2distributions(mH2distributions);
kernel->setDataSet(dataSetPtr);
block->setKernel(kernel);
}
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationCoProcessor::readDataSet end of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
delete[] dataSetArray;
@@ -1766,8 +1891,12 @@ void MPIIOMigrationCoProcessor::readDataSet(int step)
readArray(step, RelaxationFactor, std::string("/cpRelaxationFactor.bin"));
// readRelaxationFactor(step);
- if (arrPresence.isPhaseFieldPresent)
- readArray(step, PhaseField, std::string("/cpPhaseField.bin"));
+ if (arrPresence.isPhaseField1Present)
+ readArray(step, PhaseField1, std::string("/cpPhaseField1.bin"));
+
+ if (arrPresence.isPhaseField2Present)
+ readArray(step, PhaseField2, std::string("/cpPhaseField2.bin"));
+
}
void MPIIOMigrationCoProcessor::readArray(int step, Arrays arrType, std::string fname)
@@ -1776,10 +1905,10 @@ void MPIIOMigrationCoProcessor::readArray(int step, Arrays arrType, std::string
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationCoProcessor::readArray start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
if (comm->isRoot())
@@ -1799,7 +1928,8 @@ void MPIIOMigrationCoProcessor::readArray(int step, Arrays arrType, std::string
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
int maxInitLevel = this->grid->getFinestInitializedLevel();
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
grid->getBlocks(level, rank, blocksVector[level]);
blocksCount += static_cast<int>(blocksVector[level].size());
}
@@ -1807,8 +1937,7 @@ void MPIIOMigrationCoProcessor::readArray(int step, Arrays arrType, std::string
MPI_File_read_at(file_handler, (MPI_Offset)0, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
DataSetSmallMigration *dataSetSmallArray = new DataSetSmallMigration[blocksCount];
- size_t doubleCountInBlock =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ size_t doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
std::vector<double> doubleValuesArray(blocksCount * doubleCountInBlock); // double-values in all blocks
// define MPI_types depending on the block-specific information
@@ -1819,7 +1948,8 @@ void MPIIOMigrationCoProcessor::readArray(int step, Arrays arrType, std::string
MPI_Offset read_offset;
size_t sizeofOneDataSet = size_t(sizeof(DataSetSmallMigration) + doubleCountInBlock * sizeof(double));
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
{
read_offset = (MPI_Offset)(sizeof(dataSetParam) + block->getGlobalID() * sizeofOneDataSet);
@@ -1833,20 +1963,20 @@ void MPIIOMigrationCoProcessor::readArray(int step, Arrays arrType, std::string
MPI_File_close(&file_handler);
MPI_Type_free(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIOMigrationCoProcessor::readArray readArray: " << finish - start << " s");
UBLOG(logINFO, "MPIIOMigrationCoProcessor::readArray start of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
//----------------------------- restore data ---------------------------------
size_t index = 0;
- size_t nextVectorSize =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ size_t nextVectorSize = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
std::vector<double> vectorsOfValues;
- for (std::size_t n = 0; n < blocksCount; n++) {
+ for (std::size_t n = 0; n < blocksCount; n++)
+ {
SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].globalID);
vectorsOfValues.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + nextVectorSize);
@@ -1856,35 +1986,31 @@ void MPIIOMigrationCoProcessor::readArray(int step, Arrays arrType, std::string
SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> ___4DArray;
SPtr<CbArray3D<LBMReal, IndexerX3X2X1>> ___3DArray;
- switch (arrType) {
+ switch (arrType)
+ {
case AverageDensity:
___4DArray = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(
- vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2],
- dataSetParamStr.nx[3]));
+ vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
block->getKernel()->getDataSet()->setAverageDensity(___4DArray);
break;
case AverageVelocity:
___4DArray = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(
- vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2],
- dataSetParamStr.nx[3]));
+ vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
block->getKernel()->getDataSet()->setAverageVelocity(___4DArray);
break;
case AverageFluktuations:
___4DArray = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(
- vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2],
- dataSetParamStr.nx[3]));
+ vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
block->getKernel()->getDataSet()->setAverageFluctuations(___4DArray);
break;
case AverageTriple:
___4DArray = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(
- vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2],
- dataSetParamStr.nx[3]));
+ vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
block->getKernel()->getDataSet()->setAverageTriplecorrelations(___4DArray);
break;
case ShearStressVal:
___4DArray = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(
- vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2],
- dataSetParamStr.nx[3]));
+ vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
block->getKernel()->getDataSet()->setShearStressValues(___4DArray);
break;
case RelaxationFactor:
@@ -1892,21 +2018,26 @@ void MPIIOMigrationCoProcessor::readArray(int step, Arrays arrType, std::string
vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2]));
block->getKernel()->getDataSet()->setRelaxationFactor(___3DArray);
break;
- case PhaseField:
+ case PhaseField1:
___3DArray = CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(
vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2]));
block->getKernel()->getDataSet()->setPhaseField(___3DArray);
break;
+ case PhaseField2:
+ ___3DArray = CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(
+ vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2]));
+ block->getKernel()->getDataSet()->setPhaseField2(___3DArray);
+ break;
default:
UB_THROW(UbException(UB_EXARGS, "MPIIOMigrationCoProcessor::readArray : array type does not exist!"));
break;
}
}
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationCoProcessor::readArray end of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
delete[] dataSetSmallArray;
@@ -2548,10 +2679,10 @@ void MPIIOMigrationCoProcessor::readBoundaryConds(int step)
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationCoProcessor::readBoundaryConds start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
@@ -2570,7 +2701,8 @@ void MPIIOMigrationCoProcessor::readBoundaryConds(int step)
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
int maxInitLevel = this->grid->getFinestInitializedLevel();
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
grid->getBlocks(level, rank, blocksVector[level]);
blocksCount += static_cast<int>(blocksVector[level].size());
}
@@ -2585,12 +2717,12 @@ void MPIIOMigrationCoProcessor::readBoundaryConds(int step)
std::vector<int> bcindexmatrixV;
std::vector<int> indexContainerV;
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIOMigrationCoProcessor::readBoundaryConds time: " << finish - start << " s");
UBLOG(logINFO, "MPIIOMigrationCoProcessor::readBoundaryConds start of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
MPI_File_read_at(file_handler, (MPI_Offset)0, &boundCondParamStr, 1, boundCondParamType, MPI_STATUS_IGNORE);
@@ -2599,7 +2731,8 @@ void MPIIOMigrationCoProcessor::readBoundaryConds(int step)
int ic = 0;
MPI_Offset read_offset1, read_offset2;
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
{
read_offset1 = (MPI_Offset)(sizeof(boundCondParam) + block->getGlobalID() * sizeof(size_t));
@@ -2611,31 +2744,30 @@ void MPIIOMigrationCoProcessor::readBoundaryConds(int step)
intArray1 = new int[boundCondParamStr.bcindexmatrixCount];
intArray2 = new int[bcAddArray[ic].indexContainer_count];
- if (bcAddArray[ic].boundCond_count > 0) {
+ if (bcAddArray[ic].boundCond_count > 0)
+ {
MPI_File_read_at(file_handler, (MPI_Offset)(read_offset2 + sizeof(BCAddMigration)), &bcArray[0],
bcAddArray[ic].boundCond_count, boundCondType, MPI_STATUS_IGNORE);
}
- MPI_File_read_at(file_handler,
- (MPI_Offset)(read_offset2 + sizeof(BCAddMigration) +
- bcAddArray[ic].boundCond_count * sizeof(BoundaryCondition)),
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset2 + sizeof(BCAddMigration) + bcAddArray[ic].boundCond_count * sizeof(BoundaryCondition)),
&intArray1[0], 1, bcindexmatrixType, MPI_STATUS_IGNORE);
- if (bcAddArray[ic].indexContainer_count > 0) {
- MPI_File_read_at(file_handler,
- (MPI_Offset)(read_offset2 + sizeof(BCAddMigration) +
- bcAddArray[ic].boundCond_count * sizeof(BoundaryCondition) +
- boundCondParamStr.bcindexmatrixCount * sizeof(int)),
- &intArray2[0], bcAddArray[ic].indexContainer_count, MPI_INT, MPI_STATUS_IGNORE);
+ if (bcAddArray[ic].indexContainer_count > 0)
+ {
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset2 + sizeof(BCAddMigration) + bcAddArray[ic].boundCond_count * sizeof(BoundaryCondition) +
+ boundCondParamStr.bcindexmatrixCount * sizeof(int)), &intArray2[0], bcAddArray[ic].indexContainer_count, MPI_INT, MPI_STATUS_IGNORE);
}
bcindexmatrixV.resize(0);
indexContainerV.resize(0);
bcVector.resize(0);
- for (int ibc = 0; ibc < bcAddArray[ic].boundCond_count; ibc++) {
+ for (int ibc = 0; ibc < bcAddArray[ic].boundCond_count; ibc++)
+ {
SPtr<BoundaryConditions> bc;
if (memcmp(&bcArray[ibc], nullBouCond, sizeof(BoundaryCondition)) == 0)
bc = SPtr<BoundaryConditions>();
- else {
+ else
+ {
bc = SPtr<BoundaryConditions>(new BoundaryConditions);
bc->noslipBoundaryFlags = bcArray[ibc].noslipBoundaryFlags;
bc->slipBoundaryFlags = bcArray[ibc].slipBoundaryFlags;
@@ -2669,8 +2801,7 @@ void MPIIOMigrationCoProcessor::readBoundaryConds(int step)
for (int b2 = 0; b2 < bcAddArray[ic].indexContainer_count; b2++)
indexContainerV.push_back(intArray2[b2]);
- CbArray3D<int, IndexerX3X2X1> bcim(bcindexmatrixV, boundCondParamStr.nx1, boundCondParamStr.nx2,
- boundCondParamStr.nx3);
+ CbArray3D<int, IndexerX3X2X1> bcim(bcindexmatrixV, boundCondParamStr.nx1, boundCondParamStr.nx2, boundCondParamStr.nx3);
SPtr<Block3D> block1 = grid->getBlock(bcAddArray[ic].globalID);
SPtr<BCProcessor> bcProc = bcProcessor->clone(block1->getKernel());
@@ -2694,10 +2825,10 @@ void MPIIOMigrationCoProcessor::readBoundaryConds(int step)
delete nullBouCond;
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIOMigrationCoProcessor::readBoundaryConds end of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
}
diff --git a/src/cpu/VirtualFluidsCore/CoProcessors/MPIIOMigrationCoProcessor.h b/src/cpu/VirtualFluidsCore/CoProcessors/MPIIOMigrationCoProcessor.h
index ad7a93086afa379822fc7909a68fd39748dd607f..ca0de8f3e7ba315bc8a870f89063ea9f38d7b59f 100644
--- a/src/cpu/VirtualFluidsCore/CoProcessors/MPIIOMigrationCoProcessor.h
+++ b/src/cpu/VirtualFluidsCore/CoProcessors/MPIIOMigrationCoProcessor.h
@@ -25,7 +25,8 @@ public:
AverageTriple = 4,
ShearStressVal = 5,
RelaxationFactor = 6,
- PhaseField = 7
+ PhaseField1 = 7,
+ PhaseField2 = 8
};
MPIIOMigrationCoProcessor(SPtr<Grid3D> grid, SPtr<UbScheduler> s, const std::string &path, SPtr<Communicator> comm);
diff --git a/src/cpu/VirtualFluidsCore/CoProcessors/MPIIORestartCoProcessor.cpp b/src/cpu/VirtualFluidsCore/CoProcessors/MPIIORestartCoProcessor.cpp
index ed5d3b275c006700d29c43f16928d2ddc08827f0..49277be64ffca7f28a52eb3dfd3a252d1cb4c2fd 100644
--- a/src/cpu/VirtualFluidsCore/CoProcessors/MPIIORestartCoProcessor.cpp
+++ b/src/cpu/VirtualFluidsCore/CoProcessors/MPIIORestartCoProcessor.cpp
@@ -25,8 +25,7 @@
using namespace MPIIODataStructures;
-MPIIORestartCoProcessor::MPIIORestartCoProcessor(SPtr<Grid3D> grid, SPtr<UbScheduler> s, const std::string &path,
- SPtr<Communicator> comm)
+MPIIORestartCoProcessor::MPIIORestartCoProcessor(SPtr<Grid3D> grid, SPtr<UbScheduler> s, const std::string &path, SPtr<Communicator> comm)
: MPIIOCoProcessor(grid, s, path, comm)
{
memset(&boundCondParamStr, 0, sizeof(boundCondParamStr));
@@ -34,7 +33,7 @@ MPIIORestartCoProcessor::MPIIORestartCoProcessor(SPtr<Grid3D> grid, SPtr<UbSched
//------------------------- define MPI types ---------------------------------
MPI_Datatype typesDataSet[3] = { MPI_DOUBLE, MPI_INT, MPI_CHAR };
- int blocksDataSet[3] = { 2, 5, 2 };
+ int blocksDataSet[3] = { 5, 5, 2 };
MPI_Aint offsetsDatatSet[3], lbDataSet, extentDataSet;
offsetsDatatSet[0] = 0;
@@ -80,7 +79,8 @@ MPIIORestartCoProcessor::~MPIIORestartCoProcessor()
//////////////////////////////////////////////////////////////////////////
void MPIIORestartCoProcessor::process(double step)
{
- if (scheduler->isDue(step)) {
+ if (scheduler->isDue(step))
+ {
if (comm->isRoot())
UBLOG(logINFO, "MPIIORestartCoProcessor save step: " << step);
if (comm->isRoot())
@@ -130,56 +130,77 @@ void MPIIORestartCoProcessor::writeDataSet(int step)
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
int maxInitLevel = this->grid->getFinestInitializedLevel();
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
grid->getBlocks(level, rank, blocksVector[level]);
blocksCount += static_cast<int>(blocksVector[level].size());
}
dataSetParam dataSetParamStr1, dataSetParamStr2, dataSetParamStr3;
DataSetRestart *dataSetArray = new DataSetRestart[blocksCount];
- std::vector<double> doubleValuesArray; // double-values (arrays of f's) in all blocks
+ std::vector<double> doubleValuesArrayF; // double-values (arrays of f's) in all blocks Fdistribution
+ std::vector<double> doubleValuesArrayH1; // double-values (arrays of f's) in all blocks H1distribution
+ // std::vector<double> doubleValuesArrayH2; // double-values (arrays of f's) in all blocks H2distribution
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::writeDataSet start collect data rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
+ bool multiPhase = false;
DSArraysPresence arrPresence;
bool firstBlock = true;
int doubleCountInBlock = 0;
int ic = 0;
- SPtr<D3Q27EsoTwist3DSplittedVector> D3Q27EsoTwist3DSplittedVectorPtrF, D3Q27EsoTwist3DSplittedVectorPtrH;
- CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr localDistributionsF, localDistributionsH;
- CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr nonLocalDistributionsF, nonLocalDistributionsH;
- CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr zeroDistributionsF, zeroDistributionsH;
+ SPtr<D3Q27EsoTwist3DSplittedVector> D3Q27EsoTwist3DSplittedVectorPtrF, D3Q27EsoTwist3DSplittedVectorPtrH1, D3Q27EsoTwist3DSplittedVectorPtrH2;
+ CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr localDistributionsF, localDistributionsH1, localDistributionsH2;
+ CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr nonLocalDistributionsF, nonLocalDistributionsH1, nonLocalDistributionsH2;
+ CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr zeroDistributionsF, zeroDistributionsH1, zeroDistributionsH2;
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ SPtr<LBMKernel> kernel;
+
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
{
- dataSetArray[ic].x1 =
- block->getX1(); // coordinates of the block needed to find it while regenerating the grid
- dataSetArray[ic].x2 = block->getX2();
- dataSetArray[ic].x3 = block->getX3();
- dataSetArray[ic].level = block->getLevel();
- dataSetArray[ic].ghostLayerWidth = block->getKernel()->getGhostLayerWidth();
- dataSetArray[ic].collFactor = block->getKernel()->getCollisionFactor();
- dataSetArray[ic].deltaT = block->getKernel()->getDeltaT();
- dataSetArray[ic].compressible = block->getKernel()->getCompressible();
- dataSetArray[ic].withForcing = block->getKernel()->getWithForcing();
-
- D3Q27EsoTwist3DSplittedVectorPtrF = dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(
- block->getKernel()->getDataSet()->getFdistributions());
+ kernel = dynamicPointerCast<LBMKernel>(block->getKernel());
+
+ dataSetArray[ic].x1 = block->getX1(); // coordinates of the block needed to find it while regenerating the grid
+ dataSetArray[ic].x2 = block->getX2();
+ dataSetArray[ic].x3 = block->getX3();
+ dataSetArray[ic].level = block->getLevel();
+ dataSetArray[ic].ghostLayerWidth = kernel->getGhostLayerWidth();
+ dataSetArray[ic].collFactor = kernel->getCollisionFactor();
+ dataSetArray[ic].deltaT = kernel->getDeltaT();
+ dataSetArray[ic].compressible = kernel->getCompressible();
+ dataSetArray[ic].withForcing = kernel->getWithForcing();
+ dataSetArray[ic].collFactorL = kernel->getCollisionFactorL();
+ dataSetArray[ic].collFactorG = kernel->getCollisionFactorG();
+ dataSetArray[ic].densityRatio = kernel->getDensityRatio();
+
+ D3Q27EsoTwist3DSplittedVectorPtrF = dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(block->getKernel()->getDataSet()->getFdistributions());
localDistributionsF = D3Q27EsoTwist3DSplittedVectorPtrF->getLocalDistributions();
nonLocalDistributionsF = D3Q27EsoTwist3DSplittedVectorPtrF->getNonLocalDistributions();
zeroDistributionsF = D3Q27EsoTwist3DSplittedVectorPtrF->getZeroDistributions();
- D3Q27EsoTwist3DSplittedVectorPtrH = dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(
- block->getKernel()->getDataSet()->getHdistributions());
- localDistributionsH = D3Q27EsoTwist3DSplittedVectorPtrH->getLocalDistributions();
- nonLocalDistributionsH = D3Q27EsoTwist3DSplittedVectorPtrH->getNonLocalDistributions();
- zeroDistributionsH = D3Q27EsoTwist3DSplittedVectorPtrH->getZeroDistributions();
+ D3Q27EsoTwist3DSplittedVectorPtrH1 = dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(block->getKernel()->getDataSet()->getHdistributions());
+ if (D3Q27EsoTwist3DSplittedVectorPtrH1 != 0)
+ {
+ multiPhase = true;
+ localDistributionsH1 = D3Q27EsoTwist3DSplittedVectorPtrH1->getLocalDistributions();
+ nonLocalDistributionsH1 = D3Q27EsoTwist3DSplittedVectorPtrH1->getNonLocalDistributions();
+ zeroDistributionsH1 = D3Q27EsoTwist3DSplittedVectorPtrH1->getZeroDistributions();
+ }
+
+ /*D3Q27EsoTwist3DSplittedVectorPtrH2 = dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(block->getKernel()->getDataSet()->getH2distributions());
+ if (D3Q27EsoTwist3DSplittedVectorPtrH2 != 0)
+ {
+ localDistributionsH2 = D3Q27EsoTwist3DSplittedVectorPtrH2->getLocalDistributions();
+ nonLocalDistributionsH2 = D3Q27EsoTwist3DSplittedVectorPtrH2->getNonLocalDistributions();
+ zeroDistributionsH2 = D3Q27EsoTwist3DSplittedVectorPtrH2->getZeroDistributions();
+ }*/
if (firstBlock) // when first (any) valid block...
{
@@ -204,96 +225,88 @@ void MPIIORestartCoProcessor::writeDataSet(int step)
}
// ... than save some parameters that are equal in all dataSets
- dataSetParamStr1.nx1 = dataSetParamStr2.nx1 = dataSetParamStr3.nx1 =
- static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX1());
- dataSetParamStr1.nx2 = dataSetParamStr2.nx2 = dataSetParamStr3.nx2 =
- static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX2());
- dataSetParamStr1.nx3 = dataSetParamStr2.nx3 = dataSetParamStr3.nx3 =
- static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX3());
-
- // Fdistributions + Hdistributions
- doubleCountInBlock =
- (dataSetParamStr1.nx[0] * dataSetParamStr1.nx[1] * dataSetParamStr1.nx[2] * dataSetParamStr1.nx[3] +
+ dataSetParamStr1.nx1 = dataSetParamStr2.nx1 = dataSetParamStr3.nx1 = static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX1());
+ dataSetParamStr1.nx2 = dataSetParamStr2.nx2 = dataSetParamStr3.nx2 = static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX2());
+ dataSetParamStr1.nx3 = dataSetParamStr2.nx3 = dataSetParamStr3.nx3 = static_cast<int>(block->getKernel()->getDataSet()->getFdistributions()->getNX3());
+
+ doubleCountInBlock = dataSetParamStr1.nx[0] * dataSetParamStr1.nx[1] * dataSetParamStr1.nx[2] * dataSetParamStr1.nx[3] +
dataSetParamStr2.nx[0] * dataSetParamStr2.nx[1] * dataSetParamStr2.nx[2] * dataSetParamStr2.nx[3] +
- dataSetParamStr3.nx[0] * dataSetParamStr3.nx[1] * dataSetParamStr3.nx[2] * dataSetParamStr3.nx[3]) * 2;
+ dataSetParamStr3.nx[0] * dataSetParamStr3.nx[1] * dataSetParamStr3.nx[2] * dataSetParamStr3.nx[3];
- SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> averageDensityArray =
- block->getKernel()->getDataSet()->getAverageDensity();
+ SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> averageDensityArray = kernel->getDataSet()->getAverageDensity();
if (averageDensityArray)
arrPresence.isAverageDensityArrayPresent = true;
else
arrPresence.isAverageDensityArrayPresent = false;
- SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> AverageVelocityArray3DPtr =
- block->getKernel()->getDataSet()->getAverageVelocity();
+ SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> AverageVelocityArray3DPtr = kernel->getDataSet()->getAverageVelocity();
if (AverageVelocityArray3DPtr)
arrPresence.isAverageVelocityArrayPresent = true;
else
arrPresence.isAverageVelocityArrayPresent = false;
- SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> AverageFluctArray3DPtr =
- block->getKernel()->getDataSet()->getAverageFluctuations();
+ SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> AverageFluctArray3DPtr = kernel->getDataSet()->getAverageFluctuations();
if (AverageFluctArray3DPtr)
arrPresence.isAverageFluktuationsArrayPresent = true;
else
arrPresence.isAverageFluktuationsArrayPresent = false;
- SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> AverageTripleArray3DPtr =
- block->getKernel()->getDataSet()->getAverageTriplecorrelations();
+ SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> AverageTripleArray3DPtr = kernel->getDataSet()->getAverageTriplecorrelations();
if (AverageTripleArray3DPtr)
arrPresence.isAverageTripleArrayPresent = true;
else
arrPresence.isAverageTripleArrayPresent = false;
- SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> ShearStressValArray3DPtr =
- block->getKernel()->getDataSet()->getShearStressValues();
+ SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> ShearStressValArray3DPtr = kernel->getDataSet()->getShearStressValues();
if (ShearStressValArray3DPtr)
arrPresence.isShearStressValArrayPresent = true;
else
arrPresence.isShearStressValArrayPresent = false;
- SPtr<CbArray3D<LBMReal, IndexerX3X2X1>> relaxationFactor3DPtr =
- block->getKernel()->getDataSet()->getRelaxationFactor();
+ SPtr<CbArray3D<LBMReal, IndexerX3X2X1>> relaxationFactor3DPtr = kernel->getDataSet()->getRelaxationFactor();
if (relaxationFactor3DPtr)
arrPresence.isRelaxationFactorPresent = true;
else
arrPresence.isRelaxationFactorPresent = false;
- SPtr<CbArray3D<LBMReal, IndexerX3X2X1>> phaseField3DPtr =
- block->getKernel()->getDataSet()->getPhaseField();
- if (phaseField3DPtr)
- arrPresence.isPhaseFieldPresent = true;
+ SPtr<CbArray3D<LBMReal, IndexerX3X2X1>> phaseField3DPtr1 = kernel->getDataSet()->getPhaseField();
+ if (phaseField3DPtr1)
+ arrPresence.isPhaseField1Present = true;
+ else
+ arrPresence.isPhaseField1Present = false;
+
+ SPtr<CbArray3D<LBMReal, IndexerX3X2X1>> phaseField3DPtr2 = kernel->getDataSet()->getPhaseField2();
+ if (phaseField3DPtr2)
+ arrPresence.isPhaseField2Present = true;
else
- arrPresence.isPhaseFieldPresent = false;
+ arrPresence.isPhaseField2Present = false;
firstBlock = false;
}
- if (localDistributionsF && (dataSetParamStr1.nx[0] > 0) && (dataSetParamStr1.nx[1] > 0) &&
- (dataSetParamStr1.nx[2] > 0) && (dataSetParamStr1.nx[3] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), localDistributionsF->getDataVector().begin(),
- localDistributionsF->getDataVector().end());
- if (nonLocalDistributionsF && (dataSetParamStr2.nx[0] > 0) && (dataSetParamStr2.nx[1] > 0) &&
- (dataSetParamStr2.nx[2] > 0) && (dataSetParamStr2.nx[3] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), nonLocalDistributionsF->getDataVector().begin(),
- nonLocalDistributionsF->getDataVector().end());
- if (zeroDistributionsF && (dataSetParamStr3.nx[0] > 0) && (dataSetParamStr3.nx[1] > 0) &&
- (dataSetParamStr3.nx[2] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), zeroDistributionsF->getDataVector().begin(),
- zeroDistributionsF->getDataVector().end());
-
- if (localDistributionsH && (dataSetParamStr1.nx[0] > 0) && (dataSetParamStr1.nx[1] > 0) &&
- (dataSetParamStr1.nx[2] > 0) && (dataSetParamStr1.nx[3] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), localDistributionsH->getDataVector().begin(),
- localDistributionsH->getDataVector().end());
- if (nonLocalDistributionsH && (dataSetParamStr2.nx[0] > 0) && (dataSetParamStr2.nx[1] > 0) &&
- (dataSetParamStr2.nx[2] > 0) && (dataSetParamStr2.nx[3] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), nonLocalDistributionsH->getDataVector().begin(),
- nonLocalDistributionsH->getDataVector().end());
- if (zeroDistributionsH && (dataSetParamStr3.nx[0] > 0) && (dataSetParamStr3.nx[1] > 0) &&
- (dataSetParamStr3.nx[2] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), zeroDistributionsH->getDataVector().begin(),
- zeroDistributionsH->getDataVector().end());
+ if (localDistributionsF && (dataSetParamStr1.nx[0] > 0) && (dataSetParamStr1.nx[1] > 0) && (dataSetParamStr1.nx[2] > 0) && (dataSetParamStr1.nx[3] > 0))
+ doubleValuesArrayF.insert(doubleValuesArrayF.end(), localDistributionsF->getDataVector().begin(), localDistributionsF->getDataVector().end());
+ if (nonLocalDistributionsF && (dataSetParamStr2.nx[0] > 0) && (dataSetParamStr2.nx[1] > 0) && (dataSetParamStr2.nx[2] > 0) && (dataSetParamStr2.nx[3] > 0))
+ doubleValuesArrayF.insert(doubleValuesArrayF.end(), nonLocalDistributionsF->getDataVector().begin(), nonLocalDistributionsF->getDataVector().end());
+ if (zeroDistributionsF && (dataSetParamStr3.nx[0] > 0) && (dataSetParamStr3.nx[1] > 0) && (dataSetParamStr3.nx[2] > 0))
+ doubleValuesArrayF.insert(doubleValuesArrayF.end(), zeroDistributionsF->getDataVector().begin(), zeroDistributionsF->getDataVector().end());
+
+ if (multiPhase)
+ {
+ if (localDistributionsH1 && (dataSetParamStr1.nx[0] > 0) && (dataSetParamStr1.nx[1] > 0) && (dataSetParamStr1.nx[2] > 0) && (dataSetParamStr1.nx[3] > 0))
+ doubleValuesArrayH1.insert(doubleValuesArrayH1.end(), localDistributionsH1->getDataVector().begin(), localDistributionsH1->getDataVector().end());
+ if (nonLocalDistributionsH1 && (dataSetParamStr2.nx[0] > 0) && (dataSetParamStr2.nx[1] > 0) && (dataSetParamStr2.nx[2] > 0) && (dataSetParamStr2.nx[3] > 0))
+ doubleValuesArrayH1.insert(doubleValuesArrayH1.end(), nonLocalDistributionsH1->getDataVector().begin(), nonLocalDistributionsH1->getDataVector().end());
+ if (zeroDistributionsH1 && (dataSetParamStr3.nx[0] > 0) && (dataSetParamStr3.nx[1] > 0) && (dataSetParamStr3.nx[2] > 0))
+ doubleValuesArrayH1.insert(doubleValuesArrayH1.end(), zeroDistributionsH1->getDataVector().begin(), zeroDistributionsH1->getDataVector().end());
+ }
+
+ /*if (localDistributionsH2 && (dataSetParamStr1.nx[0] > 0) && (dataSetParamStr1.nx[1] > 0) && (dataSetParamStr1.nx[2] > 0) && (dataSetParamStr1.nx[3] > 0))
+ doubleValuesArrayH2.insert(doubleValuesArrayH2.end(), localDistributionsH2->getDataVector().begin(), localDistributionsH2->getDataVector().end());
+ if (nonLocalDistributionsH2 && (dataSetParamStr2.nx[0] > 0) && (dataSetParamStr2.nx[1] > 0) && (dataSetParamStr2.nx[2] > 0) && (dataSetParamStr2.nx[3] > 0))
+ doubleValuesArrayH2.insert(doubleValuesArrayH2.end(), nonLocalDistributionsH2->getDataVector().begin(), nonLocalDistributionsH2->getDataVector().end());
+ if (zeroDistributionsH2 && (dataSetParamStr3.nx[0] > 0) && (dataSetParamStr3.nx[1] > 0) && (dataSetParamStr3.nx[2] > 0))
+ doubleValuesArrayH2.insert(doubleValuesArrayH2.end(), zeroDistributionsH2->getDataVector().begin(), zeroDistributionsH2->getDataVector().end());*/
ic++;
}
@@ -303,10 +316,10 @@ void MPIIORestartCoProcessor::writeDataSet(int step)
MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
MPI_Type_commit(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::writeDataSet start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
// write to the file
@@ -315,15 +328,17 @@ void MPIIORestartCoProcessor::writeDataSet(int step)
MPI_Offset write_offset = (MPI_Offset)(size * sizeof(int));
size_t next_write_offset = 0;
- if (size > 1) {
- if (rank == 0) {
- next_write_offset = write_offset + 3 * sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetRestart) + doubleCountInBlock * sizeof(double));
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_write_offset = write_offset + 3 * sizeof(dataSetParam) + blocksCount * (sizeof(DataSetRestart) + doubleCountInBlock * sizeof(double));
MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
- } else {
+ }
+ else
+ {
MPI_Recv(&write_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
- next_write_offset = write_offset + 3 * sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetRestart) + doubleCountInBlock * sizeof(double));
+ next_write_offset = write_offset + 3 * sizeof(dataSetParam) + blocksCount * (sizeof(DataSetRestart) + doubleCountInBlock * sizeof(double));
if (rank < size - 1)
MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
}
@@ -342,7 +357,7 @@ void MPIIORestartCoProcessor::writeDataSet(int step)
#endif
MPI_File file_handler;
- std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSet.bin";
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSetF.bin";
int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
if (rc != MPI_SUCCESS)
throw UbException(UB_EXARGS, "couldn't open file " + filename);
@@ -351,27 +366,54 @@ void MPIIORestartCoProcessor::writeDataSet(int step)
MPI_File_write_at(file_handler, (MPI_Offset)(rank * sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
// each process writes common parameters of a dataSet
MPI_File_write_at(file_handler, write_offset, &dataSetParamStr1, 1, dataSetParamType, MPI_STATUS_IGNORE);
- MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam)), &dataSetParamStr2, 1,
- dataSetParamType, MPI_STATUS_IGNORE);
- MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + 2 * sizeof(dataSetParam)), &dataSetParamStr3, 1,
- dataSetParamType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam)), &dataSetParamStr2, 1, dataSetParamType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + 2 * sizeof(dataSetParam)), &dataSetParamStr3, 1, dataSetParamType, MPI_STATUS_IGNORE);
// each process writes data identifying blocks
- MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + 3 * sizeof(dataSetParam)), dataSetArray, blocksCount,
- dataSetType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + 3 * sizeof(dataSetParam)), dataSetArray, blocksCount, dataSetType, MPI_STATUS_IGNORE);
// each process writes the dataSet arrays
- if (doubleValuesArray.size() > 0)
- MPI_File_write_at(file_handler,
- (MPI_Offset)(write_offset + 3 * sizeof(dataSetParam) + blocksCount * sizeof(DataSetRestart)),
- &doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
+ if (doubleValuesArrayF.size() > 0)
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + 3 * sizeof(dataSetParam) + blocksCount * sizeof(DataSetRestart)),
+ &doubleValuesArrayF[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
MPI_File_sync(file_handler);
MPI_File_close(&file_handler);
+ //------------------------------------------------------------------------------------------------------------------
+ if (multiPhase)
+ {
+ filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSetH1.bin";
+ rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc != MPI_SUCCESS)
+ throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ // each process writes the dataSet arrays
+ if (doubleValuesArrayH1.size() > 0)
+ MPI_File_write_at(file_handler, write_offset, &doubleValuesArrayH1[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
+
+ MPI_File_sync(file_handler);
+ MPI_File_close(&file_handler);
+ }
+
+ //--------------------------------------------------------------------------------------------------------------------
+ /*filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSetH2.bin";
+ rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
+ if (rc != MPI_SUCCESS)
+ throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ // each process writes the dataSet arrays
+ if (doubleValuesArrayH1.size() > 0)
+ MPI_File_write_at(file_handler, write_offset, &doubleValuesArrayH2[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
+
+ MPI_File_sync(file_handler);
+ MPI_File_close(&file_handler);*/
+
+ //--------------------------------
MPI_Type_free(&dataSetDoubleType);
delete[] dataSetArray;
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIORestartCoProcessor::writeDataSet time: " << finish - start << " s");
}
@@ -403,8 +445,11 @@ void MPIIORestartCoProcessor::writeDataSet(int step)
if (arrPresence.isRelaxationFactorPresent)
writeRelaxationFactor(step);
- if (arrPresence.isPhaseFieldPresent)
- writePhaseField(step);
+ if (arrPresence.isPhaseField1Present)
+ writePhaseField(step, 1);
+
+ if (arrPresence.isPhaseField2Present)
+ writePhaseField(step, 2);
}
void MPIIORestartCoProcessor::writeAverageDensityArray(int step)
@@ -418,7 +463,8 @@ void MPIIORestartCoProcessor::writeAverageDensityArray(int step)
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
int maxInitLevel = this->grid->getFinestInitializedLevel();
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
grid->getBlocks(level, rank, blocksVector[level]);
blocksCount += static_cast<int>(blocksVector[level].size());
}
@@ -427,10 +473,10 @@ void MPIIORestartCoProcessor::writeAverageDensityArray(int step)
std::vector<double> doubleValuesArray; // double-values of the AverageDensityArray in all blocks
dataSetParam dataSetParamStr;
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::writeAverageDensityArray start collect data rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
bool firstBlock = true;
@@ -438,13 +484,13 @@ void MPIIORestartCoProcessor::writeAverageDensityArray(int step)
int ic = 0;
SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> averageDensityArray;
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
{
- dataSetSmallArray[ic].x1 =
- block->getX1(); // coordinates of the block needed to find it while regenerating the grid
- dataSetSmallArray[ic].x2 = block->getX2();
- dataSetSmallArray[ic].x3 = block->getX3();
+ dataSetSmallArray[ic].x1 = block->getX1(); // coordinates of the block needed to find it while regenerating the grid
+ dataSetSmallArray[ic].x2 = block->getX2();
+ dataSetSmallArray[ic].x3 = block->getX3();
dataSetSmallArray[ic].level = block->getLevel();
averageDensityArray = block->getKernel()->getDataSet()->getAverageDensity();
@@ -462,10 +508,8 @@ void MPIIORestartCoProcessor::writeAverageDensityArray(int step)
firstBlock = false;
}
- if ((dataSetParamStr.nx[0] > 0) && (dataSetParamStr.nx[1] > 0) && (dataSetParamStr.nx[2] > 0) &&
- (dataSetParamStr.nx[3] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), averageDensityArray->getDataVector().begin(),
- averageDensityArray->getDataVector().end());
+ if ((dataSetParamStr.nx[0] > 0) && (dataSetParamStr.nx[1] > 0) && (dataSetParamStr.nx[2] > 0) && (dataSetParamStr.nx[3] > 0))
+ doubleValuesArray.insert(doubleValuesArray.end(), averageDensityArray->getDataVector().begin(), averageDensityArray->getDataVector().end());
ic++;
}
@@ -475,10 +519,10 @@ void MPIIORestartCoProcessor::writeAverageDensityArray(int step)
MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
MPI_Type_commit(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::writeAverageDensityArray start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
// write to the file
@@ -487,15 +531,17 @@ void MPIIORestartCoProcessor::writeAverageDensityArray(int step)
MPI_Offset write_offset = (MPI_Offset)(size * sizeof(int));
size_t next_write_offset = 0;
- if (size > 1) {
- if (rank == 0) {
- next_write_offset = write_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
- } else {
+ }
+ else
+ {
MPI_Recv(&write_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
- next_write_offset = write_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
if (rank < size - 1)
MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
}
@@ -524,19 +570,18 @@ void MPIIORestartCoProcessor::writeAverageDensityArray(int step)
// each process writes common parameters of a dataSet
MPI_File_write_at(file_handler, write_offset, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
// each process writes data identifying blocks
- MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount,
- dataSetSmallType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount, dataSetSmallType, MPI_STATUS_IGNORE);
// each process writes the dataSet arrays
if (doubleValuesArray.size() > 0)
- MPI_File_write_at(file_handler,
- (MPI_Offset)(write_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
&doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
MPI_File_sync(file_handler);
MPI_File_close(&file_handler);
MPI_Type_free(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIORestartCoProcessor::writeAverageDensityArray time: " << finish - start << " s");
}
@@ -555,7 +600,8 @@ void MPIIORestartCoProcessor::writeAverageVelocityArray(int step)
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
int maxInitLevel = this->grid->getFinestInitializedLevel();
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
grid->getBlocks(level, rank, blocksVector[level]);
blocksCount += static_cast<int>(blocksVector[level].size());
}
@@ -564,10 +610,10 @@ void MPIIORestartCoProcessor::writeAverageVelocityArray(int step)
std::vector<double> doubleValuesArray; // double-values (arrays of f's) in all blocks
dataSetParam dataSetParamStr;
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::writeAverageVelocityArray start collect data rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
bool firstBlock = true;
@@ -575,13 +621,13 @@ void MPIIORestartCoProcessor::writeAverageVelocityArray(int step)
int ic = 0;
SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> AverageVelocityArray3DPtr;
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
{
- dataSetSmallArray[ic].x1 =
- block->getX1(); // coordinates of the block needed to find it while regenerating the grid
- dataSetSmallArray[ic].x2 = block->getX2();
- dataSetSmallArray[ic].x3 = block->getX3();
+ dataSetSmallArray[ic].x1 = block->getX1(); // coordinates of the block needed to find it while regenerating the grid
+ dataSetSmallArray[ic].x2 = block->getX2();
+ dataSetSmallArray[ic].x3 = block->getX3();
dataSetSmallArray[ic].level = block->getLevel();
AverageVelocityArray3DPtr = block->getKernel()->getDataSet()->getAverageVelocity();
@@ -593,16 +639,13 @@ void MPIIORestartCoProcessor::writeAverageVelocityArray(int step)
dataSetParamStr.nx[1] = static_cast<int>(AverageVelocityArray3DPtr->getNX2());
dataSetParamStr.nx[2] = static_cast<int>(AverageVelocityArray3DPtr->getNX3());
dataSetParamStr.nx[3] = static_cast<int>(AverageVelocityArray3DPtr->getNX4());
- doubleCountInBlock =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
firstBlock = false;
}
- if ((dataSetParamStr.nx[0] > 0) && (dataSetParamStr.nx[1] > 0) && (dataSetParamStr.nx[2] > 0) &&
- (dataSetParamStr.nx[3] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), AverageVelocityArray3DPtr->getDataVector().begin(),
- AverageVelocityArray3DPtr->getDataVector().end());
+ if ((dataSetParamStr.nx[0] > 0) && (dataSetParamStr.nx[1] > 0) && (dataSetParamStr.nx[2] > 0) && (dataSetParamStr.nx[3] > 0))
+ doubleValuesArray.insert(doubleValuesArray.end(), AverageVelocityArray3DPtr->getDataVector().begin(), AverageVelocityArray3DPtr->getDataVector().end());
ic++;
}
@@ -612,10 +655,10 @@ void MPIIORestartCoProcessor::writeAverageVelocityArray(int step)
MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
MPI_Type_commit(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::writeAverageVelocityArray start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
// write to the file
@@ -624,15 +667,16 @@ void MPIIORestartCoProcessor::writeAverageVelocityArray(int step)
MPI_Offset write_offset = (MPI_Offset)(size * sizeof(int));
size_t next_write_offset = 0;
- if (size > 1) {
- if (rank == 0) {
- next_write_offset = write_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
- } else {
+ } else
+ {
MPI_Recv(&write_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
- next_write_offset = write_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
if (rank < size - 1)
MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
}
@@ -661,19 +705,18 @@ void MPIIORestartCoProcessor::writeAverageVelocityArray(int step)
// each process writes common parameters of a dataSet
MPI_File_write_at(file_handler, write_offset, &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
// each process writes data identifying blocks
- MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount,
- dataSetSmallType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount, dataSetSmallType, MPI_STATUS_IGNORE);
// each process writes the dataSet arrays
if (doubleValuesArray.size() > 0)
- MPI_File_write_at(file_handler,
- (MPI_Offset)(write_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
&doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
MPI_File_sync(file_handler);
MPI_File_close(&file_handler);
MPI_Type_free(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIORestartCoProcessor::writeAverageVelocityArray time: " << finish - start << " s");
}
@@ -692,7 +735,8 @@ void MPIIORestartCoProcessor::writeAverageFluktuationsArray(int step)
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
int maxInitLevel = this->grid->getFinestInitializedLevel();
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
grid->getBlocks(level, rank, blocksVector[level]);
blocksCount += static_cast<int>(blocksVector[level].size());
}
@@ -701,10 +745,10 @@ void MPIIORestartCoProcessor::writeAverageFluktuationsArray(int step)
std::vector<double> doubleValuesArray; // double-values (arrays of f's) in all blocks
dataSetParam dataSetParamStr;
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::writeAverageFluktuationsArray start collect data rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
bool firstBlock = true;
@@ -712,11 +756,11 @@ void MPIIORestartCoProcessor::writeAverageFluktuationsArray(int step)
int ic = 0;
SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> AverageFluctArray3DPtr;
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
{
- dataSetSmallArray[ic].x1 =
- block->getX1(); // coordinates of the block needed to find it while regenerating the grid
+ dataSetSmallArray[ic].x1 = block->getX1(); // coordinates of the block needed to find it while regenerating the grid
dataSetSmallArray[ic].x2 = block->getX2();
dataSetSmallArray[ic].x3 = block->getX3();
dataSetSmallArray[ic].level = block->getLevel();
@@ -730,14 +774,12 @@ void MPIIORestartCoProcessor::writeAverageFluktuationsArray(int step)
dataSetParamStr.nx[1] = static_cast<int>(AverageFluctArray3DPtr->getNX2());
dataSetParamStr.nx[2] = static_cast<int>(AverageFluctArray3DPtr->getNX3());
dataSetParamStr.nx[3] = static_cast<int>(AverageFluctArray3DPtr->getNX4());
- doubleCountInBlock =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
firstBlock = false;
}
- if ((dataSetParamStr.nx[0] > 0) && (dataSetParamStr.nx[1] > 0) && (dataSetParamStr.nx[2] > 0) &&
- (dataSetParamStr.nx[3] > 0))
+ if ((dataSetParamStr.nx[0] > 0) && (dataSetParamStr.nx[1] > 0) && (dataSetParamStr.nx[2] > 0) &&(dataSetParamStr.nx[3] > 0))
doubleValuesArray.insert(doubleValuesArray.end(), AverageFluctArray3DPtr->getDataVector().begin(),
AverageFluctArray3DPtr->getDataVector().end());
@@ -749,10 +791,10 @@ void MPIIORestartCoProcessor::writeAverageFluktuationsArray(int step)
MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
MPI_Type_commit(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::writeAverageFluktuationsArray start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
// write to the file
@@ -761,15 +803,16 @@ void MPIIORestartCoProcessor::writeAverageFluktuationsArray(int step)
MPI_Offset write_offset = (MPI_Offset)(size * sizeof(int));
size_t next_write_offset = 0;
- if (size > 1) {
- if (rank == 0) {
- next_write_offset = write_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
- } else {
+ } else
+ {
MPI_Recv(&write_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
- next_write_offset = write_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
if (rank < size - 1)
MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
}
@@ -788,8 +831,7 @@ void MPIIORestartCoProcessor::writeAverageFluktuationsArray(int step)
#endif
MPI_File file_handler;
- std::string filename =
- path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageFluktuationsArray.bin";
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageFluktuationsArray.bin";
int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
if (rc != MPI_SUCCESS)
throw UbException(UB_EXARGS, "couldn't open file " + filename);
@@ -803,15 +845,15 @@ void MPIIORestartCoProcessor::writeAverageFluktuationsArray(int step)
dataSetSmallType, MPI_STATUS_IGNORE);
// each process writes the dataSet arrays
if (doubleValuesArray.size() > 0)
- MPI_File_write_at(file_handler,
- (MPI_Offset)(write_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
&doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
MPI_File_sync(file_handler);
MPI_File_close(&file_handler);
MPI_Type_free(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIORestartCoProcessor::writeAverageFluktuationsArray time: " << finish - start << " s");
}
@@ -830,7 +872,8 @@ void MPIIORestartCoProcessor::writeAverageTripleArray(int step)
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
int maxInitLevel = this->grid->getFinestInitializedLevel();
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
grid->getBlocks(level, rank, blocksVector[level]);
blocksCount += static_cast<int>(blocksVector[level].size());
}
@@ -839,10 +882,10 @@ void MPIIORestartCoProcessor::writeAverageTripleArray(int step)
std::vector<double> doubleValuesArray; // double-values (arrays of f's) in all blocks
dataSetParam dataSetParamStr;
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::writeAverageTripleArray start collect data rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
bool firstBlock = true;
@@ -850,11 +893,11 @@ void MPIIORestartCoProcessor::writeAverageTripleArray(int step)
int ic = 0;
SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> AverageTripleArray3DPtr;
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
{
- dataSetSmallArray[ic].x1 =
- block->getX1(); // coordinates of the block needed to find it while regenerating the grid
+ dataSetSmallArray[ic].x1 = block->getX1(); // coordinates of the block needed to find it while regenerating the grid
dataSetSmallArray[ic].x2 = block->getX2();
dataSetSmallArray[ic].x3 = block->getX3();
dataSetSmallArray[ic].level = block->getLevel();
@@ -868,16 +911,13 @@ void MPIIORestartCoProcessor::writeAverageTripleArray(int step)
dataSetParamStr.nx[1] = static_cast<int>(AverageTripleArray3DPtr->getNX2());
dataSetParamStr.nx[2] = static_cast<int>(AverageTripleArray3DPtr->getNX3());
dataSetParamStr.nx[3] = static_cast<int>(AverageTripleArray3DPtr->getNX4());
- doubleCountInBlock =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
firstBlock = false;
}
- if ((dataSetParamStr.nx[0] > 0) && (dataSetParamStr.nx[1] > 0) && (dataSetParamStr.nx[2] > 0) &&
- (dataSetParamStr.nx[3] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), AverageTripleArray3DPtr->getDataVector().begin(),
- AverageTripleArray3DPtr->getDataVector().end());
+ if ((dataSetParamStr.nx[0] > 0) && (dataSetParamStr.nx[1] > 0) && (dataSetParamStr.nx[2] > 0) && (dataSetParamStr.nx[3] > 0))
+ doubleValuesArray.insert(doubleValuesArray.end(), AverageTripleArray3DPtr->getDataVector().begin(), AverageTripleArray3DPtr->getDataVector().end());
ic++;
}
@@ -887,10 +927,10 @@ void MPIIORestartCoProcessor::writeAverageTripleArray(int step)
MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
MPI_Type_commit(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::writeAverageTripleArray start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
// write to the file
@@ -899,15 +939,17 @@ void MPIIORestartCoProcessor::writeAverageTripleArray(int step)
MPI_Offset write_offset = (MPI_Offset)(size * sizeof(int));
size_t next_write_offset = 0;
- if (size > 1) {
- if (rank == 0) {
- next_write_offset = write_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
- } else {
+ }
+ else
+ {
MPI_Recv(&write_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
- next_write_offset = write_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
if (rank < size - 1)
MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
}
@@ -940,8 +982,7 @@ void MPIIORestartCoProcessor::writeAverageTripleArray(int step)
dataSetSmallType, MPI_STATUS_IGNORE);
// each process writes the dataSet arrays
if (doubleValuesArray.size() > 0)
- MPI_File_write_at(file_handler,
- (MPI_Offset)(write_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
&doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
MPI_File_sync(file_handler);
@@ -967,7 +1008,8 @@ void MPIIORestartCoProcessor::writeShearStressValArray(int step)
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
int maxInitLevel = this->grid->getFinestInitializedLevel();
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
grid->getBlocks(level, rank, blocksVector[level]);
blocksCount += static_cast<int>(blocksVector[level].size());
}
@@ -976,10 +1018,10 @@ void MPIIORestartCoProcessor::writeShearStressValArray(int step)
std::vector<double> doubleValuesArray; // double-values (arrays of f's) in all blocks
dataSetParam dataSetParamStr;
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::writeShearStressValArray start collect data rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
bool firstBlock = true;
@@ -987,11 +1029,11 @@ void MPIIORestartCoProcessor::writeShearStressValArray(int step)
int ic = 0;
SPtr<CbArray4D<LBMReal, IndexerX4X3X2X1>> ShearStressValArray3DPtr;
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
{
- dataSetSmallArray[ic].x1 =
- block->getX1(); // coordinates of the block needed to find it while regenerating the grid
+ dataSetSmallArray[ic].x1 = block->getX1(); // coordinates of the block needed to find it while regenerating the grid
dataSetSmallArray[ic].x2 = block->getX2();
dataSetSmallArray[ic].x3 = block->getX3();
dataSetSmallArray[ic].level = block->getLevel();
@@ -1005,14 +1047,12 @@ void MPIIORestartCoProcessor::writeShearStressValArray(int step)
dataSetParamStr.nx[1] = static_cast<int>(ShearStressValArray3DPtr->getNX2());
dataSetParamStr.nx[2] = static_cast<int>(ShearStressValArray3DPtr->getNX3());
dataSetParamStr.nx[3] = static_cast<int>(ShearStressValArray3DPtr->getNX4());
- doubleCountInBlock =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
firstBlock = false;
}
- if ((dataSetParamStr.nx[0] > 0) && (dataSetParamStr.nx[1] > 0) && (dataSetParamStr.nx[2] > 0) &&
- (dataSetParamStr.nx[3] > 0))
+ if ((dataSetParamStr.nx[0] > 0) && (dataSetParamStr.nx[1] > 0) && (dataSetParamStr.nx[2] > 0) && (dataSetParamStr.nx[3] > 0))
doubleValuesArray.insert(doubleValuesArray.end(), ShearStressValArray3DPtr->getDataVector().begin(),
ShearStressValArray3DPtr->getDataVector().end());
@@ -1024,10 +1064,10 @@ void MPIIORestartCoProcessor::writeShearStressValArray(int step)
MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
MPI_Type_commit(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::writeShearStressValArray start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
// write to the file
@@ -1036,15 +1076,17 @@ void MPIIORestartCoProcessor::writeShearStressValArray(int step)
MPI_Offset write_offset = (MPI_Offset)(size * sizeof(int));
size_t next_write_offset = 0;
- if (size > 1) {
- if (rank == 0) {
- next_write_offset = write_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
- } else {
+ }
+ else
+ {
MPI_Recv(&write_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
- next_write_offset = write_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
if (rank < size - 1)
MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
}
@@ -1077,15 +1119,15 @@ void MPIIORestartCoProcessor::writeShearStressValArray(int step)
dataSetSmallType, MPI_STATUS_IGNORE);
// each process writes the dataSet arrays
if (doubleValuesArray.size() > 0)
- MPI_File_write_at(file_handler,
- (MPI_Offset)(write_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
&doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
MPI_File_sync(file_handler);
MPI_File_close(&file_handler);
MPI_Type_free(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIORestartCoProcessor::writeShearStressValArray time: " << finish - start << " s");
}
@@ -1104,7 +1146,8 @@ void MPIIORestartCoProcessor::writeRelaxationFactor(int step)
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
int maxInitLevel = this->grid->getFinestInitializedLevel();
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
grid->getBlocks(level, rank, blocksVector[level]);
blocksCount += static_cast<int>(blocksVector[level].size());
}
@@ -1113,10 +1156,10 @@ void MPIIORestartCoProcessor::writeRelaxationFactor(int step)
std::vector<double> doubleValuesArray; // double-values (arrays of f's) in all blocks
dataSetParam dataSetParamStr;
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::writeRelaxationFactor start collect data rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
bool firstBlock = true;
@@ -1124,11 +1167,11 @@ void MPIIORestartCoProcessor::writeRelaxationFactor(int step)
int ic = 0;
SPtr<CbArray3D<LBMReal, IndexerX3X2X1>> RelaxationFactor3DPtr;
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
{
- dataSetSmallArray[ic].x1 =
- block->getX1(); // coordinates of the block needed to find it while regenerating the grid
+ dataSetSmallArray[ic].x1 = block->getX1(); // coordinates of the block needed to find it while regenerating the grid
dataSetSmallArray[ic].x2 = block->getX2();
dataSetSmallArray[ic].x3 = block->getX3();
dataSetSmallArray[ic].level = block->getLevel();
@@ -1142,8 +1185,7 @@ void MPIIORestartCoProcessor::writeRelaxationFactor(int step)
dataSetParamStr.nx[1] = static_cast<int>(RelaxationFactor3DPtr->getNX2());
dataSetParamStr.nx[2] = static_cast<int>(RelaxationFactor3DPtr->getNX3());
dataSetParamStr.nx[3] = 1;
- doubleCountInBlock =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
firstBlock = false;
}
@@ -1160,10 +1202,10 @@ void MPIIORestartCoProcessor::writeRelaxationFactor(int step)
MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
MPI_Type_commit(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::writeRelaxationFactor start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
// write to the file
@@ -1172,15 +1214,17 @@ void MPIIORestartCoProcessor::writeRelaxationFactor(int step)
MPI_Offset write_offset = (MPI_Offset)(size * sizeof(int));
size_t next_write_offset = 0;
- if (size > 1) {
- if (rank == 0) {
- next_write_offset = write_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
- } else {
+ }
+ else
+ {
MPI_Recv(&write_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
- next_write_offset = write_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
if (rank < size - 1)
MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
}
@@ -1213,15 +1257,15 @@ void MPIIORestartCoProcessor::writeRelaxationFactor(int step)
dataSetSmallType, MPI_STATUS_IGNORE);
// each process writes the dataSet arrays
if (doubleValuesArray.size() > 0)
- MPI_File_write_at(file_handler,
- (MPI_Offset)(write_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
&doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
MPI_File_sync(file_handler);
MPI_File_close(&file_handler);
MPI_Type_free(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIORestartCoProcessor::writeRelaxationFactor time: " << finish - start << " s");
}
@@ -1229,7 +1273,7 @@ void MPIIORestartCoProcessor::writeRelaxationFactor(int step)
delete[] dataSetSmallArray;
}
-void MPIIORestartCoProcessor::writePhaseField(int step)
+void MPIIORestartCoProcessor::writePhaseField(int step, int fieldN)
{
int rank, size;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
@@ -1240,7 +1284,8 @@ void MPIIORestartCoProcessor::writePhaseField(int step)
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
int maxInitLevel = this->grid->getFinestInitializedLevel();
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
grid->getBlocks(level, rank, blocksVector[level]);
blocksCount += static_cast<int>(blocksVector[level].size());
}
@@ -1249,10 +1294,10 @@ void MPIIORestartCoProcessor::writePhaseField(int step)
std::vector<double> doubleValuesArray; // double-values (arrays of f's) in all blocks
dataSetParam dataSetParamStr;
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::writePhaseField start collect data rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
bool firstBlock = true;
@@ -1260,7 +1305,8 @@ void MPIIORestartCoProcessor::writePhaseField(int step)
int ic = 0;
SPtr<CbArray3D<LBMReal, IndexerX3X2X1>> PhaseField3DPtr;
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
for (SPtr<Block3D> block : blocksVector[level]) // blocks of the current level
{
dataSetSmallArray[ic].x1 = block->getX1(); // coordinates of the block needed to find it while regenerating the grid
@@ -1268,7 +1314,10 @@ void MPIIORestartCoProcessor::writePhaseField(int step)
dataSetSmallArray[ic].x3 = block->getX3();
dataSetSmallArray[ic].level = block->getLevel();
- PhaseField3DPtr = block->getKernel()->getDataSet()->getPhaseField();
+ if(fieldN == 1)
+ PhaseField3DPtr = block->getKernel()->getDataSet()->getPhaseField();
+ else
+ PhaseField3DPtr = block->getKernel()->getDataSet()->getPhaseField2();
if (firstBlock) // when first (any) valid block...
{
@@ -1277,28 +1326,25 @@ void MPIIORestartCoProcessor::writePhaseField(int step)
dataSetParamStr.nx[1] = static_cast<int>(PhaseField3DPtr->getNX2());
dataSetParamStr.nx[2] = static_cast<int>(PhaseField3DPtr->getNX3());
dataSetParamStr.nx[3] = 1;
- doubleCountInBlock =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
firstBlock = false;
}
-
if ((dataSetParamStr.nx[0] > 0) && (dataSetParamStr.nx[1] > 0) && (dataSetParamStr.nx[2] > 0))
- doubleValuesArray.insert(doubleValuesArray.end(), PhaseField3DPtr->getDataVector().begin(),
- PhaseField3DPtr->getDataVector().end());
+ doubleValuesArray.insert(doubleValuesArray.end(), PhaseField3DPtr->getDataVector().begin(), PhaseField3DPtr->getDataVector().end());
ic++;
}
}
-
+
// register new MPI-types depending on the block-specific information
MPI_Type_contiguous(doubleCountInBlock, MPI_DOUBLE, &dataSetDoubleType);
MPI_Type_commit(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::writePhaseField start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
// write to the file
@@ -1307,15 +1353,17 @@ void MPIIORestartCoProcessor::writePhaseField(int step)
MPI_Offset write_offset = (MPI_Offset)(size * sizeof(int));
size_t next_write_offset = 0;
- if (size > 1) {
- if (rank == 0) {
- next_write_offset = write_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
- } else {
+ }
+ else
+ {
MPI_Recv(&write_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
- next_write_offset = write_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ next_write_offset = write_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
if (rank < size - 1)
MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
}
@@ -1334,7 +1382,9 @@ void MPIIORestartCoProcessor::writePhaseField(int step)
#endif
MPI_File file_handler;
- std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpPhaseField.bin";
+ std::string filename;
+ if(fieldN == 1) filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpPhaseField1.bin";
+ else filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpPhaseField2.bin";
int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
if (rc != MPI_SUCCESS)
throw UbException(UB_EXARGS, "couldn't open file " + filename);
@@ -1348,15 +1398,15 @@ void MPIIORestartCoProcessor::writePhaseField(int step)
dataSetSmallType, MPI_STATUS_IGNORE);
// each process writes the dataSet arrays
if (doubleValuesArray.size() > 0)
- MPI_File_write_at(file_handler,
- (MPI_Offset)(write_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
&doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
MPI_File_sync(file_handler);
MPI_File_close(&file_handler);
MPI_Type_free(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIORestartCoProcessor::writePhaseField time: " << finish - start << " s");
}
@@ -1370,10 +1420,10 @@ void MPIIORestartCoProcessor::writeBoundaryConds(int step)
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::writeBoundaryConds start collect data rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
int blocksCount = 0; // quantity of blocks in the grid, max 2147483648 blocks!
@@ -1384,7 +1434,8 @@ void MPIIORestartCoProcessor::writeBoundaryConds(int step)
std::vector<SPtr<Block3D>> blocksVector[25];
int minInitLevel = this->grid->getCoarsestInitializedLevel();
int maxInitLevel = this->grid->getFinestInitializedLevel();
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
grid->getBlocks(level, rank, blocksVector[level]);
blocksCount += static_cast<int>(blocksVector[level].size());
}
@@ -1394,27 +1445,31 @@ void MPIIORestartCoProcessor::writeBoundaryConds(int step)
std::vector<int> bcindexmatrixV;
std::vector<int> indexContainerV;
bool bcindexmatrixCountNotInit = true;
- int ic = 0;
+ int ic = 0;
SPtr<BCArray3D> bcArr;
- for (int level = minInitLevel; level <= maxInitLevel; level++) {
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
for (SPtr<Block3D> block : blocksVector[level]) // all the blocks of the current level
{
bcArr = block->getKernel()->getBCProcessor()->getBCArray();
- bcAddArray[ic].x1 =
- block->getX1(); // coordinates of the block needed to find it while regenerating the grid
+ bcAddArray[ic].x1 = block->getX1(); // coordinates of the block needed to find it while regenerating the grid
bcAddArray[ic].x2 = block->getX2();
bcAddArray[ic].x3 = block->getX3();
bcAddArray[ic].level = block->getLevel();
bcAddArray[ic].boundCond_count = 0; // how many BoundaryConditions in this block
bcAddArray[ic].indexContainer_count = 0; // how many indexContainer-values in this block
- for (std::size_t bc = 0; bc < bcArr->getBCVectorSize(); bc++) {
+ for (std::size_t bc = 0; bc < bcArr->getBCVectorSize(); bc++)
+ {
BoundaryCondition *bouCond = new BoundaryCondition();
- if (bcArr->bcvector[bc] == NULL) {
+ if (bcArr->bcvector[bc] == NULL)
+ {
memset(bouCond, 0, sizeof(BoundaryCondition));
- } else {
+ }
+ else
+ {
bouCond->noslipBoundaryFlags = bcArr->bcvector[bc]->getNoSlipBoundary();
bouCond->slipBoundaryFlags = bcArr->bcvector[bc]->getSlipBoundary();
bouCond->velocityBoundaryFlags = bcArr->bcvector[bc]->getVelocityBoundary();
@@ -1445,15 +1500,15 @@ void MPIIORestartCoProcessor::writeBoundaryConds(int step)
// the quantity of elements in the bcindexmatrix array (CbArray3D<int, IndexerX3X2X1>) in bcArray(BCArray3D)
// is always equal, this will be the size of the "write-read-block" in MPI_write_.../MPI_read-functions when
// writing/reading BoundConds
- if (bcindexmatrixCountNotInit) {
+ if (bcindexmatrixCountNotInit)
+ {
boundCondParamStr.nx1 = static_cast<int>(bcArr->bcindexmatrix.getNX1());
boundCondParamStr.nx2 = static_cast<int>(bcArr->bcindexmatrix.getNX2());
boundCondParamStr.nx3 = static_cast<int>(bcArr->bcindexmatrix.getNX3());
boundCondParamStr.bcindexmatrixCount = static_cast<int>(bcArr->bcindexmatrix.getDataVector().size());
bcindexmatrixCountNotInit = false;
}
- bcindexmatrixV.insert(bcindexmatrixV.end(), bcArr->bcindexmatrix.getDataVector().begin(),
- bcArr->bcindexmatrix.getDataVector().end());
+ bcindexmatrixV.insert(bcindexmatrixV.end(), bcArr->bcindexmatrix.getDataVector().begin(), bcArr->bcindexmatrix.getDataVector().end());
indexContainerV.insert(indexContainerV.end(), bcArr->indexContainer.begin(), bcArr->indexContainer.end());
bcAddArray[ic].indexContainer_count = static_cast<int>(bcArr->indexContainer.size());
@@ -1470,7 +1525,8 @@ void MPIIORestartCoProcessor::writeBoundaryConds(int step)
int bcBlockCount = (int)(count_boundCond / BLOCK_SIZE);
if (bcBlockCount * BLOCK_SIZE < (int)count_boundCond)
bcBlockCount += 1;
- for (int i = (int)count_boundCond; i < bcBlockCount * BLOCK_SIZE; i++) {
+ for (int i = (int)count_boundCond; i < bcBlockCount * BLOCK_SIZE; i++)
+ {
BoundaryCondition *bouCond = new BoundaryCondition();
memset(bouCond, 0, sizeof(BoundaryCondition));
bcVector.push_back(*bouCond);
@@ -1485,11 +1541,15 @@ void MPIIORestartCoProcessor::writeBoundaryConds(int step)
MPI_Offset write_offset = (MPI_Offset)(size * (3 * sizeof(int) + sizeof(boundCondParam)));
size_t next_write_offset = 0;
- if (size > 1) {
- if (rank == 0) {
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
next_write_offset = write_offset + byteCount;
MPI_Send(&next_write_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
- } else {
+ }
+ else
+ {
MPI_Recv(&write_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
next_write_offset = write_offset + byteCount;
if (rank < size - 1)
@@ -1497,10 +1557,10 @@ void MPIIORestartCoProcessor::writeBoundaryConds(int step)
}
}
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::writeBoundaryConds start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
@@ -1526,14 +1586,11 @@ void MPIIORestartCoProcessor::writeBoundaryConds(int step)
// each process writes the quantity of it's blocks
MPI_File_write_at(file_handler, write_offset1, &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
// each process writes the quantity of "big blocks" of BLOCK_SIZE of boundary conditions
- MPI_File_write_at(file_handler, (MPI_Offset)(write_offset1 + sizeof(int)), &bcBlockCount, 1, MPI_INT,
- MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset1 + sizeof(int)), &bcBlockCount, 1, MPI_INT, MPI_STATUS_IGNORE);
// each process writes the quantity of indexContainer elements in all blocks
- MPI_File_write_at(file_handler, (MPI_Offset)(write_offset1 + 2 * sizeof(int)), &count_indexContainer, 1, MPI_INT,
- MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset1 + 2 * sizeof(int)), &count_indexContainer, 1, MPI_INT, MPI_STATUS_IGNORE);
// each process writes the quantity of bcindexmatrix elements in every block
- MPI_File_write_at(file_handler, (MPI_Offset)(write_offset1 + 3 * sizeof(int)), &boundCondParamStr, 1,
- boundCondParamType, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset1 + 3 * sizeof(int)), &boundCondParamStr, 1, boundCondParamType, MPI_STATUS_IGNORE);
// each process writes data identifying the blocks
MPI_File_write_at(file_handler, write_offset, bcAddArray, blocksCount, boundCondTypeAdd, MPI_STATUS_IGNORE);
@@ -1543,23 +1600,19 @@ void MPIIORestartCoProcessor::writeBoundaryConds(int step)
bcBlockCount, boundCondType1000, MPI_STATUS_IGNORE);
// each process writes bcindexmatrix values
if (bcindexmatrixV.size() > 0)
- MPI_File_write_at(file_handler,
- (MPI_Offset)(write_offset + blocksCount * sizeof(BCAddRestart) +
- bcBlockCount * BLOCK_SIZE * sizeof(BoundaryCondition)),
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + blocksCount * sizeof(BCAddRestart) + bcBlockCount * BLOCK_SIZE * sizeof(BoundaryCondition)),
&bcindexmatrixV[0], blocksCount, bcindexmatrixType, MPI_STATUS_IGNORE);
// each process writes indexContainer values
if (indexContainerV.size() > 0)
- MPI_File_write_at(file_handler,
- (MPI_Offset)(write_offset + blocksCount * sizeof(BCAddRestart) +
- bcBlockCount * BLOCK_SIZE * sizeof(BoundaryCondition) +
- blocksCount * boundCondParamStr.bcindexmatrixCount * sizeof(int)),
- &indexContainerV[0], count_indexContainer, MPI_INT, MPI_STATUS_IGNORE);
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + blocksCount * sizeof(BCAddRestart) + bcBlockCount * BLOCK_SIZE * sizeof(BoundaryCondition) +
+ blocksCount * boundCondParamStr.bcindexmatrixCount * sizeof(int)), &indexContainerV[0], count_indexContainer, MPI_INT, MPI_STATUS_IGNORE);
MPI_File_sync(file_handler);
MPI_File_close(&file_handler);
MPI_Type_free(&bcindexmatrixType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIORestartCoProcessor::writeBoundaryConds time: " << finish - start << " s");
}
@@ -1593,17 +1646,18 @@ void MPIIORestartCoProcessor::readDataSet(int step)
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::readDataSet start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
+
double start, finish;
if (comm->isRoot())
start = MPI_Wtime();
MPI_File file_handler;
- std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSet.bin";
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSetF.bin";
int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
if (rc != MPI_SUCCESS)
throw UbException(UB_EXARGS, "couldn't open file " + filename);
@@ -1611,6 +1665,7 @@ void MPIIORestartCoProcessor::readDataSet(int step)
// calculate the read offset
MPI_Offset read_offset = (MPI_Offset)(size * sizeof(int));
size_t next_read_offset = 0;
+ bool multiPhase = false;
// read count of blocks
int blocksCount = 0;
@@ -1618,122 +1673,146 @@ void MPIIORestartCoProcessor::readDataSet(int step)
MPI_File_read_at(file_handler, (MPI_Offset)(rank * sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
MPI_File_read_at(file_handler, read_offset, &dataSetParamStr1, 1, dataSetParamType, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam)), &dataSetParamStr2, 1,
- dataSetParamType, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + 2 * sizeof(dataSetParam)), &dataSetParamStr3, 1,
- dataSetParamType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam)), &dataSetParamStr2, 1, dataSetParamType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + 2 * sizeof(dataSetParam)), &dataSetParamStr3, 1, dataSetParamType, MPI_STATUS_IGNORE);
DataSetRestart *dataSetArray = new DataSetRestart[blocksCount];
- double doubleCountInBlock =
- (dataSetParamStr1.nx[0] * dataSetParamStr1.nx[1] * dataSetParamStr1.nx[2] * dataSetParamStr1.nx[3] +
+ double doubleCountInBlock = dataSetParamStr1.nx[0] * dataSetParamStr1.nx[1] * dataSetParamStr1.nx[2] * dataSetParamStr1.nx[3] +
dataSetParamStr2.nx[0] * dataSetParamStr2.nx[1] * dataSetParamStr2.nx[2] * dataSetParamStr2.nx[3] +
- dataSetParamStr3.nx[0] * dataSetParamStr3.nx[1] * dataSetParamStr3.nx[2] * dataSetParamStr3.nx[3]) * 2;
- std::vector<double> doubleValuesArray(size_t(blocksCount * doubleCountInBlock)); // double-values in all blocks
+ dataSetParamStr3.nx[0] * dataSetParamStr3.nx[1] * dataSetParamStr3.nx[2] * dataSetParamStr3.nx[3];
+ std::vector<double> doubleValuesArrayF(size_t(blocksCount * doubleCountInBlock)); // double-values in all blocks Fdistributions
+ std::vector<double> doubleValuesArrayH1; // double-values in all blocks H1distributions
+ //std::vector<double> doubleValuesArrayH2; // double-values in all blocks H2distributions
// define MPI_types depending on the block-specific information
MPI_Type_contiguous(int(doubleCountInBlock), MPI_DOUBLE, &dataSetDoubleType);
MPI_Type_commit(&dataSetDoubleType);
- if (size > 1) {
- if (rank == 0) {
- next_read_offset = read_offset + 3 * sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetRestart) + size_t(doubleCountInBlock) * sizeof(double));
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_read_offset = read_offset + 3 * sizeof(dataSetParam) + blocksCount * (sizeof(DataSetRestart) + size_t(doubleCountInBlock) * sizeof(double));
MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
- } else {
+ }
+ else
+ {
MPI_Recv(&read_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
- next_read_offset = read_offset + 3 * sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetRestart) + size_t(doubleCountInBlock) * sizeof(double));
+ next_read_offset = read_offset + 3 * sizeof(dataSetParam) + blocksCount * (sizeof(DataSetRestart) + size_t(doubleCountInBlock) * sizeof(double));
if (rank < size - 1)
MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
}
}
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + 3 * sizeof(dataSetParam)), dataSetArray, blocksCount,
- dataSetType, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler,
- (MPI_Offset)(read_offset + 3 * sizeof(dataSetParam) + blocksCount * sizeof(DataSetRestart)),
- &doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + 3 * sizeof(dataSetParam)), dataSetArray, blocksCount, dataSetType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + 3 * sizeof(dataSetParam) + blocksCount * sizeof(DataSetRestart)),
+ &doubleValuesArrayF[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
+ MPI_File_close(&file_handler);
+
+ //-------------------------------------- H1 -----------------------------
+ MPI_Offset fsize;
+ filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSetH1.bin";
+ rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
+ if (rc != MPI_SUCCESS)
+ throw UbException(UB_EXARGS, "couldn't open file " + filename);
+ int fs = MPI_File_get_size(file_handler, &fsize);
+ if (fsize > 0)
+ {
+ multiPhase = true;
+ doubleValuesArrayH1.resize(blocksCount * doubleCountInBlock);
+ MPI_File_read_at(file_handler, read_offset, &doubleValuesArrayH1[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
+ }
MPI_File_close(&file_handler);
+
+ //-------------------------------------- H2 -----------------------------
+ /*filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSetH2.bin";
+ rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
+ if (rc != MPI_SUCCESS)
+ throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ doubleValuesArrayH2.resize(blocksCount * doubleCountInBlock);
+ MPI_File_read_at(file_handler, read_offset, &doubleValuesArrayH2[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
+ MPI_File_close(&file_handler);*/
+
MPI_Type_free(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIORestartCoProcessor::readDataSet time: " << finish - start << " s");
UBLOG(logINFO, "MPIIORestartCoProcessor::readDataSet start of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
-
+
size_t index = 0;
std::vector<double> vectorsOfValuesF1, vectorsOfValuesF2, vectorsOfValuesF3;
- std::vector<double> vectorsOfValuesH1, vectorsOfValuesH2, vectorsOfValuesH3;
- size_t vectorSize1 =
- dataSetParamStr1.nx[0] * dataSetParamStr1.nx[1] * dataSetParamStr1.nx[2] * dataSetParamStr1.nx[3];
- size_t vectorSize2 =
- dataSetParamStr2.nx[0] * dataSetParamStr2.nx[1] * dataSetParamStr2.nx[2] * dataSetParamStr2.nx[3];
- size_t vectorSize3 =
- dataSetParamStr3.nx[0] * dataSetParamStr3.nx[1] * dataSetParamStr3.nx[2] * dataSetParamStr3.nx[3];
-
- for (int n = 0; n < blocksCount; n++) {
- vectorsOfValuesF1.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + vectorSize1);
- index += vectorSize1;
-
- vectorsOfValuesF2.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + vectorSize2);
- index += vectorSize2;
-
- vectorsOfValuesF3.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + vectorSize3);
- index += vectorSize3;
-
- vectorsOfValuesH1.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + vectorSize1);
+ std::vector<double> vectorsOfValuesH11, vectorsOfValuesH12, vectorsOfValuesH13;
+ //std::vector<double> vectorsOfValuesH21, vectorsOfValuesH22, vectorsOfValuesH23;
+ size_t vectorSize1 = dataSetParamStr1.nx[0] * dataSetParamStr1.nx[1] * dataSetParamStr1.nx[2] * dataSetParamStr1.nx[3];
+ size_t vectorSize2 = dataSetParamStr2.nx[0] * dataSetParamStr2.nx[1] * dataSetParamStr2.nx[2] * dataSetParamStr2.nx[3];
+ size_t vectorSize3 = dataSetParamStr3.nx[0] * dataSetParamStr3.nx[1] * dataSetParamStr3.nx[2] * dataSetParamStr3.nx[3];
+
+ for (int n = 0; n < blocksCount; n++)
+ {
+ vectorsOfValuesF1.assign(doubleValuesArrayF.data() + index, doubleValuesArrayF.data() + index + vectorSize1);
+ if (multiPhase)
+ vectorsOfValuesH11.assign(doubleValuesArrayH1.data() + index, doubleValuesArrayH1.data() + index + vectorSize1);
+ //vectorsOfValuesH21.assign(doubleValuesArrayH2.data() + index, doubleValuesArrayH2.data() + index + vectorSize1);
index += vectorSize1;
- vectorsOfValuesH2.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + vectorSize2);
+ vectorsOfValuesF2.assign(doubleValuesArrayF.data() + index, doubleValuesArrayF.data() + index + vectorSize2);
+ if (multiPhase)
+ vectorsOfValuesH12.assign(doubleValuesArrayH1.data() + index, doubleValuesArrayH1.data() + index + vectorSize2);
+ //vectorsOfValuesH22.assign(doubleValuesArrayH2.data() + index, doubleValuesArrayH2.data() + index + vectorSize2);
index += vectorSize2;
- vectorsOfValuesH3.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + vectorSize3);
+ vectorsOfValuesF3.assign(doubleValuesArrayF.data() + index, doubleValuesArrayF.data() + index + vectorSize3);
+ if (multiPhase)
+ vectorsOfValuesH13.assign(doubleValuesArrayH1.data() + index, doubleValuesArrayH1.data() + index + vectorSize3);
+ //vectorsOfValuesH23.assign(doubleValuesArrayH2.data() + index, doubleValuesArrayH2.data() + index + vectorSize3);
index += vectorSize3;
SPtr<DistributionArray3D> mFdistributions(new D3Q27EsoTwist3DSplittedVector());
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)
- ->setLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
- new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesF1, dataSetParamStr1.nx[0],
- dataSetParamStr1.nx[1], dataSetParamStr1.nx[2],
- dataSetParamStr1.nx[3])));
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)
- ->setNonLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
- new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesF2, dataSetParamStr2.nx[0],
- dataSetParamStr2.nx[1], dataSetParamStr2.nx[2],
- dataSetParamStr2.nx[3])));
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)
- ->setZeroDistributions(CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
+ new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesF1, dataSetParamStr1.nx[0], dataSetParamStr1.nx[1], dataSetParamStr1.nx[2], dataSetParamStr1.nx[3])));
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNonLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
+ new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesF2, dataSetParamStr2.nx[0], dataSetParamStr2.nx[1], dataSetParamStr2.nx[2], dataSetParamStr2.nx[3])));
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setZeroDistributions(CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(
vectorsOfValuesF3, dataSetParamStr3.nx[0], dataSetParamStr3.nx[1], dataSetParamStr3.nx[2])));
dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNX1(dataSetParamStr1.nx1);
dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNX2(dataSetParamStr1.nx2);
dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mFdistributions)->setNX3(dataSetParamStr1.nx3);
- SPtr<DistributionArray3D> mHdistributions(new D3Q27EsoTwist3DSplittedVector());
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mHdistributions)
- ->setLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
- new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesH1, dataSetParamStr1.nx[0],
- dataSetParamStr1.nx[1], dataSetParamStr1.nx[2],
- dataSetParamStr1.nx[3])));
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mHdistributions)
- ->setNonLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
- new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesH2, dataSetParamStr2.nx[0],
- dataSetParamStr2.nx[1], dataSetParamStr2.nx[2],
- dataSetParamStr2.nx[3])));
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mHdistributions)
- ->setZeroDistributions(CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(
- vectorsOfValuesH3, dataSetParamStr3.nx[0], dataSetParamStr3.nx[1], dataSetParamStr3.nx[2])));
-
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mHdistributions)->setNX1(dataSetParamStr1.nx1);
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mHdistributions)->setNX2(dataSetParamStr1.nx2);
- dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mHdistributions)->setNX3(dataSetParamStr1.nx3);
-
+ SPtr<DistributionArray3D> mH1distributions(new D3Q27EsoTwist3DSplittedVector());
+ if (multiPhase)
+ {
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH1distributions)->setLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
+ new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesH11, dataSetParamStr1.nx[0], dataSetParamStr1.nx[1], dataSetParamStr1.nx[2], dataSetParamStr1.nx[3])));
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH1distributions)->setNonLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
+ new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesH12, dataSetParamStr2.nx[0], dataSetParamStr2.nx[1], dataSetParamStr2.nx[2], dataSetParamStr2.nx[3])));
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH1distributions)->setZeroDistributions(CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(
+ vectorsOfValuesH13, dataSetParamStr3.nx[0], dataSetParamStr3.nx[1], dataSetParamStr3.nx[2])));
+
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH1distributions)->setNX1(dataSetParamStr1.nx1);
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH1distributions)->setNX2(dataSetParamStr1.nx2);
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH1distributions)->setNX3(dataSetParamStr1.nx3);
+ }
+ /*SPtr<DistributionArray3D> mH2distributions(new D3Q27EsoTwist3DSplittedVector());
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH2distributions)->setLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
+ new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesH21, dataSetParamStr1.nx[0], dataSetParamStr1.nx[1], dataSetParamStr1.nx[2], dataSetParamStr1.nx[3])));
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH2distributions)->setNonLocalDistributions(CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
+ new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValuesH22, dataSetParamStr2.nx[0], dataSetParamStr2.nx[1], dataSetParamStr2.nx[2], dataSetParamStr2.nx[3])));
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH2distributions)->setZeroDistributions(CbArray3D<LBMReal, IndexerX3X2X1>::CbArray3DPtr(new CbArray3D<LBMReal, IndexerX3X2X1>(
+ vectorsOfValuesH23, dataSetParamStr3.nx[0], dataSetParamStr3.nx[1], dataSetParamStr3.nx[2])));
+
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH2distributions)->setNX1(dataSetParamStr1.nx1);
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH2distributions)->setNX2(dataSetParamStr1.nx2);
+ dynamicPointerCast<D3Q27EsoTwist3DSplittedVector>(mH2distributions)->setNX3(dataSetParamStr1.nx3);*/
+
// find the nesessary block and fill it
- SPtr<Block3D> block =
- grid->getBlock(dataSetArray[n].x1, dataSetArray[n].x2, dataSetArray[n].x3, dataSetArray[n].level);
+ SPtr<Block3D> block = grid->getBlock(dataSetArray[n].x1, dataSetArray[n].x2, dataSetArray[n].x3, dataSetArray[n].level);
+
this->lbmKernel->setBlock(block);
SPtr<LBMKernel> kernel = this->lbmKernel->clone();
kernel->setGhostLayerWidth(dataSetArray[n].ghostLayerWidth);
@@ -1741,17 +1820,22 @@ void MPIIORestartCoProcessor::readDataSet(int step)
kernel->setDeltaT(dataSetArray[n].deltaT);
kernel->setCompressible(dataSetArray[n].compressible);
kernel->setWithForcing(dataSetArray[n].withForcing);
+ kernel->setCollisionFactorMultiphase(dataSetArray[n].collFactorL, dataSetArray[n].collFactorG);
+ kernel->setDensityRatio(dataSetArray[n].densityRatio);
+
SPtr<DataSet3D> dataSetPtr = SPtr<DataSet3D>(new DataSet3D());
dataSetPtr->setFdistributions(mFdistributions);
- dataSetPtr->setHdistributions(mHdistributions);
+ if (multiPhase)
+ dataSetPtr->setHdistributions(mH1distributions);
+ //dataSetPtr->setH2distributions(mH2distributions);
kernel->setDataSet(dataSetPtr);
block->setKernel(kernel);
}
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::readDataSet end of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
delete[] dataSetArray;
@@ -1785,8 +1869,11 @@ void MPIIORestartCoProcessor::readDataSet(int step)
if (arrPresence.isRelaxationFactorPresent)
readRelaxationFactor(step);
- if (arrPresence.isPhaseFieldPresent)
- readPhaseField(step);
+ if (arrPresence.isPhaseField1Present)
+ readPhaseField(step, 1);
+
+ if (arrPresence.isPhaseField2Present)
+ readPhaseField(step, 2);
}
void MPIIORestartCoProcessor::readAverageDensityArray(int step)
@@ -1795,10 +1882,10 @@ void MPIIORestartCoProcessor::readAverageDensityArray(int step)
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageDensityArray start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
if (comm->isRoot())
@@ -1816,12 +1903,10 @@ void MPIIORestartCoProcessor::readAverageDensityArray(int step)
memset(&dataSetParamStr, 0, sizeof(dataSetParam));
MPI_File_read_at(file_handler, (MPI_Offset)(rank * sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, (MPI_Offset)(size * sizeof(int)), &dataSetParamStr, 1, dataSetParamType,
- MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(size * sizeof(int)), &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
DataSetSmallRestart *dataSetSmallArray = new DataSetSmallRestart[blocksCount];
- int doubleCountInBlock =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ int doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
std::vector<double> doubleValuesArray(blocksCount * doubleCountInBlock); // double-values in all blocks
// define MPI_types depending on the block-specific information
@@ -1832,61 +1917,59 @@ void MPIIORestartCoProcessor::readAverageDensityArray(int step)
MPI_Offset read_offset = (MPI_Offset)(size * sizeof(int));
size_t next_read_offset = 0;
- if (size > 1) {
- if (rank == 0) {
- next_read_offset = read_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
- } else {
+ }
+ else
+ {
MPI_Recv(&read_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
- next_read_offset = read_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
if (rank < size - 1)
MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
}
}
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount,
- dataSetSmallType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount, dataSetSmallType, MPI_STATUS_IGNORE);
if (doubleCountInBlock > 0)
- MPI_File_read_at(file_handler,
- (MPI_Offset)(read_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
&doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
MPI_File_close(&file_handler);
MPI_Type_free(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageDensityArray time: " << finish - start << " s");
UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageDensityArray start of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
size_t index = 0;
- size_t nextVectorSize =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ size_t nextVectorSize = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
std::vector<double> vectorsOfValues;
- for (int n = 0; n < blocksCount; n++) {
+ for (int n = 0; n < blocksCount; n++)
+ {
vectorsOfValues.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + nextVectorSize);
index += nextVectorSize;
// fill mAverageDensity arrays
SPtr<AverageValuesArray3D> mAverageDensity;
- mAverageDensity = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
- new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1],
- dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
+ mAverageDensity = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues,
+ dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
// find the nesessary block and fill it
- SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].x1, dataSetSmallArray[n].x2, dataSetSmallArray[n].x3,
- dataSetSmallArray[n].level);
+ SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].x1, dataSetSmallArray[n].x2, dataSetSmallArray[n].x3, dataSetSmallArray[n].level);
block->getKernel()->getDataSet()->setAverageDensity(mAverageDensity);
}
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageDensityArray end of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
delete[] dataSetSmallArray;
@@ -1898,10 +1981,10 @@ void MPIIORestartCoProcessor::readAverageVelocityArray(int step)
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageVelocityArray start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
if (comm->isRoot())
@@ -1917,12 +2000,10 @@ void MPIIORestartCoProcessor::readAverageVelocityArray(int step)
int blocksCount = 0;
dataSetParam dataSetParamStr;
MPI_File_read_at(file_handler, (MPI_Offset)(rank * sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, (MPI_Offset)(size * sizeof(int)), &dataSetParamStr, 1, dataSetParamType,
- MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(size * sizeof(int)), &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
DataSetSmallRestart *dataSetSmallArray = new DataSetSmallRestart[blocksCount];
- int doubleCountInBlock =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ int doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
std::vector<double> doubleValuesArray(blocksCount * doubleCountInBlock); // double-values in all blocks
// define MPI_types depending on the block-specific information
@@ -1933,61 +2014,59 @@ void MPIIORestartCoProcessor::readAverageVelocityArray(int step)
MPI_Offset read_offset = (MPI_Offset)(size * sizeof(int));
size_t next_read_offset = 0;
- if (size > 1) {
- if (rank == 0) {
- next_read_offset = read_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
- } else {
+ }
+ else
+ {
MPI_Recv(&read_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
- next_read_offset = read_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
if (rank < size - 1)
MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
}
}
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount,
- dataSetSmallType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount, dataSetSmallType, MPI_STATUS_IGNORE);
if (doubleCountInBlock > 0)
- MPI_File_read_at(file_handler,
- (MPI_Offset)(read_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
&doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
MPI_File_close(&file_handler);
MPI_Type_free(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageVelocityArray time: " << finish - start << " s");
UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageVelocityArray start of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
size_t index = 0;
- size_t nextVectorSize =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ size_t nextVectorSize = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
std::vector<double> vectorsOfValues;
- for (int n = 0; n < blocksCount; n++) {
+ for (int n = 0; n < blocksCount; n++)
+ {
vectorsOfValues.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + nextVectorSize);
index += nextVectorSize;
// fill mAverageVelocity array
SPtr<AverageValuesArray3D> mAverageVelocity;
- mAverageVelocity = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
- new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1],
- dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
+ mAverageVelocity = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues, dataSetParamStr.nx[0],
+ dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
// find the nesessary block and fill it
- SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].x1, dataSetSmallArray[n].x2, dataSetSmallArray[n].x3,
- dataSetSmallArray[n].level);
+ SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].x1, dataSetSmallArray[n].x2, dataSetSmallArray[n].x3, dataSetSmallArray[n].level);
block->getKernel()->getDataSet()->setAverageVelocity(mAverageVelocity);
}
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageVelocityArray end of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
delete[] dataSetSmallArray;
@@ -1999,10 +2078,10 @@ void MPIIORestartCoProcessor::readAverageFluktuationsArray(int step)
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageFluktuationsArray start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
if (comm->isRoot())
@@ -2019,12 +2098,10 @@ void MPIIORestartCoProcessor::readAverageFluktuationsArray(int step)
int blocksCount = 0;
dataSetParam dataSetParamStr;
MPI_File_read_at(file_handler, (MPI_Offset)(rank * sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, (MPI_Offset)(size * sizeof(int)), &dataSetParamStr, 1, dataSetParamType,
- MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(size * sizeof(int)), &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
DataSetSmallRestart *dataSetSmallArray = new DataSetSmallRestart[blocksCount];
- int doubleCountInBlock =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ int doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
std::vector<double> doubleValuesArray(blocksCount * doubleCountInBlock); // double-values in all blocks
// define MPI_types depending on the block-specific information
@@ -2035,62 +2112,59 @@ void MPIIORestartCoProcessor::readAverageFluktuationsArray(int step)
MPI_Offset read_offset = (MPI_Offset)(size * sizeof(int));
size_t next_read_offset = 0;
- if (size > 1) {
- if (rank == 0) {
- next_read_offset = read_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
- } else {
+ }
+ else
+ {
MPI_Recv(&read_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
- next_read_offset = read_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
if (rank < size - 1)
MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
}
}
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount,
- dataSetSmallType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount, dataSetSmallType, MPI_STATUS_IGNORE);
if (doubleCountInBlock > 0)
- MPI_File_read_at(file_handler,
- (MPI_Offset)(read_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
&doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
MPI_File_close(&file_handler);
MPI_Type_free(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageFluktuationsArray time: " << finish - start << " s");
- UBLOG(logINFO,
- "MPIIORestartCoProcessor::readAverageFluktuationsArray start of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageFluktuationsArray start of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
size_t index = 0;
- size_t nextVectorSize =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ size_t nextVectorSize = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
std::vector<double> vectorsOfValues;
- for (int n = 0; n < blocksCount; n++) {
+ for (int n = 0; n < blocksCount; n++)
+ {
vectorsOfValues.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + nextVectorSize);
index += nextVectorSize;
// fill AverageFluktuations array
SPtr<AverageValuesArray3D> mAverageFluktuations;
- mAverageFluktuations = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
- new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1],
- dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
+ mAverageFluktuations = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues,
+ dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
// find the nesessary block and fill it
- SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].x1, dataSetSmallArray[n].x2, dataSetSmallArray[n].x3,
- dataSetSmallArray[n].level);
+ SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].x1, dataSetSmallArray[n].x2, dataSetSmallArray[n].x3, dataSetSmallArray[n].level);
block->getKernel()->getDataSet()->setAverageFluctuations(mAverageFluktuations);
}
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageFluktuationsArray end of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
delete[] dataSetSmallArray;
@@ -2102,10 +2176,10 @@ void MPIIORestartCoProcessor::readAverageTripleArray(int step)
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageTripleArray start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
if (comm->isRoot())
@@ -2121,12 +2195,10 @@ void MPIIORestartCoProcessor::readAverageTripleArray(int step)
int blocksCount = 0;
dataSetParam dataSetParamStr;
MPI_File_read_at(file_handler, (MPI_Offset)(rank * sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, (MPI_Offset)(size * sizeof(int)), &dataSetParamStr, 1, dataSetParamType,
- MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(size * sizeof(int)), &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
DataSetSmallRestart *dataSetSmallArray = new DataSetSmallRestart[blocksCount];
- int doubleCountInBlock =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ int doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
std::vector<double> doubleValuesArray(blocksCount * doubleCountInBlock); // double-values in all blocks
// define MPI_types depending on the block-specific information
@@ -2137,61 +2209,59 @@ void MPIIORestartCoProcessor::readAverageTripleArray(int step)
MPI_Offset read_offset = (MPI_Offset)(size * sizeof(int));
size_t next_read_offset = 0;
- if (size > 1) {
- if (rank == 0) {
- next_read_offset = read_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
- } else {
+ }
+ else
+ {
MPI_Recv(&read_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
- next_read_offset = read_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
if (rank < size - 1)
MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
}
}
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount,
- dataSetSmallType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount, dataSetSmallType, MPI_STATUS_IGNORE);
if (doubleCountInBlock > 0)
- MPI_File_read_at(file_handler,
- (MPI_Offset)(read_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
&doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
MPI_File_close(&file_handler);
MPI_Type_free(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageTripleArray time: " << finish - start << " s");
UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageTripleArray start of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
size_t index = 0;
- size_t nextVectorSize =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ size_t nextVectorSize = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
std::vector<double> vectorsOfValues;
- for (int n = 0; n < blocksCount; n++) {
+ for (int n = 0; n < blocksCount; n++)
+ {
vectorsOfValues.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + nextVectorSize);
index += nextVectorSize;
// fill AverageTriplecorrelations array
SPtr<AverageValuesArray3D> mAverageTriplecorrelations;
- mAverageTriplecorrelations = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
- new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1],
- dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
+ mAverageTriplecorrelations = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues,
+ dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
// find the nesessary block and fill it
- SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].x1, dataSetSmallArray[n].x2, dataSetSmallArray[n].x3,
- dataSetSmallArray[n].level);
+ SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].x1, dataSetSmallArray[n].x2, dataSetSmallArray[n].x3, dataSetSmallArray[n].level);
block->getKernel()->getDataSet()->setAverageTriplecorrelations(mAverageTriplecorrelations);
}
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::readAverageTripleArray end of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
delete[] dataSetSmallArray;
@@ -2203,10 +2273,10 @@ void MPIIORestartCoProcessor::readShearStressValArray(int step)
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::readShearStressValArray start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
if (comm->isRoot())
@@ -2222,12 +2292,10 @@ void MPIIORestartCoProcessor::readShearStressValArray(int step)
int blocksCount = 0;
dataSetParam dataSetParamStr;
MPI_File_read_at(file_handler, (MPI_Offset)(rank * sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, (MPI_Offset)(size * sizeof(int)), &dataSetParamStr, 1, dataSetParamType,
- MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(size * sizeof(int)), &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
DataSetSmallRestart *dataSetSmallArray = new DataSetSmallRestart[blocksCount];
- int doubleCountInBlock =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ int doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
std::vector<double> doubleValuesArray(blocksCount * doubleCountInBlock); // double-values in all blocks
// define MPI_types depending on the block-specific information
@@ -2238,61 +2306,59 @@ void MPIIORestartCoProcessor::readShearStressValArray(int step)
MPI_Offset read_offset = (MPI_Offset)(size * sizeof(int));
size_t next_read_offset = 0;
- if (size > 1) {
- if (rank == 0) {
- next_read_offset = read_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
- } else {
+ }
+ else
+ {
MPI_Recv(&read_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
- next_read_offset = read_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
if (rank < size - 1)
MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
}
}
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount,
- dataSetSmallType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount, dataSetSmallType, MPI_STATUS_IGNORE);
if (doubleCountInBlock > 0)
- MPI_File_read_at(file_handler,
- (MPI_Offset)(read_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
&doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
MPI_File_close(&file_handler);
MPI_Type_free(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIORestartCoProcessor::readShearStressValArray time: " << finish - start << " s");
UBLOG(logINFO, "MPIIORestartCoProcessor::readShearStressValArray start of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
size_t index = 0;
- size_t nextVectorSize =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ size_t nextVectorSize = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
std::vector<double> vectorsOfValues;
- for (int n = 0; n < blocksCount; n++) {
+ for (int n = 0; n < blocksCount; n++)
+ {
vectorsOfValues.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + nextVectorSize);
index += nextVectorSize;
// fill ShearStressValuesArray array
SPtr<ShearStressValuesArray3D> mShearStressValues;
- mShearStressValues = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(
- new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1],
- dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
+ mShearStressValues = CbArray4D<LBMReal, IndexerX4X3X2X1>::CbArray4DPtr(new CbArray4D<LBMReal, IndexerX4X3X2X1>(vectorsOfValues,
+ dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2], dataSetParamStr.nx[3]));
// find the nesessary block and fill it
- SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].x1, dataSetSmallArray[n].x2, dataSetSmallArray[n].x3,
- dataSetSmallArray[n].level);
+ SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].x1, dataSetSmallArray[n].x2, dataSetSmallArray[n].x3, dataSetSmallArray[n].level);
block->getKernel()->getDataSet()->setShearStressValues(mShearStressValues);
}
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::readShearStressValArray end of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
delete[] dataSetSmallArray;
@@ -2304,10 +2370,10 @@ void MPIIORestartCoProcessor::readRelaxationFactor(int step)
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::readRelaxationFactor start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
if (comm->isRoot())
@@ -2323,12 +2389,10 @@ void MPIIORestartCoProcessor::readRelaxationFactor(int step)
int blocksCount = 0;
dataSetParam dataSetParamStr;
MPI_File_read_at(file_handler, (MPI_Offset)(rank * sizeof(int)), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, (MPI_Offset)(size * sizeof(int)), &dataSetParamStr, 1, dataSetParamType,
- MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(size * sizeof(int)), &dataSetParamStr, 1, dataSetParamType, MPI_STATUS_IGNORE);
DataSetSmallRestart *dataSetSmallArray = new DataSetSmallRestart[blocksCount];
- int doubleCountInBlock =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ int doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
std::vector<double> doubleValuesArray(blocksCount * doubleCountInBlock); // double-values in all blocks
// define MPI_types depending on the block-specific information
@@ -2339,42 +2403,42 @@ void MPIIORestartCoProcessor::readRelaxationFactor(int step)
MPI_Offset read_offset = (MPI_Offset)(size * sizeof(int));
size_t next_read_offset = 0;
- if (size > 1) {
- if (rank == 0) {
- next_read_offset = read_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
- } else {
+ }
+ else
+ {
MPI_Recv(&read_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
- next_read_offset = read_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
if (rank < size - 1)
MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
}
}
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount,
- dataSetSmallType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount, dataSetSmallType, MPI_STATUS_IGNORE);
if (doubleCountInBlock > 0)
- MPI_File_read_at(file_handler,
- (MPI_Offset)(read_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
&doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
MPI_File_close(&file_handler);
MPI_Type_free(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIORestartCoProcessor::readRelaxationFactor time: " << finish - start << " s");
UBLOG(logINFO, "MPIIORestartCoProcessor::readRelaxationFactor start of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
size_t index = 0;
- size_t nextVectorSize =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ size_t nextVectorSize = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
std::vector<double> vectorsOfValues;
- for (int n = 0; n < blocksCount; n++) {
+ for (int n = 0; n < blocksCount; n++)
+ {
vectorsOfValues.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + nextVectorSize);
index += nextVectorSize;
@@ -2384,37 +2448,38 @@ void MPIIORestartCoProcessor::readRelaxationFactor(int step)
vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2]));
// find the nesessary block and fill it
- SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].x1, dataSetSmallArray[n].x2, dataSetSmallArray[n].x3,
- dataSetSmallArray[n].level);
+ SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].x1, dataSetSmallArray[n].x2, dataSetSmallArray[n].x3, dataSetSmallArray[n].level);
block->getKernel()->getDataSet()->setRelaxationFactor(mRelaxationFactor);
}
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::readRelaxationFactor end of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
delete[] dataSetSmallArray;
}
-void MPIIORestartCoProcessor::readPhaseField(int step)
+void MPIIORestartCoProcessor::readPhaseField(int step, int fieldN)
{
int rank, size;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::readPhaseField start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
if (comm->isRoot())
start = MPI_Wtime();
MPI_File file_handler;
- std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpPhaseField.bin";
+ std::string filename;
+ if(fieldN == 1) filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpPhaseField1.bin";
+ else filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpPhaseField2.bin";
int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
if (rc != MPI_SUCCESS)
throw UbException(UB_EXARGS, "couldn't open file " + filename);
@@ -2427,8 +2492,7 @@ void MPIIORestartCoProcessor::readPhaseField(int step)
MPI_STATUS_IGNORE);
DataSetSmallRestart *dataSetSmallArray = new DataSetSmallRestart[blocksCount];
- int doubleCountInBlock =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ int doubleCountInBlock = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
std::vector<double> doubleValuesArray(blocksCount * doubleCountInBlock); // double-values in all blocks
// define MPI_types depending on the block-specific information
@@ -2439,42 +2503,43 @@ void MPIIORestartCoProcessor::readPhaseField(int step)
MPI_Offset read_offset = (MPI_Offset)(size * sizeof(int));
size_t next_read_offset = 0;
- if (size > 1) {
- if (rank == 0) {
- next_read_offset = read_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
- } else {
+ }
+ else
+ {
MPI_Recv(&read_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
- next_read_offset = read_offset + sizeof(dataSetParam) +
- blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
+ next_read_offset = read_offset + sizeof(dataSetParam) + blocksCount * (sizeof(DataSetSmallRestart) + doubleCountInBlock * sizeof(double));
if (rank < size - 1)
MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
}
}
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount,
- dataSetSmallType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam)), dataSetSmallArray, blocksCount, dataSetSmallType, MPI_STATUS_IGNORE);
if (doubleCountInBlock > 0)
- MPI_File_read_at(file_handler,
- (MPI_Offset)(read_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(dataSetParam) + blocksCount * sizeof(DataSetSmallRestart)),
&doubleValuesArray[0], blocksCount, dataSetDoubleType, MPI_STATUS_IGNORE);
MPI_File_close(&file_handler);
MPI_Type_free(&dataSetDoubleType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIORestartCoProcessor::readPhaseField time: " << finish - start << " s");
UBLOG(logINFO, "MPIIORestartCoProcessor::readPhaseField start of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
size_t index = 0;
- size_t nextVectorSize =
- dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
+ size_t nextVectorSize = dataSetParamStr.nx[0] * dataSetParamStr.nx[1] * dataSetParamStr.nx[2] * dataSetParamStr.nx[3];
std::vector<double> vectorsOfValues;
- for (int n = 0; n < blocksCount; n++) {
+
+ for (int n = 0; n < blocksCount; n++)
+ {
vectorsOfValues.assign(doubleValuesArray.data() + index, doubleValuesArray.data() + index + nextVectorSize);
index += nextVectorSize;
@@ -2484,15 +2549,18 @@ void MPIIORestartCoProcessor::readPhaseField(int step)
vectorsOfValues, dataSetParamStr.nx[0], dataSetParamStr.nx[1], dataSetParamStr.nx[2]));
// find the nesessary block and fill it
- SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].x1, dataSetSmallArray[n].x2, dataSetSmallArray[n].x3,
- dataSetSmallArray[n].level);
- block->getKernel()->getDataSet()->setPhaseField(mPhaseField);
+ SPtr<Block3D> block = grid->getBlock(dataSetSmallArray[n].x1, dataSetSmallArray[n].x2, dataSetSmallArray[n].x3, dataSetSmallArray[n].level);
+ if(fieldN == 1)
+ block->getKernel()->getDataSet()->setPhaseField(mPhaseField);
+ else
+ block->getKernel()->getDataSet()->setPhaseField2(mPhaseField);
+
}
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::readPhaseField end of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
delete[] dataSetSmallArray;
@@ -2504,10 +2572,10 @@ void MPIIORestartCoProcessor::readBoundaryConds(int step)
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
UBLOG(logINFO, "MPIIORestartCoProcessor::readBoundaryConds start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
double start, finish;
if (comm->isRoot())
@@ -2527,14 +2595,11 @@ void MPIIORestartCoProcessor::readBoundaryConds(int step)
// read count of blocks
MPI_File_read_at(file_handler, read_offset1, &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
// read count of big BoundaryCondition blocks
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset1 + sizeof(int)), &dataCount1000, 1, MPI_INT,
- MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset1 + sizeof(int)), &dataCount1000, 1, MPI_INT, MPI_STATUS_IGNORE);
// read count of indexContainer values in all blocks
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset1 + 2 * sizeof(int)), &dataCount2, 1, MPI_INT,
- MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset1 + 2 * sizeof(int)), &dataCount2, 1, MPI_INT, MPI_STATUS_IGNORE);
// read count of bcindexmatrix values in every block
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset1 + 3 * sizeof(int)), &boundCondParamStr, 1,
- boundCondParamType, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset1 + 3 * sizeof(int)), &boundCondParamStr, 1, boundCondParamType, MPI_STATUS_IGNORE);
MPI_Type_contiguous(boundCondParamStr.bcindexmatrixCount, MPI_INT, &bcindexmatrixType);
MPI_Type_commit(&bcindexmatrixType);
@@ -2550,16 +2615,18 @@ void MPIIORestartCoProcessor::readBoundaryConds(int step)
MPI_Offset read_offset = (MPI_Offset)(size * (3 * sizeof(int) + sizeof(boundCondParam)));
size_t next_read_offset = 0;
- if (size > 1) {
- if (rank == 0) {
- next_read_offset = read_offset + blocksCount * sizeof(BCAddRestart) +
- dataCount * sizeof(BoundaryCondition) +
+ if (size > 1)
+ {
+ if (rank == 0)
+ {
+ next_read_offset = read_offset + blocksCount * sizeof(BCAddRestart) + dataCount * sizeof(BoundaryCondition) +
(blocksCount * boundCondParamStr.bcindexmatrixCount + dataCount2) * sizeof(int);
MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, 1, 5, MPI_COMM_WORLD);
- } else {
+ }
+ else
+ {
MPI_Recv(&read_offset, 1, MPI_LONG_LONG_INT, rank - 1, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
- next_read_offset = read_offset + blocksCount * sizeof(BCAddRestart) +
- dataCount * sizeof(BoundaryCondition) +
+ next_read_offset = read_offset + blocksCount * sizeof(BCAddRestart) + dataCount * sizeof(BoundaryCondition) +
(blocksCount * boundCondParamStr.bcindexmatrixCount + dataCount2) * sizeof(int);
if (rank < size - 1)
MPI_Send(&next_read_offset, 1, MPI_LONG_LONG_INT, rank + 1, 5, MPI_COMM_WORLD);
@@ -2567,27 +2634,21 @@ void MPIIORestartCoProcessor::readBoundaryConds(int step)
}
MPI_File_read_at(file_handler, read_offset, bcAddArray, blocksCount, boundCondTypeAdd, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + blocksCount * sizeof(BCAddRestart)), &bcArray[0],
- dataCount1000, boundCondType1000, MPI_STATUS_IGNORE);
- MPI_File_read_at(
- file_handler,
- (MPI_Offset)(read_offset + blocksCount * sizeof(BCAddRestart) + dataCount * sizeof(BoundaryCondition)),
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + blocksCount * sizeof(BCAddRestart)), &bcArray[0], dataCount1000, boundCondType1000, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + blocksCount * sizeof(BCAddRestart) + dataCount * sizeof(BoundaryCondition)),
&intArray1[0], blocksCount, bcindexmatrixType, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler,
- (MPI_Offset)(read_offset + blocksCount * sizeof(BCAddRestart) +
- dataCount * sizeof(BoundaryCondition) +
- blocksCount * boundCondParamStr.bcindexmatrixCount * sizeof(int)),
- &intArray2[0], dataCount2, MPI_INT, MPI_STATUS_IGNORE);
+ MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + blocksCount * sizeof(BCAddRestart) + dataCount * sizeof(BoundaryCondition) +
+ blocksCount * boundCondParamStr.bcindexmatrixCount * sizeof(int)), &intArray2[0], dataCount2, MPI_INT, MPI_STATUS_IGNORE);
MPI_File_close(&file_handler);
MPI_Type_free(&bcindexmatrixType);
- if (comm->isRoot()) {
+ if (comm->isRoot())
+ {
finish = MPI_Wtime();
UBLOG(logINFO, "MPIIORestartCoProcessor::readBoundaryConds time: " << finish - start << " s");
UBLOG(logINFO, "MPIIORestartCoProcessor::readBoundaryConds start of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
int index = 0, index1 = 0, index2 = 0;
@@ -2595,16 +2656,19 @@ void MPIIORestartCoProcessor::readBoundaryConds(int step)
std::vector<int> bcindexmatrixV;
std::vector<int> indexContainerV;
- for (int n = 0; n < blocksCount; n++) {
+ for (int n = 0; n < blocksCount; n++)
+ {
bcVector.resize(0);
bcindexmatrixV.resize(0);
indexContainerV.resize(0);
- for (int ibc = 0; ibc < bcAddArray[n].boundCond_count; ibc++) {
+ for (int ibc = 0; ibc < bcAddArray[n].boundCond_count; ibc++)
+ {
SPtr<BoundaryConditions> bc;
if (memcmp(&bcArray[index], nullBouCond, sizeof(BoundaryCondition)) == 0)
bc = SPtr<BoundaryConditions>();
- else {
+ else
+ {
bc = SPtr<BoundaryConditions>(new BoundaryConditions);
bc->noslipBoundaryFlags = bcArray[index].noslipBoundaryFlags;
bc->slipBoundaryFlags = bcArray[index].slipBoundaryFlags;
@@ -2639,8 +2703,7 @@ void MPIIORestartCoProcessor::readBoundaryConds(int step)
for (int b2 = 0; b2 < bcAddArray[n].indexContainer_count; b2++)
indexContainerV.push_back(intArray2[index2++]);
- CbArray3D<int, IndexerX3X2X1> bcim(bcindexmatrixV, boundCondParamStr.nx1, boundCondParamStr.nx2,
- boundCondParamStr.nx3);
+ CbArray3D<int, IndexerX3X2X1> bcim(bcindexmatrixV, boundCondParamStr.nx1, boundCondParamStr.nx2, boundCondParamStr.nx3);
SPtr<Block3D> block = grid->getBlock(bcAddArray[n].x1, bcAddArray[n].x2, bcAddArray[n].x3, bcAddArray[n].level);
SPtr<BCProcessor> bcProc = bcProcessor->clone(block->getKernel());
@@ -2661,8 +2724,7 @@ void MPIIORestartCoProcessor::readBoundaryConds(int step)
if (comm->isRoot()) {
UBLOG(logINFO, "MPIIORestartCoProcessor::readBoundaryConds end of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: "
- << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
}
//////////////////////////////////////////////////////////////////////////
diff --git a/src/cpu/VirtualFluidsCore/CoProcessors/MPIIORestartCoProcessor.h b/src/cpu/VirtualFluidsCore/CoProcessors/MPIIORestartCoProcessor.h
index cbcf8c553943aa325f415cd123ae1fbe0bf4dcf3..57f559769a06d9a87a968ada73fbaba712da789b 100644
--- a/src/cpu/VirtualFluidsCore/CoProcessors/MPIIORestartCoProcessor.h
+++ b/src/cpu/VirtualFluidsCore/CoProcessors/MPIIORestartCoProcessor.h
@@ -4,6 +4,7 @@
#include <mpi.h>
//#include <PointerDefinitions.h>
#include <string>
+#include <vector>
#include "MPIIOCoProcessor.h"
#include "MPIIODataStructures.h"
@@ -35,8 +36,8 @@ public:
void writeAverageTripleArray(int step);
void writeShearStressValArray(int step);
void writeRelaxationFactor(int step);
- void writePhaseField(int step);
- //! Writes the boundary conditions of the blocks into the file cpBC.bin
+ void writePhaseField(int step, int num);
+ //! Writes the boundary conditions of the blocks into the file cpBC.bin
void writeBoundaryConds(int step);
//! Reads the blocks of the grid from the file cpBlocks.bin
@@ -49,7 +50,7 @@ public:
void readAverageTripleArray(int step);
void readShearStressValArray(int step);
void readRelaxationFactor(int step);
- void readPhaseField(int step);
+ void readPhaseField(int step, int num);
//! Reads the boundary conditions of the blocks from the file cpBC.bin
void readBoundaryConds(int step);
//! The function sets LBMKernel
diff --git a/src/cpu/VirtualFluidsCore/Data/DataSet3D.h b/src/cpu/VirtualFluidsCore/Data/DataSet3D.h
index b8b541546f3c2cccb49ff09c859b8c97c2e22f63..e53e38a74daea2a2a40ca53eff1aa1f4febcc27a 100644
--- a/src/cpu/VirtualFluidsCore/Data/DataSet3D.h
+++ b/src/cpu/VirtualFluidsCore/Data/DataSet3D.h
@@ -91,23 +91,18 @@ public:
protected:
private:
SPtr<DistributionArray3D> fdistributions;
-
SPtr<DistributionArray3D> hdistributions;
-
//SPtr<DistributionArray3D> h1distributions;
SPtr<DistributionArray3D> h2distributions;
-
+
SPtr<AverageValuesArray3D> averageValues;
-
SPtr<AverageValuesArray3D> averageDensity;
SPtr<AverageValuesArray3D> averageVelocity;
SPtr<AverageValuesArray3D> averageFluktuations;
SPtr<AverageValuesArray3D> averageTriplecorrelations;
-
SPtr<ShearStressValuesArray3D> shearStressValues;
SPtr<RelaxationFactorArray3D> relaxationFactor;
-
SPtr<PhaseFieldArray3D> phaseField;
SPtr<PhaseFieldArray3D> phaseField2;
};
diff --git a/src/cpu/VirtualFluidsCore/Parallel/MPIIODataStructures.h b/src/cpu/VirtualFluidsCore/Parallel/MPIIODataStructures.h
index 8b284fc2f768472a4115c61cd567ce0b37b7f4e9..c8bd2d0797af86858b40a1a29a154107f04e46c8 100644
--- a/src/cpu/VirtualFluidsCore/Parallel/MPIIODataStructures.h
+++ b/src/cpu/VirtualFluidsCore/Parallel/MPIIODataStructures.h
@@ -59,6 +59,9 @@ struct dataSetParam {
struct DataSetRestart {
double collFactor;
double deltaT;
+ double collFactorL; // for Multiphase model
+ double collFactorG; // for Multiphase model
+ double densityRatio;// for Multiphase model
int x1;
int x2;
int x3;
@@ -74,6 +77,9 @@ struct DataSetRestart {
struct DataSetMigration {
double collFactor;
double deltaT;
+ double collFactorL; // for Multiphase model
+ double collFactorG; // for Multiphase model
+ double densityRatio;// for Multiphase model
int globalID;
int ghostLayerWidth;
bool compressible;
@@ -164,7 +170,8 @@ struct DSArraysPresence {
bool isAverageTripleArrayPresent;
bool isShearStressValArrayPresent;
bool isRelaxationFactorPresent;
- bool isPhaseFieldPresent;
+ bool isPhaseField1Present;
+ bool isPhaseField2Present;
};
} // namespace MPIIODataStructures
#endif
\ No newline at end of file
diff --git a/src/cpu/VirtualFluidsCore/Utilities/CheckpointConverter.cpp b/src/cpu/VirtualFluidsCore/Utilities/CheckpointConverter.cpp
index 914659afa685814842904e9622c31b875d6a2207..b66eff480e99102edf332cfd750e0d2b6965ba83 100644
--- a/src/cpu/VirtualFluidsCore/Utilities/CheckpointConverter.cpp
+++ b/src/cpu/VirtualFluidsCore/Utilities/CheckpointConverter.cpp
@@ -77,7 +77,7 @@ CheckpointConverter::CheckpointConverter(SPtr<Grid3D> grid, const std::string &p
//---------------------------------------
MPI_Datatype typesDataSetRead[3] = { MPI_DOUBLE, MPI_INT, MPI_CHAR };
- int blocksDataSetRead[3] = { 2, 5, 2 };
+ int blocksDataSetRead[3] = { 3, 5, 2 };
MPI_Aint offsetsDataSetRead[3], lbDataSetRead, extentDataSetRead;
offsetsDataSetRead[0] = 0;
@@ -358,6 +358,7 @@ void CheckpointConverter::convertDataSet(int step, int procCount)
dataSetWriteArray[nb].deltaT = dataSetReadArray[nb].deltaT;
dataSetWriteArray[nb].compressible = dataSetReadArray[nb].compressible;
dataSetWriteArray[nb].withForcing = dataSetReadArray[nb].withForcing;
+// dataSetWriteArray[nb].densityRatio = dataSetReadArray[nb].densityRatio;
write_offset = (MPI_Offset)(3 * sizeof(dataSetParam) + dataSetWriteArray[nb].globalID * sizeofOneDataSet);
MPI_File_write_at(file_handlerW, write_offset, &dataSetWriteArray[nb], 1, dataSetTypeWrite,