diff --git a/source/VirtualFluidsCore/CoProcessors/MPIIOCoProcessors.cpp b/source/VirtualFluidsCore/CoProcessors/MPIIOCoProcessor.cpp
similarity index 58%
rename from source/VirtualFluidsCore/CoProcessors/MPIIOCoProcessors.cpp
rename to source/VirtualFluidsCore/CoProcessors/MPIIOCoProcessor.cpp
index 2e9769f30b57c76d3534e979eaeb1bb0b38a0bb8..c9427060213c916f89fa941629162e2184588eb3 100644
--- a/source/VirtualFluidsCore/CoProcessors/MPIIOCoProcessors.cpp
+++ b/source/VirtualFluidsCore/CoProcessors/MPIIOCoProcessor.cpp
@@ -1,4 +1,4 @@
-#include "MPIIOCoProcessors.h"
+#include "MPIIOCoProcessor.h"
#include "Block3D.h"
#include "CoordinateTransformation3D.h"
#include "Grid3D.h"
@@ -8,6 +8,8 @@
#include "UbLogger.h"
#include "MemoryUtil.h"
+using namespace MPIIODataStructures;
+
MPIIOCoProcessor::MPIIOCoProcessor(SPtr<Grid3D> grid, SPtr<UbScheduler> s, const std::string& path, SPtr<Communicator> comm) :
CoProcessor(grid, s),
path(path),
@@ -51,15 +53,162 @@ MPIIOCoProcessor::~MPIIOCoProcessor()
MPI_Type_free(&block3dType);
}
-void MPIIOCoProcessor::readBlocks(int step)
+void MPIIOCoProcessor::writeBlocks(int step)
{
- using namespace MPIIODataStructures;
-
int rank, size;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
//MPI_Comm_size(MPI_COMM_WORLD, &size);
size = 1;
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOCoProcessor::writeBlocks start collect data rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ int blocksCount = 0; // quantity of all the blocks in the grid, max 2147483648 blocks!
+ int minInitLevel = this->grid->getCoarsestInitializedLevel();
+ int maxInitLevel = this->grid->getFinestInitializedLevel();
+
+ std::vector<SPtr<Block3D>> blocksVector[25]; // max 25 levels
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ //grid->getBlocks(level, rank, blockVector[level]);
+ grid->getBlocks(level, blocksVector[level]);
+ blocksCount += static_cast<int>(blocksVector[level].size());
+ }
+
+ GridParam* gridParameters = new GridParam;
+ gridParameters->trafoParams[0] = grid->getCoordinateTransformator()->Tx1;
+ gridParameters->trafoParams[1] = grid->getCoordinateTransformator()->Tx2;
+ gridParameters->trafoParams[2] = grid->getCoordinateTransformator()->Tx3;
+ gridParameters->trafoParams[3] = grid->getCoordinateTransformator()->Sx1;
+ gridParameters->trafoParams[4] = grid->getCoordinateTransformator()->Sx2;
+ gridParameters->trafoParams[5] = grid->getCoordinateTransformator()->Sx3;
+ gridParameters->trafoParams[6] = grid->getCoordinateTransformator()->alpha;
+ gridParameters->trafoParams[7] = grid->getCoordinateTransformator()->beta;
+ gridParameters->trafoParams[8] = grid->getCoordinateTransformator()->gamma;
+
+ gridParameters->trafoParams[9] = grid->getCoordinateTransformator()->toX1factorX1;
+ gridParameters->trafoParams[10] = grid->getCoordinateTransformator()->toX1factorX2;
+ gridParameters->trafoParams[11] = grid->getCoordinateTransformator()->toX1factorX3;
+ gridParameters->trafoParams[12] = grid->getCoordinateTransformator()->toX1delta;
+ gridParameters->trafoParams[13] = grid->getCoordinateTransformator()->toX2factorX1;
+ gridParameters->trafoParams[14] = grid->getCoordinateTransformator()->toX2factorX2;
+ gridParameters->trafoParams[15] = grid->getCoordinateTransformator()->toX2factorX3;
+ gridParameters->trafoParams[16] = grid->getCoordinateTransformator()->toX2delta;
+ gridParameters->trafoParams[17] = grid->getCoordinateTransformator()->toX3factorX1;
+ gridParameters->trafoParams[18] = grid->getCoordinateTransformator()->toX3factorX2;
+ gridParameters->trafoParams[19] = grid->getCoordinateTransformator()->toX3factorX3;
+ gridParameters->trafoParams[20] = grid->getCoordinateTransformator()->toX3delta;
+
+ gridParameters->trafoParams[21] = grid->getCoordinateTransformator()->fromX1factorX1;
+ gridParameters->trafoParams[22] = grid->getCoordinateTransformator()->fromX1factorX2;
+ gridParameters->trafoParams[23] = grid->getCoordinateTransformator()->fromX1factorX3;
+ gridParameters->trafoParams[24] = grid->getCoordinateTransformator()->fromX1delta;
+ gridParameters->trafoParams[25] = grid->getCoordinateTransformator()->fromX2factorX1;
+ gridParameters->trafoParams[26] = grid->getCoordinateTransformator()->fromX2factorX2;
+ gridParameters->trafoParams[27] = grid->getCoordinateTransformator()->fromX2factorX3;
+ gridParameters->trafoParams[28] = grid->getCoordinateTransformator()->fromX2delta;
+ gridParameters->trafoParams[29] = grid->getCoordinateTransformator()->fromX3factorX1;
+ gridParameters->trafoParams[30] = grid->getCoordinateTransformator()->fromX3factorX2;
+ gridParameters->trafoParams[31] = grid->getCoordinateTransformator()->fromX3factorX3;
+ gridParameters->trafoParams[32] = grid->getCoordinateTransformator()->fromX3delta;
+
+ gridParameters->active = grid->getCoordinateTransformator()->active;
+ gridParameters->transformation = grid->getCoordinateTransformator()->transformation;
+
+ gridParameters->deltaX = grid->getDeltaX(minInitLevel);
+ UbTupleInt3 blocknx = grid->getBlockNX();
+ gridParameters->blockNx1 = val<1>(blocknx);
+ gridParameters->blockNx2 = val<2>(blocknx);
+ gridParameters->blockNx3 = val<3>(blocknx);
+ gridParameters->nx1 = grid->getNX1();
+ gridParameters->nx2 = grid->getNX2();
+ gridParameters->nx3 = grid->getNX3();
+ gridParameters->periodicX1 = grid->isPeriodicX1();
+ gridParameters->periodicX2 = grid->isPeriodicX2();
+ gridParameters->periodicX3 = grid->isPeriodicX3();
+
+ //----------------------------------------------------------------------
+
+ Block3d* block3dArray = new Block3d[blocksCount];
+ int ic = 0;
+ for (int level = minInitLevel; level <= maxInitLevel; level++)
+ {
+ for (SPtr<Block3D> block : blocksVector[level]) // all the blocks of the current level
+ {
+ // save data describing the block
+ block3dArray[ic].x1 = block->getX1();
+ block3dArray[ic].x2 = block->getX2();
+ block3dArray[ic].x3 = block->getX3();
+ block3dArray[ic].bundle = block->getBundle();
+ block3dArray[ic].rank = block->getRank();
+ block3dArray[ic].lrank = block->getLocalRank();
+ block3dArray[ic].part = block->getPart();
+ block3dArray[ic].globalID = block->getGlobalID();
+ block3dArray[ic].localID = block->getLocalID();
+ block3dArray[ic].level = block->getLevel();
+ block3dArray[ic].interpolationFlagCF = block->getCollectionOfInterpolationFlagCF();
+ block3dArray[ic].interpolationFlagFC = block->getCollectionOfInterpolationFlagFC();
+ block3dArray[ic].counter = block->getMaxGlobalID();
+ block3dArray[ic].active = block->isActive();
+
+ ic++;
+ }
+ }
+
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOCoProcessor::writeBlocks start MPI IO rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
+
+ // write to the file
+ MPI_File file_handler;
+ MPI_Info info = MPI_INFO_NULL;
+
+ UbSystem::makeDirectory(path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step));
+ std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpBlocks.bin";
+ int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &file_handler);
+ if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
+
+ double start, finish;
+ MPI_Offset write_offset = (MPI_Offset)(size * sizeof(int));
+
+ if (comm->isRoot())
+ {
+ start = MPI_Wtime();
+
+ // each process writes the quantity of it's blocks
+ MPI_File_write_at(file_handler, 0/*rank*sizeof(int)*/, &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
+ // each process writes parameters of the grid
+ MPI_File_write_at(file_handler, write_offset, gridParameters, 1, gridParamType, MPI_STATUS_IGNORE);
+ // each process writes it's blocks
+ MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(GridParam)), &block3dArray[0], blocksCount, block3dType, MPI_STATUS_IGNORE);
+ //MPI_File_sync(file_handler);
+ }
+ MPI_File_close(&file_handler);
+
+ if (comm->isRoot())
+ {
+ finish = MPI_Wtime();
+ UBLOG(logINFO, "MPIIOCoProcessor::writeBlocks time: " << finish - start << " s");
+ }
+
+ delete[] block3dArray;
+ delete gridParameters;
+}
+
+void MPIIOCoProcessor::readBlocks(int step)
+{
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ size = 1;
+
+ if (comm->isRoot()) UBLOG(logINFO, "MPIIOCoProcessor restart step: " << step);
+ if (comm->isRoot()) UBLOG(logINFO, "Load check point - start");
+
double start, finish;
if (comm->isRoot()) start = MPI_Wtime();
@@ -76,7 +225,6 @@ void MPIIOCoProcessor::readBlocks(int step)
GridParam* gridParameters = new GridParam;
// calculate the read offset
- //MPI_Offset read_offset = (MPI_Offset)(size * sizeof(int));
MPI_Offset read_offset = (MPI_Offset)(sizeof(int));
// read parameters of the grid
@@ -92,8 +240,8 @@ void MPIIOCoProcessor::readBlocks(int step)
if (comm->isRoot())
{
finish = MPI_Wtime();
- UBLOG(logINFO, "MPIIOMigrationBECoProcessor::readBlocks time: " << finish - start << " s");
- UBLOG(logINFO, "MPIIOMigrationBECoProcessor::readBlocks start of restore of data, rank = " << rank);
+ UBLOG(logINFO, "MPIIOCoProcessor::readBlocks time: " << finish - start << " s");
+ UBLOG(logINFO, "MPIIOCoProcessor::readBlocks start of restore of data, rank = " << rank);
UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
}
@@ -173,6 +321,11 @@ void MPIIOCoProcessor::readBlocks(int step)
delete gridParameters;
delete[] block3dArray;
+ if (comm->isRoot())
+ {
+ UBLOG(logINFO, "MPIIOCoProcessor::readBlocks end of restore of data, rank = " << rank);
+ UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
+ }
}
void MPIIOCoProcessor::clearAllFiles(int step)
diff --git a/source/VirtualFluidsCore/CoProcessors/MPIIOCoProcessors.h b/source/VirtualFluidsCore/CoProcessors/MPIIOCoProcessor.h
similarity index 91%
rename from source/VirtualFluidsCore/CoProcessors/MPIIOCoProcessors.h
rename to source/VirtualFluidsCore/CoProcessors/MPIIOCoProcessor.h
index 988971ddbd7b49bbf52ce65912d2d1e2e8542cd8..93de60bff9ed52575d6c53553ed547eb35ac7bd9 100644
--- a/source/VirtualFluidsCore/CoProcessors/MPIIOCoProcessors.h
+++ b/source/VirtualFluidsCore/CoProcessors/MPIIOCoProcessor.h
@@ -21,6 +21,9 @@ public:
//! Each timestep writes the grid into the files
virtual void process(double step) = 0;
+ //! Writes the blocks of the grid into the file cpBlocks.bin
+ void writeBlocks(int step);
+
//! Reads the blocks of the grid from the file cpBlocks.bin
void readBlocks(int step);
diff --git a/source/VirtualFluidsCore/CoProcessors/MPIIOMigrationCoProcessor.cpp b/source/VirtualFluidsCore/CoProcessors/MPIIOMigrationCoProcessor.cpp
index 5813e8e95323564f3b9b8be461e1250a4bdab067..9e7684cb5633890a2f7db295d26e083168999b99 100644
--- a/source/VirtualFluidsCore/CoProcessors/MPIIOMigrationCoProcessor.cpp
+++ b/source/VirtualFluidsCore/CoProcessors/MPIIOMigrationCoProcessor.cpp
@@ -24,48 +24,12 @@
using namespace MPIIODataStructures;
-MPIIOMigrationCoProcessor::MPIIOMigrationCoProcessor(SPtr<Grid3D> grid, SPtr<UbScheduler> s,
- const std::string& path,
- SPtr<Communicator> comm) :
- CoProcessor(grid, s),
- path(path),
- comm(comm)
+MPIIOMigrationCoProcessor::MPIIOMigrationCoProcessor(SPtr<Grid3D> grid, SPtr<UbScheduler> s, const std::string& path, SPtr<Communicator> comm) : MPIIOCoProcessor(grid, s, path, comm)
{
- UbSystem::makeDirectory(path + "/mpi_io_cp");
-
memset(&boundCondParamStr, 0, sizeof(boundCondParamStr));
//------------------------- define MPI types ---------------------------------
- MPI_Datatype typesGP[3] = { MPI_DOUBLE, MPI_INT, MPI_CHAR };
- int blocksGP[3] = { 34, 6, 5 };
- MPI_Aint offsetsGP[3], lbGP, extentGP;
-
- offsetsGP[0] = 0;
- MPI_Type_get_extent(MPI_DOUBLE, &lbGP, &extentGP);
- offsetsGP[1] = blocksGP[0] * extentGP;
-
- MPI_Type_get_extent(MPI_INT, &lbGP, &extentGP);
- offsetsGP[2] = offsetsGP[1] + blocksGP[1] * extentGP;
-
- MPI_Type_create_struct(3, blocksGP, offsetsGP, typesGP, &gridParamType);
- MPI_Type_commit(&gridParamType);
-
- //-----------------------------------------------------------------------
-
- MPI_Datatype typesBlock[2] = { MPI_INT, MPI_CHAR };
- int blocksBlock[2] = { 13, 1 };
- MPI_Aint offsetsBlock[2], lbBlock, extentBlock;
-
- offsetsBlock[0] = 0;
- MPI_Type_get_extent(MPI_INT, &lbBlock, &extentBlock);
- offsetsBlock[1] = blocksBlock[0] * extentBlock;
-
- MPI_Type_create_struct(2, blocksBlock, offsetsBlock, typesBlock, &block3dType);
- MPI_Type_commit(&block3dType);
-
- //-----------------------------------------------------------------------
-
MPI_Type_contiguous(7, MPI_INT, &dataSetParamType);
MPI_Type_commit(&dataSetParamType);
@@ -124,9 +88,7 @@ MPIIOMigrationCoProcessor::MPIIOMigrationCoProcessor(SPtr<Grid3D> grid, SPtr<UbS
//////////////////////////////////////////////////////////////////////////
MPIIOMigrationCoProcessor::~MPIIOMigrationCoProcessor()
{
- MPI_Type_free(&gridParamType);
MPI_Type_free(&dataSetParamType);
- MPI_Type_free(&block3dType);
MPI_Type_free(&dataSetType);
MPI_Type_free(&dataSetSmallType);
MPI_Type_free(&boundCondParamType);
@@ -153,81 +115,6 @@ void MPIIOMigrationCoProcessor::process(double step)
if (comm->isRoot()) UBLOG(logINFO, "Save check point - end");
}
}
-////////////////////////////////////////////////////////////////////////
-void MPIIOMigrationCoProcessor::clearAllFiles(int step)
-{
- MPI_File file_handler;
- MPI_Info info = MPI_INFO_NULL;
- MPI_Offset new_size = 0;
-
- UbSystem::makeDirectory(path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step));
-
- std::string filename1 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpBlocks.bin";
- int rc1 = MPI_File_open(MPI_COMM_WORLD, filename1.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &file_handler);
- if (rc1 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename1);
- MPI_File_set_size(file_handler, new_size);
- MPI_File_close(&file_handler);
-
- std::string filename2 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpDataSet.bin";
- int rc2 = MPI_File_open(MPI_COMM_WORLD, filename2.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
- if (rc2 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename2);
- MPI_File_set_size(file_handler, new_size);
- MPI_File_close(&file_handler);
-
- std::string filename3 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpArrays.bin";
- int rc3 = MPI_File_open(MPI_COMM_WORLD, filename3.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
- if (rc3 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename3);
- MPI_File_set_size(file_handler, new_size);
- MPI_File_close(&file_handler);
-
- std::string filename4 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageDensityArray.bin";
- MPI_File_delete(filename4.c_str(), info);
- //int rc4 = MPI_File_open(MPI_COMM_WORLD, filename4.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
- //if (rc4 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename4);
- //MPI_File_set_size(file_handler, new_size);
- //MPI_File_close(&file_handler);
-
- std::string filename5 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageVelocityArray.bin";
- MPI_File_delete(filename5.c_str(), info);
- //int rc5 = MPI_File_open(MPI_COMM_WORLD, filename5.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
- //if (rc5 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename5);
- //MPI_File_set_size(file_handler, new_size);
- //MPI_File_close(&file_handler);
-
- std::string filename6 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageFluktuationsArray.bin";
- MPI_File_delete(filename6.c_str(), info);
- //int rc6 = MPI_File_open(MPI_COMM_WORLD, filename6.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
- //if (rc6 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename6);
- //MPI_File_set_size(file_handler, new_size);
- //MPI_File_close(&file_handler);
-
- std::string filename7 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpAverageTripleArray.bin";
- MPI_File_delete(filename7.c_str(), info);
- //int rc7 = MPI_File_open(MPI_COMM_WORLD, filename7.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
- //if (rc7 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename7);
- //MPI_File_set_size(file_handler, new_size);
- //MPI_File_close(&file_handler);
-
- std::string filename8 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpShearStressValArray.bin";
- MPI_File_delete(filename8.c_str(), info);
- //int rc8 = MPI_File_open(MPI_COMM_WORLD, filename8.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
- //if (rc8 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename8);
- //MPI_File_set_size(file_handler, new_size);
- //MPI_File_close(&file_handler);
-
- std::string filename9 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpRelaxationFactor.bin";
- MPI_File_delete(filename9.c_str(), info);
- //int rc9 = MPI_File_open(MPI_COMM_WORLD, filename9.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
- //if (rc9 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename9);
- //MPI_File_set_size(file_handler, new_size);
- //MPI_File_close(&file_handler);
-
- std::string filename10 = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpBC.bin";
- int rc10 = MPI_File_open(MPI_COMM_WORLD, filename10.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, info, &file_handler);
- if (rc10 != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename10);
- MPI_File_set_size(file_handler, new_size);
- MPI_File_close(&file_handler);
-}
//////////////////////////////////////////////////////////////////////////
void MPIIOMigrationCoProcessor::writeCpTimeStep(int step)
{
@@ -247,162 +134,8 @@ int MPIIOMigrationCoProcessor::readCpTimeStep()
//////////////////////////////////////////////////////////////////////////
void MPIIOMigrationCoProcessor::writeBlocks(int step)
{
- int rank, size;
- MPI_Comm_rank(MPI_COMM_WORLD, &rank);
- //MPI_Comm_size(MPI_COMM_WORLD, &size);
- size = 1;
-
grid->renumberBlockIDs();
-
- //grid->deleteBlockIDs();
- //RenumberBlockVisitor renumber;
- //grid->accept(renumber);
-
- if (comm->isRoot())
- {
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeBlocks start collect data rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
- }
-
- int blocksCount = 0; // quantity of all the blocks in the grid, max 2147483648 blocks!
- int minInitLevel = this->grid->getCoarsestInitializedLevel();
- int maxInitLevel = this->grid->getFinestInitializedLevel();
-
- std::vector<SPtr<Block3D>> blocksVector[25]; // max 25 levels
- for (int level = minInitLevel; level <= maxInitLevel; level++)
- {
- //grid->getBlocks(level, rank, blockVector[level]);
- grid->getBlocks(level, blocksVector[level]);
- blocksCount += static_cast<int>(blocksVector[level].size());
- }
-
- GridParam* gridParameters = new GridParam;
- gridParameters->trafoParams[0] = grid->getCoordinateTransformator()->Tx1;
- gridParameters->trafoParams[1] = grid->getCoordinateTransformator()->Tx2;
- gridParameters->trafoParams[2] = grid->getCoordinateTransformator()->Tx3;
- gridParameters->trafoParams[3] = grid->getCoordinateTransformator()->Sx1;
- gridParameters->trafoParams[4] = grid->getCoordinateTransformator()->Sx2;
- gridParameters->trafoParams[5] = grid->getCoordinateTransformator()->Sx3;
- gridParameters->trafoParams[6] = grid->getCoordinateTransformator()->alpha;
- gridParameters->trafoParams[7] = grid->getCoordinateTransformator()->beta;
- gridParameters->trafoParams[8] = grid->getCoordinateTransformator()->gamma;
-
- gridParameters->trafoParams[9] = grid->getCoordinateTransformator()->toX1factorX1;
- gridParameters->trafoParams[10] = grid->getCoordinateTransformator()->toX1factorX2;
- gridParameters->trafoParams[11] = grid->getCoordinateTransformator()->toX1factorX3;
- gridParameters->trafoParams[12] = grid->getCoordinateTransformator()->toX1delta;
- gridParameters->trafoParams[13] = grid->getCoordinateTransformator()->toX2factorX1;
- gridParameters->trafoParams[14] = grid->getCoordinateTransformator()->toX2factorX2;
- gridParameters->trafoParams[15] = grid->getCoordinateTransformator()->toX2factorX3;
- gridParameters->trafoParams[16] = grid->getCoordinateTransformator()->toX2delta;
- gridParameters->trafoParams[17] = grid->getCoordinateTransformator()->toX3factorX1;
- gridParameters->trafoParams[18] = grid->getCoordinateTransformator()->toX3factorX2;
- gridParameters->trafoParams[19] = grid->getCoordinateTransformator()->toX3factorX3;
- gridParameters->trafoParams[20] = grid->getCoordinateTransformator()->toX3delta;
-
- gridParameters->trafoParams[21] = grid->getCoordinateTransformator()->fromX1factorX1;
- gridParameters->trafoParams[22] = grid->getCoordinateTransformator()->fromX1factorX2;
- gridParameters->trafoParams[23] = grid->getCoordinateTransformator()->fromX1factorX3;
- gridParameters->trafoParams[24] = grid->getCoordinateTransformator()->fromX1delta;
- gridParameters->trafoParams[25] = grid->getCoordinateTransformator()->fromX2factorX1;
- gridParameters->trafoParams[26] = grid->getCoordinateTransformator()->fromX2factorX2;
- gridParameters->trafoParams[27] = grid->getCoordinateTransformator()->fromX2factorX3;
- gridParameters->trafoParams[28] = grid->getCoordinateTransformator()->fromX2delta;
- gridParameters->trafoParams[29] = grid->getCoordinateTransformator()->fromX3factorX1;
- gridParameters->trafoParams[30] = grid->getCoordinateTransformator()->fromX3factorX2;
- gridParameters->trafoParams[31] = grid->getCoordinateTransformator()->fromX3factorX3;
- gridParameters->trafoParams[32] = grid->getCoordinateTransformator()->fromX3delta;
-
- gridParameters->active = grid->getCoordinateTransformator()->active;
- gridParameters->transformation = grid->getCoordinateTransformator()->transformation;
-
- gridParameters->deltaX = grid->getDeltaX(minInitLevel);
- UbTupleInt3 blocknx = grid->getBlockNX();
- gridParameters->blockNx1 = val<1>(blocknx);
- gridParameters->blockNx2 = val<2>(blocknx);
- gridParameters->blockNx3 = val<3>(blocknx);
- gridParameters->nx1 = grid->getNX1();
- gridParameters->nx2 = grid->getNX2();
- gridParameters->nx3 = grid->getNX3();
- gridParameters->periodicX1 = grid->isPeriodicX1();
- gridParameters->periodicX2 = grid->isPeriodicX2();
- gridParameters->periodicX3 = grid->isPeriodicX3();
-
- //----------------------------------------------------------------------
-
- Block3d* block3dArray = new Block3d[blocksCount];
- int ic = 0;
- for (int level = minInitLevel; level <= maxInitLevel; level++)
- {
- for (SPtr<Block3D> block : blocksVector[level]) // all the blocks of the current level
- {
- // save data describing the block
- block3dArray[ic].x1 = block->getX1();
- block3dArray[ic].x2 = block->getX2();
- block3dArray[ic].x3 = block->getX3();
- block3dArray[ic].bundle = block->getBundle();
- block3dArray[ic].rank = block->getRank();
- block3dArray[ic].lrank = block->getLocalRank();
- block3dArray[ic].part = block->getPart();
- block3dArray[ic].globalID = block->getGlobalID();
- block3dArray[ic].localID = block->getLocalID();
- block3dArray[ic].level = block->getLevel();
- block3dArray[ic].interpolationFlagCF = block->getCollectionOfInterpolationFlagCF();
- block3dArray[ic].interpolationFlagFC = block->getCollectionOfInterpolationFlagFC();
- block3dArray[ic].counter = block->getMaxGlobalID();
- block3dArray[ic].active = block->isActive();
-
- ic++;
- }
- }
-
- if (comm->isRoot())
- {
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeBlocks start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
- }
-
- // write to the file
- MPI_File file_handler;
- MPI_Info info = MPI_INFO_NULL;
- //MPI_Info_create (&info);
- //MPI_Info_set(info,"romio_cb_write","enable");
- //MPI_Info_set(info,"cb_buffer_size","4194304");
- //MPI_Info_set(info,"striping_unit","4194304");
-
- // if (comm->isRoot())
- // {
- UbSystem::makeDirectory(path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step));
- std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpBlocks.bin";
- int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &file_handler);
- if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
- // }
-
- double start, finish;
- MPI_Offset write_offset = (MPI_Offset)(size * sizeof(int));
-
- if (comm->isRoot())
- {
- start = MPI_Wtime();
-
- // each process writes the quantity of it's blocks
- MPI_File_write_at(file_handler, 0/*rank*sizeof(int)*/, &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
- // each process writes parameters of the grid
- MPI_File_write_at(file_handler, write_offset, gridParameters, 1, gridParamType, MPI_STATUS_IGNORE);
- // each process writes it's blocks
- MPI_File_write_at(file_handler, (MPI_Offset)(write_offset + sizeof(GridParam)), &block3dArray[0], blocksCount, block3dType, MPI_STATUS_IGNORE);
- //MPI_File_sync(file_handler);
- }
- MPI_File_close(&file_handler);
-
- if (comm->isRoot())
- {
- finish = MPI_Wtime();
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::writeBlocks time: " << finish - start << " s");
- }
-
- delete[] block3dArray;
- delete gridParameters;
+ MPIIOCoProcessor::writeBlocks(step);
}
void MPIIOMigrationCoProcessor::writeDataSet(int step)
@@ -1506,141 +1239,7 @@ void MPIIOMigrationCoProcessor::restart(int step)
void MPIIOMigrationCoProcessor::readBlocks(int step)
{
- int rank, size;
- MPI_Comm_rank(MPI_COMM_WORLD, &rank);
- //MPI_Comm_size(MPI_COMM_WORLD, &size);
- size = 1;
-
- if (comm->isRoot())
- {
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::readBlocks start MPI IO rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
- }
-
- double start, finish;
- if (comm->isRoot()) start = MPI_Wtime();
-
- MPI_File file_handler;
- std::string filename = path + "/mpi_io_cp/mpi_io_cp_" + UbSystem::toString(step) + "/cpBlocks.bin";
- int rc = MPI_File_open(MPI_COMM_WORLD, filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handler);
- if (rc != MPI_SUCCESS) throw UbException(UB_EXARGS, "couldn't open file " + filename);
-
- // read count of blocks
- int blocksCount = 0;
- //MPI_File_read_at(file_handler, rank*sizeof(int), &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
- MPI_File_read_at(file_handler, 0, &blocksCount, 1, MPI_INT, MPI_STATUS_IGNORE);
- Block3d* block3dArray = new Block3d[blocksCount];
-
- GridParam* gridParameters = new GridParam;
-
- // calculate the read offset
- MPI_Offset read_offset = (MPI_Offset)(size * sizeof(int));
-
- // read parameters of the grid
- MPI_File_read_at(file_handler, read_offset, gridParameters, 1, gridParamType, MPI_STATUS_IGNORE);
- // read all the blocks
- if (comm->isRoot())
- {
- MPI_File_read_at(file_handler, (MPI_Offset)(read_offset + sizeof(GridParam)), &block3dArray[0], blocksCount, block3dType, MPI_STATUS_IGNORE);
- }
-
- MPI_Bcast(block3dArray, blocksCount, block3dType, comm->getRoot(), MPI_COMM_WORLD);
-
- MPI_File_close(&file_handler);
-
- if (comm->isRoot())
- {
- finish = MPI_Wtime();
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::readBlocks time: " << finish - start << " s");
- }
-
- if (comm->isRoot())
- {
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::readBlocks start of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
- }
-
- // clear the grid
- grid->deleteBlocks();
-
- // restore the grid
- SPtr<CoordinateTransformation3D> trafo(new CoordinateTransformation3D());
- trafo->Tx1 = gridParameters->trafoParams[0];
- trafo->Tx2 = gridParameters->trafoParams[1];
- trafo->Tx3 = gridParameters->trafoParams[2];
- trafo->Sx1 = gridParameters->trafoParams[3];
- trafo->Sx2 = gridParameters->trafoParams[4];
- trafo->Sx3 = gridParameters->trafoParams[5];
- trafo->alpha = gridParameters->trafoParams[6];
- trafo->beta = gridParameters->trafoParams[7];
- trafo->gamma = gridParameters->trafoParams[8];
-
- trafo->toX1factorX1 = gridParameters->trafoParams[9];
- trafo->toX1factorX2 = gridParameters->trafoParams[10];
- trafo->toX1factorX3 = gridParameters->trafoParams[11];
- trafo->toX1delta = gridParameters->trafoParams[12];
- trafo->toX2factorX1 = gridParameters->trafoParams[13];
- trafo->toX2factorX2 = gridParameters->trafoParams[14];
- trafo->toX2factorX3 = gridParameters->trafoParams[15];
- trafo->toX2delta = gridParameters->trafoParams[16];
- trafo->toX3factorX1 = gridParameters->trafoParams[17];
- trafo->toX3factorX2 = gridParameters->trafoParams[18];
- trafo->toX3factorX3 = gridParameters->trafoParams[19];
- trafo->toX3delta = gridParameters->trafoParams[20];
-
- trafo->fromX1factorX1 = gridParameters->trafoParams[21];
- trafo->fromX1factorX2 = gridParameters->trafoParams[22];
- trafo->fromX1factorX3 = gridParameters->trafoParams[23];
- trafo->fromX1delta = gridParameters->trafoParams[24];
- trafo->fromX2factorX1 = gridParameters->trafoParams[25];
- trafo->fromX2factorX2 = gridParameters->trafoParams[26];
- trafo->fromX2factorX3 = gridParameters->trafoParams[27];
- trafo->fromX2delta = gridParameters->trafoParams[28];
- trafo->fromX3factorX1 = gridParameters->trafoParams[29];
- trafo->fromX3factorX2 = gridParameters->trafoParams[30];
- trafo->fromX3factorX3 = gridParameters->trafoParams[31];
- trafo->fromX3delta = gridParameters->trafoParams[32];
-
- trafo->active = gridParameters->active;
- trafo->transformation = gridParameters->transformation;
-
- grid->setCoordinateTransformator(trafo);
-
- grid->setDeltaX(gridParameters->deltaX);
- grid->setBlockNX(gridParameters->blockNx1, gridParameters->blockNx2, gridParameters->blockNx3);
- grid->setNX1(gridParameters->nx1);
- grid->setNX2(gridParameters->nx2);
- grid->setNX3(gridParameters->nx3);
- grid->setPeriodicX1(gridParameters->periodicX1);
- grid->setPeriodicX2(gridParameters->periodicX2);
- grid->setPeriodicX3(gridParameters->periodicX3);
-
- // regenerate blocks
- for (int n = 0; n<blocksCount; n++)
- {
- SPtr<Block3D> block(new Block3D(block3dArray[n].x1, block3dArray[n].x2, block3dArray[n].x3, block3dArray[n].level));
- block->setActive(block3dArray[n].active);
- block->setBundle(block3dArray[n].bundle);
- block->setRank(block3dArray[n].rank);
- block->setLocalRank(block3dArray[n].lrank);
- block->setGlobalID(block3dArray[n].globalID);
- block->setLocalID(block3dArray[n].localID);
- block->setPart(block3dArray[n].part);
- block->setLevel(block3dArray[n].level);
- block->setCollectionOfInterpolationFlagCF(block3dArray[n].interpolationFlagCF);
- block->setCollectionOfInterpolationFlagFC(block3dArray[n].interpolationFlagFC);
-
- grid->addBlock(block);
- }
-
- delete gridParameters;
- delete[] block3dArray;
-
- if (comm->isRoot())
- {
- UBLOG(logINFO, "MPIIOMigrationCoProcessor::readBlocks end of restore of data, rank = " << rank);
- UBLOG(logINFO, "Physical Memory currently used by current process: " << Utilities::getPhysMemUsedByMe() / 1073741824.0 << " GB");
- }
+ MPIIOCoProcessor::readBlocks(step);
}
void MPIIOMigrationCoProcessor::readDataSet(int step)
diff --git a/source/VirtualFluidsCore/CoProcessors/MPIIOMigrationCoProcessor.h b/source/VirtualFluidsCore/CoProcessors/MPIIOMigrationCoProcessor.h
index 34852934424b94aa9ca35960d78e6560a81154c5..8644e37ef2e9ee0c632b6411005dc1a2102bac28 100644
--- a/source/VirtualFluidsCore/CoProcessors/MPIIOMigrationCoProcessor.h
+++ b/source/VirtualFluidsCore/CoProcessors/MPIIOMigrationCoProcessor.h
@@ -4,8 +4,8 @@
#include <mpi.h>
#include <string>
-#include "CoProcessor.h"
#include "MPIIODataStructures.h"
+#include "MPIIOCoProcessor.h"
class Grid3D;
class UbScheduler;
@@ -14,9 +14,9 @@ class BCProcessor;
class LBMKernel;
-//! \class MPIWriteBlocksCoProcessor
+//! \class MPIIOMigrationCoProcessor
//! \brief Writes the grid each timestep into the files and reads the grip from the files before regenerating
-class MPIIOMigrationCoProcessor : public CoProcessor
+class MPIIOMigrationCoProcessor : public MPIIOCoProcessor
{
public:
MPIIOMigrationCoProcessor(SPtr<Grid3D> grid, SPtr<UbScheduler> s, const std::string& path, SPtr<Communicator> comm);
@@ -54,20 +54,18 @@ public:
void setLBMKernel(SPtr<LBMKernel> kernel);
//!The function sets BCProcessor
void setBCProcessor(SPtr<BCProcessor> bcProcessor);
- //!The function truncates the data files
- void clearAllFiles(int step);
- //void setNu(double nu);
//!The function write a time step of last check point
void writeCpTimeStep(int step);
//!The function read a time step of last check point
int readCpTimeStep();
protected:
- std::string path;
- SPtr<Communicator> comm;
+ //std::string path;
+ //SPtr<Communicator> comm;
private:
- MPI_Datatype gridParamType, block3dType, arrayPresenceType;
+ //MPI_Datatype gridParamType, block3dType,
+ MPI_Datatype arrayPresenceType;
MPI_Datatype dataSetParamType, dataSetType, dataSetSmallType, dataSetDoubleType;
MPI_Datatype boundCondParamType, boundCondType, boundCondTypeAdd, bcindexmatrixType;
diff --git a/source/VirtualFluidsCore/Visitors/RenumberGridVisitor.h b/source/VirtualFluidsCore/Visitors/RenumberGridVisitor.h
index 86ea8d90e2034ac841cdfc6f2559a432b7719247..b7005579aefddcec1c599a35f4bf7ae5c464d430 100644
--- a/source/VirtualFluidsCore/Visitors/RenumberGridVisitor.h
+++ b/source/VirtualFluidsCore/Visitors/RenumberGridVisitor.h
@@ -13,7 +13,7 @@
class Grid3D;
-//! \brief Visitor class which renumber blocks.
+//! \brief Visitor class which renumber blocks in order: rank->level.
//! \details Visitor class which renumber blocks.
//! \author Konstantin Kutscher
class RenumberGridVisitor : public Grid3DVisitor