From 256237d2e63244cc6504ee1eb3974074d66a68fc Mon Sep 17 00:00:00 2001
From: Sven Marcus <s.marcus@outlook.de>
Date: Tue, 13 Apr 2021 12:34:12 +0200
Subject: [PATCH] AcousticScalingTest, Simulation only writes in main process
---
Python/SlurmTests/poiseuille/settings.py | 18 +--
Python/acousticscaling.py | 10 +-
Python/test_acousticscaling.py | 115 ++++++++++++++++++
.../include/simulationconfig/Simulation.h | 4 +-
src/cpu/simulationconfig/src/Simulation.cpp | 47 ++++---
5 files changed, 164 insertions(+), 30 deletions(-)
create mode 100644 Python/test_acousticscaling.py
diff --git a/Python/SlurmTests/poiseuille/settings.py b/Python/SlurmTests/poiseuille/settings.py
index 7b2f13b83..b989c5d2b 100644
--- a/Python/SlurmTests/poiseuille/settings.py
+++ b/Python/SlurmTests/poiseuille/settings.py
@@ -1,26 +1,26 @@
import os
-from acousticscaling import AcousticScaling
+from acousticscaling import OneDirectionalAcousticScaling
from pyfluids.kernel import LBMKernel, KernelType
from pyfluids.parameters import RuntimeParameters, GridParameters, PhysicalParameters
grid_params = GridParameters()
grid_params.node_distance = 1
-grid_params.number_of_nodes_per_direction = [1, 1, 16]
-grid_params.blocks_per_direction = [1, 1, 8]
+grid_params.number_of_nodes_per_direction = [2, 2, 16]
+grid_params.blocks_per_direction = [1, 1, 2]
grid_params.periodic_boundary_in_x1 = True
grid_params.periodic_boundary_in_x2 = True
physical_params = PhysicalParameters()
-physical_params.lattice_viscosity = 1e-3
+physical_params.lattice_viscosity = 1e-4
runtime_params = RuntimeParameters()
-runtime_params.number_of_threads = int(os.environ["SLURM_NTASKS"])
-runtime_params.number_of_timesteps = 2_500_000
-runtime_params.timestep_log_interval = 500_000
+runtime_params.number_of_threads = int(os.environ["PYFLUIDS_NUM_THREADS"])
+runtime_params.number_of_timesteps = 4_000_000
+runtime_params.timestep_log_interval = 1_000_000
kernel = LBMKernel(KernelType.CompressibleCumulantFourthOrderViscosity)
kernel.use_forcing = True
-kernel.forcing_in_x1 = 1e-9
+kernel.forcing_in_x1 = 5e-10
-Scaling = AcousticScaling(grid_params, physical_params, runtime_params, kernel)
+Scaling = OneDirectionalAcousticScaling(grid_params, physical_params, runtime_params, kernel)
diff --git a/Python/acousticscaling.py b/Python/acousticscaling.py
index ff10cbd49..50b81db06 100644
--- a/Python/acousticscaling.py
+++ b/Python/acousticscaling.py
@@ -2,7 +2,7 @@ from pyfluids.kernel import LBMKernel
from pyfluids.parameters import GridParameters, PhysicalParameters, RuntimeParameters
-class AcousticScaling:
+class OneDirectionalAcousticScaling:
def __init__(self, grid_parameters: GridParameters,
physical_parameters: PhysicalParameters,
@@ -55,7 +55,7 @@ class AcousticScaling:
physical_params.lattice_viscosity = self._physical_params.lattice_viscosity
if level > 0:
- physical_params.lattice_viscosity /= (level * 2)
+ physical_params.lattice_viscosity *= (level * 2)
return physical_params
@@ -78,8 +78,8 @@ class AcousticScaling:
kernel.forcing_in_x3 = self._kernel.forcing_in_x3
if level > 0:
- kernel.forcing_in_x1 *= (level * 2)
- kernel.forcing_in_x2 *= (level * 2)
- kernel.forcing_in_x3 *= (level * 2)
+ kernel.forcing_in_x1 /= (level * 2)
+ kernel.forcing_in_x2 /= (level * 2)
+ kernel.forcing_in_x3 /= (level * 2)
return kernel
diff --git a/Python/test_acousticscaling.py b/Python/test_acousticscaling.py
new file mode 100644
index 000000000..2da531452
--- /dev/null
+++ b/Python/test_acousticscaling.py
@@ -0,0 +1,115 @@
+import unittest
+from typing import List
+
+from pyfluids.kernel import LBMKernel, KernelType
+from pyfluids.parameters import GridParameters, PhysicalParameters, RuntimeParameters
+
+from acousticscaling import OneDirectionalAcousticScaling
+
+
+class OneDirectionalAcousticScalingTest(unittest.TestCase):
+
+ def setUp(self) -> None:
+ self.grid_params = self.make_grid_params()
+ self.physical_params = self.make_physical_params()
+ self.runtime_params = self.make_runtime_params()
+ self.kernel = self.make_kernel()
+
+ self.sut = OneDirectionalAcousticScaling(self.grid_params, self.physical_params, self.runtime_params,
+ self.kernel)
+
+ def test_given_sim_parameters__when_scaling_level_zero__should_return_equal_sim_parameters(self):
+ factor = 1
+ actual_params = self.sut.configuration_for_scale_level(0)
+ actual_grid_params = actual_params[0]
+ actual_physical_params = actual_params[factor]
+ actual_runtime_params = actual_params[2]
+ actual_kernel = actual_params[3]
+
+ self.assert_parameters_scaled_by_factor(actual_grid_params, actual_kernel,
+ actual_physical_params, actual_runtime_params, factor)
+
+ def test_given_sim_parameters__when_scaling_level_one__should_return_sim_parameters_scaled_by_two(self):
+ actual_params = self.sut.configuration_for_scale_level(1)
+ actual_grid_params = actual_params[0]
+ actual_physical_params = actual_params[1]
+ actual_runtime_params = actual_params[2]
+ actual_kernel = actual_params[3]
+
+ self.assert_parameters_scaled_by_factor(actual_grid_params, actual_kernel,
+ actual_physical_params, actual_runtime_params, 2)
+
+ def assert_parameters_scaled_by_factor(self, actual_grid_params, actual_kernel,
+ actual_physical_params, actual_runtime_params, factor):
+ self.assert_grid_params_scaled_by_factor(actual_grid_params, factor=factor)
+ self.assert_physical_params_scaled_by_factor(actual_physical_params, factor=factor)
+ self.assert_runtime_params_scaled_by_factor(actual_runtime_params, factor=factor)
+ self.assert_kernel_forcing_scaled_by_factor(actual_kernel, factor=factor)
+
+ def assert_grid_params_scaled_by_factor(self, actual_grid_params: GridParameters, factor: int):
+ expected_nodes_per_direction = self.scaled_list(self.grid_params.number_of_nodes_per_direction, factor)
+ expected_blocks_per_direction = self.scaled_list(self.grid_params.blocks_per_direction, factor)
+ expected_node_distance = self.grid_params.node_distance / factor
+ self.assertEqual(expected_node_distance, actual_grid_params.node_distance)
+ self.assertEqual(expected_nodes_per_direction, actual_grid_params.number_of_nodes_per_direction)
+ self.assertEqual(expected_blocks_per_direction, actual_grid_params.blocks_per_direction)
+ self.assertEqual(self.grid_params.reference_direction_index, actual_grid_params.reference_direction_index)
+ self.assertEqual(self.grid_params.periodic_boundary_in_x1, actual_grid_params.periodic_boundary_in_x1)
+ self.assertEqual(self.grid_params.periodic_boundary_in_x2, actual_grid_params.periodic_boundary_in_x2)
+ self.assertEqual(self.grid_params.periodic_boundary_in_x3, actual_grid_params.periodic_boundary_in_x3)
+
+ def assert_physical_params_scaled_by_factor(self, actual_params: PhysicalParameters, factor: int):
+ self.assertEqual(self.physical_params.lattice_viscosity * factor, actual_params.lattice_viscosity)
+ self.assertEqual(self.physical_params.bulk_viscosity_factor, actual_params.bulk_viscosity_factor)
+
+ def assert_runtime_params_scaled_by_factor(self, actual_params: RuntimeParameters, factor: int):
+ self.assertEqual(self.runtime_params.number_of_timesteps * factor, actual_params.number_of_timesteps)
+ self.assertEqual(self.runtime_params.number_of_threads, actual_params.number_of_threads)
+ self.assertEqual(self.runtime_params.timestep_log_interval, actual_params.timestep_log_interval)
+
+ def assert_kernel_forcing_scaled_by_factor(self, actual_kernel: LBMKernel, factor: int):
+ self.assertEqual(self.kernel.type, actual_kernel.type)
+ self.assertEqual(self.kernel.use_forcing, actual_kernel.use_forcing)
+ self.assertAlmostEqual(self.kernel.forcing_in_x1 / factor, actual_kernel.forcing_in_x1)
+ self.assertAlmostEqual(self.kernel.forcing_in_x2, actual_kernel.forcing_in_x2)
+ self.assertAlmostEqual(self.kernel.forcing_in_x3, actual_kernel.forcing_in_x3)
+
+ @staticmethod
+ def scaled_list(list_to_scale: List[int], factor: int) -> List[int]:
+ return [list_to_scale[0], list_to_scale[1], list_to_scale[2] * factor]
+
+ @staticmethod
+ def make_kernel():
+ kernel = LBMKernel(KernelType.CompressibleCumulantFourthOrderViscosity)
+ kernel.use_forcing = True
+ kernel.forcing_in_x1 = 5e-10
+ return kernel
+
+ @staticmethod
+ def make_runtime_params():
+ runtime_params = RuntimeParameters()
+ runtime_params.number_of_threads = 4
+ runtime_params.number_of_timesteps = 4_000_000
+ runtime_params.timestep_log_interval = 1_000_000
+ return runtime_params
+
+ @staticmethod
+ def make_physical_params():
+ physical_params = PhysicalParameters()
+ physical_params.lattice_viscosity = 1e-4
+ return physical_params
+
+ @staticmethod
+ def make_grid_params():
+ grid_params = GridParameters()
+ grid_params.node_distance = 1
+ grid_params.number_of_nodes_per_direction = [1, 1, 16]
+ grid_params.blocks_per_direction = [1, 1, 16]
+ grid_params.periodic_boundary_in_x1 = True
+ grid_params.periodic_boundary_in_x2 = True
+
+ return grid_params
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/src/cpu/simulationconfig/include/simulationconfig/Simulation.h b/src/cpu/simulationconfig/include/simulationconfig/Simulation.h
index 47f8b0e6e..f145cb546 100644
--- a/src/cpu/simulationconfig/include/simulationconfig/Simulation.h
+++ b/src/cpu/simulationconfig/include/simulationconfig/Simulation.h
@@ -63,7 +63,9 @@ public:
void run();
private:
- std::shared_ptr<GbObject3D> makeSimulationBoundingBox() const;
+ bool isMainProcess();
+
+ std::shared_ptr<GbObject3D> makeSimulationBoundingBox();
void writeBlocksToFile() const;
diff --git a/src/cpu/simulationconfig/src/Simulation.cpp b/src/cpu/simulationconfig/src/Simulation.cpp
index 6d80a1cfa..2d5bae329 100644
--- a/src/cpu/simulationconfig/src/Simulation.cpp
+++ b/src/cpu/simulationconfig/src/Simulation.cpp
@@ -65,6 +65,8 @@ Simulation::addObject(const std::shared_ptr<GbObject3D> &object, const std::shar
const bool is_in = registeredAdapters.find(bcAdapter) != registeredAdapters.end();
if (!is_in) addBCAdapter(bcAdapter);
this->interactors.push_back(lbmSystem->makeInteractor(object, this->grid, bcAdapter, state));
+ if (communicator->getProcessID() != 0) return;
+
GbSystem3D::writeGeoObject(object, writerConfig.outputPath + folderPath, writerConfig.getWriter());
}
@@ -105,7 +107,9 @@ void Simulation::run()
int &nodesInX1 = gridParameters->numberOfNodesPerDirection[0];
int &nodesInX2 = gridParameters->numberOfNodesPerDirection[1];
int &nodesInX3 = gridParameters->numberOfNodesPerDirection[2];
- logSimulationData(nodesInX1, nodesInX2, nodesInX3);
+
+ if (isMainProcess())
+ logSimulationData(nodesInX1, nodesInX2, nodesInX3);
setBlockSize(nodesInX1, nodesInX2, nodesInX3);
auto gridCube = makeSimulationBoundingBox();
@@ -132,13 +136,13 @@ void Simulation::run()
grid->accept(kernelVisitor);
intHelper.setBC();
- double bulkViscosity = physicalParameters->latticeViscosity * physicalParameters->bulkViscosityFactor;
+// double bulkViscosity = physicalParameters->latticeViscosity * physicalParameters->bulkViscosityFactor;
//auto iProcessor = std::make_shared<CompressibleOffsetMomentsInterpolationProcessor>();
//iProcessor->setBulkViscosity(physicalParameters->latticeViscosity, bulkViscosity);
- //SetConnectorsBlockVisitor setConnsVisitor(communicator, true,
- // lbmSystem->getNumberOfDirections(),
- // physicalParameters->latticeViscosity, iProcessor);
+// SetConnectorsBlockVisitor setConnsVisitor(communicator, true,
+// lbmSystem->getNumberOfDirections(),
+// physicalParameters->latticeViscosity, iProcessor);
OneDistributionSetConnectorsBlockVisitor setConnsVisitor(communicator);
grid->accept(setConnsVisitor);
@@ -163,7 +167,8 @@ void Simulation::run()
#ifdef _OPENMP
omp_set_num_threads(simulationParameters->numberOfThreads);
- UBLOG(logINFO, "OpenMP is set to run with " << omp_get_num_threads() << " threads")
+ if (isMainProcess())
+ UBLOG(logINFO, "OpenMP is set to run with " << simulationParameters->numberOfThreads << " threads")
#endif
auto calculator = std::make_shared<BasicCalculator>(grid, visualizationScheduler,
@@ -171,9 +176,14 @@ void Simulation::run()
calculator->addCoProcessor(nupsCoProcessor);
calculator->addCoProcessor(mqCoProcessor);
- UBLOG(logINFO, "Simulation-start")
+ if (isMainProcess()) UBLOG(logINFO, "Simulation-start")
calculator->calculate();
- UBLOG(logINFO, "Simulation-end")
+ if (isMainProcess()) UBLOG(logINFO, "Simulation-end")
+}
+
+bool Simulation::isMainProcess()
+{
+ return communicator->getProcessID() == 0;
}
void
@@ -248,8 +258,9 @@ void Simulation::writeBoundaryConditions() const
void Simulation::writeBlocksToFile() const
{
UBLOG(logINFO, "Write block grid to VTK-file")
+ auto scheduler = std::make_shared<UbScheduler>(1);
auto ppblocks = std::make_shared<WriteBlocksCoProcessor>(grid,
- std::make_shared<UbScheduler>(1),
+ scheduler,
writerConfig.outputPath,
writerConfig.getWriter(),
communicator);
@@ -258,17 +269,23 @@ void Simulation::writeBlocksToFile() const
}
std::shared_ptr<GbObject3D>
-Simulation::makeSimulationBoundingBox() const
+Simulation::makeSimulationBoundingBox()
{
auto box = gridParameters->boundingBox();
+ auto gridCube = std::make_shared<GbCuboid3D>(box->minX1, box->minX2, box->minX3, box->maxX1, box->maxX2,
+ box->maxX3);
+
+ if (isMainProcess())
+ {
+ UBLOG(logINFO, "Bounding box dimensions = [("
+ << box->minX1 << ", " << box->minX2 << ", " << box->minX3 << "); ("
+ << box->maxX1 << ", " << box->maxX2 << ", " << box->maxX3 << ")]")
- UBLOG(logINFO, "Bounding box dimensions = [("
- << box->minX1 << ", " << box->minX2 << ", " << box->minX3 << "); ("
- << box->maxX1 << ", " << box->maxX2 << ", " << box->maxX3 << ")]")
+ GbSystem3D::writeGeoObject(gridCube.get(), writerConfig.outputPath + "/geo/gridCube", writerConfig.getWriter());
+ }
- auto gridCube = std::make_shared<GbCuboid3D>(box->minX1, box->minX2, box->minX3, box->maxX1, box->maxX2, box->maxX3);
- GbSystem3D::writeGeoObject(gridCube.get(), writerConfig.outputPath + "/geo/gridCube", writerConfig.getWriter());
return gridCube;
}
+
Simulation::~Simulation() = default;
--
GitLab