AcousticScalingTest, Simulation only writes in main process

Python/SlurmTests/poiseuille/settings.py

 import os
-from acousticscaling import AcousticScaling
+from acousticscaling import OneDirectionalAcousticScaling
 from pyfluids.kernel import LBMKernel, KernelType
 from pyfluids.parameters import RuntimeParameters, GridParameters, PhysicalParameters
 
 
 grid_params = GridParameters()
 grid_params.node_distance = 1
-grid_params.number_of_nodes_per_direction = [1, 1, 16]
-grid_params.blocks_per_direction = [1, 1, 8]
+grid_params.number_of_nodes_per_direction = [2, 2, 16]
+grid_params.blocks_per_direction = [1, 1, 2]
 grid_params.periodic_boundary_in_x1 = True
 grid_params.periodic_boundary_in_x2 = True
 
 physical_params = PhysicalParameters()
-physical_params.lattice_viscosity = 1e-3
+physical_params.lattice_viscosity = 1e-4
 
 runtime_params = RuntimeParameters()
-runtime_params.number_of_threads = int(os.environ["SLURM_NTASKS"])
-runtime_params.number_of_timesteps = 2_500_000
-runtime_params.timestep_log_interval = 500_000
+runtime_params.number_of_threads = int(os.environ["PYFLUIDS_NUM_THREADS"])
+runtime_params.number_of_timesteps = 4_000_000
+runtime_params.timestep_log_interval = 1_000_000
 
 kernel = LBMKernel(KernelType.CompressibleCumulantFourthOrderViscosity)
 kernel.use_forcing = True
-kernel.forcing_in_x1 = 1e-9
+kernel.forcing_in_x1 = 5e-10
 
-Scaling = AcousticScaling(grid_params, physical_params, runtime_params, kernel)
+Scaling = OneDirectionalAcousticScaling(grid_params, physical_params, runtime_params, kernel)

Python/acousticscaling.py

@@ -2,7 +2,7 @@ from pyfluids.kernel import LBMKernel
 from pyfluids.parameters import GridParameters, PhysicalParameters, RuntimeParameters
 
 
-class AcousticScaling:
+class OneDirectionalAcousticScaling:
 
     def __init__(self, grid_parameters: GridParameters,
                  physical_parameters: PhysicalParameters,
@@ -55,7 +55,7 @@ class AcousticScaling:
         physical_params.lattice_viscosity = self._physical_params.lattice_viscosity
 
         if level > 0:
-            physical_params.lattice_viscosity /= (level * 2)
+            physical_params.lattice_viscosity *= (level * 2)
 
         return physical_params
 
@@ -78,8 +78,8 @@ class AcousticScaling:
         kernel.forcing_in_x3 = self._kernel.forcing_in_x3
 
         if level > 0:
-            kernel.forcing_in_x1 *= (level * 2)
-            kernel.forcing_in_x2 *= (level * 2)
-            kernel.forcing_in_x3 *= (level * 2)
+            kernel.forcing_in_x1 /= (level * 2)
+            kernel.forcing_in_x2 /= (level * 2)
+            kernel.forcing_in_x3 /= (level * 2)
 
         return kernel

Python/test_acousticscaling.py

+import unittest
+from typing import List
+
+from pyfluids.kernel import LBMKernel, KernelType
+from pyfluids.parameters import GridParameters, PhysicalParameters, RuntimeParameters
+
+from acousticscaling import OneDirectionalAcousticScaling
+
+
+class OneDirectionalAcousticScalingTest(unittest.TestCase):
+
+    def setUp(self) -> None:
+        self.grid_params = self.make_grid_params()
+        self.physical_params = self.make_physical_params()
+        self.runtime_params = self.make_runtime_params()
+        self.kernel = self.make_kernel()
+
+        self.sut = OneDirectionalAcousticScaling(self.grid_params, self.physical_params, self.runtime_params,
+                                                 self.kernel)
+
+    def test_given_sim_parameters__when_scaling_level_zero__should_return_equal_sim_parameters(self):
+        factor = 1
+        actual_params = self.sut.configuration_for_scale_level(0)
+        actual_grid_params = actual_params[0]
+        actual_physical_params = actual_params[factor]
+        actual_runtime_params = actual_params[2]
+        actual_kernel = actual_params[3]
+
+        self.assert_parameters_scaled_by_factor(actual_grid_params, actual_kernel,
+                                                actual_physical_params, actual_runtime_params, factor)
+
+    def test_given_sim_parameters__when_scaling_level_one__should_return_sim_parameters_scaled_by_two(self):
+        actual_params = self.sut.configuration_for_scale_level(1)
+        actual_grid_params = actual_params[0]
+        actual_physical_params = actual_params[1]
+        actual_runtime_params = actual_params[2]
+        actual_kernel = actual_params[3]
+
+        self.assert_parameters_scaled_by_factor(actual_grid_params, actual_kernel,
+                                                actual_physical_params, actual_runtime_params, 2)
+
+    def assert_parameters_scaled_by_factor(self, actual_grid_params, actual_kernel,
+                                           actual_physical_params, actual_runtime_params, factor):
+        self.assert_grid_params_scaled_by_factor(actual_grid_params, factor=factor)
+        self.assert_physical_params_scaled_by_factor(actual_physical_params, factor=factor)
+        self.assert_runtime_params_scaled_by_factor(actual_runtime_params, factor=factor)
+        self.assert_kernel_forcing_scaled_by_factor(actual_kernel, factor=factor)
+
+    def assert_grid_params_scaled_by_factor(self, actual_grid_params: GridParameters, factor: int):
+        expected_nodes_per_direction = self.scaled_list(self.grid_params.number_of_nodes_per_direction, factor)
+        expected_blocks_per_direction = self.scaled_list(self.grid_params.blocks_per_direction, factor)
+        expected_node_distance = self.grid_params.node_distance / factor
+        self.assertEqual(expected_node_distance, actual_grid_params.node_distance)
+        self.assertEqual(expected_nodes_per_direction, actual_grid_params.number_of_nodes_per_direction)
+        self.assertEqual(expected_blocks_per_direction, actual_grid_params.blocks_per_direction)
+        self.assertEqual(self.grid_params.reference_direction_index, actual_grid_params.reference_direction_index)
+        self.assertEqual(self.grid_params.periodic_boundary_in_x1, actual_grid_params.periodic_boundary_in_x1)
+        self.assertEqual(self.grid_params.periodic_boundary_in_x2, actual_grid_params.periodic_boundary_in_x2)
+        self.assertEqual(self.grid_params.periodic_boundary_in_x3, actual_grid_params.periodic_boundary_in_x3)
+
+    def assert_physical_params_scaled_by_factor(self, actual_params: PhysicalParameters, factor: int):
+        self.assertEqual(self.physical_params.lattice_viscosity * factor, actual_params.lattice_viscosity)
+        self.assertEqual(self.physical_params.bulk_viscosity_factor, actual_params.bulk_viscosity_factor)
+
+    def assert_runtime_params_scaled_by_factor(self, actual_params: RuntimeParameters, factor: int):
+        self.assertEqual(self.runtime_params.number_of_timesteps * factor, actual_params.number_of_timesteps)
+        self.assertEqual(self.runtime_params.number_of_threads, actual_params.number_of_threads)
+        self.assertEqual(self.runtime_params.timestep_log_interval, actual_params.timestep_log_interval)
+
+    def assert_kernel_forcing_scaled_by_factor(self, actual_kernel: LBMKernel, factor: int):
+        self.assertEqual(self.kernel.type, actual_kernel.type)
+        self.assertEqual(self.kernel.use_forcing, actual_kernel.use_forcing)
+        self.assertAlmostEqual(self.kernel.forcing_in_x1 / factor, actual_kernel.forcing_in_x1)
+        self.assertAlmostEqual(self.kernel.forcing_in_x2, actual_kernel.forcing_in_x2)
+        self.assertAlmostEqual(self.kernel.forcing_in_x3, actual_kernel.forcing_in_x3)
+
+    @staticmethod
+    def scaled_list(list_to_scale: List[int], factor: int) -> List[int]:
+        return [list_to_scale[0], list_to_scale[1], list_to_scale[2] * factor]
+
+    @staticmethod
+    def make_kernel():
+        kernel = LBMKernel(KernelType.CompressibleCumulantFourthOrderViscosity)
+        kernel.use_forcing = True
+        kernel.forcing_in_x1 = 5e-10
+        return kernel
+
+    @staticmethod
+    def make_runtime_params():
+        runtime_params = RuntimeParameters()
+        runtime_params.number_of_threads = 4
+        runtime_params.number_of_timesteps = 4_000_000
+        runtime_params.timestep_log_interval = 1_000_000
+        return runtime_params
+
+    @staticmethod
+    def make_physical_params():
+        physical_params = PhysicalParameters()
+        physical_params.lattice_viscosity = 1e-4
+        return physical_params
+
+    @staticmethod
+    def make_grid_params():
+        grid_params = GridParameters()
+        grid_params.node_distance = 1
+        grid_params.number_of_nodes_per_direction = [1, 1, 16]
+        grid_params.blocks_per_direction = [1, 1, 16]
+        grid_params.periodic_boundary_in_x1 = True
+        grid_params.periodic_boundary_in_x2 = True
+
+        return grid_params
+
+
+if __name__ == '__main__':
+    unittest.main()

src/cpu/simulationconfig/include/simulationconfig/Simulation.h

@@ -63,7 +63,9 @@ public:
     void run();
 
 private:
-    std::shared_ptr<GbObject3D> makeSimulationBoundingBox() const;
+    bool isMainProcess();
+
+    std::shared_ptr<GbObject3D> makeSimulationBoundingBox();
 
     void writeBlocksToFile() const;
 

src/cpu/simulationconfig/src/Simulation.cpp

@@ -65,6 +65,8 @@ Simulation::addObject(const std::shared_ptr<GbObject3D> &object, const std::shar
     const bool is_in = registeredAdapters.find(bcAdapter) != registeredAdapters.end();
     if (!is_in) addBCAdapter(bcAdapter);
     this->interactors.push_back(lbmSystem->makeInteractor(object, this->grid, bcAdapter, state));
+    if (communicator->getProcessID() != 0) return;
+
     GbSystem3D::writeGeoObject(object, writerConfig.outputPath + folderPath, writerConfig.getWriter());
 }
 
@@ -105,7 +107,9 @@ void Simulation::run()
     int &nodesInX1 = gridParameters->numberOfNodesPerDirection[0];
     int &nodesInX2 = gridParameters->numberOfNodesPerDirection[1];
     int &nodesInX3 = gridParameters->numberOfNodesPerDirection[2];
-    logSimulationData(nodesInX1, nodesInX2, nodesInX3);
+
+    if (isMainProcess())
+        logSimulationData(nodesInX1, nodesInX2, nodesInX3);
 
     setBlockSize(nodesInX1, nodesInX2, nodesInX3);
     auto gridCube = makeSimulationBoundingBox();
@@ -132,13 +136,13 @@ void Simulation::run()
     grid->accept(kernelVisitor);
     intHelper.setBC();
 
-    double bulkViscosity = physicalParameters->latticeViscosity * physicalParameters->bulkViscosityFactor;
+//    double bulkViscosity = physicalParameters->latticeViscosity * physicalParameters->bulkViscosityFactor;
     //auto iProcessor = std::make_shared<CompressibleOffsetMomentsInterpolationProcessor>();
     //iProcessor->setBulkViscosity(physicalParameters->latticeViscosity, bulkViscosity);
 
-    //SetConnectorsBlockVisitor setConnsVisitor(communicator, true,
-    //                                          lbmSystem->getNumberOfDirections(),
-    //                                          physicalParameters->latticeViscosity, iProcessor);
+//    SetConnectorsBlockVisitor setConnsVisitor(communicator, true,
+//                                              lbmSystem->getNumberOfDirections(),
+//                                              physicalParameters->latticeViscosity, iProcessor);
 
     OneDistributionSetConnectorsBlockVisitor setConnsVisitor(communicator);
     grid->accept(setConnsVisitor);
@@ -163,7 +167,8 @@ void Simulation::run()
 
 #ifdef _OPENMP
     omp_set_num_threads(simulationParameters->numberOfThreads);
-    UBLOG(logINFO, "OpenMP is set to run with " << omp_get_num_threads() << " threads")
+    if (isMainProcess())
+        UBLOG(logINFO, "OpenMP is set to run with " << simulationParameters->numberOfThreads << " threads")
 #endif
 
     auto calculator = std::make_shared<BasicCalculator>(grid, visualizationScheduler,
@@ -171,9 +176,14 @@ void Simulation::run()
     calculator->addCoProcessor(nupsCoProcessor);
     calculator->addCoProcessor(mqCoProcessor);
 
-    UBLOG(logINFO, "Simulation-start")
+    if (isMainProcess()) UBLOG(logINFO, "Simulation-start")
     calculator->calculate();
-    UBLOG(logINFO, "Simulation-end")
+    if (isMainProcess()) UBLOG(logINFO, "Simulation-end")
+}
+
+bool Simulation::isMainProcess()
+{
+    return communicator->getProcessID() == 0;
 }
 
 void
@@ -248,8 +258,9 @@ void Simulation::writeBoundaryConditions() const
 void Simulation::writeBlocksToFile() const
 {
     UBLOG(logINFO, "Write block grid to VTK-file")
+    auto scheduler = std::make_shared<UbScheduler>(1);
     auto ppblocks = std::make_shared<WriteBlocksCoProcessor>(grid,
-                                                             std::make_shared<UbScheduler>(1),
+                                                             scheduler,
                                                              writerConfig.outputPath,
                                                              writerConfig.getWriter(),
                                                              communicator);
@@ -258,17 +269,23 @@ void Simulation::writeBlocksToFile() const
 }
 
 std::shared_ptr<GbObject3D>
-Simulation::makeSimulationBoundingBox() const
+Simulation::makeSimulationBoundingBox()
 {
     auto box = gridParameters->boundingBox();
+    auto gridCube = std::make_shared<GbCuboid3D>(box->minX1, box->minX2, box->minX3, box->maxX1, box->maxX2,
+                                                 box->maxX3);
+
+    if (isMainProcess())
+    {
+        UBLOG(logINFO, "Bounding box dimensions = [("
+                << box->minX1 << ", " << box->minX2 << ", " << box->minX3 << "); ("
+                << box->maxX1 << ", " << box->maxX2 << ", " << box->maxX3 << ")]")
 
-    UBLOG(logINFO, "Bounding box dimensions = [("
-            << box->minX1 << ", " << box->minX2 << ", " << box->minX3 << "); ("
-            << box->maxX1 << ", " << box->maxX2 << ", " << box->maxX3 << ")]")
+        GbSystem3D::writeGeoObject(gridCube.get(), writerConfig.outputPath + "/geo/gridCube", writerConfig.getWriter());
+    }
 
-    auto gridCube = std::make_shared<GbCuboid3D>(box->minX1, box->minX2, box->minX3, box->maxX1, box->maxX2, box->maxX3);
-    GbSystem3D::writeGeoObject(gridCube.get(), writerConfig.outputPath + "/geo/gridCube", writerConfig.getWriter());
     return gridCube;
 }
 
+
 Simulation::~Simulation() = default;