implements functionality to identify communication indices

src/GridGenerator/geometries/BoundingBox/BoundingBox.cu

@@ -67,6 +67,11 @@
 	 return false;
  }
 
+ bool BoundingBox::isInside(const real x, const real y, const real z) const
+ {
+     return this->isInside(Vertex(x,y,z));
+ }
+
  bool BoundingBox::isInside(const Vertex &v) const
  {
      if (v.isXbetween(minX, maxX) && v.isYbetween(minY, maxY) && v.isZbetween(minZ, maxZ))
@@ -162,4 +167,15 @@
          && vf::Math::equal(maxZ, box.maxZ);
  }
 
+ void BoundingBox::extend(real delta)
+ {
+     this->minX -= delta;
+     this->minY -= delta;
+     this->minZ -= delta;
+
+     this->maxX += delta;
+     this->maxY += delta;
+     this->maxZ += delta;
+ }
+
 

src/GridGenerator/geometries/BoundingBox/BoundingBox.h

@@ -31,6 +31,7 @@ public:
 	void print() const;
 
 	bool isInside(const Triangle &t) const;
+	bool isInside(const real x, const real y, const real z) const;
 	bool intersect(const Triangle &t) const;
 
 	std::vector<std::vector<Vertex> > getIntersectionPoints(const BoundingBox &b) const;
@@ -38,6 +39,7 @@ public:
 
     HOST bool operator==(const BoundingBox &box) const;
     
+    void extend(real delta);
 
 private:
 	bool isInside(const Vertex &v) const;

src/GridGenerator/grid/Grid.h

@@ -14,6 +14,7 @@ struct Triangle;
 class GridStrategy;
 class GridInterface;
 class Object;
+class BoundingBox;
 
 class VF_PUBLIC Grid
 {
@@ -79,6 +80,8 @@ public:
     HOST virtual void setOddStart( bool xOddStart, bool yOddStart, bool zOddStart ) = 0;
 
     HOST virtual void findGridInterface(SPtr<Grid> grid, LbmOrGks lbmOrGks) = 0;
+
+    HOST virtual void limitToSubDomain(SPtr<BoundingBox> subDomainBox) = 0;
     
     HOST virtual void enableFindSolidBoundaryNodes() = 0;
     HOST virtual void enableComputeQs() = 0;
@@ -119,6 +122,8 @@ public:
     HOST virtual void setInnerRegionFromFinerGrid( bool innerRegionFromFinerGrid ) = 0;
 
     HOST virtual void setNumberOfLayers( uint numberOfLayers ) = 0;
+
+    virtual void findCommunicationIndices(int direction, SPtr<BoundingBox> subDomainBox) = 0;
 };
 
 #endif

src/GridGenerator/grid/GridBuilder/LevelGridBuilder.h

@@ -129,6 +129,8 @@ public:
     VF_PUBLIC void getOffsetFC(real* xOffCf, real* yOffCf, real* zOffCf, int level) override;
     VF_PUBLIC void getOffsetCF(real* xOffFc, real* yOffFc, real* zOffFc, int level) override;
 
+    VF_PUBLIC void getSendReceiveIndices( uint* sendIndices, uint* receiveIndices, int level );
+
 };
 
 #endif

src/GridGenerator/grid/GridBuilder/MultipleGridBuilder.cpp

@@ -14,9 +14,10 @@
 #include "io/GridVTKWriter/GridVTKWriter.h"
 #include <grid/BoundaryConditions/BoundaryCondition.h>
 #include <grid/BoundaryConditions/Side.h>
+#include "geometries/BoundingBox/BoundingBox.h"
 
 MultipleGridBuilder::MultipleGridBuilder(SPtr<GridFactory> gridFactory, Device device, const std::string &d3qxx) :
-    LevelGridBuilder(device, d3qxx), gridFactory(gridFactory), solidObject(nullptr), numberOfLayersFine(12), numberOfLayersBetweenLevels(8)
+    LevelGridBuilder(device, d3qxx), gridFactory(gridFactory), solidObject(nullptr), numberOfLayersFine(12), numberOfLayersBetweenLevels(8), subDomainBox(nullptr)
 {
 
 }
@@ -30,6 +31,13 @@ void MultipleGridBuilder::addCoarseGrid(real startX, real startY, real startZ, r
 {
     boundaryConditions.push_back(SPtr<BoundaryConditions>(new BoundaryConditions));
 
+    startX -= 0.5 * delta;
+    startY -= 0.5 * delta;
+    startZ -= 0.5 * delta;
+    endX   += 0.5 * delta;
+    endY   += 0.5 * delta;
+    endZ   += 0.5 * delta;
+
     const auto grid = this->makeGrid(new Cuboid(startX, startY, startZ, endX, endY, endZ), startX, startY, startZ, endX, endY, endZ, delta, 0);
     addGridToList(grid);
 }
@@ -429,7 +437,7 @@ void MultipleGridBuilder::buildGrids( LbmOrGks lbmOrGks, bool enableThinWalls )
         
         if     ( level == 0 )
             grids[level]->inital( nullptr, 0 );
-        else if( level == 0 || level == grids.size()-1 )
+        else if( level == grids.size()-1 )
             grids[level]->inital( nullptr, this->numberOfLayersFine );
         else
             grids[level]->inital( grids[level+1], this->numberOfLayersBetweenLevels );
@@ -467,6 +475,10 @@ void MultipleGridBuilder::buildGrids( LbmOrGks lbmOrGks, bool enableThinWalls )
     for (size_t i = 0; i < grids.size() - 1; i++)
         grids[i]->findGridInterface(grids[i + 1], lbmOrGks);
 
+    if( this->subDomainBox )
+        for (size_t i = 0; i < grids.size(); i++)
+            grids[i]->limitToSubDomain( this->subDomainBox );
+
 	if (lbmOrGks == LBM) {
 		for (size_t i = 0; i < grids.size() - 1; i++)
 			grids[i]->findSparseIndices(grids[i + 1]);
@@ -493,6 +505,13 @@ void MultipleGridBuilder::emitGridIsNotInCoarseGridWarning()
     *logging::out << logging::Logger::WARNING << "Grid lies not inside of coarse grid. Actual Grid is not added.\n";
 }
 
+void MultipleGridBuilder::findCommunicationIndices(int direction)
+{
+    if( this->subDomainBox )
+        for (size_t i = 0; i < grids.size(); i++)
+            grids[i]->findCommunicationIndices(direction, this->subDomainBox);
+}
+
 void MultipleGridBuilder::writeGridsToVtk(const std::string& path) const
 {
     for(uint level = 0; level < grids.size(); level++)
@@ -507,4 +526,9 @@ void MultipleGridBuilder::writeGridsToVtk(const std::string& path) const
         //    GridVTKWriter::writeInterpolationCellsToVTKXML(grids[level], nullptr       , ss.str() + ".InterpolationCells");
         //GridVTKWriter::writeSparseGridToVTK(grids[level], ss.str());
     }
-}
\ No newline at end of file
+}
+
+VF_PUBLIC void MultipleGridBuilder::setSubDomainBox(SPtr<BoundingBox> subDomainBox)
+{
+    this->subDomainBox = subDomainBox;
+}

src/GridGenerator/grid/GridBuilder/MultipleGridBuilder.h

@@ -13,7 +13,7 @@
 #include "../GridFactory.h"
 
 class Object;
-
+class BoundingBox;
 
 class MultipleGridBuilder : public LevelGridBuilder
 {
@@ -48,6 +48,8 @@ public:
 
     VF_PUBLIC void writeGridsToVtk(const std::string& path) const;
 
+    VF_PUBLIC void setSubDomainBox(SPtr<BoundingBox> subDomainBox);
+
 private:
     void addGridToList(SPtr<Grid> grid);
     real calculateDelta(uint level) const;
@@ -77,6 +79,12 @@ private:
 
     uint numberOfLayersFine;
     uint numberOfLayersBetweenLevels;
+
+    SPtr<BoundingBox> subDomainBox;
+
+public:
+
+    VF_PUBLIC void findCommunicationIndices( int direction );
 };
 
 #endif

src/GridGenerator/grid/GridImp.cu

@@ -975,6 +975,22 @@ HOST void GridImp::findGridInterface(SPtr<Grid> finerGrid, LbmOrGks lbmOrGks)
     gridStrategy->findGridInterface(shared_from_this(), std::static_pointer_cast<GridImp>(finerGrid), lbmOrGks);
 }
 
+HOST void GridImp::limitToSubDomain(SPtr<BoundingBox> subDomainBox)
+{
+    for( uint index = 0; index < this->size; index++ ){
+
+        real x, y, z;
+        this->transIndexToCoords( index, x, y, z );
+
+        BoundingBox tmpSubDomainBox = *subDomainBox;
+
+        tmpSubDomainBox.extend(this->delta);
+
+        if( !tmpSubDomainBox.isInside(x,y,z) )
+            this->setFieldEntry(index, INVALID_OUT_OF_GRID);
+    }
+}
+
 HOSTDEVICE void GridImp::findGridInterfaceCF(uint index, GridImp& finerGrid, LbmOrGks lbmOrGks)
 {
 	if (lbmOrGks == LBM)
@@ -1318,6 +1334,50 @@ HOSTDEVICE bool GridImp::checkIfAtLeastOneValidQ(const uint index, const Vertex
     return false;
 }
 
+void GridImp::findCommunicationIndices(int direction, SPtr<BoundingBox> subDomainBox)
+{
+    for( uint index = 0; index < this->size; index++ ){
+        
+        real x, y, z;
+        this->transIndexToCoords(index, x, y, z);
+    
+        if( this->getFieldEntry(index) == INVALID_OUT_OF_GRID ||
+            this->getFieldEntry(index) == INVALID_SOLID ||
+            this->getFieldEntry(index) == INVALID_COARSE_UNDER_FINE ) continue;
+
+        if( direction == CommunicationDirections::MX ) findCommunicationIndex( index, x, subDomainBox->minX, direction);
+        if( direction == CommunicationDirections::PX ) findCommunicationIndex( index, x, subDomainBox->maxX, direction);
+        if( direction == CommunicationDirections::MY ) findCommunicationIndex( index, y, subDomainBox->minY, direction);
+        if( direction == CommunicationDirections::PY ) findCommunicationIndex( index, y, subDomainBox->maxY, direction);
+        if( direction == CommunicationDirections::MZ ) findCommunicationIndex( index, z, subDomainBox->minZ, direction);
+        if( direction == CommunicationDirections::PZ ) findCommunicationIndex( index, z, subDomainBox->maxZ, direction);
+    }
+}
+
+void GridImp::findCommunicationIndex( uint index, real coordinate, real limit, int direction ){
+        
+    // negative direction get a negative sign
+    real s = ( direction % 2 == 0 ) ? ( -1.0 ) : ( 1.0 );
+
+    //if( vf::Math::equal( coordinate, limit + s * 0.5 * this->delta ) ){
+    //    this->communicationIndices[direction].receiveIndices.push_back(index);
+    //    this->setFieldEntry(index, MULTI_GPU_RECIEVE);
+    //}
+    //if( vf::Math::equal( coordinate, limit - s * 0.5 * this->delta ) ){
+    //    this->communicationIndices[direction].sendIndices.push_back(index);
+    //    this->setFieldEntry(index, MULTI_GPU_SEND);
+    //}    
+
+    if( vf::Math::equal( coordinate, limit + s * 0.5 * this->delta, 0.01 * this->delta ) ){
+        this->communicationIndices[direction].receiveIndices.push_back(index);
+        this->setFieldEntry(index, MULTI_GPU_RECIEVE);
+    }
+    if( vf::Math::equal( coordinate, limit - s * 0.5 * this->delta, 0.01 * this->delta ) ){
+        this->communicationIndices[direction].sendIndices.push_back(index);
+        this->setFieldEntry(index, MULTI_GPU_SEND);
+    }  
+} 
+
 
 // --------------------------------------------------------- //
 //                        Getter                             //

src/GridGenerator/grid/GridImp.h

 #ifndef GRID_IMP_H
 #define GRID_IMP_H
 
+#include <array>
+
 #include "GridGenerator/global.h"
 #include "distributions/Distribution.h"
 
@@ -22,6 +24,17 @@ class TriangularMeshDiscretizationStrategy;
 
 extern CONSTANT int DIRECTIONS[DIR_END_MAX][DIMENSION];
 
+namespace CommunicationDirections {
+    enum {
+        MX = 0,
+        PX = 1,
+        MY = 2,
+        PY = 3,
+        MZ = 4,
+        PZ = 5
+    };
+}
+
 class VF_PUBLIC GridImp : public enableSharedFromThis<GridImp>, public Grid
 {
 private:
@@ -105,6 +118,9 @@ public:
     HOSTDEVICE void transIndexToCoords(uint index, real &x, real &y, real &z) const override;
 
     HOST virtual void findGridInterface(SPtr<Grid> grid, LbmOrGks lbmOrGks) override;
+
+    HOST virtual void limitToSubDomain(SPtr<BoundingBox> subDomainBox) override;
+
     HOST void freeMemory() override;
 
     HOST uint getLevel(real levelNull) const;
@@ -264,10 +280,20 @@ private:
 	HOSTDEVICE void calculateQs(const uint index, const Vertex &point, const Triangle &triangle) const;
     HOSTDEVICE bool checkIfAtLeastOneValidQ(const uint index, const Vertex &point, const Triangle &triangle) const;
 
+public:
 
+    void findCommunicationIndices(int direction, SPtr<BoundingBox> subDomainBox);
+    void findCommunicationIndex( uint index, real coordinate, real limit, int direction );
 
 private:
-    //HOSTDEVICE bool isNeighborInside(const int &index) const;
+
+    struct CommunicationIndices
+    {
+        std::vector<uint> sendIndices;
+        std::vector<uint> receiveIndices;
+    };
+
+    std::array<CommunicationIndices, 6> communicationIndices;
 
 
 private:

src/GridGenerator/grid/GridMocks.h

@@ -62,6 +62,8 @@ public:
 
     virtual void findGridInterface(SPtr<Grid> grid, LbmOrGks lbmOrGks) override {}
     
+    virtual void limitToSubDomain(SPtr<BoundingBox> subDomainBox) override {}
+
     virtual void enableFindSolidBoundaryNodes() override {}
     virtual void enableComputeQs() override {}
 
@@ -98,9 +100,12 @@ public:
 	real getQValue(const uint index, const uint dir) const override { return 0.0; }
 	uint getQPatch(const uint index) const override { return 0.0; }
 
-    void setInnerRegionFromFinerGrid( bool innerRegionFromFinerGrid ) override {};
+    void setInnerRegionFromFinerGrid( bool innerRegionFromFinerGrid ) override {}
+
+    void setNumberOfLayers( uint numberOfLayers ) override {}
+
+    void findCommunicationIndices(int direction, SPtr<BoundingBox> subDomainBox) override {}
 
-    void setNumberOfLayers( uint numberOfLayers ) override {};
 };
 
 class GridStub : public GridDummy

src/GridGenerator/grid/NodeValues.h

@@ -9,6 +9,9 @@
 #define FLUID_FCC 3
 #define FLUID_FCF 4
 
+#define MULTI_GPU_SEND 10
+#define MULTI_GPU_RECIEVE 11
+
 #define BC_PRESSURE 20
 #define BC_VELOCITY 21
 #define BC_SOLID 22

targets/apps/HULC/main.cpp

@@ -278,7 +278,7 @@ void multipleLevel(const std::string& configPath)
     ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
     ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-    bool useGridGenerator = false;
+    bool useGridGenerator = true;
 
     if(useGridGenerator){
 
@@ -321,33 +321,43 @@ void multipleLevel(const std::string& configPath)
         // DLC - Golf
         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-        real dx = 0.2;
-        real vx = 0.05;
+        //real dx = 0.2;
+        //real vx = 0.05;
 
-        real z0 = 0.265+0.5*dx;
+        //real z0 = 0.265+0.5*dx;
 
-        std::vector<uint> ignorePatches = { 152, 153, 154 };
+        //std::vector<uint> ignorePatches = { 152, 153, 154 };
 
-        TriangularMesh* VW370_SERIE_STL = TriangularMesh::make("C:/Users/lenz/Desktop/Work/gridGenerator/stl/VW370_SERIE.stl", ignorePatches);
-        
-        TriangularMesh* DrivAerRefBoxSTL = TriangularMesh::make("C:/Users/lenz/Desktop/Work/gridGenerator/stl/DrivAer_REF_BOX_Adrea.stl");
+        //TriangularMesh* VW370_SERIE_STL = TriangularMesh::make("C:/Users/lenz/Desktop/Work/gridGenerator/stl/VW370_SERIE.stl", ignorePatches);
+        //
+        ////TriangularMesh* DrivAerRefBoxSTL = TriangularMesh::make("C:/Users/lenz/Desktop/Work/gridGenerator/stl/DrivAer_REF_BOX_Adrea.stl");
 
-        TriangularMesh* DLC_RefBox = TriangularMesh::make("C:/Users/lenz/Desktop/Work/gridGenerator/stl/DLC_RefBox.stl");
+        //TriangularMesh* DLC_RefBox = TriangularMesh::make("C:/Users/lenz/Desktop/Work/gridGenerator/stl/DLC_RefBox.stl");
 
-        gridBuilder->addCoarseGrid(- 5.0, -5.0, 0.0 - z0,
-                                    15.0,  5.0, 5.0 - z0, dx);
+        //TriangularMesh* DLC_RefBox_1 = TriangularMesh::make("C:/Users/lenz/Desktop/Work/gridGenerator/stl/DLC_RefBox_withWake/DLC_RefBox_withWake_4m.stl");
+        //TriangularMesh* DLC_RefBox_2 = TriangularMesh::make("C:/Users/lenz/Desktop/Work/gridGenerator/stl/DLC_RefBox_withWake/DLC_RefBox_withWake_3m.stl");
+        //TriangularMesh* DLC_RefBox_3 = TriangularMesh::make("C:/Users/lenz/Desktop/Work/gridGenerator/stl/DLC_RefBox_withWake/DLC_RefBox_withWake_2m.stl");
+        //TriangularMesh* DLC_RefBox_4 = TriangularMesh::make("C:/Users/lenz/Desktop/Work/gridGenerator/stl/DLC_RefBox_withWake/DLC_RefBox_withWake_1m.stl");
 
-        gridBuilder->setNumberOfLayers(10,8);
-        gridBuilder->addGrid(DrivAerRefBoxSTL, 4);
-        
-        Conglomerate* refinement = new Conglomerate();
-        refinement->add(DLC_RefBox);
-        refinement->add(VW370_SERIE_STL);
+        //gridBuilder->addCoarseGrid(- 5.0, -5.0, 0.0 - z0,
+        //                            15.0,  5.0, 5.0 - z0, dx);
 
-        gridBuilder->setNumberOfLayers(10,8);
-        gridBuilder->addGrid(refinement, 5);
+        //gridBuilder->setNumberOfLayers(10,8);
+        ////gridBuilder->addGrid(DLC_RefBox_1, 1);
+        ////gridBuilder->addGrid(DLC_RefBox_2, 2);
+        ////gridBuilder->addGrid(DLC_RefBox_3, 3);
+        ////gridBuilder->addGrid(DLC_RefBox_4, 4);
 
-        gridBuilder->addGeometry(VW370_SERIE_STL);
+        //gridBuilder->addGrid(DLC_RefBox_4, 4);
+        //
+        //Conglomerate* refinement = new Conglomerate();
+        //refinement->add(DLC_RefBox);
+        //refinement->add(VW370_SERIE_STL);
+
+        //gridBuilder->setNumberOfLayers(10,8);
+        //gridBuilder->addGrid(refinement, 5);
+
+        //gridBuilder->addGeometry(VW370_SERIE_STL);
 
         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
         // Wall Mounted Cube
@@ -467,6 +477,58 @@ void multipleLevel(const std::string& configPath)
 
         //gridBuilder->addGrid(new Cuboid( -0.2, -0.6, -0.2, 0.2, 0.6, 0.2 ), 1);
 
+        //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+        // Testing Multi GPU grids
+        //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+        const uint generatePart = 2;
+
+        real dx = 1.0 / 10.0;
+        real vx = 0.01;
+
+        TriangularMesh* triangularMesh = TriangularMesh::make("C:/Users/lenz/Desktop/Work/gridGenerator/stl/ShpereNotOptimal.stl");
+
+        // all
+        //gridBuilder->addCoarseGrid(-2, -2, -2,  
+        //                            4,  2,  2, dx);
+
+        real overlap = 10.0 * dx;
+
+        if( generatePart == 1 )
+            gridBuilder->addCoarseGrid(-2.0          , -2.0, -2.0,  
+                                        0.5 + overlap,  2.0,  2.0, dx);
+
+        if( generatePart == 2 )
+            gridBuilder->addCoarseGrid( 0.5 - overlap, -2.0, -2.0,  
+                                        4.0          ,  2.0,  2.0, dx);
+
+
+        gridBuilder->setNumberOfLayers(10,8);
+        gridBuilder->addGrid(triangularMesh, 1);
+
+        gridBuilder->addGeometry(triangularMesh);
+        
+        if( generatePart == 1 )
+            gridBuilder->setSubDomainBox( std::make_shared<BoundingBox>( -2.0, 0.5, 
+                                                                         -2.0, 2.0, 
+                                                                         -2.0, 2.0 ) );
+        
+        if( generatePart == 2 )
+            gridBuilder->setSubDomainBox( std::make_shared<BoundingBox>(  0.5, 4.0, 
+                                                                         -2.0, 2.0, 
+                                                                         -2.0, 2.0 ) );
+
+
+        gridBuilder->setPeriodicBoundaryCondition(false, false, false);
+
+        gridBuilder->buildGrids(LBM, true); // buildGrids() has to be called before setting the BCs!!!!
+        
+        if( generatePart == 1 )
+            gridBuilder->findCommunicationIndices(CommunicationDirections::PX);
+        
+        if( generatePart == 2 )
+            gridBuilder->findCommunicationIndices(CommunicationDirections::MX);
+
         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
         // other tests
         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -488,12 +550,6 @@ void multipleLevel(const std::string& configPath)
         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
 
-        gridBuilder->setPeriodicBoundaryCondition(false, false, false);
-        //gridBuilder->setPeriodicBoundaryCondition(true, true, true);
-
-        gridBuilder->buildGrids(LBM, true); // buildGrids() has to be called before setting the BCs!!!!
-
-
         ///////////////////////////////////////////////////////////////////////////
         //BCs
         //gridBuilder->setVelocityBoundaryCondition(SideType::PX, 0.0, 0.0, 0.0);
@@ -503,8 +559,19 @@ void multipleLevel(const std::string& configPath)
         gridBuilder->setVelocityBoundaryCondition(SideType::PZ, vx , 0.0, 0.0);
         gridBuilder->setVelocityBoundaryCondition(SideType::MZ, vx , 0.0, 0.0);
 
-        gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0);
-        gridBuilder->setVelocityBoundaryCondition(SideType::MX, vx, 0.0, 0.0);
+        //gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0);
+        //gridBuilder->setVelocityBoundaryCondition(SideType::MX, vx, 0.0, 0.0);
+
+        gridBuilder->setVelocityBoundaryCondition(SideType::GEOMETRY, 0.0, 0.0, 0.0);
+
+        if (generatePart == 1) {
+            gridBuilder->setVelocityBoundaryCondition(SideType::MX, vx, 0.0, 0.0);
+            //gridBuilder->setDomainDecompositionBoundaryCondition(SideType::PX);
+        }
+        if (generatePart == 2) {
+            gridBuilder->setPressureBoundaryCondition(SideType::PX, 0.0);
+            //gridBuilder->setDomainDecompositionBoundaryCondition(SideType::MX);
+        }
 
         ////////////////////////////////////////////////////////////////////////////
 
@@ -513,9 +580,6 @@ void multipleLevel(const std::string& configPath)
      //   gridBuilder->setNoSlipBoundaryCondition(SideType::MZ);
      //   gridBuilder->setNoSlipBoundaryCondition(SideType::PZ);
 
-
-        gridBuilder->setVelocityBoundaryCondition(SideType::GEOMETRY, 0.0, 0.0, 0.0);
-
         {
             //gridBuilder->getGeometryBoundaryCondition(gridBuilder->getNumberOfLevels() - 1)->setVelocityForPatch(0, vx, 0.0, 0.0);
 
@@ -531,26 +595,26 @@ void multipleLevel(const std::string& configPath)
             //gridBuilder->getGeometryBoundaryCondition(gridBuilder->getNumberOfLevels() - 1)->setTangentialVelocityForPatch( grid, 3, 2.793 , -2.0, 0.0,
             //                                                                                                                         2.793 ,  2.0, 0.0, -vx, 0.318);
 
-            real wheelsFrontX = -0.081;
-            real wheelsRearX  =  2.5485;
+            //real wheelsFrontX = -0.081;
+            //real wheelsRearX  =  2.5485;
 
-            real wheelsFrontZ =  0.0515;
-            real wheelsRearZ  =  0.0585;
+            //real wheelsFrontZ =  0.0515;
+            //real wheelsRearZ  =  0.0585;
 
-            real wheelsRadius =  0.318;
+            //real wheelsRadius =  0.318;
 
-            std::vector<uint> frontWheelPatches = { 71, 86, 87,  88,  89,  90,  91,  92,  93,  94,  95,  96,  97, 159 };
-            std::vector<uint> rearWheelPatches  = { 82, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 160 };
+            //std::vector<uint> frontWheelPatches = { 71, 86, 87,  88,  89,  90,  91,  92,  93,  94,  95,  96,  97, 159 };
+            //std::vector<uint> rearWheelPatches  = { 82, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 160 };
 
-            for( uint patch : frontWheelPatches ){
-                gridBuilder->getGeometryBoundaryCondition(gridBuilder->getNumberOfLevels() - 1)->setTangentialVelocityForPatch( grid, patch, wheelsFrontX, -2.0, wheelsFrontZ,
-                                                                                                                                             wheelsFrontX,  2.0, wheelsFrontZ, -vx, wheelsRadius);
-            }
+            //for( uint patch : frontWheelPatches ){
+            //    gridBuilder->getGeometryBoundaryCondition(gridBuilder->getNumberOfLevels() - 1)->setTangentialVelocityForPatch( grid, patch, wheelsFrontX, -2.0, wheelsFrontZ,
+            //                                                                                                                                 wheelsFrontX,  2.0, wheelsFrontZ, -vx, wheelsRadius);
+            //}
 
-            for( uint patch : rearWheelPatches ){
-                gridBuilder->getGeometryBoundaryCondition(gridBuilder->getNumberOfLevels() - 1)->setTangentialVelocityForPatch( grid, patch, wheelsRearX , -2.0, wheelsRearZ ,
-                                                                                                                                             wheelsRearX ,  2.0, wheelsRearZ , -vx, wheelsRadius);
-            }
+            //for( uint patch : rearWheelPatches ){
+            //    gridBuilder->getGeometryBoundaryCondition(gridBuilder->getNumberOfLevels() - 1)->setTangentialVelocityForPatch( grid, patch, wheelsRearX , -2.0, wheelsRearZ ,
+            //                                                                                                                                 wheelsRearX ,  2.0, wheelsRearZ , -vx, wheelsRadius);
+            //}
 
         }
 
@@ -563,12 +627,22 @@ void multipleLevel(const std::string& configPath)
         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+        
+
+        if (generatePart == 1) {
+            gridBuilder->writeGridsToVtk("C:/Users/lenz/Desktop/Work/gridGenerator/gridMultiGPU_Part_1/Test_");
+            gridBuilder->writeArrows    ("C:/Users/lenz/Desktop/Work/gridGenerator/gridMultiGPU_Part_1/Test_Arrow");
+        }
+        if (generatePart == 2) {
+            gridBuilder->writeGridsToVtk("C:/Users/lenz/Desktop/Work/gridGenerator/gridMultiGPU_Part_2/Test_");
+            gridBuilder->writeArrows    ("C:/Users/lenz/Desktop/Work/gridGenerator/gridMultiGPU_Part_2/Test_Arrow");
+        }
 
         //gridBuilder->writeGridsToVtk("M:/TestGridGeneration/results/ConcaveTest_");
-        gridBuilder->writeGridsToVtk("C:/Users/lenz/Desktop/Work/gridGenerator/grid/Test_");
+        //gridBuilder->writeGridsToVtk("C:/Users/lenz/Desktop/Work/gridGenerator/gridMultiGPU_Part_2/Test_");
 
         //gridBuilder->writeGridsToVtk("M:/TestGridGeneration/results/CylinderTest_");
-        gridBuilder->writeArrows("C:/Users/lenz/Desktop/Work/gridGenerator/grid/Test_Arrow");
+        //gridBuilder->writeArrows("C:/Users/lenz/Desktop/Work/gridGenerator/gridMultiGPU_Part_2/Test_Arrow");
 
         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -665,11 +739,11 @@ void multipleLevel(const std::string& configPath)
         //}
 
         //SimulationFileWriter::write("D:/GRIDGENERATION/files/", gridBuilder, FILEFORMAT::ASCII);
-        SimulationFileWriter::write("C:/Users/lenz/Desktop/Work/gridGenerator/grid/", gridBuilder, FILEFORMAT::ASCII);
+        //SimulationFileWriter::write("C:/Users/lenz/Desktop/Work/gridGenerator/grid/", gridBuilder, FILEFORMAT::ASCII);
 
-        gridGenerator = GridGenerator::make(gridBuilder, para);
+        //gridGenerator = GridGenerator::make(gridBuilder, para);
 
-        //return;
+        return;
     }
     else
     {