From e64482eaeeef8f46f79fc86e8de20bef9209a7b4 Mon Sep 17 00:00:00 2001
From: Anna Wellmann <a.wellmann@tu-bs.de>
Date: Wed, 6 Apr 2022 15:58:19 +0200
Subject: [PATCH] Write send nodes to file

for debugging
---
 .../configPhoenix1GPU.txt                     |   4 +-
 .../configPhoenix4GPU.txt                     |   4 +-
 .../VirtualFluids_GPU/Output/FileWriter.cpp   | 719 +++++++++---------
 src/gpu/VirtualFluids_GPU/Output/FileWriter.h |   1 +
 4 files changed, 371 insertions(+), 357 deletions(-)

diff --git a/apps/gpu/LBM/DrivenCavityMultiGPU/configPhoenix1GPU.txt b/apps/gpu/LBM/DrivenCavityMultiGPU/configPhoenix1GPU.txt
index 925ee2d80..e63db13b5 100644
--- a/apps/gpu/LBM/DrivenCavityMultiGPU/configPhoenix1GPU.txt
+++ b/apps/gpu/LBM/DrivenCavityMultiGPU/configPhoenix1GPU.txt
@@ -30,6 +30,6 @@ GridPath=/work/y0078217/Grids/GridDrivenCavityMultiGPU/1GPU/
 ##################################################
 #simulation parameter
 ##################################################
-TimeEnd=400000
-TimeOut=200000
+TimeEnd=1000
+TimeOut=1000
 #TimeStartOut=0
\ No newline at end of file
diff --git a/apps/gpu/LBM/DrivenCavityMultiGPU/configPhoenix4GPU.txt b/apps/gpu/LBM/DrivenCavityMultiGPU/configPhoenix4GPU.txt
index 1b6a057f4..c710922b9 100644
--- a/apps/gpu/LBM/DrivenCavityMultiGPU/configPhoenix4GPU.txt
+++ b/apps/gpu/LBM/DrivenCavityMultiGPU/configPhoenix4GPU.txt
@@ -30,8 +30,8 @@ GridPath=/work/y0078217/Grids/GridDrivenCavityMultiGPU/4GPU/
 ##################################################
 #simulation parameter
 ##################################################
-TimeEnd=1000
-TimeOut=100
+TimeEnd=1
+TimeOut=1
 #TimeStartOut=0
 
 ##################################################
diff --git a/src/gpu/VirtualFluids_GPU/Output/FileWriter.cpp b/src/gpu/VirtualFluids_GPU/Output/FileWriter.cpp
index 2006ba6a7..9f215e9ad 100644
--- a/src/gpu/VirtualFluids_GPU/Output/FileWriter.cpp
+++ b/src/gpu/VirtualFluids_GPU/Output/FileWriter.cpp
@@ -25,10 +25,10 @@
 void FileWriter::writeInit(std::shared_ptr<Parameter> para, std::shared_ptr<CudaMemoryManager> cudaManager)
 {
     unsigned int timestep = para->getTInit();
-	for (int level = para->getCoarse(); level <= para->getFine(); level++) {
-		cudaManager->cudaCopyPrint(level);
-		writeTimestep(para, timestep, level);
-	}
+    for (int level = para->getCoarse(); level <= para->getFine(); level++) {
+        cudaManager->cudaCopyPrint(level);
+        writeTimestep(para, timestep, level);
+    }
 
     this->writeCollectionFile(para, timestep);
 
@@ -51,35 +51,35 @@ void FileWriter::writeTimestep(std::shared_ptr<Parameter> para, unsigned int tim
 {
     const unsigned int numberOfParts = para->getParH(level)->size_Mat_SP / para->getlimitOfNodesForVTK() + 1;
     std::vector<std::string> fname;
-	std::vector<std::string> fnameMed;
-	for (unsigned int i = 1; i <= numberOfParts; i++)
-	{
-		fname.push_back(para->getFName() + "_bin_lev_" + StringUtil::toString<int>(level) + "_ID_" + StringUtil::toString<int>(para->getMyID()) + "_Part_" + StringUtil::toString<int>(i) + "_t_" + StringUtil::toString<int>(timestep) + ".vtk");
-		fnameMed.push_back(para->getFName() + "_bin_median_lev_" + StringUtil::toString<int>(level) + "_ID_" + StringUtil::toString<int>(para->getMyID()) + "_Part_" + StringUtil::toString<int>(i) + "_t_" + StringUtil::toString<int>(timestep) + ".vtk");
+    std::vector<std::string> fnameMed;
+    for (unsigned int i = 1; i <= numberOfParts; i++)
+    {
+        fname.push_back(para->getFName() + "_bin_lev_" + StringUtil::toString<int>(level) + "_ID_" + StringUtil::toString<int>(para->getMyID()) + "_Part_" + StringUtil::toString<int>(i) + "_t_" + StringUtil::toString<int>(timestep) + ".vtk");
+        fnameMed.push_back(para->getFName() + "_bin_median_lev_" + StringUtil::toString<int>(level) + "_ID_" + StringUtil::toString<int>(para->getMyID()) + "_Part_" + StringUtil::toString<int>(i) + "_t_" + StringUtil::toString<int>(timestep) + ".vtk");
 
         this->fileNamesForCollectionFile.push_back( fname.back() );
         this->fileNamesForCollectionFileMedian.push_back( fnameMed.back() );
-	}
-
-	if (para->getDiffOn() == true)
-		writeUnstrucuredGridLTConc(para, level, fname);
-	else
-		writeUnstrucuredGridLT(para, level, fname);
-
-	if (para->getCalcMedian())
-	{
-		if (para->getDiffOn() == true)
-			writeUnstrucuredGridMedianLTConc(para, level, fnameMed);
-		else
-			writeUnstrucuredGridMedianLT(para, level, fnameMed);
-	}
+    }
+
+    if (para->getDiffOn() == true)
+        writeUnstrucuredGridLTConc(para, level, fname);
+    else
+        writeUnstrucuredGridLT(para, level, fname);
+
+    if (para->getCalcMedian())
+    {
+        if (para->getDiffOn() == true)
+            writeUnstrucuredGridMedianLTConc(para, level, fnameMed);
+        else
+            writeUnstrucuredGridMedianLT(para, level, fnameMed);
+    }
 }
 
 bool FileWriter::isPeriodicCell(std::shared_ptr<Parameter> para, int level, unsigned int number2, unsigned int number1, unsigned int number3, unsigned int number5)
 {
-	return (para->getParH(level)->coordX_SP[number2] < para->getParH(level)->coordX_SP[number1]) ||
-		   (para->getParH(level)->coordY_SP[number3] < para->getParH(level)->coordY_SP[number1]) ||
-		   (para->getParH(level)->coordZ_SP[number5] < para->getParH(level)->coordZ_SP[number1]);
+    return (para->getParH(level)->coordX_SP[number2] < para->getParH(level)->coordX_SP[number1]) ||
+           (para->getParH(level)->coordY_SP[number3] < para->getParH(level)->coordY_SP[number1]) ||
+           (para->getParH(level)->coordZ_SP[number5] < para->getParH(level)->coordZ_SP[number1]);
 }
 
 void VIRTUALFLUIDS_GPU_EXPORT FileWriter::writeCollectionFile(std::shared_ptr<Parameter> para, unsigned int timestep)
@@ -181,7 +181,9 @@ void FileWriter::writeUnstrucuredGridLT(std::shared_ptr<Parameter> para, int lev
     nodedatanames.push_back("vx2");
     nodedatanames.push_back("vx3");
     nodedatanames.push_back("geo");
-    //nodedatanames.push_back("sendNodes");
+    nodedatanames.push_back("sendNodesX");
+    nodedatanames.push_back("sendNodesY");
+    nodedatanames.push_back("sendNodesZ");
     //nodedatanames.push_back("sparseIndex");
 
     uint firstTurbNode = (uint) nodedatanames.size();
@@ -192,7 +194,7 @@ void FileWriter::writeUnstrucuredGridLT(std::shared_ptr<Parameter> para, int lev
         nodedatanames.push_back("vxy");
         nodedatanames.push_back("vxz");
         nodedatanames.push_back("vyz");
-	}
+    }
     unsigned int number1, number2, number3, number4, number5, number6, number7, number8;
     uint dn1, dn2, dn3, dn4, dn5, dn6, dn7, dn8;
     bool neighborsAreFluid;
@@ -215,7 +217,7 @@ void FileWriter::writeUnstrucuredGridLT(std::shared_ptr<Parameter> para, int lev
         cells.clear();
         nodes.resize(sizeOfNodes);
         for (uint i = 0; i < (uint)nodedatanames.size(); i++)
-			nodedata[i].resize(sizeOfNodes);
+            nodedata[i].resize(sizeOfNodes);
 
         //////////////////////////////////////////////////////////////////////////
         for (unsigned int pos = startpos; pos < endpos; pos++)
@@ -239,11 +241,18 @@ void FileWriter::writeUnstrucuredGridLT(std::shared_ptr<Parameter> para, int lev
                 nodedata[4][dn1] = (double)para->getParH(level)->vz_SP[pos] * (double)para->getVelocityRatio();
                 nodedata[5][dn1] = (double)para->getParH(level)->geoSP[pos];
 
-                //nodedata[6][dn1] = (double) pos;
+                // nodedata[6][dn1] = (double) pos;
+                int sendNodeX = 0; // 0 - not a sendNode; 1 - sendNode; 2 - sendNode in communication after fine to coarse
+                testForSendNodeX(para, level, pos, sendNodeX); // slow and should not be done multiple times --> use for debugging only!
+                nodedata[6][dn1] = (double) sendNodeX;
+
+                int sendNodeY = 0; // 0 - not a sendNode; 1 - sendNode; 2 - sendNode in communication after fine to coarse
+                testForSendNodeY(para, level, pos, sendNodeY); // slow and should not be done multiple times --> use for debugging only!
+                nodedata[7][dn1] = (double) sendNodeY;
 
-				//int sendNode = 0; // 0 - not a sendNode; 1 - sendNode; 2 - sendNode in communication after fine to coarse
-    //            testForSendNodeZ(para, level, pos, sendNode); // slow and should not be done multiple times --> use for debugging only!
-				//nodedata[6][dn1] = (double) sendNode;
+                int sendNodeZ = 0; // 0 - not a sendNode; 1 - sendNode; 2 - sendNode in communication after fine to coarse
+                testForSendNodeZ(para, level, pos, sendNodeZ); // slow and should not be done multiple times --> use for debugging only!
+                nodedata[8][dn1] = (double) sendNodeZ;
 
                 if (para->getCalcTurbulenceIntensity()) {
                     nodedata[firstTurbNode    ][dn1] = (double)para->getParH(level)->vxx[pos];
@@ -286,10 +295,10 @@ void FileWriter::writeUnstrucuredGridLT(std::shared_ptr<Parameter> para, int lev
                 dn6 = number6 - startpos;
                 dn7 = number7 - startpos;
                 dn8 = number8 - startpos;
-				//////////////////////////////////////////////////////////////////////////
-				if (isPeriodicCell(para, level, number2, number1, number3, number5))
-					continue;
-				//////////////////////////////////////////////////////////////////////////
+                //////////////////////////////////////////////////////////////////////////
+                if (isPeriodicCell(para, level, number2, number1, number3, number5))
+                    continue;
+                //////////////////////////////////////////////////////////////////////////
                 if (neighborsAreFluid)
                     cells.push_back(makeUbTuple(dn1, dn2, dn3, dn4, dn5, dn6, dn7, dn8));
             }
@@ -298,6 +307,12 @@ void FileWriter::writeUnstrucuredGridLT(std::shared_ptr<Parameter> para, int lev
     }
 }
 
+void FileWriter::testForSendNodeX(std::shared_ptr<Parameter> &para, int level, unsigned int pos, int &sendNode)
+{
+    testForCommunicationNode(para, level, pos, sendNode, &para->getParH(level)->sendProcessNeighborX,
+                        &para->getParH(level)->sendProcessNeighborsAfterFtoCX);
+}
+
 void FileWriter::testForSendNodeY(std::shared_ptr<Parameter> &para, int level, unsigned int pos, int &sendNode)
 {
     testForCommunicationNode(para, level, pos, sendNode, &para->getParH(level)->sendProcessNeighborY,
@@ -318,10 +333,8 @@ void FileWriter::testForCommunicationNode(std::shared_ptr<Parameter> &para, int
         for (int i = 0; i < (*sendOrRecvProcessNeighbor)[direction].numberOfNodes; i++) {
             if (pos == (uint)(*sendOrRecvProcessNeighbor)[direction].index[i]) {
                 sendNode = 1;
-				//std::cout << "send/recv Node found: " << pos << std::endl;
                 if (level < para->getMaxLevel() && i < (*sendOrRecvProcessNeighborAfterFtoC)[direction].numberOfNodes) {
-                    //std::cout << "Communication send/recv Node found: " << pos << std::endl;
-					sendNode = 2;
+                    sendNode = 2;
                 }
                 return;
             }
@@ -331,327 +344,327 @@ void FileWriter::testForCommunicationNode(std::shared_ptr<Parameter> &para, int
 
 void FileWriter::writeUnstrucuredGridLTConc(std::shared_ptr<Parameter> para, int level, std::vector<std::string >& fname)
 {
-	std::vector< UbTupleFloat3 > nodes;
-	std::vector< UbTupleUInt8 > cells;
-	std::vector< std::string > nodedatanames;
-	nodedatanames.push_back("press");
-	nodedatanames.push_back("rho");
-	nodedatanames.push_back("vx1");
-	nodedatanames.push_back("vx2");
-	nodedatanames.push_back("vx3");
-	nodedatanames.push_back("geo");
-	nodedatanames.push_back("conc");
-	unsigned int number1, number2, number3, number4, number5, number6, number7, number8;
-	uint dn1, dn2, dn3, dn4, dn5, dn6, dn7, dn8;
-	bool neighborsAreFluid;
-	unsigned int startpos = 0;
-	unsigned int endpos = 0;
-	unsigned int sizeOfNodes = 0;
-	std::vector< std::vector< double > > nodedata(nodedatanames.size());
-
-	for (unsigned int part = 0; part < fname.size(); part++)
-	{
-		if (((part + 1)*para->getlimitOfNodesForVTK()) > para->getParH(level)->size_Mat_SP)
-			sizeOfNodes = para->getParH(level)->size_Mat_SP - (part * para->getlimitOfNodesForVTK());
-		else
-			sizeOfNodes = para->getlimitOfNodesForVTK();
-
-		//////////////////////////////////////////////////////////////////////////
-		startpos = part * para->getlimitOfNodesForVTK();
-		endpos = startpos + sizeOfNodes;
-		//////////////////////////////////////////////////////////////////////////
-		cells.clear();
-		nodes.resize(sizeOfNodes);
-		nodedata[0].resize(sizeOfNodes);
-		nodedata[1].resize(sizeOfNodes);
-		nodedata[2].resize(sizeOfNodes);
-		nodedata[3].resize(sizeOfNodes);
-		nodedata[4].resize(sizeOfNodes);
-		nodedata[5].resize(sizeOfNodes);
-		nodedata[6].resize(sizeOfNodes);
-		//////////////////////////////////////////////////////////////////////////
-		for (unsigned int pos = startpos; pos < endpos; pos++)
-		{
-			if (para->getParH(level)->geoSP[pos] == GEO_FLUID)
-			{
-				//////////////////////////////////////////////////////////////////////////
-				double x1 = para->getParH(level)->coordX_SP[pos];
-				double x2 = para->getParH(level)->coordY_SP[pos];
-				double x3 = para->getParH(level)->coordZ_SP[pos];
-				//////////////////////////////////////////////////////////////////////////
-				number1 = pos;
-				dn1 = pos - startpos;
-				neighborsAreFluid = true;
-				//////////////////////////////////////////////////////////////////////////
-				nodes[dn1] = (makeUbTuple((float)(x1), (float)(x2), (float)(x3)));
-				nodedata[0][dn1] = (double)para->getParH(level)->press_SP[pos] / (double)3.0 * (double)para->getDensityRatio() * (double)para->getVelocityRatio() * (double)para->getVelocityRatio();
-				nodedata[1][dn1] = (double)para->getParH(level)->rho_SP[pos] / (double)3.0 * (double)para->getDensityRatio() * (double)para->getVelocityRatio() * (double)para->getVelocityRatio();
-				nodedata[2][dn1] = (double)para->getParH(level)->vx_SP[pos] * (double)para->getVelocityRatio();
-				nodedata[3][dn1] = (double)para->getParH(level)->vy_SP[pos] * (double)para->getVelocityRatio();
-				nodedata[4][dn1] = (double)para->getParH(level)->vz_SP[pos] * (double)para->getVelocityRatio();
-				nodedata[5][dn1] = (double)para->getParH(level)->geoSP[pos];
-				nodedata[6][dn1] = (double)para->getParH(level)->Conc[pos];
-				//////////////////////////////////////////////////////////////////////////
-				number2 = para->getParH(level)->neighborX_SP[number1];
-				number3 = para->getParH(level)->neighborY_SP[number2];
-				number4 = para->getParH(level)->neighborY_SP[number1];
-				number5 = para->getParH(level)->neighborZ_SP[number1];
-				number6 = para->getParH(level)->neighborZ_SP[number2];
-				number7 = para->getParH(level)->neighborZ_SP[number3];
-				number8 = para->getParH(level)->neighborZ_SP[number4];
-				//////////////////////////////////////////////////////////////////////////
-				if (para->getParH(level)->geoSP[number2] != GEO_FLUID ||
-					para->getParH(level)->geoSP[number3] != GEO_FLUID ||
-					para->getParH(level)->geoSP[number4] != GEO_FLUID ||
-					para->getParH(level)->geoSP[number5] != GEO_FLUID ||
-					para->getParH(level)->geoSP[number6] != GEO_FLUID ||
-					para->getParH(level)->geoSP[number7] != GEO_FLUID ||
-					para->getParH(level)->geoSP[number8] != GEO_FLUID)  neighborsAreFluid = false;
-				//////////////////////////////////////////////////////////////////////////
-				if (number2 > endpos ||
-					number3 > endpos ||
-					number4 > endpos ||
-					number5 > endpos ||
-					number6 > endpos ||
-					number7 > endpos ||
-					number8 > endpos)  neighborsAreFluid = false;
-				//////////////////////////////////////////////////////////////////////////
-				dn2 = number2 - startpos;
-				dn3 = number3 - startpos;
-				dn4 = number4 - startpos;
-				dn5 = number5 - startpos;
-				dn6 = number6 - startpos;
-				dn7 = number7 - startpos;
-				dn8 = number8 - startpos;
-				//////////////////////////////////////////////////////////////////////////
-				if (isPeriodicCell(para, level, number2, number1, number3, number5))
-					continue;
-				//////////////////////////////////////////////////////////////////////////
-				if (neighborsAreFluid)
-					cells.push_back(makeUbTuple(dn1, dn2, dn3, dn4, dn5, dn6, dn7, dn8));
-			}
-		}
-		WbWriterVtkXmlBinary::getInstance()->writeOctsWithNodeData(fname[part], nodes, cells, nodedatanames, nodedata);
-	}
+    std::vector< UbTupleFloat3 > nodes;
+    std::vector< UbTupleUInt8 > cells;
+    std::vector< std::string > nodedatanames;
+    nodedatanames.push_back("press");
+    nodedatanames.push_back("rho");
+    nodedatanames.push_back("vx1");
+    nodedatanames.push_back("vx2");
+    nodedatanames.push_back("vx3");
+    nodedatanames.push_back("geo");
+    nodedatanames.push_back("conc");
+    unsigned int number1, number2, number3, number4, number5, number6, number7, number8;
+    uint dn1, dn2, dn3, dn4, dn5, dn6, dn7, dn8;
+    bool neighborsAreFluid;
+    unsigned int startpos = 0;
+    unsigned int endpos = 0;
+    unsigned int sizeOfNodes = 0;
+    std::vector< std::vector< double > > nodedata(nodedatanames.size());
+
+    for (unsigned int part = 0; part < fname.size(); part++)
+    {
+        if (((part + 1)*para->getlimitOfNodesForVTK()) > para->getParH(level)->size_Mat_SP)
+            sizeOfNodes = para->getParH(level)->size_Mat_SP - (part * para->getlimitOfNodesForVTK());
+        else
+            sizeOfNodes = para->getlimitOfNodesForVTK();
+
+        //////////////////////////////////////////////////////////////////////////
+        startpos = part * para->getlimitOfNodesForVTK();
+        endpos = startpos + sizeOfNodes;
+        //////////////////////////////////////////////////////////////////////////
+        cells.clear();
+        nodes.resize(sizeOfNodes);
+        nodedata[0].resize(sizeOfNodes);
+        nodedata[1].resize(sizeOfNodes);
+        nodedata[2].resize(sizeOfNodes);
+        nodedata[3].resize(sizeOfNodes);
+        nodedata[4].resize(sizeOfNodes);
+        nodedata[5].resize(sizeOfNodes);
+        nodedata[6].resize(sizeOfNodes);
+        //////////////////////////////////////////////////////////////////////////
+        for (unsigned int pos = startpos; pos < endpos; pos++)
+        {
+            if (para->getParH(level)->geoSP[pos] == GEO_FLUID)
+            {
+                //////////////////////////////////////////////////////////////////////////
+                double x1 = para->getParH(level)->coordX_SP[pos];
+                double x2 = para->getParH(level)->coordY_SP[pos];
+                double x3 = para->getParH(level)->coordZ_SP[pos];
+                //////////////////////////////////////////////////////////////////////////
+                number1 = pos;
+                dn1 = pos - startpos;
+                neighborsAreFluid = true;
+                //////////////////////////////////////////////////////////////////////////
+                nodes[dn1] = (makeUbTuple((float)(x1), (float)(x2), (float)(x3)));
+                nodedata[0][dn1] = (double)para->getParH(level)->press_SP[pos] / (double)3.0 * (double)para->getDensityRatio() * (double)para->getVelocityRatio() * (double)para->getVelocityRatio();
+                nodedata[1][dn1] = (double)para->getParH(level)->rho_SP[pos] / (double)3.0 * (double)para->getDensityRatio() * (double)para->getVelocityRatio() * (double)para->getVelocityRatio();
+                nodedata[2][dn1] = (double)para->getParH(level)->vx_SP[pos] * (double)para->getVelocityRatio();
+                nodedata[3][dn1] = (double)para->getParH(level)->vy_SP[pos] * (double)para->getVelocityRatio();
+                nodedata[4][dn1] = (double)para->getParH(level)->vz_SP[pos] * (double)para->getVelocityRatio();
+                nodedata[5][dn1] = (double)para->getParH(level)->geoSP[pos];
+                nodedata[6][dn1] = (double)para->getParH(level)->Conc[pos];
+                //////////////////////////////////////////////////////////////////////////
+                number2 = para->getParH(level)->neighborX_SP[number1];
+                number3 = para->getParH(level)->neighborY_SP[number2];
+                number4 = para->getParH(level)->neighborY_SP[number1];
+                number5 = para->getParH(level)->neighborZ_SP[number1];
+                number6 = para->getParH(level)->neighborZ_SP[number2];
+                number7 = para->getParH(level)->neighborZ_SP[number3];
+                number8 = para->getParH(level)->neighborZ_SP[number4];
+                //////////////////////////////////////////////////////////////////////////
+                if (para->getParH(level)->geoSP[number2] != GEO_FLUID ||
+                    para->getParH(level)->geoSP[number3] != GEO_FLUID ||
+                    para->getParH(level)->geoSP[number4] != GEO_FLUID ||
+                    para->getParH(level)->geoSP[number5] != GEO_FLUID ||
+                    para->getParH(level)->geoSP[number6] != GEO_FLUID ||
+                    para->getParH(level)->geoSP[number7] != GEO_FLUID ||
+                    para->getParH(level)->geoSP[number8] != GEO_FLUID)  neighborsAreFluid = false;
+                //////////////////////////////////////////////////////////////////////////
+                if (number2 > endpos ||
+                    number3 > endpos ||
+                    number4 > endpos ||
+                    number5 > endpos ||
+                    number6 > endpos ||
+                    number7 > endpos ||
+                    number8 > endpos)  neighborsAreFluid = false;
+                //////////////////////////////////////////////////////////////////////////
+                dn2 = number2 - startpos;
+                dn3 = number3 - startpos;
+                dn4 = number4 - startpos;
+                dn5 = number5 - startpos;
+                dn6 = number6 - startpos;
+                dn7 = number7 - startpos;
+                dn8 = number8 - startpos;
+                //////////////////////////////////////////////////////////////////////////
+                if (isPeriodicCell(para, level, number2, number1, number3, number5))
+                    continue;
+                //////////////////////////////////////////////////////////////////////////
+                if (neighborsAreFluid)
+                    cells.push_back(makeUbTuple(dn1, dn2, dn3, dn4, dn5, dn6, dn7, dn8));
+            }
+        }
+        WbWriterVtkXmlBinary::getInstance()->writeOctsWithNodeData(fname[part], nodes, cells, nodedatanames, nodedata);
+    }
 }
 
 void FileWriter::writeUnstrucuredGridMedianLT(std::shared_ptr<Parameter> para, int level, std::vector<std::string >& fname)
 {
-	std::vector< UbTupleFloat3 > nodes;
-	std::vector< UbTupleUInt8 > cells;
-	//std::vector< UbTupleUInt8 > cells2;
-	std::vector< std::string > nodedatanames;
-	nodedatanames.push_back("pressMed");
-	nodedatanames.push_back("rhoMed");
-	nodedatanames.push_back("vx1Med");
-	nodedatanames.push_back("vx2Med");
-	nodedatanames.push_back("vx3Med");
-	nodedatanames.push_back("geo");
-	unsigned int number1, number2, number3, number4, number5, number6, number7, number8;
-	unsigned int dn1, dn2, dn3, dn4, dn5, dn6, dn7, dn8;
-	bool neighborsFluid;
-	unsigned int startpos = 0;
-	unsigned int endpos = 0;
-	unsigned int sizeOfNodes = 0;
-	std::vector< std::vector< double > > nodedata(nodedatanames.size());
-
-	//printf("\n test for if... \n");
-	for (unsigned int part = 0; part < fname.size(); part++)
-	{
-		//printf("\n test in if I... \n");
-		//////////////////////////////////////////////////////////////////////////
-		if (((part + 1)*para->getlimitOfNodesForVTK()) > para->getParH(level)->size_Mat_SP)
-		{
-			sizeOfNodes = para->getParH(level)->size_Mat_SP - (part * para->getlimitOfNodesForVTK());
-		}
-		else
-		{
-			sizeOfNodes = para->getlimitOfNodesForVTK();
-		}
-		//////////////////////////////////////////////////////////////////////////
-		startpos = part * para->getlimitOfNodesForVTK();
-		endpos = startpos + sizeOfNodes;
-		//////////////////////////////////////////////////////////////////////////
-		cells.clear();
-		nodes.resize(sizeOfNodes);
-		nodedata[0].resize(sizeOfNodes);
-		nodedata[1].resize(sizeOfNodes);
-		nodedata[2].resize(sizeOfNodes);
-		nodedata[3].resize(sizeOfNodes);
-		nodedata[4].resize(sizeOfNodes);
-		nodedata[5].resize(sizeOfNodes);
-		//////////////////////////////////////////////////////////////////////////
-		//printf("\n test in if II... \n");
-		for (unsigned int pos = startpos; pos < endpos; pos++)
-		{
-			if (para->getParH(level)->geoSP[pos] == GEO_FLUID)
-			{
-				//////////////////////////////////////////////////////////////////////////
-				double x1 = para->getParH(level)->coordX_SP[pos];
-				double x2 = para->getParH(level)->coordY_SP[pos];
-				double x3 = para->getParH(level)->coordZ_SP[pos];
-				//////////////////////////////////////////////////////////////////////////
-				number1 = pos;
-				dn1 = pos - startpos;
-				neighborsFluid = true;
-				//////////////////////////////////////////////////////////////////////////
-				nodes[dn1] = (makeUbTuple((float)(x1), (float)(x2), (float)(x3)));
-				nodedata[0][dn1] = para->getParH(level)->press_SP_Med_Out[pos] / 3.0f * para->getDensityRatio() * para->getVelocityRatio() * para->getVelocityRatio();
-				nodedata[1][dn1] = para->getParH(level)->rho_SP_Med_Out[pos] / 3.0f * para->getDensityRatio() * para->getVelocityRatio() * para->getVelocityRatio();
-				nodedata[2][dn1] = para->getParH(level)->vx_SP_Med_Out[pos] * para->getVelocityRatio();
-				nodedata[3][dn1] = para->getParH(level)->vy_SP_Med_Out[pos] * para->getVelocityRatio();
-				nodedata[4][dn1] = para->getParH(level)->vz_SP_Med_Out[pos] * para->getVelocityRatio();
-				nodedata[5][dn1] = (double)para->getParH(level)->geoSP[pos];
-				//////////////////////////////////////////////////////////////////////////
-				number2 = para->getParH(level)->neighborX_SP[number1];
-				number3 = para->getParH(level)->neighborY_SP[number2];
-				number4 = para->getParH(level)->neighborY_SP[number1];
-				number5 = para->getParH(level)->neighborZ_SP[number1];
-				number6 = para->getParH(level)->neighborZ_SP[number2];
-				number7 = para->getParH(level)->neighborZ_SP[number3];
-				number8 = para->getParH(level)->neighborZ_SP[number4];
-				//////////////////////////////////////////////////////////////////////////
-				if (para->getParH(level)->geoSP[number2] != GEO_FLUID ||
-					para->getParH(level)->geoSP[number3] != GEO_FLUID ||
-					para->getParH(level)->geoSP[number4] != GEO_FLUID ||
-					para->getParH(level)->geoSP[number5] != GEO_FLUID ||
-					para->getParH(level)->geoSP[number6] != GEO_FLUID ||
-					para->getParH(level)->geoSP[number7] != GEO_FLUID ||
-					para->getParH(level)->geoSP[number8] != GEO_FLUID)  neighborsFluid = false;
-				//////////////////////////////////////////////////////////////////////////
-				if (number2 > endpos ||
-					number3 > endpos ||
-					number4 > endpos ||
-					number5 > endpos ||
-					number6 > endpos ||
-					number7 > endpos ||
-					number8 > endpos)  neighborsFluid = false;
-				//////////////////////////////////////////////////////////////////////////
-				dn2 = number2 - startpos;
-				dn3 = number3 - startpos;
-				dn4 = number4 - startpos;
-				dn5 = number5 - startpos;
-				dn6 = number6 - startpos;
-				dn7 = number7 - startpos;
-				dn8 = number8 - startpos;
-				//////////////////////////////////////////////////////////////////////////
-				if (isPeriodicCell(para, level, number2, number1, number3, number5))
-					continue;
-				//////////////////////////////////////////////////////////////////////////
-				if (neighborsFluid == true) cells.push_back(makeUbTuple(dn1, dn2, dn3, dn4, dn5, dn6, dn7, dn8));
-				//////////////////////////////////////////////////////////////////////////
-			}
-		}
-		WbWriterVtkXmlBinary::getInstance()->writeOctsWithNodeData(fname[part], nodes, cells, nodedatanames, nodedata);
-		//////////////////////////////////////////////////////////////////////////
-	}
+    std::vector< UbTupleFloat3 > nodes;
+    std::vector< UbTupleUInt8 > cells;
+    //std::vector< UbTupleUInt8 > cells2;
+    std::vector< std::string > nodedatanames;
+    nodedatanames.push_back("pressMed");
+    nodedatanames.push_back("rhoMed");
+    nodedatanames.push_back("vx1Med");
+    nodedatanames.push_back("vx2Med");
+    nodedatanames.push_back("vx3Med");
+    nodedatanames.push_back("geo");
+    unsigned int number1, number2, number3, number4, number5, number6, number7, number8;
+    unsigned int dn1, dn2, dn3, dn4, dn5, dn6, dn7, dn8;
+    bool neighborsFluid;
+    unsigned int startpos = 0;
+    unsigned int endpos = 0;
+    unsigned int sizeOfNodes = 0;
+    std::vector< std::vector< double > > nodedata(nodedatanames.size());
+
+    //printf("\n test for if... \n");
+    for (unsigned int part = 0; part < fname.size(); part++)
+    {
+        //printf("\n test in if I... \n");
+        //////////////////////////////////////////////////////////////////////////
+        if (((part + 1)*para->getlimitOfNodesForVTK()) > para->getParH(level)->size_Mat_SP)
+        {
+            sizeOfNodes = para->getParH(level)->size_Mat_SP - (part * para->getlimitOfNodesForVTK());
+        }
+        else
+        {
+            sizeOfNodes = para->getlimitOfNodesForVTK();
+        }
+        //////////////////////////////////////////////////////////////////////////
+        startpos = part * para->getlimitOfNodesForVTK();
+        endpos = startpos + sizeOfNodes;
+        //////////////////////////////////////////////////////////////////////////
+        cells.clear();
+        nodes.resize(sizeOfNodes);
+        nodedata[0].resize(sizeOfNodes);
+        nodedata[1].resize(sizeOfNodes);
+        nodedata[2].resize(sizeOfNodes);
+        nodedata[3].resize(sizeOfNodes);
+        nodedata[4].resize(sizeOfNodes);
+        nodedata[5].resize(sizeOfNodes);
+        //////////////////////////////////////////////////////////////////////////
+        //printf("\n test in if II... \n");
+        for (unsigned int pos = startpos; pos < endpos; pos++)
+        {
+            if (para->getParH(level)->geoSP[pos] == GEO_FLUID)
+            {
+                //////////////////////////////////////////////////////////////////////////
+                double x1 = para->getParH(level)->coordX_SP[pos];
+                double x2 = para->getParH(level)->coordY_SP[pos];
+                double x3 = para->getParH(level)->coordZ_SP[pos];
+                //////////////////////////////////////////////////////////////////////////
+                number1 = pos;
+                dn1 = pos - startpos;
+                neighborsFluid = true;
+                //////////////////////////////////////////////////////////////////////////
+                nodes[dn1] = (makeUbTuple((float)(x1), (float)(x2), (float)(x3)));
+                nodedata[0][dn1] = para->getParH(level)->press_SP_Med_Out[pos] / 3.0f * para->getDensityRatio() * para->getVelocityRatio() * para->getVelocityRatio();
+                nodedata[1][dn1] = para->getParH(level)->rho_SP_Med_Out[pos] / 3.0f * para->getDensityRatio() * para->getVelocityRatio() * para->getVelocityRatio();
+                nodedata[2][dn1] = para->getParH(level)->vx_SP_Med_Out[pos] * para->getVelocityRatio();
+                nodedata[3][dn1] = para->getParH(level)->vy_SP_Med_Out[pos] * para->getVelocityRatio();
+                nodedata[4][dn1] = para->getParH(level)->vz_SP_Med_Out[pos] * para->getVelocityRatio();
+                nodedata[5][dn1] = (double)para->getParH(level)->geoSP[pos];
+                //////////////////////////////////////////////////////////////////////////
+                number2 = para->getParH(level)->neighborX_SP[number1];
+                number3 = para->getParH(level)->neighborY_SP[number2];
+                number4 = para->getParH(level)->neighborY_SP[number1];
+                number5 = para->getParH(level)->neighborZ_SP[number1];
+                number6 = para->getParH(level)->neighborZ_SP[number2];
+                number7 = para->getParH(level)->neighborZ_SP[number3];
+                number8 = para->getParH(level)->neighborZ_SP[number4];
+                //////////////////////////////////////////////////////////////////////////
+                if (para->getParH(level)->geoSP[number2] != GEO_FLUID ||
+                    para->getParH(level)->geoSP[number3] != GEO_FLUID ||
+                    para->getParH(level)->geoSP[number4] != GEO_FLUID ||
+                    para->getParH(level)->geoSP[number5] != GEO_FLUID ||
+                    para->getParH(level)->geoSP[number6] != GEO_FLUID ||
+                    para->getParH(level)->geoSP[number7] != GEO_FLUID ||
+                    para->getParH(level)->geoSP[number8] != GEO_FLUID)  neighborsFluid = false;
+                //////////////////////////////////////////////////////////////////////////
+                if (number2 > endpos ||
+                    number3 > endpos ||
+                    number4 > endpos ||
+                    number5 > endpos ||
+                    number6 > endpos ||
+                    number7 > endpos ||
+                    number8 > endpos)  neighborsFluid = false;
+                //////////////////////////////////////////////////////////////////////////
+                dn2 = number2 - startpos;
+                dn3 = number3 - startpos;
+                dn4 = number4 - startpos;
+                dn5 = number5 - startpos;
+                dn6 = number6 - startpos;
+                dn7 = number7 - startpos;
+                dn8 = number8 - startpos;
+                //////////////////////////////////////////////////////////////////////////
+                if (isPeriodicCell(para, level, number2, number1, number3, number5))
+                    continue;
+                //////////////////////////////////////////////////////////////////////////
+                if (neighborsFluid == true) cells.push_back(makeUbTuple(dn1, dn2, dn3, dn4, dn5, dn6, dn7, dn8));
+                //////////////////////////////////////////////////////////////////////////
+            }
+        }
+        WbWriterVtkXmlBinary::getInstance()->writeOctsWithNodeData(fname[part], nodes, cells, nodedatanames, nodedata);
+        //////////////////////////////////////////////////////////////////////////
+    }
 }
 
 void FileWriter::writeUnstrucuredGridMedianLTConc(std::shared_ptr<Parameter> para, int level, std::vector<std::string >& fname)
 {
-	std::vector< UbTupleFloat3 > nodes;
-	std::vector< UbTupleUInt8 > cells;
-	std::vector< std::string > nodedatanames;
-	nodedatanames.push_back("concMed");
-	nodedatanames.push_back("pressMed");
-	nodedatanames.push_back("rhoMed");
-	nodedatanames.push_back("vx1Med");
-	nodedatanames.push_back("vx2Med");
-	nodedatanames.push_back("vx3Med");
-	nodedatanames.push_back("geo");
-	uint number1, number2, number3, number4, number5, number6, number7, number8;
-	uint dn1, dn2, dn3, dn4, dn5, dn6, dn7, dn8;
-	bool neighborsFluid;
-	uint startpos = 0;
-	uint endpos = 0;
-	uint sizeOfNodes = 0;
-	std::vector< std::vector< double > > nodedata(nodedatanames.size());
-
-	for (unsigned int part = 0; part < fname.size(); part++)
-	{
-		if (((part + 1)*para->getlimitOfNodesForVTK()) > para->getParH(level)->size_Mat_SP)
-			sizeOfNodes = para->getParH(level)->size_Mat_SP - (part * para->getlimitOfNodesForVTK());
-		else
-			sizeOfNodes = para->getlimitOfNodesForVTK();
-		//////////////////////////////////////////////////////////////////////////
-		startpos = part * para->getlimitOfNodesForVTK();
-		endpos = startpos + sizeOfNodes;
-		//////////////////////////////////////////////////////////////////////////
-		cells.clear();
-		nodes.resize(sizeOfNodes);
-		nodedata[0].resize(sizeOfNodes);
-		nodedata[1].resize(sizeOfNodes);
-		nodedata[2].resize(sizeOfNodes);
-		nodedata[3].resize(sizeOfNodes);
-		nodedata[4].resize(sizeOfNodes);
-		nodedata[5].resize(sizeOfNodes);
-		nodedata[6].resize(sizeOfNodes);
-		//////////////////////////////////////////////////////////////////////////
-		for (unsigned int pos = startpos; pos < endpos; pos++)
-		{
-			if (para->getParH(level)->geoSP[pos] == GEO_FLUID)
-			{
-				//////////////////////////////////////////////////////////////////////////
-				double x1 = para->getParH(level)->coordX_SP[pos];
-				double x2 = para->getParH(level)->coordY_SP[pos];
-				double x3 = para->getParH(level)->coordZ_SP[pos];
-				//////////////////////////////////////////////////////////////////////////
-				number1 = pos;
-				dn1 = pos - startpos;
-				neighborsFluid = true;
-				//////////////////////////////////////////////////////////////////////////
-				nodes[dn1] = (makeUbTuple((float)(x1), (float)(x2), (float)(x3)));
-				nodedata[0][dn1] = (double)para->getParH(level)->Conc_Med_Out[pos];
-				nodedata[1][dn1] = (double)para->getParH(level)->press_SP_Med_Out[pos] / 3.0f * para->getDensityRatio() * para->getVelocityRatio() * para->getVelocityRatio();
-				nodedata[2][dn1] = (double)para->getParH(level)->rho_SP_Med_Out[pos] / 3.0f * para->getDensityRatio() * para->getVelocityRatio() * para->getVelocityRatio();
-				nodedata[3][dn1] = (double)para->getParH(level)->vx_SP_Med_Out[pos] * para->getVelocityRatio();
-				nodedata[4][dn1] = (double)para->getParH(level)->vy_SP_Med_Out[pos] * para->getVelocityRatio();
-				nodedata[5][dn1] = (double)para->getParH(level)->vz_SP_Med_Out[pos] * para->getVelocityRatio();
-				nodedata[6][dn1] = (double)para->getParH(level)->geoSP[pos];
-				//////////////////////////////////////////////////////////////////////////
-				number2 = para->getParH(level)->neighborX_SP[number1];
-				number3 = para->getParH(level)->neighborY_SP[number2];
-				number4 = para->getParH(level)->neighborY_SP[number1];
-				number5 = para->getParH(level)->neighborZ_SP[number1];
-				number6 = para->getParH(level)->neighborZ_SP[number2];
-				number7 = para->getParH(level)->neighborZ_SP[number3];
-				number8 = para->getParH(level)->neighborZ_SP[number4];
-				//////////////////////////////////////////////////////////////////////////
-				if (para->getParH(level)->geoSP[number2] != GEO_FLUID ||
-					para->getParH(level)->geoSP[number3] != GEO_FLUID ||
-					para->getParH(level)->geoSP[number4] != GEO_FLUID ||
-					para->getParH(level)->geoSP[number5] != GEO_FLUID ||
-					para->getParH(level)->geoSP[number6] != GEO_FLUID ||
-					para->getParH(level)->geoSP[number7] != GEO_FLUID ||
-					para->getParH(level)->geoSP[number8] != GEO_FLUID)  neighborsFluid = false;
-				//////////////////////////////////////////////////////////////////////////
-				if (number2 > endpos ||
-					number3 > endpos ||
-					number4 > endpos ||
-					number5 > endpos ||
-					number6 > endpos ||
-					number7 > endpos ||
-					number8 > endpos)  neighborsFluid = false;
-				//////////////////////////////////////////////////////////////////////////
-				dn2 = number2 - startpos;
-				dn3 = number3 - startpos;
-				dn4 = number4 - startpos;
-				dn5 = number5 - startpos;
-				dn6 = number6 - startpos;
-				dn7 = number7 - startpos;
-				dn8 = number8 - startpos;
-				//////////////////////////////////////////////////////////////////////////
-				if (isPeriodicCell(para, level, number2, number1, number3, number5))
-					continue;
-				//////////////////////////////////////////////////////////////////////////
-				if (neighborsFluid) 
-					cells.push_back(makeUbTuple(dn1, dn2, dn3, dn4, dn5, dn6, dn7, dn8));
-				//////////////////////////////////////////////////////////////////////////
-			}
-		}
-		WbWriterVtkXmlBinary::getInstance()->writeOctsWithNodeData(fname[part], nodes, cells, nodedatanames, nodedata);
-		//////////////////////////////////////////////////////////////////////////
-	}
+    std::vector< UbTupleFloat3 > nodes;
+    std::vector< UbTupleUInt8 > cells;
+    std::vector< std::string > nodedatanames;
+    nodedatanames.push_back("concMed");
+    nodedatanames.push_back("pressMed");
+    nodedatanames.push_back("rhoMed");
+    nodedatanames.push_back("vx1Med");
+    nodedatanames.push_back("vx2Med");
+    nodedatanames.push_back("vx3Med");
+    nodedatanames.push_back("geo");
+    uint number1, number2, number3, number4, number5, number6, number7, number8;
+    uint dn1, dn2, dn3, dn4, dn5, dn6, dn7, dn8;
+    bool neighborsFluid;
+    uint startpos = 0;
+    uint endpos = 0;
+    uint sizeOfNodes = 0;
+    std::vector< std::vector< double > > nodedata(nodedatanames.size());
+
+    for (unsigned int part = 0; part < fname.size(); part++)
+    {
+        if (((part + 1)*para->getlimitOfNodesForVTK()) > para->getParH(level)->size_Mat_SP)
+            sizeOfNodes = para->getParH(level)->size_Mat_SP - (part * para->getlimitOfNodesForVTK());
+        else
+            sizeOfNodes = para->getlimitOfNodesForVTK();
+        //////////////////////////////////////////////////////////////////////////
+        startpos = part * para->getlimitOfNodesForVTK();
+        endpos = startpos + sizeOfNodes;
+        //////////////////////////////////////////////////////////////////////////
+        cells.clear();
+        nodes.resize(sizeOfNodes);
+        nodedata[0].resize(sizeOfNodes);
+        nodedata[1].resize(sizeOfNodes);
+        nodedata[2].resize(sizeOfNodes);
+        nodedata[3].resize(sizeOfNodes);
+        nodedata[4].resize(sizeOfNodes);
+        nodedata[5].resize(sizeOfNodes);
+        nodedata[6].resize(sizeOfNodes);
+        //////////////////////////////////////////////////////////////////////////
+        for (unsigned int pos = startpos; pos < endpos; pos++)
+        {
+            if (para->getParH(level)->geoSP[pos] == GEO_FLUID)
+            {
+                //////////////////////////////////////////////////////////////////////////
+                double x1 = para->getParH(level)->coordX_SP[pos];
+                double x2 = para->getParH(level)->coordY_SP[pos];
+                double x3 = para->getParH(level)->coordZ_SP[pos];
+                //////////////////////////////////////////////////////////////////////////
+                number1 = pos;
+                dn1 = pos - startpos;
+                neighborsFluid = true;
+                //////////////////////////////////////////////////////////////////////////
+                nodes[dn1] = (makeUbTuple((float)(x1), (float)(x2), (float)(x3)));
+                nodedata[0][dn1] = (double)para->getParH(level)->Conc_Med_Out[pos];
+                nodedata[1][dn1] = (double)para->getParH(level)->press_SP_Med_Out[pos] / 3.0f * para->getDensityRatio() * para->getVelocityRatio() * para->getVelocityRatio();
+                nodedata[2][dn1] = (double)para->getParH(level)->rho_SP_Med_Out[pos] / 3.0f * para->getDensityRatio() * para->getVelocityRatio() * para->getVelocityRatio();
+                nodedata[3][dn1] = (double)para->getParH(level)->vx_SP_Med_Out[pos] * para->getVelocityRatio();
+                nodedata[4][dn1] = (double)para->getParH(level)->vy_SP_Med_Out[pos] * para->getVelocityRatio();
+                nodedata[5][dn1] = (double)para->getParH(level)->vz_SP_Med_Out[pos] * para->getVelocityRatio();
+                nodedata[6][dn1] = (double)para->getParH(level)->geoSP[pos];
+                //////////////////////////////////////////////////////////////////////////
+                number2 = para->getParH(level)->neighborX_SP[number1];
+                number3 = para->getParH(level)->neighborY_SP[number2];
+                number4 = para->getParH(level)->neighborY_SP[number1];
+                number5 = para->getParH(level)->neighborZ_SP[number1];
+                number6 = para->getParH(level)->neighborZ_SP[number2];
+                number7 = para->getParH(level)->neighborZ_SP[number3];
+                number8 = para->getParH(level)->neighborZ_SP[number4];
+                //////////////////////////////////////////////////////////////////////////
+                if (para->getParH(level)->geoSP[number2] != GEO_FLUID ||
+                    para->getParH(level)->geoSP[number3] != GEO_FLUID ||
+                    para->getParH(level)->geoSP[number4] != GEO_FLUID ||
+                    para->getParH(level)->geoSP[number5] != GEO_FLUID ||
+                    para->getParH(level)->geoSP[number6] != GEO_FLUID ||
+                    para->getParH(level)->geoSP[number7] != GEO_FLUID ||
+                    para->getParH(level)->geoSP[number8] != GEO_FLUID)  neighborsFluid = false;
+                //////////////////////////////////////////////////////////////////////////
+                if (number2 > endpos ||
+                    number3 > endpos ||
+                    number4 > endpos ||
+                    number5 > endpos ||
+                    number6 > endpos ||
+                    number7 > endpos ||
+                    number8 > endpos)  neighborsFluid = false;
+                //////////////////////////////////////////////////////////////////////////
+                dn2 = number2 - startpos;
+                dn3 = number3 - startpos;
+                dn4 = number4 - startpos;
+                dn5 = number5 - startpos;
+                dn6 = number6 - startpos;
+                dn7 = number7 - startpos;
+                dn8 = number8 - startpos;
+                //////////////////////////////////////////////////////////////////////////
+                if (isPeriodicCell(para, level, number2, number1, number3, number5))
+                    continue;
+                //////////////////////////////////////////////////////////////////////////
+                if (neighborsFluid) 
+                    cells.push_back(makeUbTuple(dn1, dn2, dn3, dn4, dn5, dn6, dn7, dn8));
+                //////////////////////////////////////////////////////////////////////////
+            }
+        }
+        WbWriterVtkXmlBinary::getInstance()->writeOctsWithNodeData(fname[part], nodes, cells, nodedatanames, nodedata);
+        //////////////////////////////////////////////////////////////////////////
+    }
 }
 //////////////////////////////////////////////////////////////////////////
 
diff --git a/src/gpu/VirtualFluids_GPU/Output/FileWriter.h b/src/gpu/VirtualFluids_GPU/Output/FileWriter.h
index 99c82cbd7..a1712432e 100644
--- a/src/gpu/VirtualFluids_GPU/Output/FileWriter.h
+++ b/src/gpu/VirtualFluids_GPU/Output/FileWriter.h
@@ -26,6 +26,7 @@ private:
 	//void VIRTUALFLUIDS_GPU_EXPORT writeParticle(Parameter* para, unsigned int t);
     void VIRTUALFLUIDS_GPU_EXPORT writeUnstrucuredGridLT(std::shared_ptr<Parameter> para, int level,
                                                          std::vector<std::string> &fname);
+    void testForSendNodeX(std::shared_ptr<Parameter> &para, int level, unsigned int pos, int &sendNode);
     void testForSendNodeY(std::shared_ptr<Parameter> &para, int level, unsigned int pos, int &sendNode);
     void testForSendNodeZ(std::shared_ptr<Parameter> &para, int level, unsigned int pos, int &sendNode);
     void testForCommunicationNode(std::shared_ptr<Parameter> &para, int level, unsigned int pos, int &sendNode,
-- 
GitLab