Skip to content
Snippets Groups Projects
GPU_Interface.h 86.39 KiB
//  _    ___      __              __________      _     __        ______________   __
// | |  / (_)____/ /___  ______ _/ / ____/ /_  __(_)___/ /____   /  ___/ __  / /  / /
// | | / / / ___/ __/ / / / __ `/ / /_  / / / / / / __  / ___/  / /___/ /_/ / /  / /
// | |/ / / /  / /_/ /_/ / /_/ / / __/ / / /_/ / / /_/ (__  )  / /_) / ____/ /__/ / 
// |___/_/_/   \__/\__,_/\__,_/_/_/   /_/\__,_/_/\__,_/____/   \____/_/    \_____/
//
//////////////////////////////////////////////////////////////////////////
//random numbers
#include <curand.h>
#include <curand_kernel.h>
#include <cuda_runtime.h>

#include <DataTypes.h>
#include "LBM/LB.h"

#ifndef GPU_INTERFACE_H
#define GPU_INTERFACE_H

//////////////////////////////////////////////////////////////////////////
//Kernel
//////////////////////////////////////////////////////////////////////////
extern "C" void KernelCas27(unsigned int grid_nx, 
                            unsigned int grid_ny, 
                            unsigned int grid_nz, 
                            real s9,
                            unsigned int* bcMatD,
                            unsigned int* neighborX,
                            unsigned int* neighborY,
                            unsigned int* neighborZ,
                            real* DD,
                            int size_Mat,
                            bool EvenOrOdd);

extern "C" void KernelCasSP27(unsigned int numberOfThreads, 
                              real s9,
                              unsigned int* bcMatD,
                              unsigned int* neighborX,
                              unsigned int* neighborY,
                              unsigned int* neighborZ,
                              real* DD,
                              int size_Mat,
                              bool EvenOrOdd);

extern "C" void KernelCasSPMS27(unsigned int numberOfThreads, 
                                real s9,
                                unsigned int* bcMatD,
                                unsigned int* neighborX,
                                unsigned int* neighborY,
                                unsigned int* neighborZ,
                                real* DD,
                                int size_Mat,
                                bool EvenOrOdd);

extern "C" void KernelCasSPMSOHM27( unsigned int numberOfThreads, 
                                   real s9,
                                   unsigned int* bcMatD,
                                   unsigned int* neighborX,
                                   unsigned int* neighborY,
                                   unsigned int* neighborZ,
                                   real* DD,
                                   int size_Mat,
                                   bool EvenOrOdd);

extern "C" void KernelKumCompSRTSP27(
	unsigned int numberOfThreads,
	real omega,
	unsigned int* bcMatD,
	unsigned int* neighborX,
	unsigned int* neighborY,
	unsigned int* neighborZ,
	real* DDStart,
	int size_Mat,
	int level,
	real* forces,
	bool EvenOrOdd);

extern "C" void KernelCumulantD3Q27All4(unsigned int numberOfThreads,
									    real s9,
									    unsigned int* bcMatD,
									    unsigned int* neighborX,
									    unsigned int* neighborY,
									    unsigned int* neighborZ,
									    real* DD,
									    int size_Mat,
									    int level,
									    real* forces,
									    bool EvenOrOdd);

extern "C" void KernelKumAA2016CompBulkSP27(unsigned int numberOfThreads, 
											real s9,
											unsigned int* bcMatD,
											unsigned int* neighborX,
											unsigned int* neighborY,
											unsigned int* neighborZ,
											real* DD,
											int size_Mat,
											int size_Array,
											int level,
											real* forces,
											bool EvenOrOdd);

extern "C" void KernelKum1hSP27(    unsigned int numberOfThreads, 
									real omega,
									real deltaPhi,
									real angularVelocity,
									unsigned int* bcMatD,
									unsigned int* neighborX,
									unsigned int* neighborY,
									unsigned int* neighborZ,
									real* coordX,
									real* coordY,
									real* coordZ,
									real* DDStart,
									int size_Mat,
									bool EvenOrOdd);

extern "C" void KernelCascadeSP27(unsigned int numberOfThreads, 
								  real s9,
								  unsigned int* bcMatD,
								  unsigned int* neighborX,
								  unsigned int* neighborY,
								  unsigned int* neighborZ,
								  real* DD,
								  int size_Mat,
								  bool EvenOrOdd);

extern "C" void KernelKumNewSP27(   unsigned int numberOfThreads, 
									real s9,
									unsigned int* bcMatD,
									unsigned int* neighborX,
									unsigned int* neighborY,
									unsigned int* neighborZ,
									real* DD,
									int size_Mat,
									bool EvenOrOdd);


extern "C" void CumulantOnePreconditionedErrorDiffusionChimCompSP27(
	unsigned int numberOfThreads,
	real s9,
	unsigned int* bcMatD,
	unsigned int* neighborX,
	unsigned int* neighborY,
	unsigned int* neighborZ,
	real* DD,
	int size_Mat,
	int size_Array,
	int level,
	real* forces,
	bool EvenOrOdd);

extern "C" void CumulantOnePreconditionedChimCompSP27(
	unsigned int numberOfThreads,
	real s9,
	unsigned int* bcMatD,
	unsigned int* neighborX,
	unsigned int* neighborY,
	unsigned int* neighborZ,
	real* DD,
	int size_Mat,
	int size_Array,
	int level,
	real* forces,
	bool EvenOrOdd);

extern "C" void CumulantOneChimCompSP27(
	unsigned int numberOfThreads,
	real s9,
	unsigned int* bcMatD,
	unsigned int* neighborX,
	unsigned int* neighborY,
	unsigned int* neighborZ,
	real* DD,
	int size_Mat,
	int size_Array,
	int level,
	real* forces,
	bool EvenOrOdd);


extern "C" void KernelKumIsoTestSP27(unsigned int numberOfThreads, 
									 real s9,
									 unsigned int* bcMatD,
									 unsigned int* neighborX,
									 unsigned int* neighborY,
									 unsigned int* neighborZ,
									 real* DD,
									 real* dxxUx,
									 real* dyyUy,
									 real* dzzUz,
									 int size_Mat,
									 bool EvenOrOdd);

extern "C" void KernelKumCompSP27(  unsigned int numberOfThreads, 
									real s9,
									unsigned int* bcMatD,
									unsigned int* neighborX,
									unsigned int* neighborY,
									unsigned int* neighborZ,
									real* DD,
									int size_Mat,
									bool EvenOrOdd);

extern "C" void KernelWaleBySoniMalavCumAA2016CompSP27(
	unsigned int numberOfThreads,
	real s9,
	unsigned int* bcMatD,
	unsigned int* neighborX,
	unsigned int* neighborY,
	unsigned int* neighborZ,
	unsigned int* neighborWSB,
	real* veloX,
	real* veloY,
	real* veloZ,
	real* DD,
	real* turbulentViscosity,
	int size_Mat,
	int size_Array,
	int level,
	real* forces,
	bool EvenOrOdd);

extern "C" void KernelPMCumOneCompSP27(unsigned int numberOfThreads, 
									   real omega,
									   unsigned int* neighborX,
									   unsigned int* neighborY,
									   unsigned int* neighborZ,
									   real* DD,
									   int size_Mat,
									   int level,
									   real* forces,
									   real porosity,
									   real darcy,
									   real forchheimer,
									   unsigned int sizeOfPorousMedia,
									   unsigned int* nodeIdsPorousMedia, 
									   bool EvenOrOdd);

extern "C" void KernelADincomp7(   unsigned int numberOfThreads, 
								   real diffusivity,
								   unsigned int* bcMatD,
								   unsigned int* neighborX,
								   unsigned int* neighborY,
								   unsigned int* neighborZ,
								   real* DD,
								   real* DD7,
								   int size_Mat,
								   bool EvenOrOdd);

extern "C" void KernelADincomp27(   unsigned int numberOfThreads, 
									real diffusivity,
									unsigned int* bcMatD,
									unsigned int* neighborX,
									unsigned int* neighborY,
									unsigned int* neighborZ,
									real* DD,
									real* DD7,
									int size_Mat,
									bool EvenOrOdd);

extern "C" void Init27(int myid,
                       int numprocs,
                       real u0,
                       unsigned int* geoD,
                       unsigned int* neighborX,
                       unsigned int* neighborY,
                       unsigned int* neighborZ,
                       real* vParab,
                       unsigned int size_Mat,
                       unsigned int grid_nx, 
                       unsigned int grid_ny, 
                       unsigned int grid_nz, 
                       real* DD,
                       int level,
                       int maxlevel);

extern "C" void InitNonEqPartSP27(unsigned int numberOfThreads,
                                  unsigned int* neighborX,
                                  unsigned int* neighborY,
                                  unsigned int* neighborZ,
                                  unsigned int* neighborWSB,
                                  unsigned int* geoD,
                                  real* rho,
                                  real* ux,
                                  real* uy,
                                  real* uz,
                                  unsigned int size_Mat,
                                  real* DD,
                                  real omega,
                                  bool EvenOrOdd);


extern "C" void InitThS7(  unsigned int numberOfThreads,
                           unsigned int* neighborX,
                           unsigned int* neighborY,
                           unsigned int* neighborZ,
                           unsigned int* geoD,
                           real* Conc,
                           real* ux,
                           real* uy,
                           real* uz,
                           unsigned int size_Mat,
                           real* DD7,
                           bool EvenOrOdd);

extern "C" void InitADDev27( unsigned int numberOfThreads,
                           unsigned int* neighborX,
                           unsigned int* neighborY,
                           unsigned int* neighborZ,
                           unsigned int* geoD,
                           real* Conc,
                           real* ux,
                           real* uy,
                           real* uz,
                           unsigned int size_Mat,
                           real* DD27,
                           bool EvenOrOdd);

extern "C" void PostProcessorF3_2018Fehlberg(
	unsigned int numberOfThreads,
	real omega,
	unsigned int* bcMatD,
	unsigned int* neighborX,
	unsigned int* neighborY,
	unsigned int* neighborZ,
	real* rhoOut,
	real* vxOut,
	real* vyOut,
	real* vzOut,
	real* DDStart,
	real* G6,
	int size_Mat,
	int level,
	real* forces,
	bool EvenOrOdd);

extern "C" void CalcMac27( real* vxD,
                          real* vyD,
                          real* vzD,
                          real* rhoD,
                          unsigned int* geoD,
                          unsigned int* neighborX,
                          unsigned int* neighborY,
                          unsigned int* neighborZ,
                          unsigned int size_Mat,
                          unsigned int grid_nx, 
                          unsigned int grid_ny, 
                          unsigned int grid_nz, 
                          real* DD,
                          bool isEvenTimestep);

extern "C" void CalcMacSP27(real* vxD,
                            real* vyD,
                            real* vzD,
                            real* rhoD,
                            real* pressD,
                            unsigned int* geoD,
                            unsigned int* neighborX,
                            unsigned int* neighborY,
                            unsigned int* neighborZ,
                            unsigned int size_Mat,
                            unsigned int numberOfThreads, 
                            real* DD,
                            bool isEvenTimestep);

extern "C" void CalcMacCompSP27(real* vxD,
								real* vyD,
								real* vzD,
								real* rhoD,
								real* pressD,
								unsigned int* geoD,
								unsigned int* neighborX,
								unsigned int* neighborY,
								unsigned int* neighborZ,
								unsigned int size_Mat,
								unsigned int numberOfThreads, 
								real* DD,
								bool isEvenTimestep);

extern "C" void CalcMacThS7(  real* Conc,
                              unsigned int* geoD,
                              unsigned int* neighborX,
                              unsigned int* neighborY,
                              unsigned int* neighborZ,
                              unsigned int size_Mat,
                              unsigned int numberOfThreads, 
                              real* DD7,
                              bool isEvenTimestep);

extern "C" void PlaneConcThS7(real* Conc,
							  int* kPC,
							  unsigned int numberOfPointskPC,
							  unsigned int* geoD,
							  unsigned int* neighborX,
							  unsigned int* neighborY,
							  unsigned int* neighborZ,
							  unsigned int size_Mat,
							  unsigned int numberOfThreads, 
							  real* DD7,
							  bool isEvenTimestep);

extern "C" void PlaneConcThS27(real* Conc,
							   int* kPC,
							   unsigned int numberOfPointskPC,
							   unsigned int* geoD,
							   unsigned int* neighborX,
							   unsigned int* neighborY,
							   unsigned int* neighborZ,
							   unsigned int size_Mat,
							   unsigned int numberOfThreads, 
							   real* DD27,
							   bool isEvenTimestep);

extern "C" void CalcConcentration27( unsigned int numberOfThreads,
	                                 real* Conc,
                                     unsigned int* geoD,
                                     unsigned int* neighborX,
                                     unsigned int* neighborY,
                                     unsigned int* neighborZ,
                                     unsigned int size_Mat,
                                     real* DD27,
                                     bool isEvenTimestep);

extern "C" void CalcMedSP27(  real* vxD,
                              real* vyD,
                              real* vzD,
                              real* rhoD,
                              real* pressD,
                              unsigned int* geoD,
                              unsigned int* neighborX,
                              unsigned int* neighborY,
                              unsigned int* neighborZ,
                              unsigned int size_Mat,
                              unsigned int numberOfThreads, 
                              real* DD,
                              bool isEvenTimestep);

extern "C" void CalcMedCompSP27(real* vxD,
								real* vyD,
								real* vzD,
								real* rhoD,
								real* pressD,
								unsigned int* geoD,
								unsigned int* neighborX,
								unsigned int* neighborY,
								unsigned int* neighborZ,
								unsigned int size_Mat,
								unsigned int numberOfThreads, 
								real* DD,
								bool isEvenTimestep);

extern "C" void CalcMedCompAD27(
	real* vxD,
	real* vyD,
	real* vzD,
	real* rhoD,
	real* pressD,
	real* concD,
	unsigned int* geoD,
	unsigned int* neighborX,
	unsigned int* neighborY,
	unsigned int* neighborZ,
	unsigned int size_Mat,
	unsigned int numberOfThreads,
	real* DD,
	real* DD_AD,
	bool isEvenTimestep);

extern "C" void CalcMacMedSP27(  real* vxD,
                                 real* vyD,
                                 real* vzD,
                                 real* rhoD,
                                 real* pressD,
                                 unsigned int* geoD,
                                 unsigned int* neighborX,
                                 unsigned int* neighborY,
                                 unsigned int* neighborZ,
                                 unsigned int tdiff,
                                 unsigned int size_Mat,
                                 unsigned int numberOfThreads, 
                                 bool isEvenTimestep);

extern "C" void ResetMedianValuesSP27(
	real* vxD,
	real* vyD,
	real* vzD,
	real* rhoD,
	real* pressD,
	unsigned int size_Mat,
	unsigned int numberOfThreads,
	bool isEvenTimestep);

extern "C" void ResetMedianValuesAD27(
	real* vxD,
	real* vyD,
	real* vzD,
	real* rhoD,
	real* pressD,
	real* concD,
	unsigned int size_Mat,
	unsigned int numberOfThreads,
	bool isEvenTimestep);

extern "C" void Calc2ndMomentsIncompSP27(real* kxyFromfcNEQ,
										 real* kyzFromfcNEQ,
										 real* kxzFromfcNEQ,
										 real* kxxMyyFromfcNEQ,
										 real* kxxMzzFromfcNEQ,
										 unsigned int* geoD,
										 unsigned int* neighborX,
										 unsigned int* neighborY,
										 unsigned int* neighborZ,
										 unsigned int size_Mat,
										 unsigned int numberOfThreads, 
										 real* DD,
										 bool isEvenTimestep);

extern "C" void Calc2ndMomentsCompSP27(real* kxyFromfcNEQ,
									   real* kyzFromfcNEQ,
									   real* kxzFromfcNEQ,
									   real* kxxMyyFromfcNEQ,
									   real* kxxMzzFromfcNEQ,
									   unsigned int* geoD,
									   unsigned int* neighborX,
									   unsigned int* neighborY,
									   unsigned int* neighborZ,
									   unsigned int size_Mat,
									   unsigned int numberOfThreads, 
									   real* DD,
									   bool isEvenTimestep);

extern "C" void Calc3rdMomentsIncompSP27(real* CUMbbb,
										 real* CUMabc,
										 real* CUMbac,
										 real* CUMbca,
										 real* CUMcba,
										 real* CUMacb,
										 real* CUMcab,
										 unsigned int* geoD,
										 unsigned int* neighborX,
										 unsigned int* neighborY,
										 unsigned int* neighborZ,
										 unsigned int size_Mat,
										 unsigned int numberOfThreads, 
										 real* DD,
										 bool isEvenTimestep);

extern "C" void Calc3rdMomentsCompSP27(real* CUMbbb,
									   real* CUMabc,
									   real* CUMbac,
									   real* CUMbca,
									   real* CUMcba,
									   real* CUMacb,
									   real* CUMcab,
									   unsigned int* geoD,
									   unsigned int* neighborX,
									   unsigned int* neighborY,
									   unsigned int* neighborZ,
									   unsigned int size_Mat,
									   unsigned int numberOfThreads, 
									   real* DD,
									   bool isEvenTimestep);

extern "C" void CalcHigherMomentsIncompSP27(real* CUMcbb,
											real* CUMbcb,
											real* CUMbbc,
											real* CUMcca,
											real* CUMcac,
											real* CUMacc,
											real* CUMbcc,
											real* CUMcbc,
											real* CUMccb,
											real* CUMccc,
											unsigned int* geoD,
											unsigned int* neighborX,
											unsigned int* neighborY,
											unsigned int* neighborZ,
											unsigned int size_Mat,
											unsigned int numberOfThreads, 
											real* DD,
											bool isEvenTimestep);

extern "C" void CalcHigherMomentsCompSP27(real* CUMcbb,
										  real* CUMbcb,
										  real* CUMbbc,
										  real* CUMcca,
										  real* CUMcac,
										  real* CUMacc,
										  real* CUMbcc,
										  real* CUMcbc,
										  real* CUMccb,
										  real* CUMccc,
										  unsigned int* geoD,
										  unsigned int* neighborX,
										  unsigned int* neighborY,
										  unsigned int* neighborZ,
										  unsigned int size_Mat,
										  unsigned int numberOfThreads, 
										  real* DD,
										  bool isEvenTimestep);

extern "C" void LBCalcMeasurePoints27(real* vxMP,
                                      real* vyMP,
                                      real* vzMP,
                                      real* rhoMP,
                                      unsigned int* kMP,
                                      unsigned int numberOfPointskMP,
                                      unsigned int MPClockCycle,
                                      unsigned int t,
                                      unsigned int* geoD,
                                      unsigned int* neighborX,
                                      unsigned int* neighborY,
                                      unsigned int* neighborZ,
                                      unsigned int size_Mat,
                                      real* DD,
                                      unsigned int numberOfThreads, 
                                      bool isEvenTimestep);

extern "C" void BcPress27(int nx, 
                          int ny, 
                          int tz, 
                          unsigned int grid_nx, 
                          unsigned int grid_ny, 
                          unsigned int* bcMatD, 
                          unsigned int* neighborX,
                          unsigned int* neighborY,
                          unsigned int* neighborZ,
                          real* DD, 
                          unsigned int size_Mat, 
                          bool isEvenTimestep);

extern "C" void BcVel27(int nx, 
                        int ny, 
                        int nz, 
                        int itz, 
                        unsigned int grid_nx, 
                        unsigned int grid_ny, 
                        unsigned int* bcMatD, 
                        unsigned int* neighborX,
                        unsigned int* neighborY,
                        unsigned int* neighborZ,
                        real* DD, 
                        unsigned int size_Mat, 
                        bool isEvenTimestep, 
                        real u0x, 
                        real om);

extern "C" void QDev27( unsigned int numberOfThreads,
                        int nx,
                        int ny,
                        real* DD, 
                        int* k_Q, 
                        real* QQ,
                        unsigned int numberOfBCnodes, 
                        real om1, 
                        unsigned int* neighborX,
                        unsigned int* neighborY,
                        unsigned int* neighborZ,
                        unsigned int size_Mat, 
                        bool isEvenTimestep);

extern "C" void QDevComp27(unsigned int numberOfThreads,
						   int nx,
						   int ny,
						   real* DD, 
						   int* k_Q, 
						   real* QQ,
						   unsigned int numberOfBCnodes, 
						   real om1, 
						   unsigned int* neighborX,
						   unsigned int* neighborY,
						   unsigned int* neighborZ,
						   unsigned int size_Mat, 
						   bool isEvenTimestep);

extern "C" void QDevCompThinWalls27(unsigned int numberOfThreads,
									real* DD, 
									int* k_Q, 
									real* QQ,
									unsigned int numberOfBCnodes, 
									real om1, 
									unsigned int* geom,
									unsigned int* neighborX,
									unsigned int* neighborY,
									unsigned int* neighborZ,
									unsigned int* neighborWSB,
									unsigned int size_Mat, 
									bool isEvenTimestep);

extern "C" void QDev3rdMomentsComp27(  unsigned int numberOfThreads,
									   int nx,
									   int ny,
									   real* DD, 
									   int* k_Q, 
									   real* QQ,
									   unsigned int numberOfBCnodes, 
									   real om1, 
									   unsigned int* neighborX,
									   unsigned int* neighborY,
									   unsigned int* neighborZ,
									   unsigned int size_Mat, 
									   bool isEvenTimestep);

extern "C" void QDevIncompHighNu27(  unsigned int numberOfThreads,
									 real* DD, 
									 int* k_Q, 
									 real* QQ,
									 unsigned int numberOfBCnodes, 
									 real om1, 
									 unsigned int* neighborX,
									 unsigned int* neighborY,
									 unsigned int* neighborZ,
									 unsigned int size_Mat, 
									 bool isEvenTimestep);

extern "C" void QDevCompHighNu27(unsigned int numberOfThreads,
								 real* DD, 
								 int* k_Q, 
								 real* QQ,
								 unsigned int numberOfBCnodes, 
								 real om1, 
								 unsigned int* neighborX,
								 unsigned int* neighborY,
								 unsigned int* neighborZ,
								 unsigned int size_Mat, 
								 bool isEvenTimestep);

extern "C" void QVelDevicePlainBB27(unsigned int numberOfThreads,
									real* vx,
									real* vy,
									real* vz,
									real* DD,
									int* k_Q, 
									real* QQ,
									unsigned int numberOfBCnodes, 
									real om1, 
									unsigned int* neighborX,
									unsigned int* neighborY,
									unsigned int* neighborZ,
									unsigned int size_Mat, 
									bool isEvenTimestep);
	
extern "C" void QVelDeviceCouette27(unsigned int numberOfThreads,
									real* vx,
									real* vy,
									real* vz,
									real* DD,
									int* k_Q, 
									real* QQ,
									unsigned int numberOfBCnodes, 
									real om1, 
									unsigned int* neighborX,
									unsigned int* neighborY,
									unsigned int* neighborZ,
									unsigned int size_Mat, 
									bool isEvenTimestep);

extern "C" void QVelDevice1h27( unsigned int numberOfThreads,
								int nx,
								int ny,
								real* vx,
								real* vy,
								real* vz,
								real* DD, 
								int* k_Q, 
								real* QQ,
								unsigned int numberOfBCnodes, 
								real om1, 
								real Phi, 
								real angularVelocity,
								unsigned int* neighborX,
								unsigned int* neighborY,
								unsigned int* neighborZ,
								real* coordX,
								real* coordY,
								real* coordZ,
								unsigned int size_Mat, 
								bool isEvenTimestep);

extern "C" void QVelDev27(unsigned int numberOfThreads,
                          int nx,
                          int ny,
                          real* vx,
                          real* vy,
                          real* vz,
                          real* DD, 
                          int* k_Q, 
                          real* QQ,
                          unsigned int numberOfBCnodes, 
                          real om1, 
                          unsigned int* neighborX,
                          unsigned int* neighborY,
                          unsigned int* neighborZ,
                          unsigned int size_Mat, 
                          bool isEvenTimestep);

extern "C" void QVelDevCompPlusSlip27(unsigned int numberOfThreads,
									  real* vx,
									  real* vy,
									  real* vz,
									  real* DD, 
									  int* k_Q, 
									  real* QQ,
									  unsigned int numberOfBCnodes, 
									  real om1, 
									  unsigned int* neighborX,
									  unsigned int* neighborY,
									  unsigned int* neighborZ,
									  unsigned int size_Mat, 
									  bool isEvenTimestep);

extern "C" void QVelDevComp27(unsigned int numberOfThreads,
							  real* velocityX,
							  real* velocityY,
							  real* velocityZ,
							  real* distribution,
							  int* subgridDistanceIndices,
							  real* subgridDistances,
							  unsigned int numberOfBCnodes,
							  real omega,
							  unsigned int* neighborX,
							  unsigned int* neighborY,
							  unsigned int* neighborZ,
							  unsigned int numberOfLBnodes,
							  bool isEvenTimestep);

extern "C" void QVelDevCompThinWalls27(unsigned int numberOfThreads,
							           real* vx,
							           real* vy,
							           real* vz,
							           real* DD, 
							           int* k_Q, 
							           real* QQ,
							           unsigned int numberOfBCnodes, 
							           real om1, 
									   unsigned int* geom,
							           unsigned int* neighborX,
							           unsigned int* neighborY,
							           unsigned int* neighborZ,
									   unsigned int* neighborWSB,
							           unsigned int size_Mat, 
							           bool isEvenTimestep);

extern "C" void QVelDevCompZeroPress27(
    unsigned int numberOfThreads,
    real* velocityX,
    real* velocityY,
    real* velocityZ,
    real* distribution,
    int* subgridDistanceIndices,
    real* subgridDistances,
    unsigned int numberOfBCnodes,
    real omega,
    unsigned int* neighborX,
    unsigned int* neighborY,
    unsigned int* neighborZ,
    unsigned int numberOfLBnodes,
    bool isEvenTimestep);

extern "C" void QVelDevIncompHighNu27(  unsigned int numberOfThreads,
										real* vx,
										real* vy,
										real* vz,
										real* DD, 
										int* k_Q, 
										real* QQ,
										unsigned int numberOfBCnodes, 
										real om1, 
										unsigned int* neighborX,
										unsigned int* neighborY,
										unsigned int* neighborZ,
										unsigned int size_Mat, 
										bool isEvenTimestep);

extern "C" void QVelDevCompHighNu27(unsigned int numberOfThreads,
									real* vx,
									real* vy,
									real* vz,
									real* DD, 
									int* k_Q, 
									real* QQ,
									unsigned int numberOfBCnodes, 
									real om1, 
									unsigned int* neighborX,
									unsigned int* neighborY,
									unsigned int* neighborZ,
									unsigned int size_Mat, 
									bool isEvenTimestep);

extern "C" void QVeloDevEQ27(unsigned int numberOfThreads,
							 real* VeloX,
							 real* VeloY,
							 real* VeloZ,
							 real* DD, 
							 int* k_Q, 
							 int numberOfBCnodes, 
							 real om1, 
							 unsigned int* neighborX,
							 unsigned int* neighborY,
							 unsigned int* neighborZ,
							 unsigned int size_Mat, 
							 bool isEvenTimestep);

extern "C" void QVeloStreetDevEQ27(
	uint  numberOfThreads,
	real* veloXfraction,
	real* veloYfraction,
	int*  naschVelo,
	real* DD,
	int*  naschIndex,
	int   numberOfStreetNodes,
	real  velocityRatio,
	uint* neighborX,
	uint* neighborY,
	uint* neighborZ,
	uint  size_Mat,
	bool  isEvenTimestep);

extern "C" void QSlipDev27( unsigned int numberOfThreads,
							real* DD, 
							int* k_Q, 
							real* QQ,
							unsigned int numberOfBCnodes,
							real om1, 
							unsigned int* neighborX,
							unsigned int* neighborY,
							unsigned int* neighborZ,
							unsigned int size_Mat, 
							bool isEvenTimestep);

extern "C" void QSlipDevComp27(unsigned int numberOfThreads,
							   real* DD, 
							   int* k_Q, 
							   real* QQ,
							   unsigned int numberOfBCnodes,
							   real om1, 
							   unsigned int* neighborX,
							   unsigned int* neighborY,
							   unsigned int* neighborZ,
							   real* turbViscosity,
                        	   bool useTurbViscosity,
							   unsigned int size_Mat, 
							   bool isEvenTimestep);

extern "C" void QSlipGeomDevComp27( unsigned int numberOfThreads,
									real* DD, 
									int* k_Q, 
									real* QQ,
									unsigned int numberOfBCnodes,
									real om1, 
									real* NormalX,
									real* NormalY,
									real* NormalZ,
									unsigned int* neighborX,
									unsigned int* neighborY,
									unsigned int* neighborZ,
									unsigned int size_Mat, 
									bool isEvenTimestep);

extern "C" void QSlipNormDevComp27(unsigned int numberOfThreads,
								   real* DD, 
								   int* k_Q, 
								   real* QQ,
								   unsigned int numberOfBCnodes,
								   real om1, 
								   real* NormalX,
								   real* NormalY,
								   real* NormalZ,
								   unsigned int* neighborX,
								   unsigned int* neighborY,
								   unsigned int* neighborZ,
								   unsigned int size_Mat, 
								   bool isEvenTimestep);

extern "C" void QStressDevComp27(unsigned int numberOfThreads,
								real* DD, 
								int* k_Q, 
								int* k_N,
								real* QQ,
								unsigned int numberOfBCnodes,
								real om1, 
								real* turbViscosity,
								real* vx,
								real* vy,
								real* vz,
								real* normalX,
								real* normalY,
								real* normalZ,
								real* vx_el,
								real* vy_el,
								real* vz_el,
								real* vx_w_mean,
								real* vy_w_mean,
								real* vz_w_mean,
								int* samplingOffset,
								real* z0,
								bool  hasWallModelMonitor,
								real* u_star,
								real* Fx,
								real* Fy,
								real* Fz,
								unsigned int* neighborX,
								unsigned int* neighborY,
								unsigned int* neighborZ,
								unsigned int size_Mat, 
								bool isEvenTimestep);

extern "C" void BBStressDev27(  unsigned int numberOfThreads,
								real* DD, 
								int* k_Q, 
								int* k_N, 
								real* QQ,
								unsigned int numberOfBCnodes,
								real* vx,
								real* vy,
								real* vz,
								real* normalX,
								real* normalY,
								real* normalZ,
								real* vx_el,
								real* vy_el,
								real* vz_el,
								real* vx_w_mean,
								real* vy_w_mean,
								real* vz_w_mean,
								int* samplingOffset,
								real* z0,
								bool  hasWallModelMonitor,
								real* u_star,
								real* Fx,
								real* Fy,
								real* Fz,
								unsigned int* neighborX,
								unsigned int* neighborY,
								unsigned int* neighborZ,
								unsigned int size_Mat, 
								bool isEvenTimestep);

extern "C" void QPressDev27(unsigned int numberOfThreads,
                          real* rhoBC,
                          real* DD, 
                          int* k_Q, 
                          real* QQ,
                          unsigned int numberOfBCnodes, 
                          real om1, 
                          unsigned int* neighborX,
                          unsigned int* neighborY,
                          unsigned int* neighborZ,
                          unsigned int size_Mat, 
                          bool isEvenTimestep);

extern "C" void QPressDevFixBackflow27(unsigned int numberOfThreads,
                                       real* rhoBC,
                                       real* DD, 
                                       int* k_Q, 
                                       unsigned int numberOfBCnodes, 
                                       real om1, 
                                       unsigned int* neighborX,
                                       unsigned int* neighborY,
                                       unsigned int* neighborZ,
                                       unsigned int size_Mat, 
                                       bool isEvenTimestep);

extern "C" void QPressDevDirDepBot27(unsigned int numberOfThreads,
                                     real* rhoBC,
                                     real* DD, 
                                     int* k_Q, 
                                     unsigned int numberOfBCnodes, 
                                     real om1, 
                                     unsigned int* neighborX,
                                     unsigned int* neighborY,
                                     unsigned int* neighborZ,
                                     unsigned int size_Mat, 
                                     bool isEvenTimestep);

extern "C" void QPressNoRhoDev27(  unsigned int numberOfThreads,
								   real* rhoBC,
								   real* DD, 
								   int* k_Q, 
								   int* k_N, 
								   unsigned int numberOfBCnodes, 
								   real om1, 
								   unsigned int* neighborX,
								   unsigned int* neighborY,
								   unsigned int* neighborZ,
								   unsigned int size_Mat, 
								   bool isEvenTimestep);

extern "C" void QInflowScaleByPressDev27(unsigned int numberOfThreads,
										 real* rhoBC,
										 real* DD, 
										 int* k_Q, 
										 int* k_N, 
										 unsigned int numberOfBCnodes, 
										 real om1, 
										 unsigned int* neighborX,
										 unsigned int* neighborY,
										 unsigned int* neighborZ,
										 unsigned int size_Mat, 
										 bool isEvenTimestep);

extern "C" void QPressDevOld27(unsigned int numberOfThreads,
                               real* rhoBC,
                               real* DD, 
                               int* k_Q, 
                               int* k_N, 
                               unsigned int numberOfBCnodes, 
                               real om1, 
                               unsigned int* neighborX,
                               unsigned int* neighborY,
                               unsigned int* neighborZ,
                               unsigned int size_Mat, 
                               bool isEvenTimestep);

extern "C" void QPressDevIncompNEQ27(unsigned int numberOfThreads,
									 real* rhoBC,
									 real* DD, 
									 int* k_Q, 
									 int* k_N, 
									 unsigned int numberOfBCnodes, 
									 real om1, 
									 unsigned int* neighborX,
									 unsigned int* neighborY,
									 unsigned int* neighborZ,
									 unsigned int size_Mat, 
									 bool isEvenTimestep);

extern "C" void QPressDevNEQ27(unsigned int numberOfThreads,
							   real* rhoBC,
							   real* distribution, 
							   int* bcNodeIndices, 
							   int* bcNeighborIndices, 
							   unsigned int numberOfBCnodes, 
							   real omega1, 
							   unsigned int* neighborX,
							   unsigned int* neighborY,
							   unsigned int* neighborZ,
							   unsigned int size_Mat, 
							   bool isEvenTimestep);

extern "C" void QPressDevEQZ27(unsigned int numberOfThreads,
							   real* rhoBC,
							   real* DD, 
							   int* k_Q, 
							   int* k_N, 
							   real* kTestRE, 
							   unsigned int numberOfBCnodes, 
							   real om1, 
							   unsigned int* neighborX,
							   unsigned int* neighborY,
							   unsigned int* neighborZ,
							   unsigned int size_Mat, 
							   bool isEvenTimestep);

extern "C" void QPressDevZero27(unsigned int numberOfThreads,
                                real* DD, 
                                int* k_Q, 
                                unsigned int numberOfBCnodes, 
                                unsigned int* neighborX,
                                unsigned int* neighborY,
                                unsigned int* neighborZ,
                                unsigned int size_Mat, 
                                bool isEvenTimestep);

extern "C" void QPressDevFake27(   unsigned int numberOfThreads,
								   real* rhoBC,
								   real* DD, 
								   int* k_Q, 
								   int* k_N, 
								   unsigned int numberOfBCnodes, 
								   real om1, 
								   unsigned int* neighborX,
								   unsigned int* neighborY,
								   unsigned int* neighborZ,
								   unsigned int size_Mat, 
								   bool isEvenTimestep);

extern "C" void BBDev27( unsigned int numberOfThreads,
                        int nx,
                        int ny,
                        real* DD, 
                        int* k_Q, 
                        real* QQ,
                        unsigned int numberOfBCnodes, 
                        real om1, 
                        unsigned int* neighborX,
                        unsigned int* neighborY,
                        unsigned int* neighborZ,
                        unsigned int size_Mat, 
                        bool isEvenTimestep);

extern "C" void QPressDev27_IntBB(  unsigned int numberOfThreads,
									real* rho,
									real* DD, 
									int* k_Q, 
									real* QQ,
									unsigned int numberOfBCnodes, 
									real om1, 
									unsigned int* neighborX,
									unsigned int* neighborY,
									unsigned int* neighborZ,
									unsigned int size_Mat, 
									bool isEvenTimestep);

extern "C" void QPressDevAntiBB27(  unsigned int numberOfThreads,
								  real* rhoBC,
								  real* vx,
								  real* vy,
								  real* vz,
								  real* DD, 
								  int* k_Q, 
								  real* QQ,
								  int numberOfBCnodes, 
								  real om1, 
								  unsigned int* neighborX,
								  unsigned int* neighborY,
								  unsigned int* neighborZ,
								  unsigned int size_Mat, 
								  bool isEvenTimestep);

extern "C" void PressSchlaffer27(unsigned int numberOfThreads,
                                 real* rhoBC,
                                 real* DD,
                                 real* vx0,
                                 real* vy0,
                                 real* vz0,
                                 real* deltaVz0,
                                 int* k_Q, 
                                 int* k_N, 
                                 int numberOfBCnodes, 
                                 real om1, 
                                 unsigned int* neighborX,
                                 unsigned int* neighborY,
                                 unsigned int* neighborZ,
                                 unsigned int size_Mat, 
                                 bool isEvenTimestep);

extern "C" void VelSchlaffer27(  unsigned int numberOfThreads,
                                 int t,
                                 real* DD,
                                 real* vz0,
                                 real* deltaVz0,
                                 int* k_Q, 
                                 int* k_N, 
                                 int numberOfBCnodes, 
                                 real om1, 
                                 unsigned int* neighborX,
                                 unsigned int* neighborY,
                                 unsigned int* neighborZ,
                                 unsigned int size_Mat, 
                                 bool isEvenTimestep);

extern "C" void QADDev7(unsigned int numberOfThreads,
                        real* DD, 
                        real* DD7,
                        real* temp,
                        real diffusivity,
                        int* k_Q, 
                        real* QQ,
                        unsigned int numberOfBCnodes, 
                        real om1, 
                        unsigned int* neighborX,
                        unsigned int* neighborY,
                        unsigned int* neighborZ,
                        unsigned int size_Mat, 
                        bool isEvenTimestep);

//////////////////////////////////////////////////////////////////////////
//! \brief Advection Diffusion kernel
extern "C" void FactorizedCentralMomentsAdvectionDiffusionDeviceKernel(
	uint numberOfThreads,
	real omegaDiffusivity,
	uint* typeOfGridNode,
	uint* neighborX,
	uint* neighborY,
	uint* neighborZ,
	real* distributions,
	real* distributionsAD,
	int size_Mat,
	real* forces,
	bool isEvenTimestep);

//////////////////////////////////////////////////////////////////////////
//! \brief defines the behavior of a slip-AD boundary condition
extern "C" void ADSlipVelDevComp(
	uint numberOfThreads,
	real * normalX,
	real * normalY,
	real * normalZ,
	real * distributions,
	real * distributionsAD,
	int* QindexArray,
	real * Qarrays,
	uint numberOfBCnodes,
	real omegaDiffusivity,
	uint * neighborX,
	uint * neighborY,
	uint * neighborZ,
	uint size_Mat,
	bool isEvenTimestep);
	
extern "C" void QADDirichletDev27( unsigned int numberOfThreads,
								   real* DD, 
								   real* DD27,
								   real* temp,
								   real diffusivity,
								   int* k_Q, 
								   real* QQ,
								   unsigned int numberOfBCnodes, 
								   real om1, 
								   unsigned int* neighborX,
								   unsigned int* neighborY,
								   unsigned int* neighborZ,
								   unsigned int size_Mat, 
								   bool isEvenTimestep);

extern "C" void QADBBDev27(  unsigned int numberOfThreads,
							 real* DD, 
							 real* DD27,
							 real* temp,
							 real diffusivity,
							 int* k_Q, 
							 real* QQ,
							 unsigned int numberOfBCnodes, 
							 real om1, 
							 unsigned int* neighborX,
							 unsigned int* neighborY,
							 unsigned int* neighborZ,
							 unsigned int size_Mat, 
							 bool isEvenTimestep);

extern "C" void QADVelDev7(unsigned int numberOfThreads,
                           real* DD, 
                           real* DD7,
                           real* temp,
                           real* velo,
                           real diffusivity,
                           int* k_Q, 
                           real* QQ,
                           unsigned int numberOfBCnodes, 
                           real om1, 
                           unsigned int* neighborX,
                           unsigned int* neighborY,
                           unsigned int* neighborZ,
                           unsigned int size_Mat, 
                           bool isEvenTimestep);


extern "C" void QADVelDev27(  unsigned int numberOfThreads,
                              real* DD, 
                              real* DD27,
                              real* temp,
                              real* velo,
                              real diffusivity,
                              int* k_Q, 
                              real* QQ,
                              unsigned int numberOfBCnodes, 
                              real om1, 
                              unsigned int* neighborX,
                              unsigned int* neighborY,
                              unsigned int* neighborZ,
                              unsigned int size_Mat, 
                              bool isEvenTimestep);

extern "C" void QADPressDev7( unsigned int numberOfThreads,
                              real* DD, 
                              real* DD7,
                              real* temp,
                              real* velo,
                              real diffusivity,
                              int* k_Q, 
                              real* QQ,
                              unsigned int numberOfBCnodes, 
                              real om1, 
                              unsigned int* neighborX,
                              unsigned int* neighborY,
                              unsigned int* neighborZ,
                              unsigned int size_Mat, 
                              bool isEvenTimestep);

extern "C" void QADPressDev27(unsigned int numberOfThreads,
                              real* DD, 
                              real* DD27,
                              real* temp,
                              real* velo,
                              real diffusivity,
                              int* k_Q, 
                              real* QQ,
                              unsigned int numberOfBCnodes, 
                              real om1, 
                              unsigned int* neighborX,
                              unsigned int* neighborY,
                              unsigned int* neighborZ,
                              unsigned int size_Mat, 
                              bool isEvenTimestep);

extern "C" void QADPressNEQNeighborDev27(
											unsigned int numberOfThreads,
											real* DD,
											real* DD27,
											int* k_Q,
											int* k_N,
											int numberOfBCnodes,
											unsigned int* neighborX,
											unsigned int* neighborY,
											unsigned int* neighborZ,
											unsigned int size_Mat,
											bool isEvenTimestep
										);

extern "C" void QNoSlipADincompDev7(unsigned int numberOfThreads,
									real* DD, 
									real* DD7,
									real* temp,
									real diffusivity,
									int* k_Q, 
									real* QQ,
									unsigned int numberOfBCnodes, 
									real om1, 
									unsigned int* neighborX,
									unsigned int* neighborY,
									unsigned int* neighborZ,
									unsigned int size_Mat, 
									bool isEvenTimestep);

extern "C" void QNoSlipADincompDev27(unsigned int numberOfThreads,
									 real* DD, 
									 real* DD27,
									 real* temp,
									 real diffusivity,
									 int* k_Q, 
									 real* QQ,
									 unsigned int numberOfBCnodes, 
									 real om1, 
									 unsigned int* neighborX,
									 unsigned int* neighborY,
									 unsigned int* neighborZ,
									 unsigned int size_Mat, 
									 bool isEvenTimestep);

extern "C" void QADVeloIncompDev7( unsigned int numberOfThreads,
								   real* DD, 
								   real* DD7,
								   real* temp,
								   real* velo,
								   real diffusivity,
								   int* k_Q, 
								   real* QQ,
								   unsigned int numberOfBCnodes, 
								   real om1, 
								   unsigned int* neighborX,
								   unsigned int* neighborY,
								   unsigned int* neighborZ,
								   unsigned int size_Mat, 
								   bool isEvenTimestep);


extern "C" void QADVeloIncompDev27( unsigned int numberOfThreads,
									real* DD, 
									real* DD27,
									real* temp,
									real* velo,
									real diffusivity,
									int* k_Q, 
									real* QQ,
									unsigned int numberOfBCnodes, 
									real om1, 
									unsigned int* neighborX,
									unsigned int* neighborY,
									unsigned int* neighborZ,
									unsigned int size_Mat, 
									bool isEvenTimestep);

extern "C" void QADPressIncompDev7(  unsigned int numberOfThreads,
									 real* DD, 
									 real* DD7,
									 real* temp,
									 real* velo,
									 real diffusivity,
									 int* k_Q, 
									 real* QQ,
									 unsigned int numberOfBCnodes, 
									 real om1, 
									 unsigned int* neighborX,
									 unsigned int* neighborY,
									 unsigned int* neighborZ,
									 unsigned int size_Mat, 
									 bool isEvenTimestep);

extern "C" void QADPressIncompDev27(  unsigned int numberOfThreads,
									  real* DD, 
									  real* DD27,
									  real* temp,
									  real* velo,
									  real diffusivity,
									  int* k_Q, 
									  real* QQ,
									  unsigned int numberOfBCnodes, 
									  real om1, 
									  unsigned int* neighborX,
									  unsigned int* neighborY,
									  unsigned int* neighborZ,
									  unsigned int size_Mat, 
									  bool isEvenTimestep);

extern "C" void PropVelo(   unsigned int numberOfThreads,
							unsigned int* neighborX,
							unsigned int* neighborY,
							unsigned int* neighborZ,
							real* rho,
							real* ux,
							real* uy,
							real* uz,
							int* k_Q, 
							unsigned int size_Prop,
							unsigned int size_Mat,
							unsigned int* bcMatD,
							real* DD,
							bool EvenOrOdd);

extern "C" void ScaleCF27( real* DC, 
                           real* DF, 
                           unsigned int* neighborCX,
                           unsigned int* neighborCY,
                           unsigned int* neighborCZ,
                           unsigned int* neighborFX,
                           unsigned int* neighborFY,
                           unsigned int* neighborFZ,
                           unsigned int size_MatC, 
                           unsigned int size_MatF, 
                           bool isEvenTimestep,
                           unsigned int* posCSWB, 
                           unsigned int* posFSWB, 
                           unsigned int kCF, 
                           real omCoarse, 
                           real omFine, 
                           real nu, 
                           unsigned int nxC, 
                           unsigned int nyC, 
                           unsigned int nxF, 
                           unsigned int nyF,
                           unsigned int numberOfThreads);

extern "C" void ScaleFC27( real* DC, 
                           real* DF, 
                           unsigned int* neighborCX,
                           unsigned int* neighborCY,
                           unsigned int* neighborCZ,
                           unsigned int* neighborFX,
                           unsigned int* neighborFY,
                           unsigned int* neighborFZ,
                           unsigned int size_MatC, 
                           unsigned int size_MatF, 
                           bool isEvenTimestep,
                           unsigned int* posC, 
                           unsigned int* posFSWB, 
                           unsigned int kFC, 
                           real omCoarse, 
                           real omFine, 
                           real nu, 
                           unsigned int nxC, 
                           unsigned int nyC, 
                           unsigned int nxF, 
                           unsigned int nyF,
                           unsigned int numberOfThreads);

extern "C" void ScaleCFEff27(real* DC, 
                             real* DF, 
                             unsigned int* neighborCX,
                             unsigned int* neighborCY,
                             unsigned int* neighborCZ,
                             unsigned int* neighborFX,
                             unsigned int* neighborFY,
                             unsigned int* neighborFZ,
                             unsigned int size_MatC, 
                             unsigned int size_MatF, 
                             bool isEvenTimestep,
                             unsigned int* posCSWB, 
                             unsigned int* posFSWB, 
                             unsigned int kCF, 
                             real omCoarse, 
                             real omFine, 
                             real nu, 
                             unsigned int nxC, 
                             unsigned int nyC, 
                             unsigned int nxF, 
                             unsigned int nyF,
                             unsigned int numberOfThreads,
                             OffCF offCF);

extern "C" void ScaleFCEff27(real* DC, 
                             real* DF, 
                             unsigned int* neighborCX,
                             unsigned int* neighborCY,
                             unsigned int* neighborCZ,
                             unsigned int* neighborFX,
                             unsigned int* neighborFY,
                             unsigned int* neighborFZ,
                             unsigned int size_MatC, 
                             unsigned int size_MatF, 
                             bool isEvenTimestep,
                             unsigned int* posC, 
                             unsigned int* posFSWB, 
                             unsigned int kFC, 
                             real omCoarse, 
                             real omFine, 
                             real nu, 
                             unsigned int nxC, 
                             unsigned int nyC, 
                             unsigned int nxF, 
                             unsigned int nyF,
                             unsigned int numberOfThreads,
                             OffFC offFC);

extern "C" void ScaleCFLast27(real* DC, 
                              real* DF, 
                              unsigned int* neighborCX,
                              unsigned int* neighborCY,
                              unsigned int* neighborCZ,
                              unsigned int* neighborFX,
                              unsigned int* neighborFY,
                              unsigned int* neighborFZ,
                              unsigned int size_MatC, 
                              unsigned int size_MatF, 
                              bool isEvenTimestep,
                              unsigned int* posCSWB, 
                              unsigned int* posFSWB, 
                              unsigned int kCF, 
                              real omCoarse, 
                              real omFine, 
                              real nu, 
                              unsigned int nxC, 
                              unsigned int nyC, 
                              unsigned int nxF, 
                              unsigned int nyF,
                              unsigned int numberOfThreads,
                              OffCF offCF);

extern "C" void ScaleFCLast27(real* DC, 
                              real* DF, 
                              unsigned int* neighborCX,
                              unsigned int* neighborCY,
                              unsigned int* neighborCZ,
                              unsigned int* neighborFX,
                              unsigned int* neighborFY,
                              unsigned int* neighborFZ,
                              unsigned int size_MatC, 
                              unsigned int size_MatF, 
                              bool isEvenTimestep,
                              unsigned int* posC, 
                              unsigned int* posFSWB, 
                              unsigned int kFC, 
                              real omCoarse, 
                              real omFine, 
                              real nu, 
                              unsigned int nxC, 
                              unsigned int nyC, 
                              unsigned int nxF, 
                              unsigned int nyF,
                              unsigned int numberOfThreads,
                              OffFC offFC);

extern "C" void ScaleCFpress27(real* DC, 
                              real* DF, 
                              unsigned int* neighborCX,
                              unsigned int* neighborCY,
                              unsigned int* neighborCZ,
                              unsigned int* neighborFX,
                              unsigned int* neighborFY,
                              unsigned int* neighborFZ,
                              unsigned int size_MatC, 
                              unsigned int size_MatF, 
                              bool isEvenTimestep,
                              unsigned int* posCSWB, 
                              unsigned int* posFSWB, 
                              unsigned int kCF, 
                              real omCoarse, 
                              real omFine, 
                              real nu, 
                              unsigned int nxC, 
                              unsigned int nyC, 
                              unsigned int nxF, 
                              unsigned int nyF,
                              unsigned int numberOfThreads,
                              OffCF offCF);

extern "C" void ScaleFCpress27(  real* DC, 
                                 real* DF, 
                                 unsigned int* neighborCX,
                                 unsigned int* neighborCY,
                                 unsigned int* neighborCZ,
                                 unsigned int* neighborFX,
                                 unsigned int* neighborFY,
                                 unsigned int* neighborFZ,
                                 unsigned int size_MatC, 
                                 unsigned int size_MatF, 
                                 bool isEvenTimestep,
                                 unsigned int* posC, 
                                 unsigned int* posFSWB, 
                                 unsigned int kFC, 
                                 real omCoarse, 
                                 real omFine, 
                                 real nu, 
                                 unsigned int nxC, 
                                 unsigned int nyC, 
                                 unsigned int nxF, 
                                 unsigned int nyF,
                                 unsigned int numberOfThreads,
                                 OffFC offFC);

extern "C" void ScaleCF_Fix_27(real* DC, 
                              real* DF, 
                              unsigned int* neighborCX,
                              unsigned int* neighborCY,
                              unsigned int* neighborCZ,
                              unsigned int* neighborFX,
                              unsigned int* neighborFY,
                              unsigned int* neighborFZ,
                              unsigned int size_MatC, 
                              unsigned int size_MatF, 
                              bool isEvenTimestep,
                              unsigned int* posCSWB, 
                              unsigned int* posFSWB, 
                              unsigned int kCF, 
                              real omCoarse, 
                              real omFine, 
                              real nu, 
                              unsigned int nxC, 
                              unsigned int nyC, 
                              unsigned int nxF, 
                              unsigned int nyF,
                              unsigned int numberOfThreads,
                              OffCF offCF);

extern "C" void ScaleCF_Fix_comp_27(   real* DC, 
									   real* DF, 
									   unsigned int* neighborCX,
									   unsigned int* neighborCY,
									   unsigned int* neighborCZ,
									   unsigned int* neighborFX,
									   unsigned int* neighborFY,
									   unsigned int* neighborFZ,
									   unsigned int size_MatC, 
									   unsigned int size_MatF, 
									   bool isEvenTimestep,
									   unsigned int* posCSWB, 
									   unsigned int* posFSWB, 
									   unsigned int kCF, 
									   real omCoarse, 
									   real omFine, 
									   real nu, 
									   unsigned int nxC, 
									   unsigned int nyC, 
									   unsigned int nxF, 
									   unsigned int nyF,
									   unsigned int numberOfThreads,
									   OffCF offCF);

extern "C" void ScaleCF_0817_comp_27(  real* DC, 
									   real* DF, 
									   unsigned int* neighborCX,
									   unsigned int* neighborCY,
									   unsigned int* neighborCZ,
									   unsigned int* neighborFX,
									   unsigned int* neighborFY,
									   unsigned int* neighborFZ,
									   unsigned int size_MatC, 
									   unsigned int size_MatF, 
									   bool isEvenTimestep,
									   unsigned int* posCSWB, 
									   unsigned int* posFSWB, 
									   unsigned int kCF, 
									   real omCoarse, 
									   real omFine, 
									   real nu, 
									   unsigned int nxC, 
									   unsigned int nyC, 
									   unsigned int nxF, 
									   unsigned int nyF,
									   unsigned int numberOfThreads,
									   OffCF offCF,
									   CUstream_st* stream);

extern "C" void ScaleCF_comp_D3Q27F3_2018(	real* DC,
											real* DF,
											real* G6, 
											unsigned int* neighborCX,
											unsigned int* neighborCY,
											unsigned int* neighborCZ,
											unsigned int* neighborFX,
											unsigned int* neighborFY,
											unsigned int* neighborFZ,
											unsigned int size_MatC, 
											unsigned int size_MatF, 
											bool isEvenTimestep,
											unsigned int* posCSWB, 
											unsigned int* posFSWB, 
											unsigned int kCF, 
											real omCoarse, 
											real omFine, 
											real nu, 
											unsigned int nxC, 
											unsigned int nyC, 
											unsigned int nxF, 
											unsigned int nyF,
											unsigned int numberOfThreads,
											OffCF offCF);

extern "C" void ScaleCF_comp_D3Q27F3(real* DC,
									 real* DF,
									 real* G6, 
									 unsigned int* neighborCX,
									 unsigned int* neighborCY,
									 unsigned int* neighborCZ,
									 unsigned int* neighborFX,
									 unsigned int* neighborFY,
									 unsigned int* neighborFZ,
									 unsigned int size_MatC, 
									 unsigned int size_MatF, 
									 bool isEvenTimestep,
									 unsigned int* posCSWB, 
									 unsigned int* posFSWB, 
									 unsigned int kCF, 
									 real omCoarse, 
									 real omFine, 
									 real nu, 
									 unsigned int nxC, 
									 unsigned int nyC, 
									 unsigned int nxF, 
									 unsigned int nyF,
									 unsigned int numberOfThreads,
									 OffCF offCF,
									 CUstream_st *stream);

extern "C" void ScaleCF_staggered_time_comp_27( real* DC, 
												real* DF, 
												unsigned int* neighborCX,
												unsigned int* neighborCY,
												unsigned int* neighborCZ,
												unsigned int* neighborFX,
												unsigned int* neighborFY,
												unsigned int* neighborFZ,
												unsigned int size_MatC, 
												unsigned int size_MatF, 
												bool isEvenTimestep,
												unsigned int* posCSWB, 
												unsigned int* posFSWB, 
												unsigned int kCF, 
												real omCoarse, 
												real omFine, 
												real nu, 
												unsigned int nxC, 
												unsigned int nyC, 
												unsigned int nxF, 
												unsigned int nyF,
												unsigned int numberOfThreads,
												OffCF offCF);

extern "C" void ScaleCF_RhoSq_comp_27(  real* DC, 
										real* DF, 
										unsigned int* neighborCX,
										unsigned int* neighborCY,
										unsigned int* neighborCZ,
										unsigned int* neighborFX,
										unsigned int* neighborFY,
										unsigned int* neighborFZ,
										unsigned int size_MatC, 
										unsigned int size_MatF, 
										bool isEvenTimestep,
										unsigned int* posCSWB, 
										unsigned int* posFSWB, 
										unsigned int kCF, 
										real omCoarse, 
										real omFine, 
										real nu, 
										unsigned int nxC, 
										unsigned int nyC, 
										unsigned int nxF, 
										unsigned int nyF,
										unsigned int numberOfThreads,
										OffCF offCF,
                                        CUstream_st *stream);

extern "C" void ScaleCF_RhoSq_3rdMom_comp_27( real* DC, 
											  real* DF, 
											  unsigned int* neighborCX,
											  unsigned int* neighborCY,
											  unsigned int* neighborCZ,
											  unsigned int* neighborFX,
											  unsigned int* neighborFY,
											  unsigned int* neighborFZ,
											  unsigned int size_MatC, 
											  unsigned int size_MatF, 
											  bool isEvenTimestep,
											  unsigned int* posCSWB, 
											  unsigned int* posFSWB, 
											  unsigned int kCF, 
											  real omCoarse, 
											  real omFine, 
											  real nu, 
											  unsigned int nxC, 
											  unsigned int nyC, 
											  unsigned int nxF, 
											  unsigned int nyF,
											  unsigned int numberOfThreads,
											  OffCF offCF,
											  CUstream_st *stream);

extern "C" void ScaleCF_AA2016_comp_27( real* DC, 
										real* DF, 
										unsigned int* neighborCX,
										unsigned int* neighborCY,
										unsigned int* neighborCZ,
										unsigned int* neighborFX,
										unsigned int* neighborFY,
										unsigned int* neighborFZ,
										unsigned int size_MatC, 
										unsigned int size_MatF, 
										bool isEvenTimestep,
										unsigned int* posCSWB, 
										unsigned int* posFSWB, 
										unsigned int kCF, 
										real omCoarse, 
										real omFine, 
										real nu, 
										unsigned int nxC, 
										unsigned int nyC, 
										unsigned int nxF, 
										unsigned int nyF,
										unsigned int numberOfThreads,
										OffCF offCF,
										CUstream_st *stream);

extern "C" void ScaleCF_NSPress_27(real* DC, 
								  real* DF, 
								  unsigned int* neighborCX,
								  unsigned int* neighborCY,
								  unsigned int* neighborCZ,
								  unsigned int* neighborFX,
								  unsigned int* neighborFY,
								  unsigned int* neighborFZ,
								  unsigned int size_MatC, 
								  unsigned int size_MatF, 
								  bool isEvenTimestep,
								  unsigned int* posCSWB, 
								  unsigned int* posFSWB, 
								  unsigned int kCF, 
								  real omCoarse, 
								  real omFine, 
								  real nu, 
								  unsigned int nxC, 
								  unsigned int nyC, 
								  unsigned int nxF, 
								  unsigned int nyF,
								  unsigned int numberOfThreads,
								  OffCF offCF);

extern "C" void ScaleFC_Fix_27(  real* DC, 
                                 real* DF, 
                                 unsigned int* neighborCX,
                                 unsigned int* neighborCY,
                                 unsigned int* neighborCZ,
                                 unsigned int* neighborFX,
                                 unsigned int* neighborFY,
                                 unsigned int* neighborFZ,
                                 unsigned int size_MatC, 
                                 unsigned int size_MatF, 
                                 bool isEvenTimestep,
                                 unsigned int* posC, 
                                 unsigned int* posFSWB, 
                                 unsigned int kFC, 
                                 real omCoarse, 
                                 real omFine, 
                                 real nu, 
                                 unsigned int nxC, 
                                 unsigned int nyC, 
                                 unsigned int nxF, 
                                 unsigned int nyF,
                                 unsigned int numberOfThreads,
                                 OffFC offFC);

extern "C" void ScaleFC_Fix_comp_27(   real* DC, 
									   real* DF, 
									   unsigned int* neighborCX,
									   unsigned int* neighborCY,
									   unsigned int* neighborCZ,
									   unsigned int* neighborFX,
									   unsigned int* neighborFY,
									   unsigned int* neighborFZ,
									   unsigned int size_MatC, 
									   unsigned int size_MatF, 
									   bool isEvenTimestep,
									   unsigned int* posC, 
									   unsigned int* posFSWB, 
									   unsigned int kFC, 
									   real omCoarse, 
									   real omFine, 
									   real nu, 
									   unsigned int nxC, 
									   unsigned int nyC, 
									   unsigned int nxF, 
									   unsigned int nyF,
									   unsigned int numberOfThreads,
									   OffFC offFC);

extern "C" void ScaleFC_0817_comp_27(  real* DC, 
									   real* DF, 
									   unsigned int* neighborCX,
									   unsigned int* neighborCY,
									   unsigned int* neighborCZ,
									   unsigned int* neighborFX,
									   unsigned int* neighborFY,
									   unsigned int* neighborFZ,
									   unsigned int size_MatC, 
									   unsigned int size_MatF, 
									   bool isEvenTimestep,
									   unsigned int* posC, 
									   unsigned int* posFSWB, 
									   unsigned int kFC, 
									   real omCoarse, 
									   real omFine, 
									   real nu, 
									   unsigned int nxC, 
									   unsigned int nyC, 
									   unsigned int nxF, 
									   unsigned int nyF,
									   unsigned int numberOfThreads,
									   OffFC offFC,
									   CUstream_st *stream);
extern "C" void ScaleFC_comp_D3Q27F3_2018(real* DC,
										  real* DF,
										  real* G6,
										  unsigned int* neighborCX,
										  unsigned int* neighborCY,
										  unsigned int* neighborCZ,
										  unsigned int* neighborFX,
										  unsigned int* neighborFY,
										  unsigned int* neighborFZ,
										  unsigned int size_MatC, 
										  unsigned int size_MatF, 
										  bool isEvenTimestep,
										  unsigned int* posC, 
										  unsigned int* posFSWB, 
										  unsigned int kFC, 
										  real omCoarse, 
										  real omFine, 
										  real nu, 
										  unsigned int nxC, 
										  unsigned int nyC, 
										  unsigned int nxF, 
										  unsigned int nyF,
										  unsigned int numberOfThreads,
										  OffFC offFC);

extern "C" void ScaleFC_comp_D3Q27F3( real* DC,
									  real* DF,
									  real* G6,
									  unsigned int* neighborCX,
									  unsigned int* neighborCY,
									  unsigned int* neighborCZ,
									  unsigned int* neighborFX,
									  unsigned int* neighborFY,
									  unsigned int* neighborFZ,
									  unsigned int size_MatC, 
									  unsigned int size_MatF, 
									  bool isEvenTimestep,
									  unsigned int* posC, 
									  unsigned int* posFSWB, 
									  unsigned int kFC, 
									  real omCoarse, 
									  real omFine, 
									  real nu, 
									  unsigned int nxC, 
									  unsigned int nyC, 
									  unsigned int nxF, 
									  unsigned int nyF,
									  unsigned int numberOfThreads,
									  OffFC offFC,
									  CUstream_st *stream);

extern "C" void ScaleFC_staggered_time_comp_27( real* DC, 
												real* DF, 
												unsigned int* neighborCX,
												unsigned int* neighborCY,
												unsigned int* neighborCZ,
												unsigned int* neighborFX,
												unsigned int* neighborFY,
												unsigned int* neighborFZ,
												unsigned int size_MatC, 
												unsigned int size_MatF, 
												bool isEvenTimestep,
												unsigned int* posC, 
												unsigned int* posFSWB, 
												unsigned int kFC, 
												real omCoarse, 
												real omFine, 
												real nu, 
												unsigned int nxC, 
												unsigned int nyC, 
												unsigned int nxF, 
												unsigned int nyF,
												unsigned int numberOfThreads,
												OffFC offFC);

extern "C" void ScaleFC_RhoSq_comp_27(  real* DC, 
										real* DF, 
										unsigned int* neighborCX,
										unsigned int* neighborCY,
										unsigned int* neighborCZ,
										unsigned int* neighborFX,
										unsigned int* neighborFY,
										unsigned int* neighborFZ,
										unsigned int size_MatC, 
										unsigned int size_MatF, 
										bool isEvenTimestep,
										unsigned int* posC, 
										unsigned int* posFSWB, 
										unsigned int kFC, 
										real omCoarse, 
										real omFine, 
										real nu, 
										unsigned int nxC, 
										unsigned int nyC, 
										unsigned int nxF, 
										unsigned int nyF,
										unsigned int numberOfThreads, 
	                                    OffFC offFC,
                                        CUstream_st *stream);

extern "C" void ScaleFC_RhoSq_3rdMom_comp_27( real* DC, 
											  real* DF, 
											  unsigned int* neighborCX,
											  unsigned int* neighborCY,
											  unsigned int* neighborCZ,
											  unsigned int* neighborFX,
											  unsigned int* neighborFY,
											  unsigned int* neighborFZ,
											  unsigned int size_MatC, 
											  unsigned int size_MatF, 
											  bool isEvenTimestep,
											  unsigned int* posC, 
											  unsigned int* posFSWB, 
											  unsigned int kFC, 
											  real omCoarse, 
											  real omFine, 
											  real nu, 
											  unsigned int nxC, 
											  unsigned int nyC, 
											  unsigned int nxF, 
											  unsigned int nyF,
											  unsigned int numberOfThreads,
											  OffFC offFC,
											  CUstream_st *stream);

extern "C" void ScaleFC_AA2016_comp_27( real* DC, 
										real* DF, 
										unsigned int* neighborCX,
										unsigned int* neighborCY,
										unsigned int* neighborCZ,
										unsigned int* neighborFX,
										unsigned int* neighborFY,
										unsigned int* neighborFZ,
										unsigned int size_MatC, 
										unsigned int size_MatF, 
										bool isEvenTimestep,
										unsigned int* posC, 
										unsigned int* posFSWB, 
										unsigned int kFC, 
										real omCoarse, 
										real omFine, 
										real nu, 
										unsigned int nxC, 
										unsigned int nyC, 
										unsigned int nxF, 
										unsigned int nyF,
										unsigned int numberOfThreads,
										OffFC offFC,
										CUstream_st *stream);

extern "C" void ScaleFC_NSPress_27(  real* DC, 
									 real* DF, 
									 unsigned int* neighborCX,
									 unsigned int* neighborCY,
									 unsigned int* neighborCZ,
									 unsigned int* neighborFX,
									 unsigned int* neighborFY,
									 unsigned int* neighborFZ,
									 unsigned int size_MatC, 
									 unsigned int size_MatF, 
									 bool isEvenTimestep,
									 unsigned int* posC, 
									 unsigned int* posFSWB, 
									 unsigned int kFC, 
									 real omCoarse, 
									 real omFine, 
									 real nu, 
									 unsigned int nxC, 
									 unsigned int nyC, 
									 unsigned int nxF, 
									 unsigned int nyF,
									 unsigned int numberOfThreads,
									 OffFC offFC);

extern "C" void ScaleCFThS7(  real* DC, 
                              real* DF, 
                              real* DD7C, 
                              real* DD7F,
                              unsigned int* neighborCX,
                              unsigned int* neighborCY,
                              unsigned int* neighborCZ,
                              unsigned int* neighborFX,
                              unsigned int* neighborFY,
                              unsigned int* neighborFZ,
                              unsigned int size_MatC, 
                              unsigned int size_MatF, 
                              bool isEvenTimestep,
                              unsigned int* posCSWB, 
                              unsigned int* posFSWB, 
                              unsigned int kCF, 
                              real nu,
                              real diffusivity_fine,
                              unsigned int numberOfThreads);

extern "C" void ScaleFCThS7(  real* DC, 
                              real* DF,
                              real* DD7C, 
                              real* DD7F,
                              unsigned int* neighborCX,
                              unsigned int* neighborCY,
                              unsigned int* neighborCZ,
                              unsigned int* neighborFX,
                              unsigned int* neighborFY,
                              unsigned int* neighborFZ,
                              unsigned int size_MatC, 
                              unsigned int size_MatF, 
                              bool isEvenTimestep,
                              unsigned int* posC, 
                              unsigned int* posFSWB, 
                              unsigned int kFC, 
                              real nu,
                              real diffusivity_coarse,
                              unsigned int numberOfThreads);

extern "C" void ScaleCFThSMG7(   real* DC, 
                                 real* DF,
                                 real* DD7C, 
                                 real* DD7F,
                                 unsigned int* neighborCX,
                                 unsigned int* neighborCY,
                                 unsigned int* neighborCZ,
                                 unsigned int* neighborFX,
                                 unsigned int* neighborFY,
                                 unsigned int* neighborFZ,
                                 unsigned int size_MatC, 
                                 unsigned int size_MatF, 
                                 bool isEvenTimestep,
                                 unsigned int* posCSWB, 
                                 unsigned int* posFSWB, 
                                 unsigned int kCF, 
                                 real nu,
                                 real diffusivity_fine,
                                 unsigned int numberOfThreads,
                                 OffCF offCF);

extern "C" void ScaleFCThSMG7(real* DC, 
                              real* DF,
                              real* DD7C, 
                              real* DD7F,
                              unsigned int* neighborCX,
                              unsigned int* neighborCY,
                              unsigned int* neighborCZ,
                              unsigned int* neighborFX,
                              unsigned int* neighborFY,
                              unsigned int* neighborFZ,
                              unsigned int size_MatC, 
                              unsigned int size_MatF, 
                              bool isEvenTimestep,
                              unsigned int* posC, 
                              unsigned int* posFSWB, 
                              unsigned int kFC, 
                              real nu,
                              real diffusivity_coarse,
                              unsigned int numberOfThreads,
                              OffFC offFC);

extern "C" void ScaleCFThS27( real* DC, 
                              real* DF, 
                              real* DD27C, 
                              real* DD27F,
                              unsigned int* neighborCX,
                              unsigned int* neighborCY,
                              unsigned int* neighborCZ,
                              unsigned int* neighborFX,
                              unsigned int* neighborFY,
                              unsigned int* neighborFZ,
                              unsigned int size_MatC, 
                              unsigned int size_MatF, 
                              bool isEvenTimestep,
                              unsigned int* posCSWB, 
                              unsigned int* posFSWB, 
                              unsigned int kCF, 
                              real nu,
                              real diffusivity_fine,
							  unsigned int numberOfThreads,
							  OffCF offCF);

extern "C" void ScaleFCThS27( real* DC, 
                              real* DF,
                              real* DD27C, 
                              real* DD27F,
                              unsigned int* neighborCX,
                              unsigned int* neighborCY,
                              unsigned int* neighborCZ,
                              unsigned int* neighborFX,
                              unsigned int* neighborFY,
                              unsigned int* neighborFZ,
                              unsigned int size_MatC, 
                              unsigned int size_MatF, 
                              bool isEvenTimestep,
                              unsigned int* posC, 
                              unsigned int* posFSWB, 
                              unsigned int kFC, 
                              real nu,
                              real diffusivity_coarse,
							  unsigned int numberOfThreads,
							  OffFC offFC);

extern "C" void DragLiftPostD27(real* DD, 
								int* k_Q, 
								real* QQ,
								int numberOfBCnodes, 
								double *DragX,
								double *DragY,
								double *DragZ,
								unsigned int* neighborX,
								unsigned int* neighborY,
								unsigned int* neighborZ,
								unsigned int size_Mat, 
								bool isEvenTimestep,
								unsigned int numberOfThreads);

extern "C" void DragLiftPreD27( real* DD, 
								int* k_Q, 
								real* QQ,
								int numberOfBCnodes, 
								double *DragX,
								double *DragY,
								double *DragZ,
								unsigned int* neighborX,
								unsigned int* neighborY,
								unsigned int* neighborZ,
								unsigned int size_Mat, 
								bool isEvenTimestep,
								unsigned int numberOfThreads);

extern "C" void CalcCPtop27(real* DD, 
							int* cpIndex, 
							int nonCp, 
							double *cpPress,
							unsigned int* neighborX,
							unsigned int* neighborY,
							unsigned int* neighborZ,
							unsigned int size_Mat, 
							bool isEvenTimestep,
							unsigned int numberOfThreads);

extern "C" void CalcCPbottom27(real* DD, 
							   int* cpIndex, 
							   int nonCp, 
							   double *cpPress,
							   unsigned int* neighborX,
							   unsigned int* neighborY,
							   unsigned int* neighborZ,
							   unsigned int size_Mat, 
							   bool isEvenTimestep,
							   unsigned int numberOfThreads);

extern "C" void GetSendFsPreDev27(real* DD,
								  real* bufferFs,
								  int* sendIndex,
								  int buffmax,
								  unsigned int* neighborX,
								  unsigned int* neighborY,
								  unsigned int* neighborZ,
								  unsigned int size_Mat, 
								  bool isEvenTimestep,
								  unsigned int numberOfThreads, 
	                              cudaStream_t stream = CU_STREAM_LEGACY);

extern "C" void GetSendFsPostDev27(real* DD,
								   real* bufferFs,
								   int* sendIndex,
								   int buffmax,
								   unsigned int* neighborX,
								   unsigned int* neighborY,
								   unsigned int* neighborZ,
								   unsigned int size_Mat, 
								   bool isEvenTimestep,
								   unsigned int numberOfThreads, 
	                               cudaStream_t stream = CU_STREAM_LEGACY);

extern "C" void SetRecvFsPreDev27(real* DD,
								  real* bufferFs,
								  int* recvIndex,
								  int buffmax,
								  unsigned int* neighborX,
								  unsigned int* neighborY,
								  unsigned int* neighborZ,
								  unsigned int size_Mat, 
								  bool isEvenTimestep, unsigned int numberOfThreads, 
	                              cudaStream_t stream = CU_STREAM_LEGACY);

extern "C" void SetRecvFsPostDev27(real* DD,
								   real* bufferFs,
								   int* recvIndex,
								   int buffmax,
								   unsigned int* neighborX,
								   unsigned int* neighborY,
								   unsigned int* neighborZ,
								   unsigned int size_Mat, 
								   bool isEvenTimestep,
								   unsigned int numberOfThreads,
                                   cudaStream_t stream = CU_STREAM_LEGACY);

extern "C" void getSendGsDevF3(
	real* G6,
	real* bufferGs,
	int* sendIndex,
	int buffmax,
	unsigned int* neighborX,
	unsigned int* neighborY,
	unsigned int* neighborZ,
	unsigned int size_Mat,
	bool isEvenTimestep,
	unsigned int numberOfThreads);

extern "C" void setRecvGsDevF3(
	real* G6,
	real* bufferGs,
	int* recvIndex,
	int buffmax,
	unsigned int* neighborX,
	unsigned int* neighborY,
	unsigned int* neighborZ,
	unsigned int size_Mat,
	bool isEvenTimestep,
	unsigned int numberOfThreads);

extern "C" void WallFuncDev27(unsigned int numberOfThreads,
							  real* vx,
							  real* vy,
							  real* vz,
							  real* DD, 
							  int* k_Q, 
							  real* QQ,
							  unsigned int numberOfBCnodes, 
							  real om1, 
							  unsigned int* neighborX,
							  unsigned int* neighborY,
							  unsigned int* neighborZ,
							  unsigned int size_Mat, 
							  bool isEvenTimestep);

extern "C" void SetOutputWallVelocitySP27(unsigned int numberOfThreads,
										  real* vxD,
										  real* vyD,
										  real* vzD,
										  real* vxWall,
										  real* vyWall,
										  real* vzWall,
										  int numberOfWallNodes, 
										  int* kWallNodes, 
										  real* rhoD,
										  real* pressD,
										  unsigned int* geoD,
										  unsigned int* neighborX,
										  unsigned int* neighborY,
										  unsigned int* neighborZ,
										  unsigned int size_Mat,
										  real* DD,
										  bool isEvenTimestep);

extern "C" void GetVelotoForce27(unsigned int numberOfThreads,
								 real* DD, 
								 int* bcIndex, 
								 int nonAtBC, 
								 real* Vx,
								 real* Vy,
								 real* Vz,
								 unsigned int* neighborX,
								 unsigned int* neighborY,
								 unsigned int* neighborZ,
								 unsigned int size_Mat, 
								 bool isEvenTimestep);

extern "C" void InitParticlesDevice(real* coordX,
									real* coordY,
									real* coordZ, 
									real* coordParticleXlocal,
									real* coordParticleYlocal,
									real* coordParticleZlocal, 
									real* coordParticleXglobal,
									real* coordParticleYglobal,
									real* coordParticleZglobal,
									real* veloParticleX,
									real* veloParticleY,
									real* veloParticleZ,
									real* randArray,
									unsigned int* particleID,
									unsigned int* cellBaseID,
									unsigned int* bcMatD,
									unsigned int* neighborX,
									unsigned int* neighborY,
									unsigned int* neighborZ,
									unsigned int* neighborWSB,
									int level,
									unsigned int numberOfParticles, 
									unsigned int size_Mat,
									unsigned int numberOfThreads);

extern "C" void MoveParticlesDevice(real* coordX,
									real* coordY,
									real* coordZ, 
									real* coordParticleXlocal,
									real* coordParticleYlocal,
									real* coordParticleZlocal, 
									real* coordParticleXglobal,
									real* coordParticleYglobal,
									real* coordParticleZglobal,
									real* veloParticleX,
									real* veloParticleY,
									real* veloParticleZ,
									real* DD,
									real  omega,
									unsigned int* particleID,
									unsigned int* cellBaseID,
									unsigned int* bcMatD,
									unsigned int* neighborX,
									unsigned int* neighborY,
									unsigned int* neighborZ,
									unsigned int* neighborWSB,
									int level,
									unsigned int timestep, 
									unsigned int numberOfTimesteps, 
									unsigned int numberOfParticles, 
									unsigned int size_Mat,
									unsigned int numberOfThreads,
									bool isEvenTimestep);

extern "C" void initRandomDevice(curandState* state,
								 unsigned int size_Mat,
								 unsigned int numberOfThreads);

extern "C" void generateRandomValuesDevice(curandState* state,
										   unsigned int size_Mat,
										   real* randArray,
										   unsigned int numberOfThreads);

extern "C" void CalcTurbulenceIntensityDevice(
   real* vxx,
   real* vyy,
   real* vzz,
   real* vxy,
   real* vxz,
   real* vyz,
   real* vx_mean,
   real* vy_mean,
   real* vz_mean,
   real* DD, 
   uint *typeOfGridNode, 
   unsigned int* neighborX,
   unsigned int* neighborY,
   unsigned int* neighborZ,
   unsigned int size_Mat, 
   bool isEvenTimestep,
   uint numberOfThreads);

#endif