.gitlab-ci.yml

# #######################################################################################
# ____          ____    __    ______     __________   __      __       __        __
# \    \       |    |  |  |  |   _   \  |___    ___| |  |    |  |     /  \      |  |
#  \    \      |    |  |  |  |  |_)   |     |  |     |  |    |  |    /    \     |  |
#   \    \     |    |  |  |  |   _   /      |  |     |  |    |  |   /  /\  \    |  |
#    \    \    |    |  |  |  |  | \  \      |  |     |   \__/   |  /  ____  \   |  |____
#     \    \   |    |  |__|  |__|  \__\     |__|      \________/  /__/    \__\  |_______|
#      \    \  |    |   ________________________________________________________________
#       \    \ |    |  |  ______________________________________________________________|
#        \    \|    |  |  |         __          __     __     __     ______      _______
#         \         |  |  |_____   |  |        |  |   |  |   |  |   |   _  \    /  _____)
#          \        |  |   _____|  |  |        |  |   |  |   |  |   |  | \  \   \_______
#           \       |  |  |        |  |_____   |   \_/   |   |  |   |  |_/  /    _____  |
#            \ _____|  |__|        |________|   \_______/    |__|   |______/    (_______/
#
#  This file is part of VirtualFluids. VirtualFluids is free software: you can
#  redistribute it and/or modify it under the terms of the GNU General Public
#  License as published by the Free Software Foundation, either version 3 of
#  the License, or (at your option) any later version.
#
#  VirtualFluids is distributed in the hope that it will be useful, but WITHOUT
#  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
#  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
#  for more details.
#
#  You should have received a copy of the GNU General Public License along
#  with VirtualFluids (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>
# #######################################################################################
image: git.rz.tu-bs.de:4567/irmb/virtualfluids/ubuntu22_04:1.1

stages:
  - build
  - test
  - analyze
  - deploy

workflow:
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: never  # disable detached merge request pipelines
    - when: always # add all jobs to normal pipeline. This can be overwritten by the rules of the jobs.

###############################################################################
##                                Builds                                     ##
###############################################################################
.gnu_build_template:
  stage: build

  tags:
    - gpu
    - linux

  cache:
    key: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"
    paths:
      - $BUILD_FOLDER
      - $CI_PROJECT_DIR/cache

  artifacts:
    expire_in: 1 hrs
    paths:
      - $BUILD_FOLDER

  script:
    - export CCACHE_BASEDIR=$CI_PROJECT_DIR
    - export CCACHE_DIR=$CI_PROJECT_DIR/cache
    - export CCACHE_COMPILERCHECK=content
    - ccache --zero-stats
    - ccache --show-stats
    - $CXX --version
    - $CC --version
    - cmake --version
    - mpirun --version
    - mkdir -p $CI_PROJECT_DIR/$BUILD_FOLDER
    - cd $CI_PROJECT_DIR/$BUILD_FOLDER
    - rm -r -f ./*
    - cmake .. -LAH
      --preset=make_all
      -DVF_ENABLE_WARNINGS_AS_ERRORS=ON
      -DCMAKE_CUDA_ARCHITECTURES=70
    - make -j4
    - ccache --show-stats

  variables:
    BUILD_FOLDER: "build"

###############################################################################
gcc_12:
  extends: .gnu_build_template

  before_script:
    - export CC=gcc
    - export CXX=g++

###############################################################################
gcc_12_float:
  extends: .gnu_build_template

  before_script:
    - export CC=gcc
    - export CXX=g++

  script:
    - export CCACHE_BASEDIR=$CI_PROJECT_DIR
    - export CCACHE_DIR=$CI_PROJECT_DIR/cache
    - export CCACHE_COMPILERCHECK=content
    - ccache --zero-stats
    - ccache --show-stats
    - $CXX --version
    - $CC --version
    - cmake --version
    - mpirun --version
    - mkdir -p $CI_PROJECT_DIR/$BUILD_FOLDER
    - cd $CI_PROJECT_DIR/$BUILD_FOLDER
    - rm -r -f ./*
    - cmake .. -LAH
      --preset=make_all
      -DVF_ENABLE_DOUBLE_ACCURACY=OFF
      -DVF_ENABLE_WARNINGS_AS_ERRORS=ON
      -DCMAKE_CUDA_ARCHITECTURES=70
    - make -j4
    - ccache --show-stats

###############################################################################
clang_15:
  extends: .gnu_build_template

  before_script:
    - export CC=clang
    - export CXX=clang++

###############################################################################
msvc_17:
  stage: build

  tags:
    - win
    - gpu

  variables:
    BUILD_CONFIGURATION: "Release"
    BUILD_FOLDER: "build"

  # add cmake and MSBuild.exe to the path.
  # This Needs to be adapted when moved to a new build machine.
  before_script:
    - git --version
    - $env:Path += ";C:\Program Files\CMake\bin\"
    - cmake --version
    - $env:Path += ";C:\Program Files\Microsoft Visual Studio\2022\Community\MSBuild\Current\Bin"
    - MSBuild.exe -version

  script:
    - cd $CI_PROJECT_DIR
    - md -force $env:BUILD_FOLDER
    - cd $env:BUILD_FOLDER
    - cmake .. --preset=msvc_all -DVF_ENABLE_PYTHON_BINDINGS=OFF -DCMAKE_CUDA_ARCHITECTURES=61 -DVF_ENABLE_WARNINGS_AS_ERRORS=ON
    - MSBuild.exe VirtualFluids.sln /property:Configuration=$env:BUILD_CONFIGURATION /verbosity:minimal /maxcpucount:4

  artifacts:
    expire_in: 1 hrs
    paths:
      - $CI_PROJECT_DIR/$env:BUILD_FOLDER/


###############################################################################
gcc_12_python:
  stage: build

  cache:
    key: "gcc_12-$CI_COMMIT_REF_SLUG"
    paths:
      - pybuild/

  artifacts:
    expire_in: 1 hrs
    paths:
      - pybuild/ # build in separate folder to avoid conflicts with c++ build

  before_script:
    - export CCACHE_BASEDIR=$CI_PROJECT_DIR
    - export CCACHE_DIR=$CI_PROJECT_DIR/cache

  script:
    - export SKBUILD_BUILD_DIR="pybuild"
    - export SKBUILD_CMAKE_ARGS="--preset=make_all;-G=Ninja"
    - pip install . -v

###############################################################################
##                                Tests                                      ##
###############################################################################
gcc_12_unit_tests:
  stage: test

  needs: ["gcc_12"]

  before_script:
    - cd $CI_PROJECT_DIR/build

  script:
    - ctest --output-on-failure

###############################################################################
msvc_17_unit_tests:
  stage: test

  tags:
    - win
    - gpu

  needs: ["msvc_17"]

  before_script:
    - $env:Path += ";C:\Program Files\CMake\bin\"
    - ctest --version

  script:
    - cd $CI_PROJECT_DIR/build
    - ctest --output-on-failure -C Release

###############################################################################
gcc_12_python_bindings_test:
  stage: test

  needs: ["gcc_12_python"]

  before_script:
    - pip3 install -r Python/requirements.txt

  script:
    - python3 -m unittest discover -s Python -v

###############################################################################
build-regression-tests-ci:
  image: python:3.10
  stage: test

  before_script:
    - pip install -r utilities/ci-regression-tests/requirements.txt

  script:
    - python3 utilities/ci-regression-tests/generate-ci.py

  artifacts:
    expire_in: 1 week
    paths:
      - generated/

  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
      when: always
    - when: manual
      allow_failure: true

  needs: []

trigger-regression-tests:
  stage: test
  needs:
    - build-regression-tests-ci
  trigger:
    include:
      - artifact: generated/regression-tests-ci.yml
        job: build-regression-tests-ci
    strategy: depend
  variables:
    PARENT_PIPELINE_ID: $CI_PIPELINE_ID

###############################################################################
regression_test_4gpu:
  image: python:latest
  stage: test
  needs: []

  rules:
    - if: $REMOTE_USER && $REMOTE_HOST && $PRIVATE_KEY && $CI_PIPELINE_SOURCE == "schedule"
      when: always
    - when: manual
      allow_failure: true

  before_script:
    - pip install hpc-rocket==0.6.0
    - pip install "fieldcompare[all]"

  script:
    - hpc-rocket launch regression-tests/multigpu_test/rocket4GPU.yml --watch --save-jobid hpcrocket4GPU.log
    - cat output/4GPU/slurm4GPU.out
    - git clone --depth 1 --filter=blob:none --sparse https://github.com/irmb/test_data
    - cd test_data
    - git sparse-checkout set regression_tests/gpu/DrivenCavity_4GPU_2Levels regression_tests/gpu/SphereMultiGPU_4GPU_2Levels
    - cd ..
    - fieldcompare dir output/4GPU test_data/regression_tests/gpu/DrivenCavity_4GPU_2Levels --include-files "DrivenCavityMultiGPU*.vtu"
    - fieldcompare dir output/4GPU test_data/regression_tests/gpu/SphereMultiGPU_4GPU_2Levels --include-files "SphereMultiGPU*.vtu"

  after_script:
    - hpc-rocket cancel regression-tests/multigpu_test/rocket4GPU.yml --read-jobid-from hpcrocket4GPU.log

  artifacts:
    when: always
    paths:
      - output/4GPU/*
      - hpcrocket4GPU.log
    expire_in: 1 week
###############################################################################
regression_test_8gpu:
  image: python:latest
  stage: test
  needs: []

  rules:
    - if: $REMOTE_USER && $REMOTE_HOST && $PRIVATE_KEY && $CI_PIPELINE_SOURCE == "schedule"
      when: always
    - when: manual
      allow_failure: true

  before_script:
    - pip install hpc-rocket==0.6.0
    - pip install "fieldcompare[all]"

  script:
    - hpc-rocket launch regression-tests/multigpu_test/rocket8GPU.yml --watch --save-jobid hpcrocket8GPU.log
    - cat output/8GPU/slurm8GPU.out
    - git clone --depth 1 --filter=blob:none --sparse https://github.com/irmb/test_data
    - cd test_data
    - git sparse-checkout set regression_tests/gpu/DrivenCavity_8GPU_2Levels regression_tests/gpu/SphereMultiGPU_8GPU_2Levels
    - cd ..
    - fieldcompare dir output/8GPU test_data/regression_tests/gpu/DrivenCavity_8GPU_2Levels --include-files "DrivenCavityMultiGPU*.vtu"
    - fieldcompare dir output/8GPU test_data/regression_tests/gpu/SphereMultiGPU_8GPU_2Levels --include-files "SphereMultiGPU*.vtu"
  
  after_script:
    - hpc-rocket cancel regression-tests/multigpu_test/rocket8GPU.yml --read-jobid-from hpcrocket8GPU.log

  artifacts:
    when: always
    paths:
      - output/8GPU/*
      - hpcrocket8GPU.log
    expire_in: 1 week

###############################################################################
nvidia_env_test:
  stage: test

  image: nvidia/cuda:12.1.1-devel-ubuntu22.04

  needs: []

  tags:
    - gpu
    - linux

  script:
  - echo NVIDIA_VISIBLE_DEVICES=${NVIDIA_VISIBLE_DEVICES}
  - nvidia-smi

###############################################################################
gpu_numerical_tests:
  stage: test

  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
      when: always
    - when: manual
      allow_failure: true

  needs: []

  tags:
    - gpu
    - linux

  before_script:
    - cd /tmp
    - git clone --depth 1 --filter=blob:none --sparse https://github.com/irmb/test_data
    - cd test_data
    - git sparse-checkout set numerical_tests_gpu/grids numerical_tests_gpu/grids
    - export CCACHE_BASEDIR=$CI_PROJECT_DIR
    - export CCACHE_DIR=$CI_PROJECT_DIR/cache
    - ccache -s
    - mkdir -p $CI_PROJECT_DIR/build
    - cd $CI_PROJECT_DIR/build
    - rm -rf ./*
    - cmake ..
      --preset=make_numerical_tests_gpu
      -DCMAKE_CUDA_ARCHITECTURES=60
    - make -j4
    - ccache -s

  script:
    - echo 'Numerical Tests execution is disabled, because the K17CompressibleNavierStokes produces NaNs in the tests. https://git.rz.tu-bs.de/irmb/VirtualFluids_dev/-/issues/168'
    # - $CI_PROJECT_DIR/build/bin/NumericalTests $CI_PROJECT_DIR/apps/gpu/tests/NumericalTests/configK17chim_nu10tm3.txt /tmp/test_data/numerical_tests_gpu/ 2>&1 | tee -a numerical_tests_gpu_results.txt

  cache:
    key: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"
    paths:
      - $CI_PROJECT_DIR/cache

  artifacts:
    paths:
      - $CI_PROJECT_DIR/numerical_tests_gpu_results.txt

###############################################################################
trigger-hpc-regression-tests:
  stage: test

  needs: []

  rules:
    - when: manual
      allow_failure: true
  
  trigger:
    include:
      - local: regression-tests/hpc-test/.gitlab-ci-hpc-test.yml
  
  variables:
    PARENT_PIPELINE_ID: $CI_PIPELINE_ID


###############################################################################
##                        Code analysis                                      ##
###############################################################################
.analyze_template:
  stage: analyze

  needs: []

  tags:
    - gpu
    - linux

  rules:
    - if: '$CI_PROJECT_NAMESPACE == "irmb"'
      when: always
    - if: '$CI_PROJECT_NAMESPACE != "irmb"'
      when: manual
      allow_failure: true


clang_build_analyzer:
  extends: .analyze_template

  before_script:
    - export CC=clang
    - export CXX=clang++
    - $CXX --version
    - cd /tmp
    - git clone https://github.com/aras-p/ClangBuildAnalyzer.git
    - cd ClangBuildAnalyzer
    - cmake .
    - make
    - export PATH+=:$(pwd)

  script:
    - mkdir -p $CI_PROJECT_DIR/build
    - cd $CI_PROJECT_DIR/build
    - cmake ..
      --preset=make_all
      -DCMAKE_CUDA_ARCHITECTURES=60
      -DCMAKE_CXX_FLAGS=-ftime-trace
    - ClangBuildAnalyzer --start .
    - make
    - ClangBuildAnalyzer --stop . CBA
    - ClangBuildAnalyzer --analyze CBA

###############################################################################
include_what_you_use_clang_15:
  extends: .analyze_template

  before_script:
    - apt-get update && apt-get install -y libclang-15-dev llvm-15-dev
    - export CC=clang
    - export CXX=clang++
    - $CXX --version
    - cd /tmp
    - git clone https://github.com/include-what-you-use/include-what-you-use.git
    - cd include-what-you-use
    - git checkout clang_15
    - cmake . -DCMAKE_PREFIX_PATH=/usr/lib/llvm-15
    - make
    - export PATH+=:$(pwd)/bin

  script:
    - mkdir -p $CI_PROJECT_DIR/build
    - cd $CI_PROJECT_DIR/build
    - cmake ..
      --preset=make_all
      -DCMAKE_CUDA_ARCHITECTURES=60
      -DVF_ENABLE_INCLUDE_WHAT_YOU_USE=ON
    - make

###############################################################################
cppcheck:
  extends: .analyze_template

  before_script:
    - apt-get update && apt-get install -y libpcre3-dev
    - chmod +x utilities/install-cppcheck.sh 
    - ./utilities/install-cppcheck.sh
    - cppcheck --version

  script:
    - cd $CI_PROJECT_DIR
    - cppcheck src --enable=all --xml 2> cppcheck.xml

  artifacts:
    expire_in: 1 week
    paths:
      - cppcheck.xml

###############################################################################
# lizard - Cyclomatic Complexity Analyzer
# Ignore warnings is manually set to 185. This job will fail when new warnings are added.
lizard:
  extends: .analyze_template

  before_script:
    - lizard --version

  script:
    - cd $CI_PROJECT_DIR
    - lizard -l cpp src/ > lizard.txt --warnings_only --ignore_warnings 187

  artifacts:
    expire_in: 1 week
    paths:
      - lizard.txt

###############################################################################
# code coverage
gcov_gcc:
  stage: analyze

  extends: .analyze_template

  before_script:
    - gcovr --version

  script:
    - mkdir -p $CI_PROJECT_DIR/build
    - cd $CI_PROJECT_DIR/build
    - cmake ..
      --preset=make_all
      -DCMAKE_BUILD_TYPE=PROFILE
      -DCMAKE_CUDA_ARCHITECTURES=70
    - make -j4
    - ctest --output-on-failure
    - cd ..
    - mkdir coverage
    - gcovr -r $CI_PROJECT_DIR -k build -f "src" --print-summary --html coverage/coverage.html --html-details --xml coverage/coverage.xml

  artifacts:
    expire_in: 1 week
    paths:
      - coverage/

    reports:
      coverage_report:
        coverage_format: cobertura
        path: coverage/coverage.xml

  cache:
    key: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"
    paths:
      - build

###############################################################################
# this job also produces a compile commands json file.
clang-tidy:
  extends: .analyze_template

  before_script:
    - run-clang-tidy-15 -h

  script:
    - mkdir -p $CI_PROJECT_DIR/build
    - cd $CI_PROJECT_DIR/build
    - cmake ..
      -DVF_ENABLE_CPU=ON
      -DVF_ENABLE_DOUBLE_ACCURACY=ON
      -DVF_ENABLE_GPU=OFF
    - python3 ../utilities/filterCompileCommands.py compile_commands.json
    - run-clang-tidy-15 -quiet > clangtidy.txt

  artifacts:
    when: always
    expire_in: 1 week
    paths:
      - build/clangtidy.txt
      - build/compile_commands.json


###############################################################################
##                            Sanitizer                                      ##
###############################################################################
sanitizer-address-leak-ub:
  extends: .analyze_template
  script:
    - mkdir -p build && cd build
    - cmake ..
      --preset=make_all
      -DVF_ENABLE_PYTHON_BINDINGS=OFF
      -DCMAKE_CUDA_ARCHITECTURES=70
      -DVF_ENABLE_SANITIZER_ADDRESS=ON
      -DVF_ENABLE_SANITIZER_LEAK=ON
      -DVF_ENABLE_SANITIZER_UNDEFINED_BEHAVIOR=ON
    - make -j8
    - ctest --output-on-failure

sanitizer-memory:
  extends: .analyze_template
  script:
    - mkdir -p build && cd build
    - cmake ..
      --preset=make_all
      -DCMAKE_CUDA_ARCHITECTURES=70
      -DVF_ENABLE_SANITIZER_MEMORY=ON
    - make -j8
    - ctest --output-on-failure



###############################################################################
##                               Deploy                                      ##
###############################################################################
# doxgen
# This job only runs on the main branch of the parent repository, not on forks!
pages:
  stage: deploy

  image: alpine

  only:
    - main@irmb/VirtualFluids

  needs: []

  script:
  - apk update && apk add doxygen
  - doxygen docs/Doxyfile
  - mv docs/build/html/ public/

  artifacts:
    expire_in: 1 hrs
    paths:
    - public

###############################################################################

# sonar-scanner runs sonar-project.properties
# the reports in this file need to match the artifacts.
# This job only runs on the main branch of the parent repository, not on forks!
sonar-scanner:
  stage: deploy

  tags:
    - linux

  only:
    - main@irmb/VirtualFluids

  variables:
    SONAR_HOST_URL: "http://gitlab-runner01.irmb.bau.tu-bs.de/sonarqube/"

  needs: ["cppcheck","clang-tidy","gcov_gcc"]

  before_script:
    - cd /tmp
    - apt-get install unzip
    - wget https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-4.0.0.1744-linux.zip
    - unzip sonar-scanner-cli-4.0.0.1744-linux.zip
    - cd sonar-scanner-4.0.0.1744-linux/
    - echo "sonar.host.url=$SONAR_HOST_URL" >> conf/sonar-scanner.properties
    - echo "sonar.sourceEncoding=UTF-8" >> conf/sonar-scanner.properties
    - cat conf/sonar-scanner.properties
    - export PATH+=:$(pwd)/bin
    - sonar-scanner -v

  script:
    - cd $CI_PROJECT_DIR
    - sonar-scanner -X -Dsonar.verbose=true -Dsonar.login=$SONAR_SECURITY_TOKEN