.gitlab-ci.yml

###############################################################################
##                       VirtualFluids CI Pipeline                           ##
###############################################################################
image: git.rz.tu-bs.de:4567/irmb/virtualfluids/ubuntu20_04:1.3

stages:
  - build
  - build_python
  - container_upload
  - test
  - benchmark
  - analyze
  - deploy
  - release


###############################################################################
##                                Builds                                     ##
###############################################################################
.gnu_build_template:
  stage: build

  tags:
    - gpu
    - linux

  cache:
    key: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"
    paths:
      - $BUILD_FOLDER
      - $CI_PROJECT_DIR/cache

  artifacts:
    expire_in: 1 hrs
    paths:
      - $BUILD_FOLDER

  script:
    - export CCACHE_BASEDIR=$CI_PROJECT_DIR
    - export CCACHE_DIR=$CI_PROJECT_DIR/cache
    - export CCACHE_COMPILERCHECK=content
    - ccache --zero-stats
    - ccache --show-stats
    - $CXX --version
    - $CC --version
    - cmake --version
    - mpirun --version
    - mkdir -p $CI_PROJECT_DIR/$BUILD_FOLDER
    - cd $CI_PROJECT_DIR/$BUILD_FOLDER
    - rm -r -f ./*
    - cmake .. -LAH
      --preset=all_make
      -DBUILD_WARNINGS_AS_ERRORS=ON
      -DCMAKE_CUDA_ARCHITECTURES=60
    - make -j4
    - ccache --show-stats

  variables:
    BUILD_FOLDER: "build"

###############################################################################
gcc_9:
  extends: .gnu_build_template

  before_script:
    - export CC=gcc
    - export CXX=g++

###############################################################################
clang_10:
  extends: .gnu_build_template

  before_script:
    - export CC=clang
    - export CXX=clang++

###############################################################################
msvc_16:
  stage: build

  tags:
    - win
    - gpu

  variables:
    BUILD_CONFIGURATION: "Release"
    BUILD_FOLDER: "build"

  # add cmake and MSBuild.exe to the path.
  # This Needs to be adapted when moved to a new build machine.
  before_script:
    - git --version
    - $env:Path += ";C:\Program Files\CMake\bin\"
    - cmake --version
    - $env:Path += ";C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin"
    - MSBuild.exe -version

  script:
    - cd $CI_PROJECT_DIR
    - md -force $env:BUILD_FOLDER
    - cd $env:BUILD_FOLDER
    - cmake .. --preset=all_msvc -DCMAKE_CUDA_ARCHITECTURES=61 -DBUILD_WARNINGS_AS_ERRORS=ON
    - MSBuild.exe VirtualFluids.sln /property:Configuration=$env:BUILD_CONFIGURATION /verbosity:minimal /maxcpucount:4

  cache:
    key: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"
    paths:
      - $CI_PROJECT_DIR/$env:BUILD_FOLDER/

  artifacts:
    expire_in: 1 hrs
    paths:
      - $CI_PROJECT_DIR/$env:BUILD_FOLDER/


###############################################################################
##                             Build Python                                  ##
###############################################################################
gcc_9_python:
  stage: build_python

  needs: ["gcc_9"]

  cache:
    key: "gcc_9-$CI_COMMIT_REF_SLUG"
    paths:
      - build

  artifacts:
    expire_in: 1 hrs
    paths:
      - build/
      - dist/

  before_script:
    - export CCACHE_BASEDIR=$CI_PROJECT_DIR
    - export CCACHE_DIR=$CI_PROJECT_DIR/cache

  script:
    - python3 setup.py bdist_wheel build_ext --build-temp=build

###############################################################################
##                            Container Upload                               ##
###############################################################################
build_singularity_image:
  stage: container_upload

  needs:
    - gcc_9_python

  tags:
    - linux
    - privileged

  rules:
    - if: $CI_COMMIT_TAG

  script:
    - singularity build Containers/VirtualFluidsPython.sif Containers/VirtualFluidsPython.def
    - singularity push --docker-username "${CI_REGISTRY_USER}" --docker-password "${CI_REGISTRY_PASSWORD}" Containers/VirtualFluidsPython.sif oras://"$CI_REGISTRY_IMAGE"/"$CI_PROJECT_NAME":"$CI_COMMIT_TAG"

###############################################################################
##                                Tests                                      ##
###############################################################################
gcc_9_unit_tests:
  stage: test

  needs: ["gcc_9"]

  before_script:
    - cd $CI_PROJECT_DIR/build

  script:
    - ctest

###############################################################################
msvc_16_unit_tests:
  stage: test

  tags:
    - win
    - gpu

  needs: ["msvc_16"]

  before_script:
    - $env:Path += ";C:\Program Files\CMake\bin\"
    - ctest --version

  script:
    - cd $CI_PROJECT_DIR/build
    - ctest -C Release

###############################################################################
gcc_9_python_bindings_test:
  stage: test



  needs: ["gcc_9_python"]

  before_script:
    - export PYTHONPATH="Python"
    - export VF_WHEEL=$(find dist/*.whl)
    - pip3 install $VF_WHEEL
    - pip3 install -r Python/requirements.txt

  script:
    - python3 -m unittest discover -s Python -v

###############################################################################
gcc_9_python_slurm_test:
  stage: test



  needs: ["gcc_9_python"]

  rules:
    - if: $PHOENIX_PRIVATE_KEY

  tags:
    - linux
    - privileged

  variables:
    SSH_KEY: "$PHOENIX_PRIVATE_KEY"
    HOST: "$PHOENIX_HOSTNAME"
    USER: "$PHOENIX_USER"

  before_script:
    - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )'
    - apt-get install -y rsync
    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh
    - eval $(ssh-agent -s)
    - echo "$SSH_KEY" | tr -d '\r' | ssh-add -
    - echo $SSH_KEY >> ansible/private_key
    - ssh-keyscan -t rsa $HOST >> ~/.ssh/known_hosts
    - ssh $USER@$HOST "rm -rf output-*"
    - ssh $USER@$HOST "rm -f *.out"
    - pip3 install git+git://github.com/SvenMarcus/ssh-slurm-runner

  script:
    - singularity build PoiseuilleTestContainer.sif Python/SlurmTests/poiseuille/PoiseuilleTestContainer.def
    - scp PoiseuilleTestContainer.sif $USER@$HOST:PoiseuilleTestContainer.sif
    - scp Python/SlurmTests/poiseuille/slurm.job $USER@$HOST:slurm.job
    - python3 -m ssh_slurm_runner slurm.job --host $HOST --user $USER --keyfile ansible/private_key
    - ssh $USER@$HOST "rm -rf output-*"
    - ssh $USER@$HOST "rm -f *.out"
    - ssh $USER@$HOST "rm PoiseuilleTestContainer.sif"
    - ssh $USER@$HOST "rm slurm.job"

###############################################################################
##                            Benchmark                                      ##
###############################################################################
nvidia_test:
  stage: benchmark

  image: nvidia/cuda:11.1.1-devel-ubuntu20.04

  needs: []

  tags:
    - gpu
    - linux

  script:
  - echo NVIDIA_VISIBLE_DEVICES=${NVIDIA_VISIBLE_DEVICES}
  - nvidia-smi

###############################################################################
gpu_numerical_tests:
  stage: benchmark

  when: manual

  needs: []

  tags:
    - gpu
    - linux

  before_script:
    - cd /tmp
    - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@git.rz.tu-bs.de/irmb/test_data.git
    - export CCACHE_BASEDIR=$CI_PROJECT_DIR
    - export CCACHE_DIR=$CI_PROJECT_DIR/cache
    - ccache -s
    - mkdir -p $CI_PROJECT_DIR/build
    - cd $CI_PROJECT_DIR/build
    - rm -r -f ./*
    - cmake ..
      --preset=gpu_numerical_tests_make
      -DCMAKE_CUDA_ARCHITECTURES=60
      -DPATH_NUMERICAL_TESTS=/tmp/test_data/numerical_tests_gpu
    - make -j4
    - ccache -s

  script:
    - cd $CI_PROJECT_DIR
    - ./build/bin/NumericalTests $CI_PROJECT_DIR/apps/gpu/tests/NumericalTests/configK15_nu10tm2.txt 2>&1 | tee -a numerical_tests_gpu_results.txt

  cache:
    key: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"
    paths:
      - $CI_PROJECT_DIR/cache

  artifacts:
    expire_in: 1 hrs
    paths:
      - $CI_PROJECT_DIR/numerical_tests_gpu_results.txt


###############################################################################
##                        Code analysis                                      ##
###############################################################################
clang_build_analyzer_clang_10:
  stage: analyze

  only: ["schedules"]

  needs: []

  before_script:
    - export CC=clang
    - export CXX=clang++
    - $CXX --version
    - cd /tmp
    - git clone https://github.com/aras-p/ClangBuildAnalyzer.git
    - cd ClangBuildAnalyzer
    - cmake .
    - make
    - export PATH+=:$(pwd)

  script:
    - mkdir -p $CI_PROJECT_DIR/build
    - cd $CI_PROJECT_DIR/build
    - cmake ..
      -DBUILD_VF_CPU=ON
      -DBUILD_VF_GPU=ON
      -DCMAKE_CUDA_ARCHITECTURES=60
      -DCMAKE_CXX_FLAGS=-ftime-trace
    - ClangBuildAnalyzer --start .
    - make
    - ClangBuildAnalyzer --stop . CBA
    - ClangBuildAnalyzer --analyze CBA

###############################################################################
include_what_you_use_clang_10:
  stage: analyze

  only: ["schedules"]

  needs: []

  before_script:
    - export CC=clang
    - export CXX=clang++
    - $CXX --version
    - cd /tmp
    - git clone https://github.com/include-what-you-use/include-what-you-use.git
    - cd include-what-you-use
    - git checkout clang_10
    - cmake . -DCMAKE_PREFIX_PATH=/usr/lib/llvm-10
    - make
    - export PATH+=:$(pwd)/bin

  script:
    - mkdir -p $CI_PROJECT_DIR/build
    - cd $CI_PROJECT_DIR/build
    - cmake ..
      -DBUILD_VF_CPU=ON
      -DBUILD_VF_GPU=ON
      -DCMAKE_CUDA_ARCHITECTURES=60
      -DBUILD_VF_INCLUDE_WHAT_YOU_USE=ON
    - make

###############################################################################
.analyze_template:

  rules: # dont create MR pipeline, run always on irmb/develop and manual in all other cases
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      when: never
    - if: '$CI_PROJECT_NAMESPACE == "irmb" && $CI_COMMIT_BRANCH == "develop"'
      when: always
    - when: manual

cppcheck:
  stage: analyze

  extends: .analyze_template

  needs: []

  before_script:
    - cppcheck --version

  script:
    - cd $CI_PROJECT_DIR
    - cppcheck src --enable=all --xml 2> cppcheck.xml

  artifacts:
    expire_in: 1 week
    paths:
      - cppcheck.xml

###############################################################################
# lizard - Cyclomatic Complexity Analyzer
# Ignore warnings is manually set to 191. This job will fail when new warnings are added.
lizard:
  stage: analyze

  extends: .analyze_template

  needs: []

  before_script:
    - cd /tmp
    - git clone https://github.com/terryyin/lizard.git
    - cd lizard
    - python3 setup.py install
    - lizard --version

  script:
    - cd $CI_PROJECT_DIR
    - lizard -l cpp src/ > lizard.txt --warnings_only --ignore_warnings 400

  artifacts:
    expire_in: 1 week
    paths:
      - lizard.txt

###############################################################################
# code coverage
gcov_gcc_9:
  stage: analyze

  extends: .analyze_template

  needs: []

  before_script:
    - gcovr --version

  script:
    - mkdir -p $CI_PROJECT_DIR/build
    - cd $CI_PROJECT_DIR/build
    - cmake ..
      --preset=all_make
      -DCMAKE_CUDA_ARCHITECTURES=60
      -DBUILD_VF_COVERAGE=ON
    - make -j4
    - ./bin/basicsTests
    - cd ..
    - mkdir coverage
    - gcovr -r $CI_PROJECT_DIR -k build -f "src" --print-summary --html coverage/coverage.html --html-details --xml coverage/coverage.xml

  artifacts:
    expire_in: 1 week
    paths:
      - coverage/

    reports:
      cobertura: coverage/coverage.xml

  cache:
    key: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"
    paths:
      - build

###############################################################################
# this job also produces a compile commands json file.
clang-tidy:
  stage: analyze

  extends: .analyze_template

  needs: []

  allow_failure: true

  before_script:
    - run-clang-tidy -h

  script:
    - mkdir -p $CI_PROJECT_DIR/build
    - cd $CI_PROJECT_DIR/build
    - cmake ..
      -DBUILD_VF_CPU=ON
      -DBUILD_VF_GPU=OFF
    - python3 ../utilities/filterCompileCommands.py compile_commands.json
    - run-clang-tidy -quiet > clangtidy.txt

  artifacts:
    when: always
    expire_in: 1 week
    paths:
      - build/clangtidy.txt
      - build/compile_commands.json


###############################################################################
# doxgen
pages:
  stage: analyze

  image: alpine

  only:
    - develop@irmb/VirtualFluids_dev

  needs: []

  script:
  - apk update && apk add doxygen
  - doxygen docs/Doxyfile
  - mv docs/build/html/ public/

  artifacts:
    expire_in: 1 hrs
    paths:
    - public


###############################################################################
##                               Deploy                                      ##
###############################################################################
.deploy_template:
  stage: deploy



  before_script:
    - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )'
    - apt-get install -y rsync
    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh
    - eval $(ssh-agent -s)
    - echo "$SSH_KEY" | tr -d '\r' | ssh-add -
    - echo $SSH_KEY >> ansible/private_key
    - ssh-keyscan -t rsa $HOST >> ~/.ssh/known_hosts
    - pip3 install ansible

  variables:
    SSH_KEY: ""
    HOST: ""

###############################################################################
vf_to_phoenix:
  extends: .deploy_template
  stage: deploy
  needs: ["gcc_9_python", "gcc_9_unit_tests", "gcc_9_python_bindings_test"]

  when: manual

  variables:
    SSH_KEY: "$SSH_PRIVATE_KEY"
    HOST: "phoenix.hlr.rz.tu-bs.de"

  script:
    - ansible-playbook -i ansible/hosts.cfg -u $REMOTE_USER ansible/playbook_vf_deploy.yml

  parallel:
    matrix:
      - ANSIBLE_MATRIX: 0
        REMOTE_USER:
          - y0054816

###############################################################################
vf_wheel_to_jupyterhub:
  extends: .deploy_template
  stage: deploy

  only: ["manual"]

  needs: ["gcc_9_python", "gcc_9_unit_tests", "gcc_9_python_bindings_test"]

  variables:
    HOST: "gitlab-runner01.irmb.bau.tu-bs.de"
    SSH_KEY: "$SSH_PRIVATE_KEY"
    REMOTE_USER: "runner"
    jupyter_host: "runner"

  script:
    - ansible-playbook -i ansible/hosts.cfg -u $REMOTE_USER ansible/playbook_jupyter_update.yml

###############################################################################
# sonar-scanner runs sonar-project.properties
# the reports in this file needs to match the artifacts.
# This job only run on the development branch of the parent repository, not on forks!
sonar-scanner:
  stage: deploy

  tags:
    - linux

  only:
    #- develop
    - develop@irmb/VirtualFluids_dev

  variables:
    SONAR_HOST_URL: "http://gitlab-runner01.irmb.bau.tu-bs.de/sonarqube/"

  needs: ["cppcheck","clang-tidy","gcov_gcc_9"]

  before_script:
    - cd /tmp
    - apt-get install unzip
    - wget https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-4.0.0.1744-linux.zip
    - unzip sonar-scanner-cli-4.0.0.1744-linux.zip
    - cd sonar-scanner-4.0.0.1744-linux/
    - echo "sonar.host.url=$SONAR_HOST_URL" >> conf/sonar-scanner.properties
    - echo "sonar.sourceEncoding=UTF-8" >> conf/sonar-scanner.properties
    - cat conf/sonar-scanner.properties
    - export PATH+=:$(pwd)/bin
    - sonar-scanner -v

  script:
    - cd $CI_PROJECT_DIR
    - sonar-scanner -X -Dsonar.verbose=true -Dsonar.login=$SONAR_SECURITY_TOKEN

###############################################################################
##                              Release                                      ##
###############################################################################
create_release:
  stage: release

  image: registry.gitlab.com/gitlab-org/release-cli:latest

  needs: ["build_singularity_image"]

  rules:
    - if: $CI_COMMIT_TAG

  script:
    - echo "Creating release with tag $CI_COMMIT_TAG"
    - release-cli create --name "VirtualFluids $CI_COMMIT_TAG" \
      --description "VirtualFluids CFD Simulator" \
      --tag-name "$CI_COMMIT_TAG" \
      --ref "$CI_COMMIT_SHA" \
      --job-token "$CI_JOB_TOKEN" \
      --assets-link="{'name':'VirtualFluidsSingularityImage_OpenMPI','url':'','type':'other','filepath':'Containers/VirtualFluidsOpenMPI.sif'}"
    - build/bin/basicsTests