.gitlab-ci.yml

###############################################################################
##                       VirtualFluids CI Pipeline                           ##
###############################################################################
image: git.rz.tu-bs.de:4567/irmb/virtualfluids/ubuntu22_04:1.1

stages:
  - build
  - build_python
  - container_upload
  - test
  - benchmark
  - analyze
  - deploy
  - release

workflow:
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: never  # disable detached merge request pipelines
    - when: always # add all jobs to normal pipeline. This can be overwritten by the rules of the jobs.

###############################################################################
##                                Builds                                     ##
###############################################################################
.gnu_build_template:
  stage: build

  tags:
    - gpu
    - linux

  cache:
    key: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"
    paths:
      - $BUILD_FOLDER
      - $CI_PROJECT_DIR/cache

  artifacts:
    expire_in: 1 hrs
    paths:
      - $BUILD_FOLDER

  script:
    - export CCACHE_BASEDIR=$CI_PROJECT_DIR
    - export CCACHE_DIR=$CI_PROJECT_DIR/cache
    - export CCACHE_COMPILERCHECK=content
    - ccache --zero-stats
    - ccache --show-stats
    - $CXX --version
    - $CC --version
    - cmake --version
    - mpirun --version
    - mkdir -p $CI_PROJECT_DIR/$BUILD_FOLDER
    - cd $CI_PROJECT_DIR/$BUILD_FOLDER
    - rm -r -f ./*
    - cmake .. -LAH
      --preset=make_all
      -DBUILD_WARNINGS_AS_ERRORS=ON
      -DCMAKE_CUDA_ARCHITECTURES=70
      -DBUILD_VF_ALL_SAMPLES=ON
    - make -j4
    - ccache --show-stats

  variables:
    BUILD_FOLDER: "build"

###############################################################################
gcc_12:
  extends: .gnu_build_template

  before_script:
    - export CC=gcc
    - export CXX=g++

###############################################################################
clang_15:
  extends: .gnu_build_template

  before_script:
    - export CC=clang
    - export CXX=clang++

###############################################################################
msvc_17:
  stage: build

  tags:
    - win
    - gpu

  variables:
    BUILD_CONFIGURATION: "Release"
    BUILD_FOLDER: "build"

  # add cmake and MSBuild.exe to the path.
  # This Needs to be adapted when moved to a new build machine.
  before_script:
    - git --version
    - $env:Path += ";C:\Program Files\CMake\bin\"
    - cmake --version
    - $env:Path += ";C:\Program Files\Microsoft Visual Studio\2022\Community\MSBuild\Current\Bin"
    - MSBuild.exe -version

  script:
    - cd $CI_PROJECT_DIR
    - md -force $env:BUILD_FOLDER
    - cd $env:BUILD_FOLDER
    - cmake .. --preset=msvc_all -DCMAKE_CUDA_ARCHITECTURES=61 -DBUILD_WARNINGS_AS_ERRORS=ON -DBUILD_VF_ALL_SAMPLES=ON
    - MSBuild.exe VirtualFluids.sln /property:Configuration=$env:BUILD_CONFIGURATION /verbosity:minimal /maxcpucount:4

  artifacts:
    expire_in: 1 hrs
    paths:
      - $CI_PROJECT_DIR/$env:BUILD_FOLDER/


###############################################################################
gcc_12_python:
  stage: build

  cache:
    key: "gcc_12-$CI_COMMIT_REF_SLUG"
    paths:
      - pybuild/

  artifacts:
    expire_in: 1 hrs
    paths:
      - pybuild/ # build in separate folder to avoid conflicts with c++ build

  before_script:
    - export CCACHE_BASEDIR=$CI_PROJECT_DIR
    - export CCACHE_DIR=$CI_PROJECT_DIR/cache

  script:
    - export SKBUILD_BUILD_DIR="pybuild"
    - export SKBUILD_CMAKE_ARGS="-DBUILD_VF_CPU=ON;-DBUILD_VF_GPU=ON;-DCMAKE_CUDA_ARCHITECTURES=70;-DBUILD_VF_DOUBLE_ACCURACY=ON;-DCMAKE_CXX_COMPILER_LAUNCHER=ccache;-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache;-DCMAKE_C_COMPILER_LAUNCHER=ccache;-G=Ninja"
    - pip install . -v

###############################################################################
##                            Container Upload                               ##
###############################################################################
build_poiseuille_test_container:
  image: 
    name: quay.io/singularity/singularity:v3.10.2
    entrypoint: [""]

  stage: container_upload

  rules:
    - if: $REMOTE_USER && $REMOTE_HOST && $PRIVATE_KEY && $CI_PIPELINE_SOURCE == "schedule"
      when: always
    - when: manual
      allow_failure: true

  tags:
    - linux
    - privileged

  artifacts:
    expire_in: 1 hrs
    paths:
      - Containers/PoiseuilleTestContainer.sif

  script:
    - singularity build "Containers/PoiseuilleTestContainer.sif" "Python/SlurmTests/poiseuille/PoiseuilleTestContainer.def"

###############################################################################
##                                Tests                                      ##
###############################################################################
gcc_12_unit_tests:
  stage: test

  needs: ["gcc_12"]

  before_script:
    - cd $CI_PROJECT_DIR/build

  script:
    - ctest --output-on-failure

###############################################################################
msvc_17_unit_tests:
  stage: test

  tags:
    - win
    - gpu

  needs: ["msvc_17"]

  before_script:
    - $env:Path += ";C:\Program Files\CMake\bin\"
    - ctest --version

  script:
    - cd $CI_PROJECT_DIR/build
    - ctest --output-on-failure -C Release

###############################################################################
gcc_12_python_bindings_test:
  stage: test

  needs: ["gcc_12_python"]

  before_script:
    - pip3 install -r Python/requirements.txt

  script:
    - python3 -m unittest discover -s Python -v

###############################################################################
gcc_12_python_hpc_test:
  image: python:latest
  stage: test

  needs: ["build_poiseuille_test_container"]

  rules:
    - if: $REMOTE_USER && $REMOTE_HOST && $PRIVATE_KEY && $CI_PIPELINE_SOURCE == "schedule"
      when: always
    - when: manual
      allow_failure: true

  before_script:
    - pip install hpc-rocket

  script:
    - hpc-rocket launch --watch Python/SlurmTests/poiseuille/rocket.yml

###############################################################################
build-regression-tests-ci:
  image: python:3.10
  stage: test

  before_script:
    - pip install -r utilities/ci-regression-tests/requirements.txt

  script:
    - python3 utilities/ci-regression-tests/generate-ci.py

  artifacts:
    expire_in: 1 week
    paths:
      - generated/

  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
      when: always
    - when: manual
      allow_failure: true

  needs: []

trigger-regression-tests:
  stage: test
  needs:
    - build-regression-tests-ci
  trigger:
    include:
      - artifact: generated/regression-tests-ci.yml
        job: build-regression-tests-ci
    strategy: depend
  variables:
    PARENT_PIPELINE_ID: $CI_PIPELINE_ID

###############################################################################
regression_test_4gpu:
  image: python:latest
  stage: test
  needs: []

  rules:
    - if: $REMOTE_USER && $REMOTE_HOST && $PRIVATE_KEY && $CI_PIPELINE_SOURCE == "schedule"
      when: always
    - when: manual
      allow_failure: true

  before_script:
    - pip install hpc-rocket
    - pip install "fieldcompare[all]"

  script:
    - hpc-rocket launch regression-tests/multigpu_test/rocket4GPU.yml |& tee hpcrocket4GPU.log
    - hpc-rocket watch regression-tests/multigpu_test/rocket4GPU.yml $(python regression-tests/multigpu_test/utilities/parsejobid.py hpcrocket4GPU.log)
    - hpc-rocket finalize regression-tests/multigpu_test/rocket4GPU.yml
    - cat output/4GPU/slurm4GPU.out
    - git clone --depth 1 --filter=blob:none --sparse https://github.com/irmb/test_data
    - cd test_data
    - git sparse-checkout set regression_tests/gpu/DrivenCavity_4GPU_2Levels regression_tests/gpu/SphereScaling_4GPU_2Levels
    - cd ..
    - fieldcompare dir output/4GPU test_data/regression_tests/gpu/DrivenCavity_4GPU_2Levels --include-files "DrivenCavityMultiGPU*.vtu"
    - fieldcompare dir output/4GPU test_data/regression_tests/gpu/SphereScaling_4GPU_2Levels --include-files "SphereScaling*.vtu"

  after_script:
    - hpc-rocket cancel regression-tests/multigpu_test/rocket4GPU.yml $(python regression-tests/multigpu_test/utilities/parsejobid.py hpcrocket4GPU.log)

  artifacts:
    when: always
    paths:
      - output/4GPU/slurm4GPU.out
    expire_in: 1 week
###############################################################################
regression_test_8gpu:
  image: python:latest
  stage: test
  needs: []

  rules:
    - if: $REMOTE_USER && $REMOTE_HOST && $PRIVATE_KEY && $CI_PIPELINE_SOURCE == "schedule"
      when: always
    - when: manual
      allow_failure: true

  before_script:
    - pip install hpc-rocket
    - pip install "fieldcompare[all]"

  script:
    - hpc-rocket launch regression-tests/multigpu_test/rocket8GPU.yml |& tee hpcrocket8GPU.log
    - hpc-rocket watch regression-tests/multigpu_test/rocket8GPU.yml $(python regression-tests/multigpu_test/utilities/parsejobid.py hpcrocket8GPU.log)
    - hpc-rocket finalize regression-tests/multigpu_test/rocket8GPU.yml
    - cat output/8GPU/slurm8GPU.out
    - git clone --depth 1 --filter=blob:none --sparse https://github.com/irmb/test_data
    - cd test_data
    - git sparse-checkout set regression_tests/gpu/DrivenCavity_8GPU_2Levels regression_tests/gpu/SphereScaling_8GPU_2Levels
    - cd ..
    - fieldcompare dir output/8GPU test_data/regression_tests/gpu/DrivenCavity_8GPU_2Levels --include-files "DrivenCavityMultiGPU*.vtu"
    - fieldcompare dir output/8GPU test_data/regression_tests/gpu/SphereScaling_8GPU_2Levels --include-files "SphereScaling*.vtu"
  
  after_script:
    - hpc-rocket cancel regression-tests/multigpu_test/rocket8GPU.yml $(python regression-tests/multigpu_test/utilities/parsejobid.py hpcrocket8GPU.log)

  artifacts:
    when: always
    paths:
      - output/8GPU/slurm8GPU.out
    expire_in: 1 week
###############################################################################
##                            Benchmark                                      ##
###############################################################################
nvidia_test:
  stage: benchmark

  image: nvidia/cuda:12.1.1-devel-ubuntu22.04

  needs: []

  tags:
    - gpu
    - linux

  script:
  - echo NVIDIA_VISIBLE_DEVICES=${NVIDIA_VISIBLE_DEVICES}
  - nvidia-smi

###############################################################################
gpu_numerical_tests:
  stage: benchmark

  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
      when: always
    - when: manual
      allow_failure: true

  needs: []

  tags:
    - gpu
    - linux

  before_script:
    - cd /tmp
    - git clone --depth 1 --filter=blob:none --sparse https://github.com/irmb/test_data
    - cd test_data
    - git sparse-checkout set numerical_tests_gpu/grids numerical_tests_gpu/grids
    - export CCACHE_BASEDIR=$CI_PROJECT_DIR
    - export CCACHE_DIR=$CI_PROJECT_DIR/cache
    - ccache -s
    - mkdir -p $CI_PROJECT_DIR/build
    - cd $CI_PROJECT_DIR/build
    - rm -rf ./*
    - cmake ..
      --preset=make_numerical_tests_gpu
      -DCMAKE_CUDA_ARCHITECTURES=60
    - make -j4
    - ccache -s

  script:
    - $CI_PROJECT_DIR/build/bin/NumericalTests $CI_PROJECT_DIR/apps/gpu/tests/NumericalTests/configK17chim_nu10tm3.txt /tmp/test_data/numerical_tests_gpu/ 2>&1 | tee -a numerical_tests_gpu_results.txt

  cache:
    key: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"
    paths:
      - $CI_PROJECT_DIR/cache

  artifacts:
    paths:
      - $CI_PROJECT_DIR/numerical_tests_gpu_results.txt

###############################################################################
##                        Code analysis                                      ##
###############################################################################
.analyze_template:
  stage: analyze

  needs: []

  tags:
    - gpu
    - linux

  rules:
    - if: '$CI_PROJECT_NAMESPACE == "irmb" && $CI_COMMIT_BRANCH == "develop"'
      when: always
    - if: '$CI_PROJECT_NAMESPACE != "irmb"'
      when: manual
      allow_failure: true


clang_build_analyzer:
  extends: .analyze_template

  before_script:
    - export CC=clang
    - export CXX=clang++
    - $CXX --version
    - cd /tmp
    - git clone https://github.com/aras-p/ClangBuildAnalyzer.git
    - cd ClangBuildAnalyzer
    - cmake .
    - make
    - export PATH+=:$(pwd)

  script:
    - mkdir -p $CI_PROJECT_DIR/build
    - cd $CI_PROJECT_DIR/build
    - cmake ..
      --preset=make_all
      -DCMAKE_CUDA_ARCHITECTURES=60
      -DCMAKE_CXX_FLAGS=-ftime-trace
    - ClangBuildAnalyzer --start .
    - make
    - ClangBuildAnalyzer --stop . CBA
    - ClangBuildAnalyzer --analyze CBA

###############################################################################
include_what_you_use_clang_15:
  extends: .analyze_template

  before_script:
    - apt-get update && apt-get install -y libclang-15-dev llvm-15-dev
    - export CC=clang
    - export CXX=clang++
    - $CXX --version
    - cd /tmp
    - git clone https://github.com/include-what-you-use/include-what-you-use.git
    - cd include-what-you-use
    - git checkout clang_15
    - cmake . -DCMAKE_PREFIX_PATH=/usr/lib/llvm-15
    - make
    - export PATH+=:$(pwd)/bin

  script:
    - mkdir -p $CI_PROJECT_DIR/build
    - cd $CI_PROJECT_DIR/build
    - cmake ..
      --preset=make_all
      -DCMAKE_CUDA_ARCHITECTURES=60
      -DBUILD_VF_INCLUDE_WHAT_YOU_USE=ON
    - make

###############################################################################
cppcheck:
  extends: .analyze_template

  before_script:
    - apt-get update && apt-get install -y libpcre3-dev
    - chmod +x utilities/install-cppcheck.sh 
    - ./utilities/install-cppcheck.sh
    - cppcheck --version

  script:
    - cd $CI_PROJECT_DIR
    - cppcheck src --enable=all --xml 2> cppcheck.xml

  artifacts:
    expire_in: 1 week
    paths:
      - cppcheck.xml

###############################################################################
# lizard - Cyclomatic Complexity Analyzer
# Ignore warnings is manually set to 191. This job will fail when new warnings are added.
lizard:
  extends: .analyze_template

  before_script:
    - lizard --version

  script:
    - cd $CI_PROJECT_DIR
    - lizard -l cpp src/ > lizard.txt --warnings_only --ignore_warnings 400

  artifacts:
    expire_in: 1 week
    paths:
      - lizard.txt

###############################################################################
# code coverage
gcov_gcc:
  stage: analyze

  extends: .analyze_template

  before_script:
    - gcovr --version
    - export CC=/usr/bin/gcc-11
    - export CXX=/usr/bin/g++-11

  script:
    - mkdir -p $CI_PROJECT_DIR/build
    - cd $CI_PROJECT_DIR/build
    - cmake ..
      --preset=make_all
      -DCMAKE_BUILD_TYPE=PROFILE
      -DCMAKE_CUDA_ARCHITECTURES=70
    - make -j4
    - ctest --output-on-failure
    - cd ..
    - mkdir coverage
    - gcovr -r $CI_PROJECT_DIR -k build -f "src" --print-summary --html coverage/coverage.html --html-details --xml coverage/coverage.xml

  artifacts:
    expire_in: 1 week
    paths:
      - coverage/

    reports:
      coverage_report:
        coverage_format: cobertura
        path: coverage/coverage.xml

  cache:
    key: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"
    paths:
      - build

###############################################################################
# this job also produces a compile commands json file.
clang-tidy:
  extends: .analyze_template

  before_script:
    - run-clang-tidy-15 -h

  script:
    - mkdir -p $CI_PROJECT_DIR/build
    - cd $CI_PROJECT_DIR/build
    - cmake ..
      -DBUILD_VF_CPU=ON
      -DBUILD_VF_DOUBLE_ACCURACY=ON
      -DBUILD_VF_GPU=OFF
    - python3 ../utilities/filterCompileCommands.py compile_commands.json
    - run-clang-tidy-15 -quiet > clangtidy.txt

  artifacts:
    when: always
    expire_in: 1 week
    paths:
      - build/clangtidy.txt
      - build/compile_commands.json


###############################################################################
##                            Sanitizer                                      ##
###############################################################################
sanitizer-address-leak-ub:
  extends: .analyze_template
  script:
    - mkdir -p build && cd build
    - cmake ..
      --preset=make_all
      -DCMAKE_CUDA_ARCHITECTURES=70
      -DENABLE_SANITIZER_ADDRESS=ON
      -DENABLE_SANITIZER_LEAK=ON
      -DENABLE_SANITIZER_UNDEFINED_BEHAVIOR=ON
    - make -j8
    - ctest --output-on-failure

sanitizer-memory:
  extends: .analyze_template
  script:
    - mkdir -p build && cd build
    - cmake ..
      --preset=make_all
      -DCMAKE_CUDA_ARCHITECTURES=70
      -DENABLE_SANITIZER_MEMORY=ON
    - make -j8
    - ctest --output-on-failure

###############################################################################
# doxgen
pages:
  stage: analyze

  image: alpine

  only:
    - develop@irmb/VirtualFluids_dev

  needs: []

  script:
  - apk update && apk add doxygen
  - doxygen docs/Doxyfile
  - mv docs/build/html/ public/

  artifacts:
    expire_in: 1 hrs
    paths:
    - public


###############################################################################
##                               Deploy                                      ##
###############################################################################
.deploy_template:
  stage: deploy

  before_script:
    - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )'
    - apt-get install -y rsync
    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh
    - eval $(ssh-agent -s)
    - echo "$SSH_KEY" | tr -d '\r' | ssh-add -
    - echo $SSH_KEY >> ansible/private_key
    - ssh-keyscan -t rsa $HOST >> ~/.ssh/known_hosts
    - pip3 install ansible

  variables:
    SSH_KEY: ""
    HOST: ""

###############################################################################
vf_to_phoenix:
  extends: .deploy_template
  stage: deploy
  needs: ["gcc_12_python", "gcc_12_unit_tests", "gcc_12_python_bindings_test"]

  when: manual

  variables:
    SSH_KEY: "$SSH_PRIVATE_KEY"
    HOST: "phoenix.hlr.rz.tu-bs.de"

  script:
    - ansible-playbook -i ansible/hosts.cfg -u $REMOTE_USER ansible/playbook_vf_deploy.yml

  parallel:
    matrix:
      - ANSIBLE_MATRIX: 0
        REMOTE_USER:
          - y0054816

###############################################################################
vf_wheel_to_jupyterhub:
  extends: .deploy_template
  stage: deploy

  only: ["manual"]

  needs: ["gcc_12_python", "gcc_12_unit_tests", "gcc_12_python_bindings_test"]

  variables:
    HOST: "gitlab-runner01.irmb.bau.tu-bs.de"
    SSH_KEY: "$SSH_PRIVATE_KEY"
    REMOTE_USER: "runner"
    jupyter_host: "runner"

  script:
    - ansible-playbook -i ansible/hosts.cfg -u $REMOTE_USER ansible/playbook_jupyter_update.yml

###############################################################################
# sonar-scanner runs sonar-project.properties
# the reports in this file need to match the artifacts.
# This job only runs on the development branch of the parent repository, not on forks!
sonar-scanner:
  stage: deploy

  tags:
    - linux

  only:
    #- develop
    - develop@irmb/VirtualFluids_dev

  variables:
    SONAR_HOST_URL: "http://gitlab-runner01.irmb.bau.tu-bs.de/sonarqube/"

  needs: ["cppcheck","clang-tidy","gcov_gcc"]

  before_script:
    - cd /tmp
    - apt-get install unzip
    - wget https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-4.0.0.1744-linux.zip
    - unzip sonar-scanner-cli-4.0.0.1744-linux.zip
    - cd sonar-scanner-4.0.0.1744-linux/
    - echo "sonar.host.url=$SONAR_HOST_URL" >> conf/sonar-scanner.properties
    - echo "sonar.sourceEncoding=UTF-8" >> conf/sonar-scanner.properties
    - cat conf/sonar-scanner.properties
    - export PATH+=:$(pwd)/bin
    - sonar-scanner -v

  script:
    - cd $CI_PROJECT_DIR
    - sonar-scanner -X -Dsonar.verbose=true -Dsonar.login=$SONAR_SECURITY_TOKEN