Merge branch 'feature/singularity-ci' into 'develop'

Clean CI Pipeline

See merge request irmb/VirtualFluids_dev!272

.gitlab-ci.yml

@@ -5,13 +5,9 @@ image: git.rz.tu-bs.de:4567/irmb/virtualfluids/ubuntu22_04:1.1
 
 stages:
   - build
-  - build_python
-  - container_upload
   - test
-  - benchmark
   - analyze
   - deploy
-  - release
 
 workflow:
   rules:
@@ -137,34 +133,6 @@ gcc_12_python:
     - export SKBUILD_CMAKE_ARGS="-DBUILD_VF_CPU=ON;-DBUILD_VF_GPU=ON;-DCMAKE_CUDA_ARCHITECTURES=70;-DBUILD_VF_DOUBLE_ACCURACY=ON;-DCMAKE_CXX_COMPILER_LAUNCHER=ccache;-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache;-DCMAKE_C_COMPILER_LAUNCHER=ccache;-G=Ninja"
     - pip install . -v
 
-###############################################################################
-##                            Container Upload                               ##
-###############################################################################
-build_poiseuille_test_container:
-  image: 
-    name: quay.io/singularity/singularity:v3.10.2
-    entrypoint: [""]
-
-  stage: container_upload
-
-  rules:
-    - if: $REMOTE_USER && $REMOTE_HOST && $PRIVATE_KEY && $CI_PIPELINE_SOURCE == "schedule"
-      when: always
-    - when: manual
-      allow_failure: true
-
-  tags:
-    - linux
-    - privileged
-
-  artifacts:
-    expire_in: 1 hrs
-    paths:
-      - Containers/PoiseuilleTestContainer.sif
-
-  script:
-    - singularity build "Containers/PoiseuilleTestContainer.sif" "Python/SlurmTests/poiseuille/PoiseuilleTestContainer.def"
-
 ###############################################################################
 ##                                Tests                                      ##
 ###############################################################################
@@ -209,25 +177,6 @@ gcc_12_python_bindings_test:
   script:
     - python3 -m unittest discover -s Python -v
 
-###############################################################################
-gcc_12_python_hpc_test:
-  image: python:latest
-  stage: test
-
-  needs: ["build_poiseuille_test_container"]
-
-  rules:
-    - if: $REMOTE_USER && $REMOTE_HOST && $PRIVATE_KEY && $CI_PIPELINE_SOURCE == "schedule"
-      when: always
-    - when: manual
-      allow_failure: true
-
-  before_script:
-    - pip install hpc-rocket
-
-  script:
-    - hpc-rocket launch --watch Python/SlurmTests/poiseuille/rocket.yml
-
 ###############################################################################
 build-regression-tests-ci:
   image: python:3.10
@@ -336,11 +285,10 @@ regression_test_8gpu:
     paths:
       - output/8GPU/slurm8GPU.out
     expire_in: 1 week
+
 ###############################################################################
-##                            Benchmark                                      ##
-###############################################################################
-nvidia_test:
-  stage: benchmark
+nvidia_env_test:
+  stage: test
 
   image: nvidia/cuda:12.1.1-devel-ubuntu22.04
 
@@ -356,7 +304,7 @@ nvidia_test:
 
 ###############################################################################
 gpu_numerical_tests:
-  stage: benchmark
+  stage: test
 
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"
@@ -399,6 +347,24 @@ gpu_numerical_tests:
     paths:
       - $CI_PROJECT_DIR/numerical_tests_gpu_results.txt
 
+###############################################################################
+trigger-hpc-regression-tests:
+  stage: test
+
+  needs: []
+
+  rules:
+    - when: manual
+      allow_failure: true
+  
+  trigger:
+    include:
+      - local: regression-tests/hpc-test/.gitlab-ci-hpc-test.yml
+  
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+
+
 ###############################################################################
 ##                        Code analysis                                      ##
 ###############################################################################
@@ -601,10 +567,14 @@ sanitizer-memory:
     - make -j8
     - ctest --output-on-failure
 
+
+
+###############################################################################
+##                               Deploy                                      ##
 ###############################################################################
 # doxgen
 pages:
-  stage: analyze
+  stage: deploy
 
   image: alpine
 
@@ -623,68 +593,8 @@ pages:
     paths:
     - public
 
-
-###############################################################################
-##                               Deploy                                      ##
-###############################################################################
-.deploy_template:
-  stage: deploy
-
-  before_script:
-    - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )'
-    - apt-get install -y rsync
-    - mkdir -p ~/.ssh
-    - chmod 700 ~/.ssh
-    - eval $(ssh-agent -s)
-    - echo "$SSH_KEY" | tr -d '\r' | ssh-add -
-    - echo $SSH_KEY >> ansible/private_key
-    - ssh-keyscan -t rsa $HOST >> ~/.ssh/known_hosts
-    - pip3 install ansible
-
-  variables:
-    SSH_KEY: ""
-    HOST: ""
-
-###############################################################################
-vf_to_phoenix:
-  extends: .deploy_template
-  stage: deploy
-  needs: ["gcc_12_python", "gcc_12_unit_tests", "gcc_12_python_bindings_test"]
-
-  when: manual
-
-  variables:
-    SSH_KEY: "$SSH_PRIVATE_KEY"
-    HOST: "phoenix.hlr.rz.tu-bs.de"
-
-  script:
-    - ansible-playbook -i ansible/hosts.cfg -u $REMOTE_USER ansible/playbook_vf_deploy.yml
-
-  parallel:
-    matrix:
-      - ANSIBLE_MATRIX: 0
-        REMOTE_USER:
-          - y0054816
-
 ###############################################################################
-vf_wheel_to_jupyterhub:
-  extends: .deploy_template
-  stage: deploy
-
-  only: ["manual"]
-
-  needs: ["gcc_12_python", "gcc_12_unit_tests", "gcc_12_python_bindings_test"]
 
-  variables:
-    HOST: "gitlab-runner01.irmb.bau.tu-bs.de"
-    SSH_KEY: "$SSH_PRIVATE_KEY"
-    REMOTE_USER: "runner"
-    jupyter_host: "runner"
-
-  script:
-    - ansible-playbook -i ansible/hosts.cfg -u $REMOTE_USER ansible/playbook_jupyter_update.yml
-
-###############################################################################
 # sonar-scanner runs sonar-project.properties
 # the reports in this file need to match the artifacts.
 # This job only runs on the development branch of the parent repository, not on forks!

regression-tests/driven_cavity_test.sh

@@ -16,4 +16,3 @@ RESULT_DATA_DIR=output/DrivenCavity
 
 
 run_regression_test "$REFERENCE_DATA_DIR" "$CMAKE_FLAGS" "$APPLICATION" "$RESULT_DATA_DIR"
-

regression-tests/hpc-test/.gitlab-ci-hpc-test.yml

+###############################################################################
+# 1. builds the singularity container using a docker container
+# Needs a priviliged gitlab runner.
+# Input: Containers/rockylinux9-mpich.def
+# Output: Containers/rockylinux9-mpich.sif
+###############################################################################
+build-singularity-image:
+  image:
+    name: quay.io/singularity/singularity:v3.10.4
+    entrypoint: [""]
+
+  stage: build
+
+  tags:
+    - linux
+    - privileged
+
+  variables:
+    SINGULARITY_IMAGE: "regression-tests/hpc-test/Rockylinux9.sif"
+    SINGULARITY_FILE: "regression-tests/hpc-test/Rockylinux9.def"
+
+  artifacts:
+    expire_in: 1 hrs
+    paths:
+      - $SINGULARITY_IMAGE
+
+  cache:
+    key:
+      files:
+        - $SINGULARITY_FILE
+    paths:
+      - regression-tests/hpc-test/
+
+  script:
+    - |
+      if [[ -f $SINGULARITY_IMAGE ]]; then
+        echo "Cached version of $SINGULARITY_IMAGE used. Clear cache to force rebuild."
+        exit 0
+      fi
+    - singularity build $SINGULARITY_IMAGE $SINGULARITY_FILE
+
+
+###############################################################################
+# 2. runs the container on the hpc cluster using hpc-rocket
+# rocket.yml used the configuration file rocket.yml, copies the defined data
+# to the cluster, submits a slurm job and copies data back
+# Output: result data located in results-regression-test/
+###############################################################################
+run-simulation:
+  image: python:3.10
+
+  stage: test
+
+  needs: ["build-singularity-image"]
+
+  before_script:
+    - pip install hpc-rocket==0.4.0
+
+  script:
+    - hpc-rocket launch --watch regression-tests/hpc-test/rocket.yml
+    - cat results-regression-test/virtualfluids.out
+
+  artifacts:
+    expire_in: 1 hrs
+    paths:
+      - results-regression-test/
+
+###############################################################################
+# 3. runs a regression data using fieldcompare
+# Fieldcompare compares the in the last job produced data from results-regression-test/
+# with a set of reference files located in reference_data/
+###############################################################################
+validate-simulation:
+  image: python:3.10
+
+  stage: test
+
+  needs: ["run-simulation"]
+
+  before_script:
+    - pip install "fieldcompare[all]"
+    - fieldcompare --version
+    - rm -rf reference_data && mkdir -p reference_data
+    - git clone --depth 1 --filter=blob:none --sparse https://github.com/irmb/test_data reference_data
+    - cd reference_data && git sparse-checkout add regression_tests/cpu/FlowAroundCylinder4processes && cd ..
+
+  script:
+     - fieldcompare dir results-regression-test/FlowAroundCylinder reference_data/regression_tests/cpu/FlowAroundCylinder4processes --include-files "*.vtu"

regression-tests/hpc-test/Rockylinux9.def

+BootStrap: docker
+From: rockylinux:9
+Stage: build
+ 
+%files
+    src src
+    CMakeLists.txt CMakeLists.txt
+    CMakePresets.json CMakePresets.json
+    apps apps
+    CMake CMake
+    3rdParty 3rdParty
+    pythonbindings pythonbindings
+    Python Python
+    pyproject.toml pyproject.toml
+    cpu.cmake cpu.cmake
+    gpu.cmake gpu.cmake
+
+
+%post
+    yum update -y                            && \
+    yum group install -y "Development Tools" && \
+    yum install -y                              \
+    mpich                                       \
+    mpich-devel                                 \
+    cmake                                    && \
+    source /etc/profile.d/modules.sh         && \
+    module load mpi                          && \
+    mkdir build && cd build && cmake .. -DBUILD_VF_CPU=ON -DBUILD_VF_DOUBLE_ACCURACY=ON && make -j8
+
+
+BootStrap: docker
+From: rockylinux:9
+Stage: runtime
+
+%files from build
+    build build
+
+%environment
+    export MPI_DIR=/cluster/mpi/mpich
+    export PATH="$MPI_DIR/bin:$PATH"
+    export LD_LIBRARY_PATH="$MPI_DIR/lib:$LD_LIBRARY_PATH"
+
+%post
+    yum update -y         && \
+    yum install -y           \
+    gcc-toolset-12           \
+    compat-libgfortran-48
\ No newline at end of file

regression-tests/hpc-test/rocket.yml

+host: $PHOENIX_REMOTE_HOST
+user: $PHOENIX_REMOTE_USER
+private_keyfile: $PHOENIX_PRIVATE_KEY
+
+copy:
+  - from: regression-tests/hpc-test/test.job.sh
+    to: virtualfluids-mpich-bind/test.job
+    overwrite: true
+
+  - from: regression-tests/hpc-test/Rockylinux9.sif
+    to: virtualfluids-mpich-bind/Rockylinux9.sif
+    overwrite: true
+
+  - from: apps/cpu/FlowAroundCylinder/cylinder.cfg
+    to: virtualfluids-mpich-bind/cylinder.cfg
+    overwrite: true
+
+collect:
+  - from: virtualfluids-mpich-bind/output/
+    to: results-regression-test/
+    overwrite: true
+
+  - from: virtualfluids-mpich-bind/virtualfluids.out
+    to: results-regression-test/virtualfluids.out
+    overwrite: true
+
+
+#clean:
+#  - laplace2d/*
+
+sbatch: virtualfluids-mpich-bind/test.job
+continue_if_job_fails: true

regression-tests/hpc-test/test.job.sh

+#!/bin/bash -l
+
+#SBATCH --partition=standard
+#SBATCH --nodes=1
+#SBATCH --time=01:00:00
+#SBATCH --job-name=VirtualFluids
+#SBATCH --ntasks-per-node=4
+#SBATCH --output=virtualfluids-mpich-bind/virtualfluids.out
+
+date
+
+echo "SLURM_JOBID="$SLURM_JOBID
+echo "SLURM_JOB_NODELIST"=$SLURM_JOB_NODELIST
+echo "SLURM_NNODES"=$SLURM_NNODES
+echo "SLURM_TASKS_PER_NODE"=$SLURM_TASKS_PER_NODE
+echo "working directory = "$SLURM_SUBMIT_DIR
+echo ""
+
+module purge
+module load singularity/3.9.9
+module load mpi/mpich/mpich_3.2
+module list
+
+cd virtualfluids-mpich-bind/
+mkdir -p results
+
+export MPI_DIR="/cluster/mpi/mpich"
+srun --mpi=pmi2 singularity exec --bind "$MPI_DIR" rockylinux9-mpich-bind.sif /build/bin/FlowAroundCylinder cylinder.cfg