Merge branch 'feature/ci-regression' into 'develop'

dynamically create CI pipeline for regression tests.

See merge request irmb/VirtualFluids_dev!218

.gitignore

@@ -27,6 +27,7 @@ pythonbindings/pymuparser/bindings*
 output/
 logs/
 reference_data/
+generated/
 
 # grid
 .grid/

.gitlab-ci.yml

@@ -214,7 +214,6 @@ gcc_9_python_bindings_test:
   script:
     - python3 -m unittest discover -s Python -v
 
-
 ###############################################################################
 gcc_9_python_hpc_test:
   image: python:latest
@@ -236,6 +235,44 @@ gcc_9_python_hpc_test:
   script:
     - hpc-rocket launch --watch Python/SlurmTests/poiseuille/rocket.yml
 
+###############################################################################
+build-regression-tests-ci:
+  image: python:3.10
+  stage: test
+
+  before_script:
+    - pip install -r utilities/ci-regression-tests/requirements.txt
+
+  script:
+    - python3 utilities/ci-regression-tests/generate-ci.py
+
+  artifacts:
+    expire_in: 1 week
+    paths:
+      - generated/
+
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: always
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: never
+    - when: manual
+      allow_failure: true
+
+  needs: []
+
+trigger-regression-tests:
+  stage: test
+  needs:
+    - build-regression-tests-ci
+  trigger:
+    include:
+      - artifact: generated/regression-tests-ci.yml
+        job: build-regression-tests-ci
+    strategy: depend
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+
 ###############################################################################
 regression_test_4gpu:
   image: python:latest
@@ -354,31 +391,6 @@ gpu_numerical_tests:
     paths:
       - $CI_PROJECT_DIR/numerical_tests_gpu_results.txt
 
-###############################################################################
-run-regression-tests:
-  stage: test
-
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-      when: always
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      when: never
-    - when: manual
-      allow_failure: true
-
-  needs: []
-
-  tags:
-    - gpu
-    - linux
-
-  before_script:
-    - apt-get update && apt-get -y install python3.8-venv
-    - chmod +x ./regression-tests/*
-
-  script:
-    - ./regression-tests/regression-tests.sh
-
 ###############################################################################
 ##                        Code analysis                                      ##
 ###############################################################################
@@ -598,8 +610,6 @@ pages:
 .deploy_template:
   stage: deploy
 
-
-
   before_script:
     - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )'
     - apt-get install -y rsync

regression-tests/README.md

+# Adding Regression Tests.
+
+How to add regression test is described in [this document](https://git.rz.tu-bs.de/irmb/virtualfluids/-/wikis/Regression-Tests).
\ No newline at end of file

regression-tests/__regression_test_executer.sh

+
+
+download_reference_data () {
+    rm -rf reference_data && mkdir -p reference_data
+    git clone --depth 1 --filter=blob:none --sparse https://github.com/irmb/test_data reference_data
+    cd reference_data
+    git sparse-checkout add $1
+    cd ..
+}
+
+
+# run regression test - arguments:
+# 1. REFERENCE_DATA_DIR - to download the reference data and compare against
+# 2. CMAKE_FLAGS - cmake flags for the build of VirtualFluids
+# 3. APPLICATION - the application to be executed
+# 4. RESULT_DATA_DIR - the path to the produced data to be compared
+run_regression_test () {
+    download_reference_data $1
+
+    rm -rf build && mkdir -p build
+    cmake -B build $2
+    cmake --build build --parallel 8
+
+    # execute the application
+    $3
+
+    # execute fieldcompare (A more comprehensive manual can be found here https://gitlab.com/dglaeser/fieldcompare)
+    fieldcompare dir $4 reference_data/$1 --include-files "*.vtu"
+}
\ No newline at end of file

regression-tests/driven_cavity_test.sh

 #!/bin/bash
+source ./regression-tests/__regression_test_executer.sh
 
-#################################
-# Driven Cavity Regression Test
-#################################
 
-# build VirtualFluids accordingly to our specific test scenario.
-# in this case adding -DUSER_APPS="apps/gpu/LBM/DrivenCavity to the cmake command is not necessary, because the DrivenCavity is added to VirtualFluids by default.
-mkdir -p build
-cmake -B build --preset=make_gpu -DCMAKE_BUILD_TYPE=Release -DCMAKE_CUDA_ARCHITECTURES=75 #-DUSER_APPS="apps/gpu/LBM/DrivenCavity"
-cmake --build build --parallel 8
+# 1. set reference data directory (must match the folder structure in https://github.com/irmb/test_data)
+REFERENCE_DATA_DIR=regression_tests/gpu/DrivenCavity_2Levels
 
-# execute VirtualFluids
-./build/bin/DrivenCavity
+# 2. set cmake flags for the build of VirtualFluids
+CMAKE_FLAGS="--preset=make_gpu -DCMAKE_BUILD_TYPE=Release -DCMAKE_CUDA_ARCHITECTURES=75"
 
-# set the path to the produced data
-PATH_TO_DIR=output/DrivenCavity
+# 3. define the application to be executed
+APPLICATION=./build/bin/DrivenCavity
 
-# set the path to the reference data.
-# `regression-tests/reference_data` is fix `regression_tests/gpu/DrivenCavity_2Levels` must match the structure in https://github.com/irmb/test_data:
-PATH_TO_REFERENCE_DIR=reference_data/regression_tests/gpu/DrivenCavity_2Levels
+# 4. set the path to the produced data
+RESULT_DATA_DIR=output/DrivenCavity
+
+
+run_regression_test "$REFERENCE_DATA_DIR" "$CMAKE_FLAGS" "$APPLICATION" "$RESULT_DATA_DIR"
 
-# execute fieldcompare (A more comprehensive manual can be found here https://gitlab.com/dglaeser/fieldcompare)
-fieldcompare dir $PATH_TO_DIR $PATH_TO_REFERENCE_DIR --include-files "*.vtu"

regression-tests/driven_cavity_uniform_test.sh

 #!/bin/bash
 
-#################################
-# Driven Cavity Regression Test
-#################################
+source ./regression-tests/__regression_test_executer.sh
 
-# build VirtualFluids accordingly to our specific test scenario.
-# in this case adding -DUSER_APPS="apps/gpu/LBM/DrivenCavity to the cmake command is not necessary, because the DrivenCavity is added to VirtualFluids by default.
-mkdir -p build
-cmake -B build --preset=make_gpu -DCMAKE_BUILD_TYPE=Release -DCMAKE_CUDA_ARCHITECTURES=75 -DUSER_APPS="apps/gpu/LBM/DrivenCavityUniform"
-cmake --build build --parallel 8
 
-# execute VirtualFluids
-./build/bin/DrivenCavityUniform
+# 1. set reference data directory (must match the folder structure in https://github.com/irmb/test_data)
+REFERENCE_DATA_DIR=regression_tests/gpu/DrivenCavity_uniform
 
+# 2. set cmake flags for the build of VirtualFluids
+CMAKE_FLAGS="--preset=make_gpu -DCMAKE_BUILD_TYPE=Release -DCMAKE_CUDA_ARCHITECTURES=75 -DUSER_APPS="apps/gpu/LBM/DrivenCavityUniform""
 
-# set the path to the produced data
-PATH_TO_DIR=output/DrivenCavity_uniform
+# 3. define the application to be executed
+APPLICATION=./build/bin/DrivenCavityUniform
 
-# set the path to the reference data.
-# `regression-tests/reference_data` is fix `regression_tests/gpu/DrivenCavity_uniform_2022_12_16` must match the structure in https://github.com/irmb/test_data:
-PATH_TO_REFERENCE_DIR=reference_data/regression_tests/gpu/DrivenCavity_uniform
+# 4. set the path to the produced data
+RESULT_DATA_DIR=output/DrivenCavity_uniform
 
-# execute fieldcompare (A more comprehensive manual can be found here https://gitlab.com/dglaeser/fieldcompare)
-fieldcompare dir $PATH_TO_DIR $PATH_TO_REFERENCE_DIR --include-files "*.vtu"
+
+run_regression_test "$REFERENCE_DATA_DIR" "$CMAKE_FLAGS" "$APPLICATION" "$RESULT_DATA_DIR"

regression-tests/regression-tests.sh

-#!/bin/bash
-
-#################################
-# VirtualFludis regression tests
-#################################
-
-
-# 1. Cloning the reference data from github
-rm -r reference_data && mkdir -p reference_data
-git clone https://github.com/irmb/test_data reference_data
-
-# 2. set up the python environnement
-#    by cloning our meshio patch and fieldcompare into a venv
-python3 -m venv .venv
-source .venv/bin/activate
-pip install fieldcompare
-
-# 3. Running the specific tests
-./regression-tests/driven_cavity_uniform_test.sh
-./regression-tests/driven_cavity_test.sh
-
-
-
-# How to add a new regression test?
-# 1. setup the specfic simulation and run it to create reference data.
-# 2. fork https://github.com/irmb/test_data and create a pull request containing the reference data.
-# 3. copy ./regression-tests/driven_cavity_test.sh and adjust the file accordingly to the new test scenario.
-# 4. execute this file from here accordingly to #3.
\ No newline at end of file

utilities/ci-regression-tests/generate-ci.py

+from jinja2 import Template
+from pathlib import Path
+
+TEMPLATES_DIR = Path(__file__).parent
+REGRESSION_CI_TEMPLATE = TEMPLATES_DIR / "regression-tests-ci.yml.j2"
+GENERATED_DIR = Path("generated")
+REGRESSION_CI_FILE = GENERATED_DIR / "regression-tests-ci.yml"
+TEST_FILE_DIR = Path("regression-tests")
+
+
+def build_regression_job_string(regression_tests: list[str]) -> str:
+    template = Template(REGRESSION_CI_TEMPLATE.read_text())
+    return template.render(regression_tests=regression_tests)
+
+
+def main():
+    regression_tests_files = [item.stem for item in TEST_FILE_DIR.glob("*_test.sh")]
+    regression_tests_ci_file = build_regression_job_string(regression_tests_files)
+    REGRESSION_CI_FILE.write_text(regression_tests_ci_file)
+
+if __name__ == "__main__":
+    GENERATED_DIR.mkdir(parents=True, exist_ok=True)
+    main()
\ No newline at end of file

utilities/ci-regression-tests/regression-tests-ci.yml.j2

+image: git.rz.tu-bs.de:4567/irmb/virtualfluids/ubuntu20_04:1.5
+
+stages: 
+  - test
+
+.regression-test:
+  stage: test
+
+  needs: []
+
+  tags:
+    - gpu
+    - linux
+
+  before_script:
+    - chmod +x ./regression-tests/*
+    - pip install fieldcompare
+
+{% for regression_test in regression_tests %}
+run-regression-test-{{ regression_test }}:
+  extends: .regression-test
+
+  script:
+    - ./regression-tests/{{ regression_test }}.sh
+
+{% endfor %}

utilities/ci-regression-tests/requirements.txt

+Jinja2==3.1.2
\ No newline at end of file