From 90e544d162578126fd70947e0eeb7cada8a5358d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gr=C3=A9goire?= <gregoire.kubler@proton.me>
Date: Tue, 30 Jul 2024 23:47:42 +0200
Subject: [PATCH] feat : created code quality stage to host formatting, static
 analysis & coverage jobs

---
 .gitlab/ci/code_quality/cpp.gitlab-ci.yml     | 88 +++++++++++++++++++
 .gitlab/ci/code_quality/python.gitlab-ci.yml  | 88 +++++++++++++++++++
 .gitlab/ci/default.gitlab-ci.yml              |  6 +-
 .gitlab/ci/rules.gitlab-ci.yml                | 23 ++++-
 .gitlab/ci/static_analysis/cpp.gitlab-ci.yml  | 26 ------
 .../ci/static_analysis/python.gitlab-ci.yml   | 22 -----
 .gitlab/ci/ubuntu_cpp.gitlab-ci.yml           | 34 +------
 .gitlab/ci/ubuntu_python.gitlab-ci.yml        | 38 +-------
 .gitlab/ci/windows_cpp.gitlab-ci.yml          |  3 +-
 .gitlab/ci/windows_python.gitlab-ci.yml       |  3 +-
 10 files changed, 204 insertions(+), 127 deletions(-)
 create mode 100644 .gitlab/ci/code_quality/cpp.gitlab-ci.yml
 create mode 100644 .gitlab/ci/code_quality/python.gitlab-ci.yml
 delete mode 100644 .gitlab/ci/static_analysis/cpp.gitlab-ci.yml
 delete mode 100644 .gitlab/ci/static_analysis/python.gitlab-ci.yml

diff --git a/.gitlab/ci/code_quality/cpp.gitlab-ci.yml b/.gitlab/ci/code_quality/cpp.gitlab-ci.yml
new file mode 100644
index 0000000..24c1d8b
--- /dev/null
+++ b/.gitlab/ci/code_quality/cpp.gitlab-ci.yml
@@ -0,0 +1,88 @@
+include:
+  - local: '/.gitlab/ci/default.gitlab-ci.yml'
+
+code_quality:static_analysis:cpp:
+  stage: code_quality
+  extends: 
+    - .rules:code_quality:cpp
+  tags:
+    - static_analysis
+  before_script : 
+    - !reference [.retrieve_deps:apt, script]
+    - apt-get install -y cppcheck python-is-python3
+    - python -m pip install Pygments
+    - python -m pip install -U cppcheck_codequality
+
+  script:
+    - mkdir -p $CI_COMMIT_REF_NAME
+    - cppcheck -j 4 --enable=all --inconclusive --force --xml --xml-version=2 . 2> cppcheck-result.xml
+    - cppcheck-htmlreport --file=cppcheck-result.xml --report-dir=$CI_COMMIT_REF_NAME --source-dir=src
+    - cppcheck-codequality --input-file=cppcheck-result.xml --output-file=cppcheck.json
+    - mkdir -p public/cpp
+    - mv $CI_COMMIT_REF_NAME public/cpp/
+  artifacts:
+    paths: 
+      - public
+    reports:
+      codequality: cppcheck.json
+
+code_quality:coverage:ubuntu_cpp:
+  stage: code_quality
+  needs: ["build:ubuntu_cpp"]
+  extends:
+    - .build:variable:aidge_install 
+    - .rules:code_quality:cpp
+  tags:
+    - docker
+  before_script:
+    - !reference [.retrieve_deps:apt, script]
+    - apt-get install -qq -y gcovr
+  script:
+    - cd $BUILD_DIR
+    - ctest --output-on-failure
+    # HTML report for visualization
+    - gcovr --html-details --exclude-unreachable-branches -o coverage.html --root ${CI_PROJECT_DIR} --filter '\.\./include/' --filter '\.\./src/'
+    # Coberta XML report for Gitlab integration
+    - gcovr --xml-pretty --exclude-unreachable-branches --print-summary -o coverage.xml --root ${CI_PROJECT_DIR} --filter '\.\./include/' --filter '\.\./src/'
+  coverage: /^\s*lines:\s*\d+.\d+\%/
+  artifacts:
+    name: ${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHA}
+    expire_in: 2 days
+    reports:
+      coverage_report:
+        coverage_format: cobertura
+        path: $BUILD_DIR/coverage.xml
+
+        
+code_quality:code_format:cpp:
+  stage: code_quality
+  extends: .rules:code_quality:cpp
+  needs: []
+  variables:
+    CLANG_VERSION: 20
+  tags:
+    - docker
+  before_script:
+    - !reference [.retrieve_deps:apt, script]
+    - apt install -y -qq wget git software-properties-common lsb-release
+    - wget https://apt.llvm.org/llvm.sh
+    - chmod u+x llvm.sh
+    - ./llvm.sh $CLANG_VERSION
+    - apt-get install -y -qq clang-format-$CLANG_VERSION
+  script: 
+    - echo "Ensuring that all cpp files modified in previous commit are correctly formatted. If not, this job will fail."
+    - modified_files=$(git diff --name-only HEAD origin/dev) # change "dev" to $CI_DEFAULT_BRANCH when whole codebase is formatted
+    - modified_CXX_files=$(echo "$modified_files" | grep -Ei '\.(h|c|hpp|cpp|cu)$' || true)
+    - >
+      if [[ -n "$modified_CXX_files" ]]; then
+        CLANG_OUTPUT=$(clang-format-$CLANG_VERSION --style=file:.clang-format --dry-run $modified_CXX_files || true )
+      fi
+    - >
+      if [[ "$CLANG_OUTPUT" != "" ]]; then
+        echo "You pushed unformatted files, setup a code formatter before pushing your code."
+        echo "For more informations about code formatting head to the dedicated wiki page :"
+        echo "https://gitlab.eclipse.org/groups/eclipse/aidge/-/wikis/code-quality-:-formatting-&-linting"
+        exit 1
+      else 
+        echo "No files to format, Job successful!"
+      fi
diff --git a/.gitlab/ci/code_quality/python.gitlab-ci.yml b/.gitlab/ci/code_quality/python.gitlab-ci.yml
new file mode 100644
index 0000000..684332e
--- /dev/null
+++ b/.gitlab/ci/code_quality/python.gitlab-ci.yml
@@ -0,0 +1,88 @@
+include:
+  - local: '/.gitlab/ci/default.gitlab-ci.yml'
+
+code_quality:static_analysis:python:
+  stage: code_quality
+  extends: .rules:code_quality:python
+  needs: ["build:ubuntu_python"]
+  tags:
+    - static_analysis
+  before_script: 
+    - apt-get update
+    - apt-get install -y python-is-python3 python3-pip
+    - python -m pip install pylint pylint-gitlab
+  script:
+    - pylint --exit-zero --output-format=pylint_gitlab.GitlabCodeClimateReporter ${CI_PROJECT_NAME}/ > codeclimate.json
+    - pylint --exit-zero --output-format=pylint_gitlab.GitlabPagesHtmlReporter ${CI_PROJECT_NAME}/ > pylint.html
+    - mkdir -p public/python/$CI_COMMIT_REF_NAME
+    - mv pylint.html public/python/$CI_COMMIT_REF_NAME/
+  artifacts:
+    paths:
+      - public
+    reports:
+      codequality: codeclimate.json
+
+     
+code_quality:coverage:ubuntu_python:
+  stage: code_quality
+  needs: ["build:ubuntu_python"]
+  extends: .rules:code_quality:python
+  tags:
+    - docker
+
+  before_script: 
+    - !reference [.retrieve_deps:apt, script]
+    - source venv/bin/activate
+    - which python
+    - python -m pip list
+    - python -m pip install numpy coverage requests unittest-xml-reporting
+    - >
+      if [[ "$CI_PROJECT_NAME" == "aidge_onnx" ]]; then
+        python -m pip install pytest onnxruntime>=1.18.0
+      fi
+  script:
+    - cd ${CI_PROJECT_NAME}
+    # Retrieve the installation path of the module, since it is installed with pip.
+    - export MODULE_LOCATION=`python -c "import ${CI_PROJECT_NAME} as _; print(_.__path__[0])"`
+    - python -m coverage run --source=$MODULE_LOCATION -m unittest discover -s unit_tests/ -v -b
+    - python -m coverage report
+    - python -m coverage xml
+  coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
+  artifacts:
+    expire_in: 3 days
+    reports:
+      coverage_report:
+        coverage_format: cobertura
+        path: ${CI_PROJECT_NAME}/coverage.xml
+
+code_quality:code_format:python:
+  stage: code_quality
+  extends: .rules:code_quality:python
+  needs: []
+  tags:
+    - docker
+  before_script:
+    - apt-get update -qq 
+    - apt-get install -y -qq python-is-python3 python3-pip git 
+    - pip install ruff
+
+  script: 
+    - echo "Ensuring that all python files modified in previous commit are correctly formatted. If not, this job will fail."
+    - modified_files=$(git diff --name-only HEAD origin/dev) # change "dev" to $CI_DEFAULT_BRANCH when whole codebase is formatted
+    - modified_python_files=$(echo $modified_files | grep ".py$\|.ipynb$" || true )
+    - >
+      if [[ -n $modified_python_files ]]; then
+        python_files_to_format=$(ruff format --check $modified_python_files)
+        python_files_to_format=$(( $python_files_to_format != "" ? $python_files_to_format : ""))
+        echo "PYTHON FILES TO FORMAT :"
+        echo "$python_files_to_format"
+      fi
+    - >
+      if [[ "$python_files_to_format" != "" ]]; then
+        echo "You pushed unformatted files, setup a code formatter before pushing your code."
+        echo "For more informations about code formatting head to the dedicated wiki page :"
+        echo "https://gitlab.eclipse.org/groups/eclipse/aidge/-/wikis/code-quality-:-formatting-&-linting"
+        exit 1
+      else 
+        echo "No files to format, congratulations!"
+      fi
diff --git a/.gitlab/ci/default.gitlab-ci.yml b/.gitlab/ci/default.gitlab-ci.yml
index df934d8..0b23130 100644
--- a/.gitlab/ci/default.gitlab-ci.yml
+++ b/.gitlab/ci/default.gitlab-ci.yml
@@ -40,9 +40,9 @@ variables:
     - apt-get update -qq 
     - apt-get install -y -qq cmake git unzip curl jq
     - >
-      if [[ $CI_JOB_NAME == *"python"* || $CI_JOB_NAME == *"pip"* ]]; then
-        apt-get install -y -qq python-is-python3 python3-pip  
-      fi
+        if echo $CI_JOB_NAME | grep -q -E "python|pip|format" ; then
+          apt-get install -y -qq python-is-python3 python3-pip  
+        fi
     - >
       case $CI_PROJECT_NAME in 
         "aidge_backend_cuda" )
diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml
index a1d1de9..55ac4bc 100644
--- a/.gitlab/ci/rules.gitlab-ci.yml
+++ b/.gitlab/ci/rules.gitlab-ci.yml
@@ -1,11 +1,27 @@
-.rules:static_analysis_coverage:
+.rules:code_quality:cpp:
+  rules:
+    - if: '$CI_MERGE_REQUEST_TITLE !~ /^Draft:/'
+      changes: 
+        paths: ["CMakeLists.txt", ".clang-format","**/*.h", "**/*.hpp", "**/*.c", "**/*.cpp", "**/*.cu"]
+        compare_to: 'refs/heads/dev'
+      when: on_success
+      allow_failure: false
+    - if: '$CI'
+      when: manual
+      allow_failure: true
+
+.rules:code_quality:python:
   rules:
     - if: $CI_MERGE_REQUEST_TITLE !~ /^Draft:/
+      changes: 
+        paths: ["**/*.py", "**/*.ipynb"]
+        compare_to: 'refs/heads/dev'
       when: on_success
       allow_failure: false
-    - if: $CI
+    - if: '$CI'
       when: manual
       allow_failure: true
+      
   
 .rules:build:merge_pipeline_or_branch_dev_main:
   rules:
@@ -17,6 +33,7 @@
       when: always
       allow_failure: false
     - if: $CI
+      when: manual
       allow_failure: true
 
 # creating release jobs only when merging to main
@@ -36,7 +53,6 @@
   when: manual
   allow_failure: false
 
-
 # deploying created releases
 # when creating a commit tag & previous release job was successful
 .rules:deploy:
@@ -44,3 +60,4 @@
     - if: $CI_COMMIT_TAG 
   allow_failure: true
   when: on_success
+
diff --git a/.gitlab/ci/static_analysis/cpp.gitlab-ci.yml b/.gitlab/ci/static_analysis/cpp.gitlab-ci.yml
deleted file mode 100644
index 3e1c35b..0000000
--- a/.gitlab/ci/static_analysis/cpp.gitlab-ci.yml
+++ /dev/null
@@ -1,26 +0,0 @@
-include:
-  - local: '/.gitlab/ci/default.gitlab-ci.yml'
-
-static_analysis:cpp:
-  stage: static_analysis
-  extends: .rules:static_analysis_coverage
-  tags:
-    - static_analysis
-  before_script : 
-    - apt-get update
-    - apt-get install -y cppcheck python-is-python3 python3-pip
-    - python -m pip install Pygments
-    - python -m pip install -U cppcheck_codequality
-
-  script:
-    - mkdir -p $CI_COMMIT_REF_NAME
-    - cppcheck -j 4 --enable=all --inconclusive --force --xml --xml-version=2 . 2> cppcheck-result.xml
-    - cppcheck-htmlreport --file=cppcheck-result.xml --report-dir=$CI_COMMIT_REF_NAME --source-dir=src
-    - cppcheck-codequality --input-file=cppcheck-result.xml --output-file=cppcheck.json
-    - mkdir -p public/cpp
-    - mv $CI_COMMIT_REF_NAME public/cpp/
-  artifacts:
-    paths: 
-      - public
-    reports:
-      codequality: cppcheck.json
diff --git a/.gitlab/ci/static_analysis/python.gitlab-ci.yml b/.gitlab/ci/static_analysis/python.gitlab-ci.yml
deleted file mode 100644
index d7636fa..0000000
--- a/.gitlab/ci/static_analysis/python.gitlab-ci.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-include:
-  - local: '/.gitlab/ci/default.gitlab-ci.yml'
-
-static_analysis:python:
-  stage: static_analysis
-  extends: .rules:static_analysis_coverage
-  tags:
-    - static_analysis
-  before_script: 
-    - apt-get update
-    - apt-get install -y python-is-python3 python3-pip
-    - python -m pip install pylint pylint-gitlab
-  script:
-    - pylint --exit-zero --output-format=pylint_gitlab.GitlabCodeClimateReporter ${CI_PROJECT_NAME}/ > codeclimate.json
-    - pylint --exit-zero --output-format=pylint_gitlab.GitlabPagesHtmlReporter ${CI_PROJECT_NAME}/ > pylint.html
-    - mkdir -p public/python/$CI_COMMIT_REF_NAME
-    - mv pylint.html public/python/$CI_COMMIT_REF_NAME/
-  artifacts:
-    paths:
-      - public
-    reports:
-      codequality: codeclimate.json
diff --git a/.gitlab/ci/ubuntu_cpp.gitlab-ci.yml b/.gitlab/ci/ubuntu_cpp.gitlab-ci.yml
index f74c429..08669e3 100644
--- a/.gitlab/ci/ubuntu_cpp.gitlab-ci.yml
+++ b/.gitlab/ci/ubuntu_cpp.gitlab-ci.yml
@@ -1,6 +1,6 @@
 include:
   - local: '/.gitlab/ci/default.gitlab-ci.yml'
-  - local: '/.gitlab/ci/static_analysis/cpp.gitlab-ci.yml'
+  - local: '/.gitlab/ci/code_quality/cpp.gitlab-ci.yml'
 
 ####################################################################################################
 #                                               BUILD
@@ -89,7 +89,7 @@ build:ubuntu_cpp:clang:
 ####################################################################################################
 test:ubuntu_cpp:
   stage: test
-  needs: ["build:ubuntu_cpp"]
+  needs: [build:ubuntu_cpp]
   extends: .build:variable:aidge_install 
   rules:
     - when : on_success
@@ -106,33 +106,3 @@ test:ubuntu_cpp:
   artifacts:
     reports:
       junit: $BUILD_DIR/ctest-results.xml
-
-####################################################################################################
-#                                               COVERAGE
-####################################################################################################
-coverage:ubuntu_cpp:
-  stage: coverage
-  needs: ["build:ubuntu_cpp"]
-  extends:
-    - .build:variable:aidge_install 
-    - .rules:static_analysis_coverage
-  tags:
-    - docker
-  before_script:
-    - !reference [.retrieve_deps:apt, script]
-    - apt-get install -qq -y gcovr
-  script:
-    - cd $BUILD_DIR
-    - ctest --output-on-failure
-    # HTML report for visualization
-    - gcovr --html-details --exclude-unreachable-branches -o coverage.html --root ${CI_PROJECT_DIR} --filter '\.\./include/' --filter '\.\./src/'
-    # Coberta XML report for Gitlab integration
-    - gcovr --xml-pretty --exclude-unreachable-branches --print-summary -o coverage.xml --root ${CI_PROJECT_DIR} --filter '\.\./include/' --filter '\.\./src/'
-  coverage: /^\s*lines:\s*\d+.\d+\%/
-  artifacts:
-    name: ${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHA}
-    expire_in: 3 days
-    reports:
-      coverage_report:
-        coverage_format: cobertura
-        path: $BUILD_DIR/coverage.xml
diff --git a/.gitlab/ci/ubuntu_python.gitlab-ci.yml b/.gitlab/ci/ubuntu_python.gitlab-ci.yml
index 5cdd941..87712a6 100644
--- a/.gitlab/ci/ubuntu_python.gitlab-ci.yml
+++ b/.gitlab/ci/ubuntu_python.gitlab-ci.yml
@@ -1,6 +1,6 @@
 include:
   - local: '/.gitlab/ci/default.gitlab-ci.yml'
-  - local: '/.gitlab/ci/static_analysis/python.gitlab-ci.yml'
+  - local: '/.gitlab/ci/code_quality/python.gitlab-ci.yml'
 
 ####################################################################################################
 #                                               BUILD
@@ -60,39 +60,3 @@ test:ubuntu_python:
   artifacts:
     reports:
       junit: ${CI_PROJECT_NAME}/xmlrunner-results.xml
-
-####################################################################################################
-#                                               COVERAGE
-####################################################################################################
-coverage:ubuntu_python:
-  stage: coverage
-  needs: ["build:ubuntu_python"]
-  extends: .rules:static_analysis_coverage
-  tags:
-    - docker
-
-  before_script: 
-    - !reference [.retrieve_deps:apt, script]
-    - source venv/bin/activate
-    - which python
-    - python -m pip list
-    - python -m pip install numpy coverage requests unittest-xml-reporting 
-    - >
-      if [[ "$CI_PROJECT_NAME" == "aidge_onnx" ]]; then
-        python -m pip install pytest onnxruntime>=1.18.0
-      fi
-  script:
-    - cd ${CI_PROJECT_NAME}
-    # Retrieve the installation path of the module, since it is installed with pip.
-    - export MODULE_LOCATION=`python -c "import ${CI_PROJECT_NAME} as _; print(_.__path__[0])"`
-    - python -m coverage run --source=$MODULE_LOCATION -m unittest discover -s unit_tests/ -v -b
-    - python -m coverage report
-    - python -m coverage xml
-  coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
-  artifacts:
-    expire_in: 3 days
-    reports:
-      coverage_report:
-        coverage_format: cobertura
-        path: ${CI_PROJECT_NAME}/coverage.xml
-
diff --git a/.gitlab/ci/windows_cpp.gitlab-ci.yml b/.gitlab/ci/windows_cpp.gitlab-ci.yml
index 6a3a04a..1288607 100644
--- a/.gitlab/ci/windows_cpp.gitlab-ci.yml
+++ b/.gitlab/ci/windows_cpp.gitlab-ci.yml
@@ -1,7 +1,6 @@
 include:
   - local: '/.gitlab/ci/default.gitlab-ci.yml'
-  # Static analysis job
-  - local: '/.gitlab/ci/static_analysis/cpp.gitlab-ci.yml'
+  - local: '/.gitlab/ci/code_quality/cpp.gitlab-ci.yml'
 
 build:windows_cpp:
   stage: build
diff --git a/.gitlab/ci/windows_python.gitlab-ci.yml b/.gitlab/ci/windows_python.gitlab-ci.yml
index 5a2554a..32ec828 100644
--- a/.gitlab/ci/windows_python.gitlab-ci.yml
+++ b/.gitlab/ci/windows_python.gitlab-ci.yml
@@ -1,7 +1,6 @@
 include:
   - local: '/.gitlab/ci/default.gitlab-ci.yml'
-  # Static analysis job
-  - local: '/.gitlab/ci/static_analysis/python.gitlab-ci.yml'
+  - local: '/.gitlab/ci/code_quality/python.gitlab-ci.yml'
 
 ####################################################################################################
 #                                               BUILD
-- 
GitLab