Skip to content
Snippets Groups Projects
Commit e878aa46 authored by Grégoire Kubler's avatar Grégoire Kubler
Browse files

Merge branch 'fix/quantization_onnx_dependecy' of...

Merge branch 'fix/quantization_onnx_dependecy' of gitlab.eclipse.org:eclipse/aidge/gitlab_shared_files into fix/quantization_onnx_dependecy
+
list dependencies switch case explicited for windows
parents 451e6bf4 6082b141
No related branches found
Tags R3_3_1
1 merge request!8multiple fixes
......@@ -35,15 +35,15 @@ variables:
# This job is here to handle dependencies install package specific dependencies
.retrieve_deps:apt:
script:
- set -x
- apt-get update -qq
- apt-get install -y -qq cmake
- >
if [[ $CI_JOB_NAME == *"python"* || $CI_JOB_NAME == *"pip"* ]]; then
apt-get install -y -qq python-is-python3 python3-pip
fi
- >
case $CI_PROJECT_NAME in
"aidge_core" )
apt-get install -y -qq cmake
if [[ $CI_JOB_NAME == *"python"* ]]; then
apt-get install -y -qq python-is-python3 pip
fi
;;
"aidge_backend_cuda" )
apt-get install -y -qq libcudnn8-dev
;;
......@@ -53,3 +53,14 @@ variables:
* )
;;
esac
# install nvidia container toolkit for cuda repo releases
- >
if $CI_JOB_NAME ~= "release" && $CI_PROJECT_NAME ~= "cuda" ; then
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \
&& curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
sudo apt-get update
sudo apt-get install -y nvidia-container-toolkit
fi
- set +x
......@@ -7,10 +7,10 @@ include:
###################################################################################################
.ubuntu:download:setup:
script:
- GITLAB_ECLIPSE_URL="https://gitlab.eclipse.org"
- API_URL="$GITLAB_ECLIPSE_URL/api/v4"
- apt-get update -qq
- apt-get install -y -qq git unzip curl jq
- GITLAB_ECLIPSE_URL="https://gitlab.eclipse.org"
- API_URL="$GITLAB_ECLIPSE_URL/api/v4"
- apt-get update -qq
- apt-get install -y -qq git unzip curl jq
.ubuntu:download:artifacts:
# This script will retrieve the artifacts of latests successful specified job in every dependency.
......@@ -22,15 +22,14 @@ include:
# - DEPENDENCY_JOB: the name of the specific job that will serve as reference.
before_script :
- set -x
- !reference [.ubuntu:download:setup, script]
- !reference [.ubuntu:download:list_dependencies, script]
- !reference [.ubuntu:download:select_jobs, script]
- !reference [.ubuntu:download:artifacts, script]
- set +x
script:
- set -x
- !reference [.ubuntu:download:setup, script] # sometimes only the script must be called & not the before_script hence setup must be here.
- !reference [.ubuntu:download:setup, script] # setup must be executed if
- !reference [.ubuntu:download:select_jobs, script]
- echo $DEPS_NAMES[@]
- set +x
- echo "Retrieving artifacts of selected jobs."
......@@ -61,14 +60,13 @@ include:
before_script :
- set -x
- !reference [.ubuntu:download:setup, script]
- !reference [.ubuntu:download:list_dependencies, script]
- !reference [.ubuntu:download:select_jobs, script]
- !reference [.ubuntu:download:repositories, script]
- set +x
script:
- !reference [.ubuntu:download:setup, script] # sometimes only the script must be called & not the before_script hence setup must be here.
- !reference [.ubuntu:download:setup, script]
- !reference [.ubuntu:download:select_jobs, script]
- echo "Cloning each dependency and checking them out at the commit of selected job."
- >
for i in "${!DEPS_NAMES[@]}"; do
......@@ -138,11 +136,11 @@ include:
.windows:download:artifacts:
before_script:
- !reference [.windows:download:setup, script]
- !reference [.windows:download:list_dependencies, script]
- !reference [.windows:download:select_jobs, script]
- !reference [.windows:download:artifacts, script]
script:
- !reference [.windows:download:setup, script]
- !reference [.windows:download:select_jobs, script]
- >
for($i = 0; $i -lt $DEPS_SELECTED_JOBS.Count ; $i++) {
$DEP_ID=$DEPS_SELECTED_JOBS[$i].pipeline.project_id
......@@ -156,8 +154,10 @@ include:
.windows:download:repositories:
before_script:
- !reference [.windows:download:setup, script]
- !reference [.windows:download:list_dependencies, script]
- !reference [.windows:download:repositories, script]
script:
- !reference [.windows:download:setup, script]
- !reference [.windows:download:select_jobs, script]
- !reference [.windows:download:repositories, script]
script:
......
......@@ -12,16 +12,37 @@
- set -x
- >
case "$CI_PROJECT_NAME" in
"aidge_backend_cpu" | "aidge_backend_opencv" )
"aidge_backend_cpu" )
DEPS_NAMES=("aidge_core")
;;
# aidge_onnx needs backend_cpu for testing
"aidge_onnx" | "aidge_backend_cuda" | "aidge_learning" | "aidge_quantization" | "aidge_export_cpp" )
"aidge_backend_cuda" )
DEPS_NAMES=("aidge_core" "aidge_backend_cpu")
;;
"aidge_backend_opencv" )
DEPS_NAMES=("aidge_core")
;;
"aidge_core" )
DEPS_NAMES=()
;;
"aidge_export_arm_cortexm" )
DEPS_NAMES=("aidge_core" "aidge_backend_cpu")
;;
"aidge_export_cpp" )
DEPS_NAMES=("aidge_core" "aidge_backend_cpu")
;;
"aidge_interop_torch" )
DEPS_NAMES=("aidge_core" "aidge_backend_cpu" "aidge_onnx")
;;
"aidge_learning" )
DEPS_NAMES=("aidge_core" "aidge_backend_cpu")
;;
"aidge_onnx" )
# aidge_onnx needs backend_cpu for testing
DEPS_NAMES=("aidge_core" "aidge_backend_cpu")
;;
"aidge_quantization" )
DEPS_NAMES=("aidge_core" "aidge_backend_cpu")
;;
* )
echo "****************************************************************************************************************************"
echo "****************************************************************************************************************************"
......@@ -31,7 +52,7 @@
echo "** If this is an error please head to"
echo "** REPO : aidge/gitlab_shared_files repo"
echo "** URL : https://gitlab.eclipse.org/eclipse/aidge/gitlab_shared_files"
echo "** FILE : .gitlab/ci/common/download.gitlab-ci.yml"
echo "** FILE : .gitlab/ci/common/download/list_dependencies.gitlab-ci.yml"
echo "** Modify the dependencie list in the following job :"
echo "** .ubuntu:download:list_dependencies"
echo "**"
......@@ -51,37 +72,55 @@
- Write-Host "CI_PROJECT_NAME = $CI_PROJECT_NAME"
- >
switch ($CI_PROJECT_NAME) {
{"aidge_backend_cpu" -or
"aidge_backend_opencv"} {
$DEPS_NAMES = "aidge_core"
}
{"aidge_onnx" -or
"aidge_backend_cuda" -or
"aidge_learning" -or
"aidge_quantization" -or
"aidge_export_cpp"} {
$DEPS_NAMES = "aidge_core", "aidge_backend_cpu"
}
{"aidge_interop_torch"} {
$DEPS_NAMES = "aidge_core", "aidge_backend_cpu", "aidge_onnx"
"aidge_backend_cpu" {
$DEPS_NAMES = "aidge_core"
}
"aidge_backend_cuda" {
$DEPS_NAMES = "aidge_core","aidge_backend_cpu"
}
"aidge_backend_opencv" {
$DEPS_NAMES = "aidge_core"
}
"aidge_core" {
# empty list bc no dependencies
$DEPS_NAMES=New-Object System.Collections.ArrayList
}
"aidge_export_cpp" {
$DEPS_NAMES = "aidge_core", "aidge_backend_cpu"
}
"aidge_export_arm_cortexm" {
$DEPS_NAMES = "aidge_core", "aidge_backend_cpu"
}
"aidge_interop_torch" {
$DEPS_NAMES = "aidge_core", "aidge_backend_cpu", "aidge_onnx"
}
"aidge_onnx" {
$DEPS_NAMES = "aidge_core","aidge_backend_cpu"
}
"aidge_learning" {
$DEPS_NAMES = "aidge_core","aidge_backend_cpu"
}
"aidge_quantization" {
$DEPS_NAMES = "aidge_core","aidge_backend_cpu"
}
default {
Write-Host "****************************************************************************************************************************"
Write-Host "****************************************************************************************************************************"
Write-Host "**"
Write-Host "** WARNING"
Write-Host "** THE PROJECT $CI_PROJECT_NAME HAS NO LISTED DEPENDENCY FOR WINDOWS."
Write-Host "** If this is an error please head to"
Write-Host "** REPO : aidge/gitlab_shared_files repo"
Write-Host "** URL : https://gitlab.eclipse.org/eclipse/aidge/gitlab_shared_files"
Write-Host "** FILE : .gitlab/ci/common/download.gitlab-ci.yml"
Write-Host "** Modify the dependency list in the following job :"
Write-Host "** .windows:download:list_dependencies"
Write-Host "**"
Write-Host "****************************************************************************************************************************"
Write-Host "****************************************************************************************************************************"
$DEPS_NAMES=New-Object System.Collections.ArrayList
}
}
default {
Write-Host "****************************************************************************************************************************"
Write-Host "****************************************************************************************************************************"
Write-Host "**"
Write-Host "** WARNING"
Write-Host "** THE PROJECT $CI_PROJECT_NAME HAS NO LISTED DEPENDENCY FOR WINDOWS."
Write-Host "** If this is an error please head to"
Write-Host "** REPO : aidge/gitlab_shared_files repo"
Write-Host "** URL : https://gitlab.eclipse.org/eclipse/aidge/gitlab_shared_files"
Write-Host "** FILE : .gitlab/ci/common/download.gitlab-ci.yml"
Write-Host "** Modify the dependency list in the following job :"
Write-Host "** .windows:download:list_dependencies"
Write-Host "**"
Write-Host "****************************************************************************************************************************"
Write-Host "****************************************************************************************************************************"
$DEPS_NAMES=New-Object System.Collections.ArrayList
}
- Write-Host 'DEPS_NAMES = $(DEPS_NAMES -join ", ")'
......@@ -25,16 +25,19 @@
echo "****************************************************************************************************************************"
echo "****************************************************************************************************************************"
echo "**"
echo "** RETRIEVING DEPENDENCY \"$DEP_NAME\" ID"
echo "** RETRIEVING \$DEP_NAME\'S PROJECT ID"
echo "**"
echo "****************************************************************************************************************************"
echo "****************************************************************************************************************************"
if [[ $CI_PROJECT_NAMESPACE =~ "eclipse/aidge" ]]; then # nominal case : we are in the official project
DEP_ID=$(curl "$API_URL/groups/$CI_PROJECT_NAMESPACE_ID/projects?search=${DEP_NAME}" | jq -r '.[0].id')
echo "curling from \"$API_URL/groups/$CI_PROJECT_NAMESPACE_ID/projects?search=${DEP_NAME}\""
DEP_ID=$(curl -s "$API_URL/groups/$CI_PROJECT_NAMESPACE_ID/projects?search=${DEP_NAME}" | jq -r '.[0].id')
else # case for fork
echo "Retrieving build_artifacts from user forked project."
echo "curling from \"$API_URL/users?username=$CI_PROJECT_ROOT_NAMESPACE\""
USER=$(curl -s "$API_URL/users?username=$CI_PROJECT_ROOT_NAMESPACE")
USER_ID=$(jq -r '.[0].id' <<<$USER)
echo "curling from \"$API_URL/users/$USER_ID/projects?search=${DEP_NAME}\""
PROJ=$(curl -s "$API_URL/users/$USER_ID/projects?search=${DEP_NAME}")
DEP_ID=$(jq -r '.[0].id' <<<$PROJ)
fi
......@@ -49,7 +52,9 @@
echo "**"
echo "****************************************************************************************************************************"
echo "****************************************************************************************************************************"
BRANCH_TO_PULL="dev" # default branch to pull
DEFAULT_BRANCH="dev" # default branch to pull
BRANCH_TO_PULL=$DEFAULT_BRANCH # default branch to pull
echo "Default branch to pull : $BRANCH_TO_PULL"
echo "Retrieving branches of $DEP_NAME \"$API_URL/projects/$DEP_ID/repository/branches\""
DEP_BRANCHES=$(curl -s "$API_URL/projects/$DEP_ID/repository/branches")
echo "Choosing a branch to pull dependending if we are in a Non draft merge request or not."
......@@ -60,6 +65,7 @@
echo "NON DRAFT MERGE REQUEST Pipeline detected : Pulling from CI_MERGE_REQUEST_TARGET_BRANCH_NAME \"$CI_MERGE_REQUEST_TARGET_BRANCH_NAME\"."
echo "Checking if CI_MERGE_REQUEST_TARGET_BRANCH_NAME \"$CI_MERGE_REQUEST_TARGET_BRANCH_NAME\" branch exists, otherwise pull from default."
FILTERED_BRANCHES=$(jq --arg branch "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" 'map(select( .name == $branch ))' <<< "$DEP_BRANCHES")
if [[ $(jq length <<< $FILTERED_BRANCHES) -gt 0 ]]; then
echo "Found MR target branch in \"$DEP_NAME\"."
echo "Pulling from merge request target branch \"$CI_MERGE_REQUEST_TARGET_BRANCH_NAME\""
......@@ -108,13 +114,17 @@
MR_PIPELINES="[]"
fi
set -x
PIPELINES=$(
jq \
-n \
--argjson pipelines "$PIPELINES" \
--argjson mr_pipelines "$MR_PIPELINES" \
' $pipelines + $mr_pipelines | sort_by(.updated_at) | reverse'
)
PIPELINES=$(
jq \
-n \
--argjson pipelines "$PIPELINES" \
--argjson mr_pipelines "$MR_PIPELINES" \
--arg project_namespace "$CI_PROJECT_NAMESPACE" \
'$pipelines + $mr_pipelines
| map(select(.web_url | test($project_namespace)))
| sort_by(.updated_at)
| reverse'
)
# case for a release : we only retrieve pipelines whose status is "success"
if [[ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "$CI_DEFAULT_BRANCH" ]] || \
......@@ -162,19 +172,14 @@
.[] |
{
repo : $repo_name,
id,
status,
name,
ref,
branch_to_pull : $branch_name,
created_at,
updated_at,
id, status, name, ref, created_at, updated_at,
user : { id : .user.id, username : .user.username },
commit : { id : .commit.id, message : .commit.message, authored_date : .commit.authored_date },
commit : { author : .commit.author_name, id : .commit.id, message : .commit.message, authored_date : .commit.authored_date },
pipeline : { id : .pipeline.id, iid : .pipeline.iid, project_id : .pipeline.project_id, ref : .pipeline.ref},
}
]' <<<"$JOBS")"
if [[ $(jq length <<<$JOBS) > 0 ]]; then
if [[ $(jq length <<< $JOBS) > 0 ]]; then
FOUND_JOB=1
echo "Successful job \"$DEPENDENCY_JOB\" found in branch \"$BRANCH_TO_PULL\" for project \"$DEP_NAME\""
echo "Appending this jobs to the list of selected jobs that will serve as reference to retrieve artifact or specific commit."
......@@ -262,7 +267,8 @@
Write-Host "**"
Write-Host "****************************************************************************************************************************"
Write-Host "****************************************************************************************************************************"
$BRANCH_TO_PULL="dev" # default branch to pull
$DEFAULT_BRANCH="dev"
$BRANCH_TO_PULL=$DEFAULT_BRANCH # default branch to pull
Write-Host "Retrieving branches of $DEP_NAME $API_URL/projects/$DEP_ID/repository/branches"
$DEP_BRANCHES=$($(Invoke-RestMethod -Uri "$API_URL/projects/$DEP_ID/repository/branches" -Method Get) | Select-Object name)
Write-Host "Found $($DEP_BRANCHES.Count) branches."
......@@ -314,8 +320,10 @@
$MR_ID=(Invoke-RestMethod -Uri "$API_URL/projects/$DEP_ID/merge_requests?source_branch=$BRANCH_TO_PULL" -Method Get)[0].iid
$MR_PIPELINES=(Invoke-RestMethod -Uri "$API_URL/projects/$DEP_ID/merge_requests/$MR_ID/pipelines" -Method Get)
$PIPELINES+=$MR_PIPELINES
## Sort pipelines by "updated_at" property to retrieve the more recent pipelines first
$PIPELINES = $PIPELINES | Sort-Object -Property updated_at -Descending
## Sort pipelines by "updated_at" from earliest to latest and only retrieve the ones whose url contain our project namespaec.
$PIPELINES = $PIPELINES |
Where-Object { $_.web_url -match $CI_PROJECT_NAMESPACE } |
Sort-Object -Property updated_at -Descending
Write-Host "****************************************************************************************************************************"
Write-Host "****************************************************************************************************************************"
......
......@@ -20,8 +20,7 @@ release:pip:ubuntu:
DOCKER_TLS_CERTDIR: ""
before_script:
- apt-get update
- apt-get install -qq -y cmake python-is-python3 python3-pip
- !reference [.retrieve_deps:apt, script]
- DEPENDENCY_JOB="build:ubuntu_python"
- !reference [.ubuntu:download:repositories, before_script] # located in common.gitlab-ci.yml
......
......@@ -16,7 +16,6 @@ release:pip:ubuntu:
- apt-get install -y -qq python-is-python3 pip git curl
script:
- ls venv/bin/activate
- source venv/bin/activate
- set -x
- python -m pip wheel . -w wheelhouse
......@@ -29,22 +28,15 @@ release:pip:ubuntu:
python3 -m pip install requests
fi
- echo "Uninstalling $CI_PROJECT_NAME to install the wheel created"
- export WHEEL_NAME=$(find wheelhouse/ -name "$CI_PROJECT_NAME*.whl")
- echo "Wheel name = ${WHEEL_NAME}"
- python -m pip uninstall -y $CI_PROJECT_NAME
- python -m pip install $WHEEL_NAME
- python -m pip list
###############################################################################
# Repo name is changed the repo name with mv in order to perform the test
# on the installed package and not on the source code
# https://stackoverflow.com/questions/56908227/how-to-test-python-wheels-on-ci
###############################################################################
- echo "Current working directory = $(pwd)"
- rm -rf temp_src
- mv $CI_PROJECT_NAME temp_src
- python -m pytest temp_src/unit_tests/
- mv temp_src $CI_PROJECT_NAME
- export WHEELS=$(find wheelhouse/ -name "$CI_PROJECT_NAME*.whl")
- echo "Reinstalling each found wheels = ${WHEELS}"
- >
for wheel in $WHEELS; do # not sure if a for loop is needed here but just in case multiple wheels are found
python -m pip install --force-reinstall $wheel
python -m pip list
cd $CI_PROJECT_NAME # entering package so python doesn't import local package and in
python -m pytest unit_tests/*
done
deploy:pip:ubuntu:
extends:
......
......@@ -3,29 +3,32 @@
- if: $CI_MERGE_REQUEST_TITLE !~ /^Draft:/
when: on_success
allow_failure: false
when: manual
allow_failure: true
- if: $CI
when: manual
allow_failure: true
.rules:build:merge_pipeline_or_branch_dev_main:
rules:
- if: $CI_COMMIT_BRANCH == 'dev' || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
when: always
allow_failure: false
- if: $CI_MERGE_REQUEST_TITLE !~ /^Draft:/
allow_failure: false
when: always
when: manual
allow_failure: false
allow_failure: false
- if: $CI
allow_failure: true
# creating release jobs only when merging to main
.rules:release:
rules:
- if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH
when: always
when: on_success
allow_failure: false
- if: $CI_MERGE_REQUEST_ID # if you are in a MR you can run release jobs
allow_failure: true
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "schedule"
when: always
when: on_success
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_TAG
when: always
when: manual
......
......@@ -18,8 +18,6 @@ include:
- docker
before_script:
# Download dependencies
- apt-get update -q -y
- apt-get install -qq -y cmake python-is-python3 pip
- !reference [.retrieve_deps:apt, script]
- DEPENDENCY_JOB="$CI_JOB_NAME"
......@@ -100,8 +98,6 @@ test:ubuntu_cpp:
- docker
before_script :
# Download dependencies
- apt-get update -q -y
- apt-get install -qq -y cmake
- !reference [.retrieve_deps:apt, script]
script:
......@@ -123,9 +119,8 @@ coverage:ubuntu_cpp:
tags:
- docker
before_script:
- apt-get update -q -y
- apt-get install -qq -y cmake gcovr
- !reference [.retrieve_deps:apt, script]
- apt-get install -qq -y gcovr
script:
- cd $BUILD_DIR
- ctest --output-on-failure
......
......@@ -11,8 +11,6 @@ build:ubuntu_python:
tags:
- docker
before_script:
- apt-get update -q
- apt-get install -qq -y cmake python-is-python3 pip
- !reference [.retrieve_deps:apt, script]
- DEPENDENCY_JOB="build:ubuntu_python"
- !reference [.ubuntu:download:artifacts, before_script]
......@@ -44,20 +42,19 @@ test:ubuntu_python:
tags:
- docker
before_script:
- apt-get update -q
- apt-get install -y -qq python-is-python3 pip git curl
- !reference [.retrieve_deps:apt, script]
script:
- source venv/bin/activate
- python -m pip install unittest-xml-reporting
- which python
- python -m pip list
- >
if [[ "$CI_PROJECT_NAME" == "aidge_onnx" ]]; then
python3 -m pip install requests
python -m pip install requests
fi
script:
- cd ${CI_PROJECT_NAME}
# Run on discovery all tests located in project/unit_tests/python and discard the stdout
# only to show the errors/warnings and the results of the tests
- python -m pip list
- python -m xmlrunner discover -s unit_tests/ -v -b --output-file xmlrunner-results.xml
artifacts:
......@@ -75,9 +72,10 @@ coverage:ubuntu_python:
- docker
before_script:
- apt-get update
- apt-get install -y python-is-python3 python3-pip cmake # cmake for aidge_core tests
- !reference [.retrieve_deps:apt, script]
- source venv/bin/activate
- which python
- python -m pip list
- python -m pip install numpy coverage requests unittest-xml-reporting
script:
- cd ${CI_PROJECT_NAME}
......
# aidge_shared_files
[[_TOC_]]
[_TOC_]
This repository holds files common to the repositories of the Aidge group.
For more informations on how the ci works [head to the dedicated page in the wiki.](https://gitlab.eclipse.org/groups/eclipse/aidge/-/wikis/Understand-the-CI-CD-Pipelines)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment