From 52cff541d457e3ce9ff707be929165dcde34e227 Mon Sep 17 00:00:00 2001 From: Maxence Naud <maxence.naud@cea.fr> Date: Tue, 26 Nov 2024 19:29:00 +0000 Subject: [PATCH] convert indentation to spaces --- .../download/list_dependencies.gitlab-ci.yml | 6 +- .gitlab/ci/download/select_jobs.gitlab-ci.yml | 242 +++++++++--------- 2 files changed, 124 insertions(+), 124 deletions(-) diff --git a/.gitlab/ci/download/list_dependencies.gitlab-ci.yml b/.gitlab/ci/download/list_dependencies.gitlab-ci.yml index 218d07b..73f1884 100644 --- a/.gitlab/ci/download/list_dependencies.gitlab-ci.yml +++ b/.gitlab/ci/download/list_dependencies.gitlab-ci.yml @@ -32,7 +32,7 @@ ;; "aidge_interop_torch" ) DEPS_NAMES=("aidge_core" "aidge_backend_cpu" "aidge_onnx") - ;; + ;; "aidge_learning" ) DEPS_NAMES=("aidge_core" "aidge_backend_cpu") ;; @@ -43,7 +43,7 @@ "aidge_quantization" ) DEPS_NAMES=("aidge_core" "aidge_backend_cpu" "aidge_backend_cuda") ;; - * ) + * ) echo "****************************************************************************************************************************" echo "****************************************************************************************************************************" echo "**" @@ -58,7 +58,7 @@ echo "**" echo "****************************************************************************************************************************" echo "****************************************************************************************************************************" - DEPS_NAMES=() + DEPS_NAMES=() esac - echo "DEPS_NAMES = ${DEPS_NAMES[@]}" diff --git a/.gitlab/ci/download/select_jobs.gitlab-ci.yml b/.gitlab/ci/download/select_jobs.gitlab-ci.yml index 2421a87..2073666 100644 --- a/.gitlab/ci/download/select_jobs.gitlab-ci.yml +++ b/.gitlab/ci/download/select_jobs.gitlab-ci.yml @@ -17,7 +17,7 @@ - DEPS_SELECTED_JOBS='[]' - > for DEP_NAME in "${DEPS_NAMES[@]}"; do - echo "current dep $DEP_NAME" + echo "current dep $DEP_NAME" ###################################################################################################### # Retrieve the ID of the project @@ -30,21 +30,21 @@ echo "*************************************************************************" echo "*************************************************************************" if [[ $CI_PROJECT_NAMESPACE =~ "eclipse/aidge" ]]; then # nominal case : we are in the official project - echo "curling from \"$API_URL/groups/$CI_PROJECT_NAMESPACE_ID/projects?search=${DEP_NAME}\"" - DEP_ID=$(curl -s "$API_URL/groups/$CI_PROJECT_NAMESPACE_ID/projects?search=${DEP_NAME}" | jq -r '.[0].id') + echo "curling from \"$API_URL/groups/$CI_PROJECT_NAMESPACE_ID/projects?search=${DEP_NAME}\"" + DEP_ID=$(curl -s "$API_URL/groups/$CI_PROJECT_NAMESPACE_ID/projects?search=${DEP_NAME}" | jq -r '.[0].id') else # case for fork - echo "Retrieving build_artifacts from user forked project." - echo "curling from \"$API_URL/users?username=$CI_PROJECT_ROOT_NAMESPACE\"" - USER=$(curl -s "$API_URL/users?username=$CI_PROJECT_ROOT_NAMESPACE") - USER_ID=$(jq -r '.[0].id' <<<$USER) - echo "curling from \"$API_URL/users/$USER_ID/projects?search=${DEP_NAME}\"" - PROJ=$(curl -s "$API_URL/users/$USER_ID/projects?search=${DEP_NAME}") - DEP_ID=$(jq -r '.[0].id' <<<$PROJ) + echo "Retrieving build_artifacts from user forked project." + echo "curling from \"$API_URL/users?username=$CI_PROJECT_ROOT_NAMESPACE\"" + USER=$(curl -s "$API_URL/users?username=$CI_PROJECT_ROOT_NAMESPACE") + USER_ID=$(jq -r '.[0].id' <<<$USER) + echo "curling from \"$API_URL/users/$USER_ID/projects?search=${DEP_NAME}\"" + PROJ=$(curl -s "$API_URL/users/$USER_ID/projects?search=${DEP_NAME}") + DEP_ID=$(jq -r '.[0].id' <<<$PROJ) fi if [[ "$DEP_ID" == "null" ]]; then - echo "$DEP_NAME not found within $API_URL/users/$USER_ID/projects." - echo "Please make sure there is no typo in the writing and, if you are using a fork, make sure you have forked $DEP_NAME." - exit 1 + echo "$DEP_NAME not found within $API_URL/users/$USER_ID/projects." + echo "Please make sure there is no typo in the writing and, if you are using a fork, make sure you have forked $DEP_NAME." + exit 1 fi echo "Retrieved DEP_ID=$DEP_ID" DEP_API_URL="$API_URL/projects/$DEP_ID" @@ -71,19 +71,19 @@ # CASE NON DRAFT MERGE REQUEST if [[ ! -z "$CI_MERGE_REQUEST_ID" && "$CI_MERGE_REQUEST_TITLE" != *'Draft'* ]]; then # case we are in a merge request echo "Retrieving branches of $DEP_NAME \"$DEP_API_URL/repository/branches?per_page=100\"" - DEP_BRANCHES=$(curl -s "$DEP_API_URL/repository/branches?per_page=100") - # If the MR is not in draft: pulling from target branch (if exists otherwise dev) - echo "NON DRAFT MERGE REQUEST Pipeline detected : Pulling from CI_MERGE_REQUEST_TARGET_BRANCH_NAME \"$CI_MERGE_REQUEST_TARGET_BRANCH_NAME\"." - echo "Checking if CI_MERGE_REQUEST_TARGET_BRANCH_NAME \"$CI_MERGE_REQUEST_TARGET_BRANCH_NAME\" branch exists, otherwise pull from default." - FILTERED_BRANCHES=$(jq --arg branch "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" 'map(select( .name == $branch ))' <<<"$DEP_BRANCHES") - if [[ $(jq length <<<$FILTERED_BRANCHES) -gt 0 ]]; then - echo "Found MR target branch in \"$DEP_NAME\"." - echo "Pulling from merge request target branch \"$CI_MERGE_REQUEST_TARGET_BRANCH_NAME\"" - BRANCH_TO_PULL=$CI_MERGE_REQUEST_TARGET_BRANCH_NAME - else - echo "Branch \"$CI_MERGE_REQUEST_TARGET_BRANCH_NAME\" not found on repo \"$DEP_NAME\"." - echo "Pulling from default branch : \"$BRANCH_TO_PULL\"" - fi + DEP_BRANCHES=$(curl -s "$DEP_API_URL/repository/branches?per_page=100") + # If the MR is not in draft: pulling from target branch (if exists otherwise dev) + echo "NON DRAFT MERGE REQUEST Pipeline detected : Pulling from CI_MERGE_REQUEST_TARGET_BRANCH_NAME \"$CI_MERGE_REQUEST_TARGET_BRANCH_NAME\"." + echo "Checking if CI_MERGE_REQUEST_TARGET_BRANCH_NAME \"$CI_MERGE_REQUEST_TARGET_BRANCH_NAME\" branch exists, otherwise pull from default." + FILTERED_BRANCHES=$(jq --arg branch "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" 'map(select( .name == $branch ))' <<<"$DEP_BRANCHES") + if [[ $(jq length <<<$FILTERED_BRANCHES) -gt 0 ]]; then + echo "Found MR target branch in \"$DEP_NAME\"." + echo "Pulling from merge request target branch \"$CI_MERGE_REQUEST_TARGET_BRANCH_NAME\"" + BRANCH_TO_PULL=$CI_MERGE_REQUEST_TARGET_BRANCH_NAME + else + echo "Branch \"$CI_MERGE_REQUEST_TARGET_BRANCH_NAME\" not found on repo \"$DEP_NAME\"." + echo "Pulling from default branch : \"$BRANCH_TO_PULL\"" + fi ###################################### # CASE WHERE A COMMIT TAG IS CREATED @@ -92,7 +92,7 @@ TAG_MAJOR=$(echo "$CI_COMMIT_TAG" | awk -F'[v.]' '{print $2}') TAG_MINOR=$(echo "$CI_COMMIT_TAG" | awk -F'[v.]' '{print $3}') TAG_FIX=$(echo "$CI_COMMIT_TAG" | awk -F'[v.]' '{print $4}') - echo "Retrieving tags of $DEP_NAME via url $DEP_API_URL/repository/tags?per_page=100." + echo "Retrieving tags of $DEP_NAME via url $DEP_API_URL/repository/tags?per_page=100." DEP_TAGS=$(jq --slurp ' .[] | sort_by(.commit.created_at) @@ -115,30 +115,30 @@ 'map(select(.name | startswith("v\($major).")))' <<< "$DEP_TAGS") fi if [[ $(jq length <<< $SELECTED_DEPS_TAGS) > 0 ]]; then - BRANCH_TO_PULL=$(jq -r '.[0].name' <<<"$SELECTED_DEPS_TAGS") + BRANCH_TO_PULL=$(jq -r '.[0].name' <<<"$SELECTED_DEPS_TAGS") echo "Found a release with corresponding $BRANCH_TO_PULL" else echo "No release v$TAG_MAJOR.*.* found. Simply retrieving latest release v*.*.* in $DEP_NAME." - BRANCH_TO_PULL=$(jq -r '.[0].name' <<<"$DEP_TAGS") + BRANCH_TO_PULL=$(jq -r '.[0].name' <<<"$DEP_TAGS") fi echo "Found release : $BRANCH_TO_PULL. Using it as \$BRANCH_TO_PULL." ########################################### # CASE CASUAL COMMIT or Draft MR PIPELINE else - # checking if a branch with same name exists in $DEP_NAME, otherwise, pulling from DEV + # checking if a branch with same name exists in $DEP_NAME, otherwise, pulling from DEV echo "Retrieving branches of $DEP_NAME \"$DEP_API_URL/repository/branches?per_page=100\"" - DEP_BRANCHES=$(curl -s "$DEP_API_URL/repository/branches?per_page=100") - echo "Looking for branch $CI_COMMIT_BRANCH in repo $DEP_NAME" - FILTERED_BRANCHES=$(jq --arg branch "$CI_COMMIT_REF_NAME" 'map(select( .name == $branch ))' <<<"$DEP_BRANCHES") - if [[ $(jq length <<<$FILTERED_BRANCHES) -gt 0 ]]; then - echo "Found branch \"$CI_COMMIT_REF_NAME\" for repo \"$DEP_NAME\"." - echo "Pulling from branch \"$CI_COMMIT_REF_NAME\"" - BRANCH_TO_PULL=$CI_COMMIT_REF_NAME - else - echo "No branch \"$CI_COMMIT_REF_NAME\" found for repo \"$DEP_NAME\"." - echo "Pulling from branch \"$BRANCH_TO_PULL\"" - fi + DEP_BRANCHES=$(curl -s "$DEP_API_URL/repository/branches?per_page=100") + echo "Looking for branch $CI_COMMIT_BRANCH in repo $DEP_NAME" + FILTERED_BRANCHES=$(jq --arg branch "$CI_COMMIT_REF_NAME" 'map(select( .name == $branch ))' <<<"$DEP_BRANCHES") + if [[ $(jq length <<<$FILTERED_BRANCHES) -gt 0 ]]; then + echo "Found branch \"$CI_COMMIT_REF_NAME\" for repo \"$DEP_NAME\"." + echo "Pulling from branch \"$CI_COMMIT_REF_NAME\"" + BRANCH_TO_PULL=$CI_COMMIT_REF_NAME + else + echo "No branch \"$CI_COMMIT_REF_NAME\" found for repo \"$DEP_NAME\"." + echo "Pulling from branch \"$BRANCH_TO_PULL\"" + fi fi echo "*************************************************************************" @@ -156,34 +156,34 @@ echo "Regular pipeline curled from \"$DEP_API_URL/pipelines?ref=${BRANCH_TO_PULL}\"" PIPELINES=$(curl -s "$DEP_API_URL/pipelines?ref=${BRANCH_TO_PULL}") if [[ -z "$PIPELINES" ]]; then - echo "Failed to retrieve any pipeline, ensure there is a pipeline with ref $BRANCH_TO_PULL in $DEP_NAME" - exit 1 + echo "Failed to retrieve any pipeline, ensure there is a pipeline with ref $BRANCH_TO_PULL in $DEP_NAME" + exit 1 fi echo "Found $(jq length <<<$PIPELINES) regular pipelines with $BRANCH_TO_PULL as ref." echo "Retrieve first affiliated MR by curling \"$DEP_API_URL/merge_requests?source_branch=$BRANCH_TO_PULL&state=opened\"" MR_ID=$(curl -s "$DEP_API_URL/merge_requests?source_branch=$BRANCH_TO_PULL&state=opened" | jq '.[0].iid') if [[ $MR_ID != null ]]; then - echo "MR pipeline curled from \"$DEP_API_URL/merge_requests/$MR_ID/pipelines\"" - MR_PIPELINES=$(curl -s "$DEP_API_URL/merge_requests/$MR_ID/pipelines") - echo "Found $(jq length <<<$MR_PIPELINES) Merge Request pipelines with $BRANCH_TO_PULL as source_branch." + echo "MR pipeline curled from \"$DEP_API_URL/merge_requests/$MR_ID/pipelines\"" + MR_PIPELINES=$(curl -s "$DEP_API_URL/merge_requests/$MR_ID/pipelines") + echo "Found $(jq length <<<$MR_PIPELINES) Merge Request pipelines with $BRANCH_TO_PULL as source_branch." else - echo "No merge request with source_branch = \"$BRANCH_TO_PULL\" found." - MR_PIPELINES="[]" + echo "No merge request with source_branch = \"$BRANCH_TO_PULL\" found." + MR_PIPELINES="[]" fi set -x PIPELINES=$( - jq -n \ - --argjson pipelines "$PIPELINES" \ - --argjson mr_pipelines "$MR_PIPELINES" \ - --arg project_namespace "$CI_PROJECT_NAMESPACE" \ - '$pipelines + $mr_pipelines - | map(select(.web_url | test($project_namespace))) - | sort_by(.updated_at) - | reverse' + jq -n \ + --argjson pipelines "$PIPELINES" \ + --argjson mr_pipelines "$MR_PIPELINES" \ + --arg project_namespace "$CI_PROJECT_NAMESPACE" \ + '$pipelines + $mr_pipelines + | map(select(.web_url | test($project_namespace))) + | sort_by(.updated_at) + | reverse' ) if [[ ! -z "$CI_COMMIT_TAG" ]]; then - PIPELINES=$(jq 'map(select(.status == "success"))' <<<"$PIPELINES") + PIPELINES=$(jq 'map(select(.status == "success"))' <<<"$PIPELINES") fi echo "*************************************************************************" @@ -198,29 +198,29 @@ FOUND_JOB=0 # iterate over each pipeline and get the job given job for i in $(seq 0 $(($(jq length <<<$PIPELINES) - 1))); do - PIPELINE_ID=$(jq --argjson idx $i '.[$idx].id' <<<$PIPELINES) - echo "Current pipeline ID : $PIPELINE_ID" - echo "Retrieving jobs of curr pipeline from : \"$DEP_API_URL/pipelines/$PIPELINE_ID/jobs\"" - JOBS=$(curl -s "$DEP_API_URL/pipelines/$PIPELINE_ID/jobs") - if [[ $(jq -r '.message' <<<$JOBS) == "404 Not found" ]]; then - echo "WARNING: NO JOBS FOUND FOR PIPELINE $PIPELINE_ID OF PROJECT $DEP_NAME." - echo "WARNING: This might happen if on $DEP_NAME / $BRANCH_TO_PULL:" - echo "WARNING: - There is an ongoing pipeline" - echo "WARNING: - A branch was jsut merged in just merged to $BRANCH_TO_PULL." - echo "WARNING: IGNORING THIS PIPELINE." - echo "WARNING: To solve this issue either wait for pipeline to finish or re-run a pipeline on given branch." - continue - fi - # Retrieve jobs only if they are successful - # Only retrieve part of the json for readability - echo "Retrieving only $DEPENDENCY_JOB that are successful." - echo "Retrieving only part of the job json for readbility" - set -x - JOBS="$(jq --arg job_name "$DEPENDENCY_JOB" \ - --arg branch_name "$BRANCH_TO_PULL" \ - --arg repo_name "$DEP_NAME" \ - --arg repo_id "$DEP_ID" \ - '[ + PIPELINE_ID=$(jq --argjson idx $i '.[$idx].id' <<<$PIPELINES) + echo "Current pipeline ID : $PIPELINE_ID" + echo "Retrieving jobs of curr pipeline from : \"$DEP_API_URL/pipelines/$PIPELINE_ID/jobs\"" + JOBS=$(curl -s "$DEP_API_URL/pipelines/$PIPELINE_ID/jobs") + if [[ $(jq -r '.message' <<<$JOBS) == "404 Not found" ]]; then + echo "WARNING: NO JOBS FOUND FOR PIPELINE $PIPELINE_ID OF PROJECT $DEP_NAME." + echo "WARNING: This might happen if on $DEP_NAME / $BRANCH_TO_PULL:" + echo "WARNING: - There is an ongoing pipeline" + echo "WARNING: - A branch was jsut merged in just merged to $BRANCH_TO_PULL." + echo "WARNING: IGNORING THIS PIPELINE." + echo "WARNING: To solve this issue either wait for pipeline to finish or re-run a pipeline on given branch." + continue + fi + # Retrieve jobs only if they are successful + # Only retrieve part of the json for readability + echo "Retrieving only $DEPENDENCY_JOB that are successful." + echo "Retrieving only part of the job json for readbility" + set -x + JOBS="$(jq --arg job_name "$DEPENDENCY_JOB" \ + --arg branch_name "$BRANCH_TO_PULL" \ + --arg repo_name "$DEP_NAME" \ + --arg repo_id "$DEP_ID" \ + '[ map(select((.name == $job_name) and (.status == "success"))) | .[] | { @@ -233,36 +233,36 @@ pipeline : { id : .pipeline.id, iid : .pipeline.iid, project_id : .pipeline.project_id, ref : .pipeline.ref}, } ]' <<<"$JOBS")" - if [[ $(jq length <<<$JOBS) > 0 ]]; then - FOUND_JOB=1 - echo "Successful job \"$DEPENDENCY_JOB\" found in branch \"$BRANCH_TO_PULL\" for project \"$DEP_NAME\"" - echo "Appending this jobs to the list of selected jobs that will serve as reference to retrieve artifact or specific commit." - # concatenate json - DEPS_SELECTED_JOBS=$(jq --argjson result_json "$(jq '.[0]' <<<$JOBS)" '. + [$result_json]' <<<"$DEPS_SELECTED_JOBS") - break - fi + if [[ $(jq length <<<$JOBS) > 0 ]]; then + FOUND_JOB=1 + echo "Successful job \"$DEPENDENCY_JOB\" found in branch \"$BRANCH_TO_PULL\" for project \"$DEP_NAME\"" + echo "Appending this jobs to the list of selected jobs that will serve as reference to retrieve artifact or specific commit." + # concatenate json + DEPS_SELECTED_JOBS=$(jq --argjson result_json "$(jq '.[0]' <<<$JOBS)" '. + [$result_json]' <<<"$DEPS_SELECTED_JOBS") + break + fi done if [[ $FOUND_JOB == 0 ]]; then - echo "ERROR : no successful job \"$DEPENDENCY_JOB\" found in branch \"$BRANCH_TO_PULL\" for project \"$CI_PROJECT_NAMESPACE/$DEP_NAME\"" - if [[ ! -z "$CI_MERGE_REQUEST_ID" && $CI_MERGE_REQUEST_TITLE != *'Draft'* ]]; then - echo "*************************************************************************" - echo "*************************************************************************" - echo "**" - echo "** WARNING: This is a NON DRAFT Merge request Pipeline:" - echo "** WARNING: Pulled branch:" - echo "** WARNING: CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_MERGE_REQUEST_TARGET_BRANCH_NAME" - echo "** WARNING: If the Merge Request were in draft, pulled branch would have been:" - echo "** WARNING: CI_MERGE_REQUEST_SOURCE_BRANCH_NAME $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME." - echo "** WARNING:" - echo "** WARNING: If you encounter issues running this pipeline:" - echo "** WARNING: 1. Make sure that dependencies' merge requests are merged (if any)" - echo "** WARNING: 2. Pass your MR in draft if they are not merged." - echo "**" - echo "*************************************************************************" - echo "*************************************************************************" - fi - exit -1 + echo "ERROR : no successful job \"$DEPENDENCY_JOB\" found in branch \"$BRANCH_TO_PULL\" for project \"$CI_PROJECT_NAMESPACE/$DEP_NAME\"" + if [[ ! -z "$CI_MERGE_REQUEST_ID" && $CI_MERGE_REQUEST_TITLE != *'Draft'* ]]; then + echo "*************************************************************************" + echo "*************************************************************************" + echo "**" + echo "** WARNING: This is a NON DRAFT Merge request Pipeline:" + echo "** WARNING: Pulled branch:" + echo "** WARNING: CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_MERGE_REQUEST_TARGET_BRANCH_NAME" + echo "** WARNING: If the Merge Request were in draft, pulled branch would have been:" + echo "** WARNING: CI_MERGE_REQUEST_SOURCE_BRANCH_NAME $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME." + echo "** WARNING:" + echo "** WARNING: If you encounter issues running this pipeline:" + echo "** WARNING: 1. Make sure that dependencies' merge requests are merged (if any)" + echo "** WARNING: 2. Pass your MR in draft if they are not merged." + echo "**" + echo "*************************************************************************" + echo "*************************************************************************" + fi + exit -1 fi done - echo "*************************************************************************" @@ -469,21 +469,21 @@ if ( $FOUND_JOB -eq 0 ) { Write-Host "ERROR : no successful job $DEPENDENCY_JOB found in branch $BRANCH_TO_PULL for project $CI_PROJECT_NAMESPACE/$DEP_NAME" if ( ! "$CI_MERGE_REQUEST_ID" -and ! "$CI_MERGE_REQUEST_TITLE" -match '.+Draft.+' ){ - Write-Host "*************************************************************************" - Write-Host "*************************************************************************" - Write-Host "**" - Write-Host "** WARNING: This is a NON DRAFT Merge request Pipeline:" - Write-Host "** WARNING: Pulled branch :" - Write-Host "** WARNING: CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_MERGE_REQUEST_TARGET_BRANCH_NAME" - Write-Host "** WARNING: If the merge request were in draft, pulled branch would have been:" - Write-Host "** WARNING: CI_MERGE_REQUEST_SOURCE_BRANCH_NAME $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME." - Write-Host "** WARNING:" - Write-Host "** WARNING: If you encounter issues running this pipeline:" + Write-Host "*************************************************************************" + Write-Host "*************************************************************************" + Write-Host "**" + Write-Host "** WARNING: This is a NON DRAFT Merge request Pipeline:" + Write-Host "** WARNING: Pulled branch :" + Write-Host "** WARNING: CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_MERGE_REQUEST_TARGET_BRANCH_NAME" + Write-Host "** WARNING: If the merge request were in draft, pulled branch would have been:" + Write-Host "** WARNING: CI_MERGE_REQUEST_SOURCE_BRANCH_NAME $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME." + Write-Host "** WARNING:" + Write-Host "** WARNING: If you encounter issues running this pipeline:" Write-Host "** WARNING: 1. Make sure that dependencies' merge requests are merged (if any)" - Write-Host "** WARNING: 2. Pass your MR in draft if they are not merged." - Write-Host "**" - Write-Host "*************************************************************************" - Write-Host "*************************************************************************" + Write-Host "** WARNING: 2. Pass your MR in draft if they are not merged." + Write-Host "**" + Write-Host "*************************************************************************" + Write-Host "*************************************************************************" } exit -1 } -- GitLab