diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 86fa5d96e91c0c776bf19a7e8c20ee5bec43635f..c024ba24c56feac777791a1c9ba853a055351e94 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -32,45 +32,73 @@ reuse: # Oniro specific variables used during CI/CD process. Variables in this group # should use meaningful names and avoid abbreviations, if possible. # -# CI_ONIRO_BB_LOCAL_CONF_*: -# Mechanism for injecting variables into Bitbake configuration files. -# Variables in this group represent existing Bitbake/Yocto variables and -# retain their original name, apart from the prefix. +# CI_ONIRO_MANIFEST_REPO_{URL,REV}: +# URL and revision of the manifest repository. +# +# CI_ONIRO_MANIFEST_MIRROR_REPO_{URL,REV,DIR}: +# URL, revision and directory path of the mirror of manifest repository. +# This repository is used to speed up construction of repo workspace. +# +# CI_ONIRO_REPO_WORKSPACE_DIR: +# Directory path of repo workspace. # # CI_*: # Third party variables used during CI/CD process, defined by GitLab. # Variables in this group are defined by GitLab and retain their original # name. # -# GIT_STRATEGY, CACHE_COMPRESSOIN_LEVEL: +# GIT_STRATEGY, CACHE_COMPRESSION_LEVEL: # Part of GitLab interface. .oniro-repo-workspace: interruptible: true image: name: registry.ostc-eu.org/ostc/oniro/bitbake-builder:latest variables: - GIT_STRATEGY: none + # URL and branch or revision of the oniro.git repository which contains a + # repo manifest file. + # + # The revision should be bumped during the major release of Oniro but both + # variables can be changed to CI_PROJECT_URL/CI_COMMIT_SHA when testing + # changes landing to oniro.git. + CI_ONIRO_MANIFEST_REPO_URL: https://gitlab.eclipse.org/eclipse/oniro-core/oniro.git + CI_ONIRO_MANIFEST_REPO_REV: kirkstone + # URL and branch used with repo "repo --mirror" to speed up workspace + # construction. + # + # Those are distinct from CI_ONIRO_MANIFEST_REPO_{URL,REV} because the + # former variables can be redirected to CI_PROJECT_URL and CI_COMMIT_SHA, + # while those two stay fixed. + # + # The revision should _only_ be bumped during the major release of Oniro. + CI_ONIRO_MANIFEST_MIRROR_REPO_URL: https://gitlab.eclipse.org/eclipse/oniro-core/oniro.git + CI_ONIRO_MANIFEST_MIRROR_REPO_REV: kirkstone + # Directory where repo mirror is constructed. This location is covered by + # GitLab cache system, and will be reused between pipelines of the same + # project. Note that the .cache directory name is special. + CI_ONIRO_MANIFEST_MIRROR_REPO_DIR: $CI_PROJECT_DIR/.cache/repo-mirror + # Directory where repo workspace is constructed. + CI_ONIRO_REPO_WORKSPACE_DIR: $CI_PROJECT_DIR/.tmp/repo-workspace + # Use fastest cache compression algorithm, as bulk of the cache is + # already-compressed git history. CACHE_COMPRESSION_LEVEL: fastest - CI_ONIRO_REPO_WORKSPACE_URL: https://gitlab.eclipse.org/eclipse/oniro-core/oniro.git - CI_ONIRO_REPO_WORKSPACE_REV: kirkstone - CI_ONIRO_REPO_WORKSPACE_DIR: $CI_PROJECT_DIR/.cache/repo-workspace - CI_ONIRO_REPO_WORKSPACE_CACHE_PREFIX: workspace - CI_ONIRO_REPO_WORKSPACE_LOCAL_MANIFEST: > - <?xml version="1.0" encoding="UTF-8"?> - <manifest> - <!-- remove original docs project entry --> - <remove-project name="oniro-core/docs.git" /> - <!-- add remote representing the project --> - <remote name="oniro-override" fetch="${CI_PROJECT_URL}/../" /> - <!-- add docs at the exact version are testing --> - <project name="${CI_PROJECT_NAME}" path="docs" remote="oniro-override" revision="${CI_COMMIT_SHA}" /> - </manifest> + # Ask GitLab _not_ to check out the git repository associated with the + # project. This is, in a way, pointless, since we use repo, not pure git, + # to construct the workspace. Due to the fact that oniro is + # self-referential (the manifest refers to the repository that contains the + # manifest). This requires custom logic to behave correctly in scenarios + # that modify oniro.git in any way (e.g. a branch, a pull request or merge + # train). + GIT_STRATEGY: none cache: - - key: $CI_ONIRO_REPO_WORKSPACE_CACHE_PREFIX-$CI_ONIRO_REPO_WORKSPACE_REV - paths: [$CI_ONIRO_REPO_WORKSPACE_DIR] - when: always + - key: + prefix: repo-mirror-$CI_ONIRO_MANIFEST_MIRROR_REPO_REV + files: + - default.xml + paths: + - $CI_ONIRO_MANIFEST_MIRROR_REPO_DIR before_script: - | + # Define helper functions to generate GitLab fold markers. function gl_section_open() { printf '\e[0K''section_start'':%s:%s\r\e[0K%s\n' "$(date +%s)" "$1" "$2" @@ -84,6 +112,9 @@ reuse: printf '\e[0K''section_end'':%s:%s\r\e[0K\n' "$(date +%s)" "$1" } + # Query system information. This is mostly useful for forensics, when + # something goes wrong and access to basic information of this type can + # help to uncover the problem. - gl_section_open_collapsed system_info "Querying system information" - id - uname -a @@ -93,23 +124,55 @@ reuse: - env | grep -E '^CI_ONIRO' | sort - gl_section_close system_info + # Set up Git with bot identity. Eclipse ECA check allows this user to + # create and send commits. - gl_section_open_collapsed setup_git "Setting up git" - git config --global --add safe.directory "$CI_PROJECT_DIR" - git config --global user.name "Oniro Core Project Bot" - git config --global user.email "oniro-core-bot@eclipse.org" - gl_section_close setup_git - - gl_section_open_collapsed workspace_setup "Setting up repo workspace" - - mkdir -p "$CI_ONIRO_REPO_WORKSPACE_DIR" + # Since CI_PROJECT_DIR is set to 'none', GitLab runner does not perform any + # cleanup operations on CI_PROJECT_DIR. In consequence, repo can observe + # junk brought in by previous executions on the same runner, and get + # confused. Perform manual cleanup by removing all top-level items, other + # than .cache, where the cache items are strategically located, before + # proceeding. + - gl_section_open_collapsed cleanup_project_dir "Clean-up project directory" + - find "$CI_PROJECT_DIR" -mindepth 1 -maxdepth 1 ! -name .cache -exec rm -rf {} \; + - ls -l "$CI_PROJECT_DIR" + - gl_section_close cleanup_project_dir + + # Create and update a mirror for repo, using the semi-fixed manifest mirror + # repo URL and revision. Since this is cached, the "repo init" part is + # rarely executed (see the test command below), and only the forced + # synchronization is executed. + # + # Note that the location of the mirror is stored in GitLab cache using the + # repo revision as cache key, allowing multiple releases to co-exist + # efficiently. + - gl_section_open_collapsed repo_mirror_setup "Setting up repo mirror" + - mkdir -p "$CI_ONIRO_MANIFEST_MIRROR_REPO_DIR" + - pushd "$CI_ONIRO_MANIFEST_MIRROR_REPO_DIR" + - echo "Initializing repository mirror from $CI_ONIRO_MANIFEST_MIRROR_REPO_URL and $CI_ONIRO_MANIFEST_MIRROR_REPO_REV" + - test ! -e .repo && repo init --mirror --manifest-url "$CI_ONIRO_MANIFEST_MIRROR_REPO_URL" --manifest-branch "$CI_ONIRO_MANIFEST_MIRROR_REPO_REV" --no-clone-bundle + - echo "Synchronizing repository mirror" + - repo sync --force-sync || ( rm -rf .repo && repo init --mirror --manifest-url "$CI_ONIRO_MANIFEST_MIRROR_REPO_URL" --manifest-branch "$CI_ONIRO_MANIFEST_MIRROR_REPO_REV" --no-clone-bundle && repo sync) + - gl_section_close repo_mirror_setup + + # Create a repo workspace using the mirror as reference. This is fairly + # efficient, as repo will hardlink files (assuming they live on the same + # filesystem) and avoid bulk of the network traffic. + - gl_section_open_collapsed repo_workspace_setup "Setting up repo workspace" + - rm -rf "$CI_ONIRO_REPO_WORKSPACE_DIR" && mkdir -p "$CI_ONIRO_REPO_WORKSPACE_DIR" - pushd "$CI_ONIRO_REPO_WORKSPACE_DIR" - - echo "Initializing repository workspace from $CI_ONIRO_REPO_WORKSPACE_URL and $CI_ONIRO_REPO_WORKSPACE_REV" - - repo init --manifest-url "$CI_ONIRO_REPO_WORKSPACE_URL" --manifest-branch "$CI_ONIRO_REPO_WORKSPACE_REV" --no-clone-bundle + - echo "Initializing repository workspace from $CI_ONIRO_MANIFEST_REPO_URL and $CI_ONIRO_MANIFEST_REPO_REV" + - repo init --reference "$CI_ONIRO_MANIFEST_MIRROR_REPO_DIR" --manifest-url "$CI_ONIRO_MANIFEST_REPO_URL" --manifest-branch "$CI_ONIRO_MANIFEST_REPO_REV" --no-clone-bundle - mkdir -p "${CI_ONIRO_REPO_WORKSPACE_DIR}/.repo/local_manifests" - - test -n "${CI_ONIRO_REPO_WORKSPACE_LOCAL_MANIFEST-}" && echo "$CI_ONIRO_REPO_WORKSPACE_LOCAL_MANIFEST" | tee "${CI_ONIRO_REPO_WORKSPACE_DIR}/.repo/local_manifests/local.xml" + - test -n "${CI_ONIRO_REPO_WORKSPACE_LOCAL_MANIFEST:-}" && echo "$CI_ONIRO_REPO_WORKSPACE_LOCAL_MANIFEST" | tee "${CI_ONIRO_REPO_WORKSPACE_DIR}/.repo/local_manifests/local.xml" - echo "Synchronizing repository workspace" - repo sync --force-sync - - gl_section_close workspace_setup - + - gl_section_close repo_workspace_setup build-docs: extends: [.oniro-repo-workspace] @@ -122,8 +185,26 @@ build-docs: artifacts: paths: - build + variables: + # When the workspace is created, substitute the "docs" repository that + # described by the manifest with the project being tested. This works for + # forks and branches but not for merge requests. For that look at the build + # rule below. + CI_ONIRO_REPO_WORKSPACE_LOCAL_MANIFEST: > + <?xml version="1.0" encoding="UTF-8"?> + <manifest> + <!-- remove original docs project entry --> + <remove-project name="oniro-core/docs.git" /> + <!-- add remote representing the project --> + <remote name="oniro-override" fetch="${CI_PROJECT_URL}/../" /> + <!-- add docs at the exact version are testing --> + <project name="${CI_PROJECT_NAME}" path="docs" remote="oniro-override" revision="${CI_COMMIT_SHA}" /> + </manifest> rules: - # Build the docs when a merge request is created. + # During the merge request, substitute the "docs" repository that is + # described by the manifest with the project that is the source of the + # merge request. This does not test the merged result but is the next best + # thing we can do right now. - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' variables: CI_ONIRO_REPO_WORKSPACE_LOCAL_MANIFEST: >