From b43dedcff6fc614048b2f89fe9799ea38b4e1c18 Mon Sep 17 00:00:00 2001
From: Zygmunt Krynicki <zygmunt.krynicki@huawei.com>
Date: Thu, 15 Sep 2022 13:17:59 +0000
Subject: [PATCH] ci: prototype new CI infrastructure
The documentation repository relies on a series of symbolic links that
connect it to the oniro.git repository, when used inside a checked-out
repo workspace.
Ever since we switched to the symlink-based override that supports
branch selection for merge requests and merge trains, the docs
repository stopped to build correctly.
This is caused by now-broken symbolic links that once traversed, no
longer point to the files in oniro.git, due to the way the redirection
code works.
An entirely new pipeline was created to avoid this problem, and to unify
other repo-hacking systems that we introduced over time. A prototype of
this new pipeline is now available here, in the docs repository.
The main difference is that local manifests are now used to edit the
repo composition before "repo sync" is used. This allows us to achieve
the same effect we had before, but using first-party systems. The
secondary difference is that the docs pipeline is now separated from
changes (including breaking changes) in the oniro.git pipeline.
Signed-off-by: Zygmunt Krynicki <zygmunt.krynicki@huawei.com>
---
.gitlab-ci.yml | 127 ++++++++++++++++++++++++++++++++++++++-----------
1 file changed, 100 insertions(+), 27 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 47363af..82f7b28 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -13,7 +13,6 @@ include:
file:
- .oniro-ci/dco.yaml
- .oniro-ci/reuse.yaml
- - .oniro-ci/build-generic.yaml
dco:
extends: .dco
@@ -26,49 +25,123 @@ reuse:
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
-# Customize the .workspace job to set the path of the git repository to deviate
-# from what the git-repo manifest prepares. This effectively allows testing
-# incoming changes that match the repository holding this CI pipeline.
-.workspace:
+
+# Naming scheme for variables is defined as follows:
+#
+# CI_ONIRO_*:
+# Oniro specific variables used during CI/CD process. Variables in this group
+# should use meaningful names and avoid abbreviations, if possible.
+#
+# CI_ONIRO_BB_LOCAL_CONF_*:
+# Mechanism for injecting variables into Bitbake configuration files.
+# Variables in this group represent existing Bitbake/Yocto variables and
+# retain their original name, apart from the prefix.
+#
+# CI_*:
+# Third party variables used during CI/CD process, defined by GitLab.
+# Variables in this group are defined by GitLab and retain their original
+# name.
+#
+# GIT_STRATEGY, CACHE_COMPRESSOIN_LEVEL:
+# Part of GitLab interface.
+.oniro-repo-workspace:
+ interruptible: true
+ image:
+ name: registry.ostc-eu.org/ostc/oniro/bitbake-builder:latest
variables:
- CI_ONIRO_GIT_REPO_PATH: docs
+ GIT_STRATEGY: none
+ CACHE_COMPRESSION_LEVEL: fastest
+ CI_ONIRO_REPO_WORKSPACE_URL: https://gitlab.eclipse.org/eclipse/oniro-core/oniro.git
+ CI_ONIRO_REPO_WORKSPACE_REV: kirkstone
+ CI_ONIRO_REPO_WORKSPACE_DIR: $CI_PROJECT_DIR/.cache/repo-workspace
+ CI_ONIRO_REPO_WORKSPACE_CACHE_PREFIX: workspace
+ CI_ONIRO_REPO_WORKSPACE_LOCAL_MANIFEST: >
+ <?xml version="1.0" encoding="UTF-8"?>
+ <manifest>
+ <!-- remove original docs project entry -->
+ <remove-project name="oniro-core/docs.git" />
+ <!-- add remote representing the project -->
+ <remote name="oniro-override" fetch="${CI_PROJECT_URL}/../" />
+ <!-- add docs at the exact version are testing -->
+ <project name="${CI_PROJECT_NAME}" path="docs" remote="oniro-override" revision="${CI_COMMIT_SHA}" />
+ </manifest>
+ cache:
+ - key: $CI_ONIRO_REPO_WORKSPACE_CACHE_PREFIX-$CI_ONIRO_REPO_WORKSPACE_REV
+ paths: [$CI_ONIRO_REPO_WORKSPACE_DIR]
+ when: always
+ before_script:
+ - |
+
+ function gl_section_open() {
+ printf '\e[0K''section_start'':%s:%s\r\e[0K%s\n' "$(date +%s)" "$1" "$2"
+ }
+
+ function gl_section_open_collapsed() {
+ printf '\e[0K''section_start'':%s:%s[collapsed=true]\r\e[0K%s\n' "$(date +%s)" "$1" "$2"
+ }
+
+ function gl_section_close() {
+ printf '\e[0K''section_end'':%s:%s\r\e[0K\n' "$(date +%s)" "$1"
+ }
+
+ - gl_section_open_collapsed system_info "Querying system information"
+ - uname -a
+ - cat /etc/os-release
+ - free -m
+ - lscpu
+ - env | grep -E '^CI_ONIRO' | sort
+ - gl_section_close system_info
+
+ - gl_section_open_collapsed setup_git "Setting up git"
+ - git config --global --add safe.directory "$CI_PROJECT_DIR"
+ - git config --global user.name "CI/CD Automation"
+ - git config --global user.email "ci-cd@example.org"
+ - gl_section_close setup_git
+
+ - gl_section_open_collapsed workspace_setup "Setting up repo workspace"
+ - mkdir -p "$CI_ONIRO_REPO_WORKSPACE_DIR"
+ - pushd "$CI_ONIRO_REPO_WORKSPACE_DIR"
+ - echo "Initializing repository workspace from $CI_ONIRO_REPO_WORKSPACE_URL and $CI_ONIRO_REPO_WORKSPACE_REV"
+ - repo init --manifest-url "$CI_ONIRO_REPO_WORKSPACE_URL" --manifest-branch "$CI_ONIRO_REPO_WORKSPACE_REV" --no-clone-bundle
+ - mkdir -p "${CI_ONIRO_REPO_WORKSPACE_DIR}/.repo/local_manifests"
+ - test -n "${CI_ONIRO_REPO_WORKSPACE_LOCAL_MANIFEST-}" && echo "$CI_ONIRO_REPO_WORKSPACE_LOCAL_MANIFEST" | tee "${CI_ONIRO_REPO_WORKSPACE_DIR}/.repo/local_manifests/local.xml"
+ - echo "Synchronizing repository workspace"
+ - repo sync --force-sync
+ - gl_section_close workspace_setup
-# Disable all the bitbake jobs, since we are not building any code here.
-.bitbake-workspace:
- rules:
- - when: never
-# Define a build-docs job that extends both the .workspace, for the general
-# workspace setup, and .build-docs, for the documentation build logic. The
-# script first assembles the workspace and then proceeds to build the
-# documentation.
-#
-# The job extends more than one parent, with the order being relevant for,
-# among others, the "rules" section.
build-docs:
- extends: [.workspace, .build-docs]
- variables:
- CI_ONIRO_INSTANCE_SIZE: s3.large.2
+ extends: [.oniro-repo-workspace]
+ interruptible: true
+ image:
+ name: registry.ostc-eu.org/ostc/oniro/docs-builder:latest
script:
- - !reference [.workspace, script]
- - !reference [.build-docs, script]
- # Artifacts are relative to CI_PROJECT_DIR so we need to provide the build
- # docs there.
- - mv "$SCRATCH_DIR"/docs/build/ "$CI_PROJECT_DIR" || true
+ - make -C docs
+ - mv docs/build "$CI_PROJECT_DIR"
artifacts:
paths:
- build
rules:
# Build the docs when a merge request is created.
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+ variables:
+ CI_ONIRO_REPO_WORKSPACE_LOCAL_MANIFEST: >
+ <?xml version="1.0" encoding="UTF-8"?>
+ <manifest>
+ <!-- remove original docs project entry -->
+ <remove-project name="oniro-core/docs.git" />
+ <!-- add remote representing the project -->
+ <remote name="oniro-override" fetch="${CI_MERGE_REQUEST_SOURCE_PROJECT_URL}/../" />
+ <!-- add docs at the exact version are testing -->
+ <project name="${CI_PROJECT_NAME}" path="docs" remote="oniro-override" revision="${CI_COMMIT_SHA}" />
+ </manifest>
# Or when things land.
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
deploy:
- extends: .workspace
+ extends: .oniro-repo-workspace
stage: deploy
script:
- - !reference [.workspace, script]
# We are in the root of the git-repo workspace.
- git clone https://user:$CI_ONIRO_AGGREGATED_DOCS_TOKEN@gitlab.eclipse.org/eclipse/oniro-core/oniro-readthedocs-aggregated.git aggregated
- find aggregated -maxdepth 1 -not -path aggregated/.git -not -path aggregated -exec rm -rvf {} \;
--
GitLab