Improved CI at the level of meta-ohos repository

- Use re-usable .workspace job from pipelines repository
- Less repetition with more targets supported
- Higher uniformity with the pipeline at the manifest repository.

Closes: https://git.ostc-eu.org/OSTC/OHOS/meta-ohos/-/issues/22


Signed-off-by: Zygmunt Krynicki <zygmunt.krynicki@huawei.com>

.ostc-ci/gitlab-ci.yml

@@ -14,235 +14,168 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Abuse stages as a visual construct to make the pipeline more comprehensive.
 stages:
- - compliance
- - test
- - Linux
- - Zephyr
- - FreeRTOS
+  - compliance
+  - build
+  - test
 
 include:
  - project: 'OSTC/infrastructure/pipelines'
-   file: 'reuse.yaml'
+   file:
+    - 'dco.yaml'
+    - 'reuse.yaml'
+    - 'workspace.yaml'
+
+dco:
+  extends: .dco
 
 reuse:
   extends: .reuse
   # FIXME: https://git.ostc-eu.org/OSTC/OHOS/meta-ohos/-/issues/19
   allow_failure: true
 
-# This is a pipeline job that is used via the "extends" mechanism below.
-# For reference see https://docs.gitlab.com/ee/ci/yaml/README.html
-.workspace:
-  # This pipeline relies on a container with additional pre-installed software:
-  # - git and git-repo program to process the manifest
-  # - all of the dependencies of bitbake (basic toolchain, python, many tools)
-  # Precise, machine readable description of this container can be found
-  # in https://git.ostc-eu.org/OSTC/containers/-/blob/master/ostc-builder/Dockerfile
-  image:
-    name: registry.ostc-eu.org/ostc/containers/ostc-builder:latest
-  # The pipeline relies on being scheduled to a GitLab worker with the
-  # following properties:
-  # - sufficient amount of disk space (~ 100GB will do).
-  # - non-ephemeral disk mounted at /var/shared with even more space (~500GB)
-  #   that is shared between runs of this pipeline. This is where the bitbake
-  #   download directory and sstate-cache are configured below.
-  # - additional CPU cores as the build process is very long.
-  tags: [large-disk]
-  # Conservative timeout in case the build machine is busy and the cache is cold.
-  timeout: 3 hours
-  variables:
-    # The location of the manifest repository.
-    OSTC_MANIFEST_URL: https://git.ostc-eu.org/OSTC/manifest
-    # The develop manifest follows corresponding HEAD branches of all the
-    # repositories managed in OSTC, making it more practical for
-    # component-level CI.
-    OSTC_MANIFEST: develop.xml
-    # This variable needs to be defined by the job.
-    OHOS_BUILD_FLAVOUR: ""
-  before_script: &workspace-before
-    # Bitbake requires a non-root user to operate.
-    # The container should have a non-root user by default.
-    - test "$(id -u)" -ne 0 || ( echo "precondition failed - this job cannot run as root" && exit 1 )
-
-    # Check if the job is configured properly.
-    - test -n "$OHOS_BUILD_FLAVOUR" || ( echo "precondition failed - set OHOS_BUILD_FLAVOUR to \"flavour\" of the build to use (e.g. linux)" && exit 1 )
-
-    # Bitbake is configured to use /var/shared/bitbake directory
-    # for both the download directory and the sstate-cache.
-    - test -w /var/shared/bitbake || ( echo "precondition failed - expected /var/shared/bitbake to be writable" && exit 1 )
 
-    # Log available disk space on the persistent shared disk.
-    - df -h /var/shared/bitbake
-
-    # Create scratch space, being careful not to pollute the working directory.
-    - SCRATCH_DIR="$(mktemp -d)"
-    - echo "$SCRATCH_DIR" > "$CI_PROJECT_DIR"/.scratch-dir-name
-
-    # Create a git-repo workspace with all the files checked out.
-    #
-    # The checkout uses a mirror that is maintained in a separate pipeline
-    # https://git.ostc-eu.org/OSTC/infrastructure/ostc-manifest-mirror
-    # Even if the mirror is out-of-date, the sync command succeeds and
-    # downloads any delta required. This lowers the load on community servers
-    # and our traffic bill.
-    - mkdir "$SCRATCH_DIR"/workspace
-    - ( cd "$SCRATCH_DIR"/workspace && repo init --reference /var/shared/git-repo-mirrors/ostc-develop --manifest-url "$OSTC_MANIFEST_URL" --manifest-name "$OSTC_MANIFEST" )
-    - ( cd "$SCRATCH_DIR"/workspace && time repo sync --no-clone-bundle )
-    - du -sh "$SCRATCH_DIR"/workspace
-
-    # We subsequently rely on the sources/ directory that must be described by
-    # the manifest file.
-    - test -d "$SCRATCH_DIR"/workspace/sources || ( echo "assumption violated - expected the workspace to contain the sources directory" && ls "$SCRATCH_DIR"/workspace && exit 1 )
-
-  script: &workspace-do
-    # Reload the value of SCRATCH_DIR set in the before_script phase. Those run
-    # in separate shell processes and do not share environment variables.
-    - SCRATCH_DIR="$(cat "$CI_PROJECT_DIR"/.scratch-dir-name)"
-
-    # Initialize bitbake build environment by sourcing the oe-init-build-env
-    # into the running bash process. This has the side-effect of changing the
-    # current working directory and populating the $SCRATCH_DIR/workspace/build
-    # sub-directory with default configuration.
-    - ( cd "$SCRATCH_DIR"/workspace && TEMPLATECONF=../sources/meta-ohos/flavours/"$OHOS_BUILD_FLAVOUR" . ./sources/poky/oe-init-build-env build )
-
-    # Point to https://example.net instead of the default https://example.com.
-    # The OSTC cloud provider has misconfigured DNS which resolves the latter incorrectly.
-    - echo 'CONNECTIVITY_CHECK_URIS = "https://example.net/"' >> "$SCRATCH_DIR"/workspace/build/conf/local.conf
-
-    # Re-configure the created build directory to use our shared download cache
-    # and sstate-cache. Those are shared among all CI jobs running on our build
-    # cluster. This is what enables efficient builds and avoids (some) network
-    # problems that may be encountered when downloading third party source
-    # archives.
-    - echo 'DL_DIR = "/var/shared/bitbake/downloads"' >> "$SCRATCH_DIR"/workspace/build/conf/local.conf
-    - echo 'SSTATE_DIR = "/var/shared/bitbake/sstate-cache"' >> "$SCRATCH_DIR"/workspace/build/conf/local.conf
-
-    # Collect stats just before the build.
-    - du -sh "$SCRATCH_DIR"/workspace/build/*
-
-    - cd "$SCRATCH_DIR"/workspace && . ./sources/poky/oe-init-build-env build
-    # xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
-    # NOTE: From now on, we are running inside "$SCRATCH_DIR"/workspace/build
-    # with bash modified by oe-init-build-env. We now have access to bitbake,
-    # devtool and other related tools.
-    # xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
-
-  after_script:
-    # If the primary script failed early enough, the scratch dir may not have
-    # been created yet. Check for that to avoid confusing errors.
-    - test -f "$CI_PROJECT_DIR"/.scratch-dir-name || exit 0
-    # Reload the value of SCRATCH_DIR set in the before_script phase.
-    - SCRATCH_DIR="$(cat "$CI_PROJECT_DIR"/.scratch-dir-name)"
-
-    # Collect stats after the build.
-    - du -sh "$SCRATCH_DIR"/workspace/build/* || true
-
-    # Clean up after ourselves.
-    - rm -f "$CI_PROJECT_DIR"/.scratch-dir-name
-    - rm -rf "$SCRATCH_DIR"
-
-# This is a pipeline job that is used via the "extends" mechanism below.
-# For reference see https://docs.gitlab.com/ee/ci/yaml/README.html
 .build:
+  # Extend the workspace job. This effectively gives us an assembled tree and
+  # initialized bitbake. Some essential variables are defined in later jobs,
+  # like .build-linux, which sets OHOS_BUILD_FLAVOUR, which is inherited from
+  # the .workspace job.
   extends: .workspace
+  stage: build
   # Conservative timeout in case the build machine is busy and the cache is cold.
   timeout: 3 hours
-  # Set needs to an empty list to de-couple this job from anything preceding
-  # it in the set of stages. This is an optimization which removes the
-  # artificially created dependency between elements in different stages.
-  #
-  # TODO: depend on "reuse" when the allow_failure attribute is removed.
-  needs: []
   variables:
-    # Variables that are required by this job.
+    # Name of the bitbake recipe to build. This must be set by specific jobs
+    # which extend the .build job template.
     OHOS_RECIPE_NAME: ""
-    OHOS_GIT_REPO_PATH: ""
+    # Set to non-empty value to accept Freescale EULA.
+    OHOS_ACCEPT_FSL_EULA: ""
+    # The path of the git repository to deviate from what the git-repo manifest
+    # prepares. This effectively allows testing incoming changes that match the
+    # repository holding this CI pipeline.
+    #
+    # The path is relative to the checked out "sources/" directory.
+    OHOS_GIT_REPO_PATH: "meta-ohos"
   before_script:
     # Check if the job is configured properly.
-    - test -n "$OHOS_RECIPE_NAME" || ( echo "precondition failed - set OHOS_RECIPE_NAME to the name of the recipe to build" && exit 1 )
-    - test -n "$OHOS_GIT_REPO_PATH" || ( echo "precondition failed - set OHOS_GIT_REPO_PATH to the path of the git repository as described by the manifest" && exit 1 )
-    - *workspace-before
+    - test -n "$OHOS_RECIPE_NAME" || (
+        echo "precondition failed - set OHOS_RECIPE_NAME to the name of the recipe to build"
+        && exit 1 )
+    - test -n "$OHOS_GIT_REPO_PATH" || (
+        echo "precondition failed - set OHOS_GIT_REPO_PATH to the path of the git repository as described by the manifest"
+        && exit 1 )
+    - !reference [.workspace, before_script]
     # Switch the git repository which is being tested to the revision described
     # by the CI environment variables. This effectively performs the update
     # corresponding to the layer landing in either stable manifest or the
     # development manifest.
-    - ( cd "$SCRATCH_DIR"/workspace/sources/"$OHOS_GIT_REPO_PATH" && git checkout "$CI_COMMIT_REF_NAME" )
+    - ( cd "$SCRATCH_DIR"/sources/"$OHOS_GIT_REPO_PATH" && git checkout "$CI_COMMIT_SHA" )
   script:
-    - *workspace-do
+    - !reference [.workspace, script]
+    # Accept Freescale EULA if required.
+    - if [ -n "$OHOS_ACCEPT_FSL_EULA" ]; then
+         echo 'ACCEPT_FSL_EULA = "1"' >> conf/local.conf;
+      fi
     # Build the desired recipe.
     - time bitbake "$OHOS_RECIPE_NAME"
+    # Move artifacts for recovery, which only considers $CI_PROJECT_DIR and
+    # subdirectories.
+    - mkdir -p "$CI_PROJECT_DIR"/artifacts
+    - mv tmp/deploy/images/ "$CI_PROJECT_DIR"/artifacts || true
+    - mv tmp/deploy/licenses/ "$CI_PROJECT_DIR"/artifacts || true
+    # TODO: copy all build logs as well.
+  artifacts:
+    paths:
+      - artifacts/
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH
+
+##
+## Templates build jobs for each OpenHarmony flavor.
+##
 
 .build-linux:
   extends: .build
-  # Abuse the stage concept to put all Linux builds in one visual column.
-  stage: Linux
   variables:
-    OHOS_BUILD_FLAVOUR: "linux"
-    OHOS_RECIPE_NAME: "openharmony-image-base-tests"
-    OHOS_GIT_REPO_PATH: "meta-ohos"
+    OHOS_BUILD_FLAVOUR: linux
+    OHOS_RECIPE_NAME: openharmony-image-base-tests
 
-"Qemu x86-64 (Linux)":
-  extends: .build-linux
+.build-zephyr:
+  extends: .build
   variables:
-    MACHINE: "qemux86-64"
+    OHOS_BUILD_FLAVOUR: zephyr
+    OHOS_RECIPE_NAME: zephyr-philosophers
 
-"Seco Intel B68 (Linux)":
-  extends: .build-linux
+.build-freertos:
+  extends: .build
   variables:
-    MACHINE: seco-intel-b68
+    OHOS_BUILD_FLAVOUR: freertos
+    OHOS_RECIPE_NAME: freertos-demo
+
+##
+## Template build jobs for develop.xml
+##
+## This manifest is "floating" and follows default branches of OSTC-specific repositories
+## it includes.
+##
 
-"Seco i.MX8MM C61 (Linux)":
+.build-develop-linux:
   extends: .build-linux
   variables:
-    MACHINE: seco-imx8mm-c61
-  script:
-    - *workspace-do
-    # Accept NXP license only for building targets that require proprietary
-    # resources to boot.
-    - echo 'ACCEPT_FSL_EULA = "1"' >> "$SCRATCH_DIR"/workspace/build/conf/local.conf
-    # Build the desired recipe.
-    - time bitbake "$OHOS_RECIPE_NAME"
+    OHOS_MANIFEST_NAME: develop.xml
 
-"96Boards Avenger96 (Linux)":
-  extends: .build-linux
+.build-develop-zephyr:
+  extends: .build-zephyr
   variables:
-    MACHINE: stm32mp1-av96
+    OHOS_MANIFEST_NAME: develop.xml
 
-.build-zephyr:
-  extends: .build
-  # Abuse the stage concept to put all Zephyr builds in one visual column.
-  stage: Zephyr
+.build-develop-freertos:
+  extends: .build-freertos
   variables:
-    OHOS_BUILD_FLAVOUR: "zephyr"
-    OHOS_RECIPE_NAME: "zephyr-philosophers"
-    OHOS_GIT_REPO_PATH: "meta-ohos"
+    OHOS_MANIFEST_NAME: develop.xml
 
-"Qemu x86 (Zephyr)":
-  extends: .build-zephyr
+##
+## Build jobs for develop.xml
+##
+
+develop-linux-qemu-x86_64:
+  extends: .build-develop-linux
   variables:
-    MACHINE: "qemu-x86"
+    MACHINE: qemux86-64
 
-"96Boards Nitrogen (Zephyr)":
-  extends: .build-zephyr
+develop-linux-seco-intel-b68:
+  extends: .build-develop-linux
   variables:
-    MACHINE: "96b-nitrogen"
+    MACHINE: seco-intel-b68
 
-"96Boards Avenger96 (Zephyr)":
-  extends: .build-zephyr
+develop-linux-seco-imx8mm-c61:
+  extends: .build-develop-linux
   variables:
-    MACHINE: "96b-avenger96"
+    MACHINE: seco-imx8mm-c61
+    # This platform requires proprietary resources to boot.
+    OHOS_ACCEPT_FSL_EULA: 1
 
-.build-freertos:
-  extends: .build
-  # Abuse the stage concept to put all FreeRTOS builds in one visual column.
-  stage: FreeRTOS
+develop-linux-stm32mp1-av96:
+  extends: .build-develop-linux
   variables:
-    OHOS_BUILD_FLAVOUR: "freertos"
-    OHOS_RECIPE_NAME: "freertos-demo"
-    OHOS_GIT_REPO_PATH: "meta-ohos"
+    MACHINE: stm32mp1-av96
 
-"ARMv5 (FreeRTOS)":
-  extends: .build-freertos
+develop-zephyr-qemu-x86:
+  extends: .build-develop-zephyr
+  variables:
+    MACHINE: qemu-x86
+
+develop-zephyr-96b-nitrogen:
+  extends: .build-develop-zephyr
+  variables:
+    MACHINE: 96b-nitrogen
+
+develop-zephyr-96b-avenger:
+  extends: .build-develop-zephyr
+  variables:
+    MACHINE: 96b-avenger96
+
+develop-freertos-armv5:
+  extends: .build-develop-freertos
   variables:
-    MACHINE: "qemuarmv5"
+    MACHINE: qemuarmv5