Skip to content
Snippets Groups Projects
Commit b26828a2 authored by Zygmunt Krynicki's avatar Zygmunt Krynicki
Browse files

.oniro-ci,docs: merge CI from former manifest repo 


Most things are moved unchanged. The relatively few changes are to the
documentation, as the split between the meta-ohos and manifest is gone,
as well as to the actual CI definitions for the workspace directory,
since the sources/ sub-directory is gone, and there was some
backwards-compatibility code that no longer makes sense to maintain.

Signed-off-by: default avatarZygmunt Krynicki <zygmunt.krynicki@huawei.com>
parent ccfaa8d3
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 1144 additions and 24 deletions
.gitlab-ci.yml
+ 9
− 16
View file @ b26828a2
......@@ -26,23 +26,16 @@ include:
file:
- 'dco.yaml'
- 'reuse.yaml'
- project: 'OSTC/OHOS/manifest'
file: '.oniro-ci/build-generic.yaml'
- project: 'OSTC/OHOS/manifest'
file: '.oniro-ci/test-generic.yaml'
# By default, the CI pipeline is invoked using an explicit git ref (branch
# name) in the manifest. This include is a HACK to allow the CI pipeline to get
# changes to the machine-and-flavours.yaml picked up directly from the incoming
# branch in the merge request instead of the explicit branch name in the
# manifest.
- local: '/.oniro-ci/machines-and-flavours.yaml'
- local: '/.oniro-ci/build-generic.yaml'
- local: '/.oniro-ci/test-generic.yaml'
dco:
extends: .dco
reuse:
extends: .reuse
# FIXME: https://git.ostc-eu.org/OSTC/OHOS/meta-ohos/-/issues/19
# FIXME: https://git.ostc-eu.org/distro/oniro/-/issues/19
allow_failure: true
build-docs:
......@@ -53,11 +46,11 @@ aggregate-docs:
needs: [build-docs]
# Customize the .workspace job to set the path of the git repository to deviate
# from what the git-repo manifest prepares. This effectively allows testing
# incoming changes that match the repository holding this CI pipeline.
# from what the git-repo prepares. This effectively allows testing incoming
# changes that match the repository holding this CI pipeline.
.workspace:
variables:
CI_ONIRO_GIT_REPO_PATH: sources/meta-ohos
CI_ONIRO_GIT_REPO_PATH: oniro
# Customize the .bitbake-workspace job to set the rules governing when a build
# is attempted to: modifications (changes) to the pipeline, meta-layers,
......@@ -75,7 +68,7 @@ aggregate-docs:
- assets/**/*
- flavours/**/*
- meta-*/**/*
- .ostc-ci/*
- .oniro-ci/*
- if: '$CI_COMMIT_TAG'
# Mimic the updated rules for lava-test from the bitbake-workspace.
......@@ -89,7 +82,7 @@ aggregate-docs:
- assets/**/*
- flavours/**/*
- meta-*/**/*
- .ostc-ci/*
- .oniro-ci/*
- if: '$CI_COMMIT_TAG'
# Mimic the updated rules for lava-report from the bitbake-workspace.
......@@ -103,7 +96,7 @@ aggregate-docs:
- assets/**/*
- flavours/**/*
- meta-*/**/*
- .ostc-ci/*
- .oniro-ci/*
- if: '$CI_COMMIT_TAG'
##
......
.oniro-ci/build-generic.yaml 0 → 100644
+ 329
− 0
View file @ b26828a2
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Huawei Inc.
# This job is documented in docs/ci/hidden-jobs/workspace.rst
.workspace:
interruptible: true
image:
name: registry.ostc-eu.org/ostc/containers/ostc-builder:latest
tags: [large-disk, $CI_ONIRO_RUNNER_TAG]
variables:
CI_ONIRO_RUNNER_TAG : ""
CI_ONIRO_MANIFEST_URL: https://git.ostc-eu.org/distro/oniro
CI_ONIRO_MANIFEST_BRANCH: develop
CI_ONIRO_MANIFEST_NAME: default.xml
CI_ONIRO_MANIFEST_MIRROR: ostc-develop
CI_ONIRO_GIT_REPO_PATH: ""
before_script:
- test ! -e "$CI_PROJECT_DIR"/.scratch-dir-name || (
echo "precondition failed - concurrent modification of $CI_PROJECT_DIR"
&& env | grep CI_ | sort
&& ls -l
&& exit 1 )
# Create scratch space, being careful not to pollute the working directory.
# This is done so that we are not attached to the contents of
# $CI_PROJECT_DIR which contains something that GitLab CI prepared for us.
- SCRATCH_DIR="$(mktemp -p /tmp -d workspace.XXXXXXXXXX)"
- echo "$SCRATCH_DIR" > "$CI_PROJECT_DIR"/.scratch-dir-name
- cd "$SCRATCH_DIR"
- |
echo "Building repo workspace with the following properties:"
echo "CI_ONIRO_MANIFEST_URL: $CI_ONIRO_MANIFEST_URL"
echo "CI_ONIRO_MANIFEST_NAME: $CI_ONIRO_MANIFEST_NAME"
echo "CI_ONIRO_MANIFEST_BRANCH: $CI_ONIRO_MANIFEST_BRANCH"
repo init --reference \
/var/shared/pub/git-repo-mirrors/"$CI_ONIRO_MANIFEST_MIRROR" \
--manifest-url "$CI_ONIRO_MANIFEST_URL" \
--manifest-name "$CI_ONIRO_MANIFEST_NAME" \
--manifest-branch "$CI_ONIRO_MANIFEST_BRANCH"
- time repo sync --no-clone-bundle
# See the documentation of CI_ONIRO_GIT_REPO_PATH
- |
if [ -n "$CI_ONIRO_GIT_REPO_PATH" ]; then
if [ -n "$CI_MERGE_REQUEST_SOURCE_PROJECT_URL" ]; then
echo "CI: Bootstrapping '$CI_PROJECT_DIR' as 'incoming-merged' remote in '$CI_ONIRO_GIT_REPO_PATH'";
( cd "$CI_ONIRO_GIT_REPO_PATH" \
&& git remote add incoming-merged "$CI_PROJECT_DIR" \
&& git fetch incoming-merged HEAD );
fi;
echo "CI: Switching $CI_ONIRO_GIT_REPO_PATH to $CI_COMMIT_SHA";
( cd "$CI_ONIRO_GIT_REPO_PATH" && git checkout "$CI_COMMIT_SHA" );
fi
script:
# Reload the value of SCRATCH_DIR set in the before_script phase. Those run
# in separate shell processes and do not share environment variables.
- SCRATCH_DIR="$(cat "$CI_PROJECT_DIR"/.scratch-dir-name)"
- cd "$SCRATCH_DIR"
after_script:
# If the primary script failed early enough, the scratch dir may not have
# been created yet. Check for that to avoid confusing errors.
- test -f "$CI_PROJECT_DIR"/.scratch-dir-name || exit 0
# Reload the value of SCRATCH_DIR set in the before_script phase.
- SCRATCH_DIR="$(cat "$CI_PROJECT_DIR"/.scratch-dir-name)"
# Clean up after ourselves.
- rm -f "$CI_PROJECT_DIR"/.scratch-dir-name
- rm -rf "$SCRATCH_DIR"
# This job is documented in docs/ci/hidden-jobs/bitbake-workspace.rst
.bitbake-workspace:
extends: .workspace
stage: build
timeout: 6h
variables:
CI_ONIRO_BUILD_FLAVOUR: ""
CI_ONIRO_BUILD_CACHE: "private"
CI_ONIRO_BB_LOCAL_CONF_CONNECTIVITY_CHECK_URIS: "https://example.net/"
CI_ONIRO_BB_LOCAL_CONF_DL_DIR: /var/shared/$CI_ONIRO_BUILD_CACHE/bitbake/downloads
CI_ONIRO_BB_LOCAL_CONF_SSTATE_DIR: /var/shared/$CI_ONIRO_BUILD_CACHE/bitbake/sstate-cache
CI_ONIRO_BB_LOCAL_CONF_IMAGE_VERSION_SUFFIX: ""
CI_ONIRO_BB_LOCAL_CONF_CVE_CHECK_DB_DIR: "$${TMPDIR}/CVE_CHECK/"
CI_ONIRO_BB_LOCAL_CONF_plus_equals_INHERIT: cve-check
CI_ONIRO_BB_LOCAL_CONF_plus_equals_USER_CLASSES: "buildstats buildstats-summary"
CI_ONIRO_DEVTOOL_RECIPE_NAME: ""
CI_ONIRO_DEVTOOL_LAYER_PATH: ""
before_script:
# Bitbake requires a non-root user to operate.
# The container should have a non-root user by default.
- test "$(id -u)" -ne 0 || (
echo "precondition failed - this job cannot run as root"
&& exit 1 )
# Check if the job is configured properly.
- test -n "$CI_ONIRO_BUILD_FLAVOUR" || (
echo "precondition failed - set CI_ONIRO_BUILD_FLAVOUR to \"flavour\" of the build to use (e.g. linux)"
&& exit 1 )
# Check devtool operation is enabled and configured properly.
- |
if { [ -n "$CI_ONIRO_DEVTOOL_RECIPE_NAME" ] && [ -z "$CI_ONIRO_DEVTOOL_LAYER_PATH" ]; } || { [ -z "$CI_ONIRO_DEVTOOL_RECIPE_NAME" ] && [ -n "$CI_ONIRO_DEVTOOL_LAYER_PATH" ]; } then
echo "precondition failed - cannot define just one of CI_ONIRO_DEVTOOL_RECIPE_NAME and CI_ONIRO_DEVTOOL_LAYER_PATH"
exit 1
fi
# Bitbake is configured to use /var/shared/$CI_ONIRO_BUILD_CACHE/bitbake
# directory for both the download directory and the sstate-cache.
- test -w /var/shared/$CI_ONIRO_BUILD_CACHE/bitbake || (
echo "precondition failed - expected /var/shared/$CI_ONIRO_BUILD_CACHE/bitbake to be writable"
&& exit 1 )
# Log available disk space on the persistent shared disk.
- df -h /var/shared/$CI_ONIRO_BUILD_CACHE/bitbake
- !reference [.workspace, before_script]
script:
- !reference [.workspace, script]
# Initialize bitbake build environment by sourcing the oe-init-build-env
# into the running bash process. This has the side-effect of changing the
# current working directory and populating the $SCRATCH_DIR/build
# sub-directory with default configuration.
- TEMPLATECONF=../oniro/flavours/"$CI_ONIRO_BUILD_FLAVOUR"
. ./oe-core/oe-init-build-env build
# xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
# NOTE: From now on, we are running inside "$SCRATCH_DIR"/build
# with bash modified by oe-init-build-env. We now have access to bitbake,
# devtool and other related tools.
# xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
# Generic local.conf customization point. All environment variables with a
# given prefix are converted to local.conf variables, with the prefix
# dropped. Some extra sed processing is used to quote the variables.
# Everything runs in a sub-shell with bash pipefail disabled, as otherwise
# lack of variables matching the grep pattern causes the execution to stop.
- |
( set +o pipefail;
env \
| grep -E '^CI_ONIRO_BB_LOCAL_CONF_[A-Z_0-9]+=' \
| sed -e 's/^CI_ONIRO_BB_LOCAL_CONF_//g' -e 's/"/\\"/g' -e 's/=/ = "/g' -e 's/$/"/g' \
| sort \
| tee -a conf/local.conf )
( set +o pipefail;
env \
| grep -E '^CI_ONIRO_BB_LOCAL_CONF_plus_equals_[A-Z_0-9]+=' \
| sed -e 's/^CI_ONIRO_BB_LOCAL_CONF_plus_equals_//g' -e 's/"/\\"/g' -e 's/=/ += "/g' -e 's/$/"/g' \
| sort \
| tee -a conf/local.conf )
# Sanity check: disallow using public build cache with a specific setting
# in local.conf. The list of settings may grow over time.
- |
if [ "$CI_ONIRO_BUILD_CACHE" != private ]; then
if grep -qF 'ACCEPT_FSL_EULA = "1"' conf/local.conf; then
echo "cannot use CI_ONIRO_BUILD_CACHE=$CI_ONIRO_BUILD_CACHE with ACCEPT_FSL_EULA = \"1\""
exit 1
fi
fi
# Update a specific bitbake recipe, which must use git, to point to the
# commit that is being tested here.
- |
if [ -n "$CI_ONIRO_DEVTOOL_RECIPE_NAME" ] && [ -n "$CI_ONIRO_DEVTOOL_LAYER_PATH" ]; then
echo "CI: Switching $CI_ONIRO_DEVTOOL_RECIPE_NAME from $CI_ONIRO_DEVTOOL_LAYER_PATH to $CI_COMMIT_SHA";
# Upgrade the recipe to point to the sources in the project directory.
# Pass --no-patch to skip applying patches that are present in the
# recipe.
#
# NOTE: This requires the git repository specified in the recipe to
# have access to the SHA presented here.
time devtool upgrade --no-patch --srcrev "$CI_COMMIT_SHA" --srcbranch "$CI_COMMIT_REF_NAME" "$CI_ONIRO_DEVTOOL_RECIPE_NAME";
# Switch the detached head created by devtool upgrade above, to a
# scratch branch. This is required for the devtool finish to work.
( cd workspace/"$CI_ONIRO_DEVTOOL_RECIPE_NAME" && git checkout -b scratch );
# Finish the upgrade process and commit the change back to the layer.
# Because our recipe may have had patches, this step is required to
# correctly discard them ("devtool upgrade --no-patch" simply skipped
# them).
time devtool finish --remove-work --force "$CI_ONIRO_DEVTOOL_RECIPE_NAME" "$(basename "$CI_ONIRO_DEVTOOL_LAYER_PATH")";
fi
rules:
# Run the job when a merge request is created.
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
# Run the job when a tag is placed.
- if: '$CI_COMMIT_TAG'
# Run the build for scheduled pipelines.
- if: '$CI_PIPELINE_SOURCE == "schedule"'
# This job is documented in docs/ci/hidden-jobs/build-linux.rst
.build-linux:
extends: .bitbake-workspace
variables:
CI_ONIRO_BUILD_FLAVOUR: linux
CI_ONIRO_BITBAKE_TARGETS: "oniro-image-base-tests oniro-image-extra-tests"
# Linux builds can be extremely large. Use rm_work to remove the temporary
# build data as soon as possible to reclaim space.
CI_ONIRO_BB_LOCAL_CONF_plus_equals_INHERIT: "rm_work"
script:
- test -n "$CI_ONIRO_BITBAKE_TARGETS" || (
echo "nothing to build - CI_ONIRO_BITBAKE_TARGETS is empty"
&& exit 1 )
- !reference [.bitbake-workspace, script]
- |
for target in $CI_ONIRO_BITBAKE_TARGETS; do
time bitbake "$target"
du -sh tmp
done
# This job is documented in docs/ci/hidden-jobs/build-linux-marix.rst
.build-linux-matrix:
extends: .build-linux
parallel:
matrix:
- CI_ONIRO_BITBAKE_TARGETS: ["oniro-image-base-tests", "oniro-image-extra-tests"]
# This job is documented in docs/ci/hidden-jobs/build-zephyr.rst
.build-zephyr:
extends: .bitbake-workspace
variables:
CI_ONIRO_BUILD_FLAVOUR: zephyr
CI_ONIRO_BITBAKE_TARGETS: "zephyr-philosophers"
script:
- test -n "$CI_ONIRO_BITBAKE_TARGETS" || (
echo "nothing to build - CI_ONIRO_BITBAKE_TARGETS is empty"
&& exit 1 )
- !reference [.bitbake-workspace, script]
- |
for target in $CI_ONIRO_BITBAKE_TARGETS; do
time bitbake "$target"
done
# This job is documented in docs/ci/hidden-jobs/build-freertos.rst
.build-freertos:
extends: .bitbake-workspace
variables:
CI_ONIRO_BUILD_FLAVOUR: freertos
script:
- !reference [.bitbake-workspace, script]
- time bitbake freertos-demo
# This job is documented in docs/ci/hidden-jobs/build-recipe.rst
.build-recipe:
extends: .bitbake-workspace
variables:
CI_ONIRO_RECIPE_NAME: ""
script:
- !reference [.bitbake-workspace, script]
- time bitbake "$CI_ONIRO_RECIPE_NAME"
# This job is documented in docs/ci/hidden-jobs/build-image.rst
.build-image:
extends: .build-recipe
script:
- !reference [.build-recipe, script]
# Move artifacts for recovery, which only considers $CI_PROJECT_DIR and
# subdirectories.
- mkdir -p "$CI_PROJECT_DIR"/artifacts
# The name of the build-specific tmp directory may vary. Ask bitbake
# instead of hard-coding it. This is currently only relevant for Zephyr,
# which uses tmp-newlib.
- eval "$(bitbake -e | grep ^TMPDIR= | sed -e 's/^TMPDIR/BITBAKE_TMPDIR/g')"
# Uncompressed wic image is not needed in CI. We cannot remove it from
# meta-oniro-core/classes/ohos-image.bbclass as runqemu needs it for local
# build and boot.
- find "$BITBAKE_TMPDIR"/deploy/images/ -name *.wic -exec rm -rf {} \;
- mv "$BITBAKE_TMPDIR"/deploy/images/ "$CI_PROJECT_DIR"/artifacts || true
- mv "$BITBAKE_TMPDIR"/deploy/licenses/ "$CI_PROJECT_DIR"/artifacts || true
artifacts:
paths:
- artifacts/
.build-wic-image:
extends: .build-image
script:
- !reference [.build-image, script]
# Remove everything _except_ for the .wic.* and *.bmap files. Most of the
# remaining files are redundant and quite large. Our Linux builds really
# only care about the wic files.
- echo "Removing non-wic/bmap image files"
- find "$CI_PROJECT_DIR"/artifacts/images/ \( -type f -o -type l \) -a ! -name "*.wic.*" -print -delete
- echo "Pruning empty directories"
- find "$CI_PROJECT_DIR"/artifacts/ -type d -exec rmdir --verbose --ignore-fail-on-non-empty {} \; 2>/dev/null || true
# This job is currently used to customize the behavior in oniro and xts-acts.
# It will be removed when that is safe to do so. It is not documented.
.build:
extends: .build-recipe
# This job is documented in docs/ci/hidden-jobs/build-docs.rst
.build-docs:
interruptible: true
image:
name: registry.ostc-eu.org/ostc/containers/ostc-docs-builder
stage: build
script:
- make -C docs
artifacts:
paths:
- docs/build
rules:
# Build the docs when a merge request is created.
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
changes:
# React to changes to the docs directory.
- docs/**/*
# Run this job in case the pipeline changes.
- .oniro-ci/*.yml
- .gitlab-ci.yml
# Or when things land.
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
changes:
# React to changes to the docs directory.
- docs/**/*
# Run this job in case the pipeline changes.
- .oniro-ci/*.yml
- .gitlab-ci.yml
# This job is documented in docs/ci/hidden-jobs/aggregate-docs.rst
.aggregate-docs:
stage: deploy
trigger: OSTC/OHOS/docs
rules:
# Update the documentation when things land in the default branch.
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
changes:
# React to changes to the docs directory.
- docs/**/*
# Run this job in case the pipeline changes.
- .oniro-ci/*.yml
- .gitlab-ci.yml
.oniro-ci/machines-and-flavours.yaml
+ 4
− 8
View file @ b26828a2
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Huawei Inc.
# NOTE: This file is included from the manifest repository. It is not
# self-sufficient. It is here only because the meta-ohos repository also
# contains all the flavour and machine definitions and having both in one place
# allows one to add a new MACHINE and the associated build jobs in one go.
#
# Please do not include this file directly.
# The following jobs are documented in docs/ci/shared-jobs.rst
# Please do not include this file directly. It should be included through the
# build-generic.yaml only.
# The following jobs are documented in docs/ci/machines-and-flavours.rst
linux-qemu-x86:
extends: .build-wic-image
......
.oniro-ci/test-generic.yaml 0 → 100644
+ 115
− 0
View file @ b26828a2
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Huawei Inc.
##
## Submits a job to LAVA with CI variables
##
.lava-test:
interruptible: true
image:
name: registry.ostc-eu.org/ostc/containers/ostc-builder:latest
before_script:
# Check if the job is configured properly.
- test -n "$CI_LAVA_JOB_DEFINITION" || (
echo "precondition failed - set CI_LAVA_JOB_DEFINITION to the URL of the LAVA test job definition"
&& exit 1 )
- test -n "$CI_BUILD_JOB_NAME" || (
echo "precondition failed - set CI_BUILD_JOB_NAME to the appropriate job name from which LAVA will pick up build artifact"
&& exit 1 )
- test -n "$CI_REPORT_JOB_NAME" || (
echo "precondition failed - set CI_REPORT_JOB_NAME to the CI job name which will gather results back from LAVA"
&& exit 1 )
script:
# Build callback URL for the "report" job
- curl --silent "https://git.ostc-eu.org/api/v4/projects/$CI_PROJECT_ID/pipelines/$CI_PIPELINE_ID/jobs?per_page=100" > jobs-manual.json
- job_id="$(jq -r ".[] | select(.name == \"$CI_REPORT_JOB_NAME\") | .id" jobs-manual.json)"
- build_job_id="$(jq -r ".[] | select(.name == \"$CI_BUILD_JOB_NAME\") | .id" jobs-manual.json)"
- CALLBACK_URL="https://git.ostc-eu.org/api/v4/projects/$CI_PROJECT_ID/jobs/${job_id}/play"
# Get the job definition from remote source.
- curl --silent "$CI_LAVA_JOB_DEFINITION" > job_def.yaml
# Update the job definition with CI data.
- sed -i -e 's/$ci_job_id/'"$CI_JOB_ID"'/'
-e 's/$ci_project_id/'"$CI_PROJECT_ID"'/'
-e 's/$ci_pipeline_id/'"$CI_PIPELINE_ID"'/'
-e 's,$ci_pipeline_url,'"$CI_PIPELINE_URL"','
-e 's/$build_job_id/'"$build_job_id"'/'
-e 's,$callback_url,'"$CALLBACK_URL"',' job_def.yaml
# Generate job definitions for zephyr build
- |
mkdir lava_jobs
if echo "${CI_BUILD_JOB_NAME}" | grep -i "zephyr"; then
for image in $(find artifacts/ -name "*.elf" -exec basename {} \;); do
job_name=$(basename "${image}" ".elf")
echo "--- Generating lava job definition ${job_name}.yaml ---"
sed "s/\$elf_file/$image/" job_def.yaml | tee lava_jobs/"${job_name}".yaml
done
else
mv job_def.yaml lava_jobs/
fi
# Submit the job to LAVA.
- |
for job_def in $(find lava_jobs/ -name "*.yaml"); do
lava_job_id=$(curl -X POST -H "Authorization: Token $CI_LAVA_TOKEN" -F "definition=$(<${job_def})" "$CI_LAVA_INSTANCE/api/v0.2/jobs/" | jq ".job_ids" | tr -d "[\n ]")
if [ -n "${lava_job_id}" ]; then
echo "$CI_LAVA_INSTANCE/scheduler/job/$lava_job_id"
echo "$lava_job_id" >> job_ids_"${CI_JOB_NAME}_${lava_job_id}".txt
else
echo "Failed to submit LAVA job"
exit 1
fi
done
artifacts:
paths:
- job_ids_*.txt
rules:
# Run the build when it is scheduled.
- if: $CI_PIPELINE_SOURCE == "schedule"
# Do not run pipelines for draft merge requests unless manually triggered.
- if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TITLE =~ /^(wip|draft):.*/i'
when: manual
# Run the build when a merge request is created.
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
# Run the build when a tag is placed.
- if: '$CI_COMMIT_TAG'
.lava-report:
interruptible: true
image:
name: registry.ostc-eu.org/ostc/containers/ostc-builder:latest
stage: report
script:
- |
for file in $(find ./ -name "job_ids_*.txt"); do
echo "Job file: $file"
while read -r p; do
echo "Job ID: $p"
mkdir "$p"
# Echo LAVA job health and link.
link="$CI_LAVA_INSTANCE/scheduler/job/$p"
health="$(curl --silent "$CI_LAVA_INSTANCE/api/v0.2/jobs/$p/" | jq '.health' | tr -d '"')"
echo "[$health] $link"
# Get the JUnit export from LAVA.
curl --silent -o job_$p.xml "$CI_LAVA_INSTANCE/api/v0.2/jobs/$p/junit/"
echo "Report file: $(ls job_$p.xml)"
done < "$file"
done
artifacts:
when: always
paths:
- job_*.xml
reports:
junit:
- job_*.xml
rules:
# Run the build when it is scheduled.
- if: $CI_PIPELINE_SOURCE == "schedule"
when: manual
# Do not run pipelines for draft merge requests unless manually triggered.
- if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TITLE =~ /^(wip|draft):.*/i'
when: manual
# Run the build when a merge request is created.
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: manual
# Run the build when a tag is placed.
- if: '$CI_COMMIT_TAG'
when: manual
docs/ci/hidden-jobs/aggregate-docs.rst 0 → 100644
+ 43
− 0
View file @ b26828a2
.. SPDX-FileCopyrightText: Huawei Inc.
..
.. SPDX-License-Identifier: CC-BY-4.0
===============
.aggregate-docs
===============
The ``.aggregate-docs`` job triggers a build of the aggregated documentation of
All Scenarios OS.
All Scenarios OS documentation is maintained in an unusual way. Individual
repositories contain dedicated documentation that can be built with
:doc`build-docs` job. A special *centralized* documentation project aggregates
documentation from multiple git repositories, as checked out by *git-repo* based
on the *manifest* file, to build a standalone document.
This job encapsulates knowledge on how to trigger the aggregated build upon
changes to documentation specific to a given project.
Usage Guide
===========
.. warning::
If a project is pinned in the *git-repo manifest* then using
this job makes no sense, as the resulting aggregate will not
change without a prior modification of the manifest.
To use this job in your pipeline include the generic build definition file,
``build-generic.yaml`` and define the ``aggregate-docs`` job in addition to the
:doc:`build-docs` job.
.. code-block:: yaml
# Build documentation present in the repository.
build-docs:
extends: .build-docs
# Trigger aggregation of All Scenarios OS documentation.
aggregate-docs:
extends: .aggregate-docs
needs: [build-docs]
docs/ci/hidden-jobs/bitbake-workspace.rst 0 → 100644
+ 176
− 0
View file @ b26828a2
.. SPDX-FileCopyrightText: Huawei Inc.
..
.. SPDX-License-Identifier: CC-BY-4.0
==================
.bitbake-workspace
==================
The ``.bitbake-workspace`` job extends the :doc:`workspace` job to configure
and initialize BitBake for the desired build operations. The job does not build
anything by itself, that functionality is available in the ``.build-recipe``
job.
Job Variables
=============
The ``.bitbake-workspace`` job defines several variables as a way to customize
the way BitBake is configured.
CI_ONIRO_BUILD_FLAVOUR
----------------------
The name of the *flavour* of All Scenarios OS, which effectively picks the kernel
type. This is used to select the initial BitBake configuration template.
Templates are stored in the **meta-ohos** repository.
Available values are ``linux``, ``zephyr`` and ``freertos``. There is no
default value. This variable must be set by a derivative job, it is usually set
by the three ``.build-linux``, ``.build-zephyr`` and ``.build-freertos`` jobs.
Specific build jobs, in turn, extend those.
CI_ONIRO_BUILD_CACHE
--------------------
The set of build caches to use.
Currently there are only two sets ``private`` or ``pub``. The names directly
correspond to file system path where the cache is stored.
By default all builds are assumed to be tainted, and use the private cache. If
a given build configuration is not legally problematic, in the sense of pulling
in code or blobs that are non-redistributable, this attribute can be set to
``pub``.
Public build cache is exposed as `<https://cache.ostc-eu.org/>`_ and can be
used by third parties to speed up local builds.
The default value is ``private``.
CI_ONIRO_DEVTOOL_RECIPE_NAME
----------------------------
Name of the BitBake recipe to upgrade.
This may be set by jobs which extend or re-define the ``.bitbake-workspace``
job. It must be used in tandem with ``CI_ONIRO_DEVTOOL_LAYER_PATH``. The side
effects are discussed below.
CI_ONIRO_DEVTOOL_LAYER_PATH
---------------------------
Path of the layer containing the recipe to upgrade.
This may be set by jobs which extend or re-define the ``.bitbake-workspace``
job. It must be used in tandem with ``CI_ONIRO_DEVTOOL_RECIPE_NAME``.
Before the build is attempted, the recipe is updated to ignore any existing
patches and point to the code corresponding to the repository and commit being
tested.
.. code-block:: bash
devtool upgrade \
--no-patch \
--srcrev "$CI_COMMIT_SHA" \
--srcbranch "$CI_COMMIT_REF_NAME" \
"$CI_ONIRO_DEVTOOL_RECIPE_NAME"
devtool finish \
--remove-work \
--force \
"$CI_ONIRO_DEVTOOL_RECIPE_NAME" \
"$(basename "$CI_ONIRO_DEVTOOL_LAYER_PATH")"
This functionality is useful for testing incoming changes to repositories that
contain source code that is already packaged one of the layers.
Configuring BitBake
===================
The ``local.conf`` file can define numerous variables that influence the
BitBake build process. This job offers a declarative method of doing that. Job
variables with have the prefix ``CI_ONIRO_BB_LOCAL_CONF_`` are converted to
``attr = "value"`` and those with prefix
``CI_ONIRO_BB_LOCAL_CONF_plus_equals_`` are converted to ``attr += "value"``.
This method is friendly to job inheritance and re-definition. Derivative jobs
can add or re-define variables without having to duplicate any imperative logic
or maintaining synchronized settings across distinct jobs.
This mechanism is used to set the following BitBake variables
CONNECTIVITY_CHECK_URIS
-----------------------
BitBake contains a connectivity check system. That system relies on access to
``https://example.com/``. OSTC cloud provider has a DNS configuration problem,
where that specific domain is not resolved correctly. For a compatible
workaround the connectivity check URL is set to ``https://example.net/``.
This is implemented using the ``CI_ONIRO_BB_LOCAL_CONF_`` system.
DL_DIR
------
BitBake downloads source archives and git repositories in the through the
``fetch`` task of many recipes. Those are all stored in the local file system
and can be shared between separate builds for a great speed up, as the files
are obtained from multiple third party servers and their connectivity varies.
To optimize the build process, the download directory is set to point at the
shared NFS volume that persists between job execution, and is more efficient
than the artifact system, that copies all the data regardless of the need to
actually use that data in practice.
The default location is changed to
``/var/shared/$CI_ONIRO_BUILD_CACHE/bitbake/downloads``, but this should
be treated as an implementation detail. The location may change in the future.
The download cache is not automatically purged yet. In the future it may be
purged periodically, if space becomes an issue.
Note that the location relies on the value of ``$CI_ONIRO_BUILD_CACHE``
discussed above.
SSTATE_CACHE
------------
BitBake relies on an elaborate cache system, that can be used to avoid
duplicating work at the level of a specific recipe. The dependencies and
side-effects of each recipe are recorded in the cache, and are reused whenever
possible.
Having access to a persistent cache has a dramatic effect on the performance of
the CI system as, in the fast-path, it can avoid virtually all compilation
tasks and simply assemble the desired system image out of intermediate files
present in the cache.
The default location is changed to
``/var/shared/$CI_ONIRO_BUILD_CACHE/bitbake/sstate-cache``, but this
should be treated as an implementation detail. The location may change in the
future. The sstate cache is not automatically purged yet. It can be purged
periodically with the only caveat, that initial builds will be much slower.
Cache Considerations
====================
The ``.bitbake-workspace`` job configures BitBake to use a persistent directory
that is shared between CI jobs, for the location of the ``download`` directory
as well as the ``sstate-cache`` directory.
The job is using GitLab runner tags to schedule jobs in the environment where
that shared storage is available. When a new dependency is added or when the
layers and recipes are changed or updated, the download is automatically
populated with the necessary source archives. Similarly ``sstate-cache`` is
populated by all the build jobs present throughout the CI system.
Due to legal restrictions, the caches are split into two pairs, public and private.
The public cache is automatically published in https://cache.ostc-eu.org/bitbake/
The private cache, which is used by default, is available on the same volume but it is
not shared anywhere.
In case the cache is fed with a software package that is, in retrospective
somehow problematic, for example, by not being freely redistributable, the cache
can be purged at will.
For details on how cache selection and BitBake configuration looks like, please
refer to the pipeline source code.
docs/ci/hidden-jobs/build-docs.rst 0 → 100644
+ 53
− 0
View file @ b26828a2
.. SPDX-FileCopyrightText: Huawei Inc.
..
.. SPDX-License-Identifier: CC-BY-4.0
===========
.build-docs
===========
The ``.build-docs`` job builds reStructuredText documentation.
This job offers no configuration and works by convention. This is done to both
simplify its use and improve uniformity of developer experience. The job
automatically reacts to changes in the ``docs/`` directory as well as the
definition of the pipeline. Built documentation is collected as job artifacts.
Usage Guide
===========
To use this job in your pipeline include the generic build definition file,
``build-generic.yaml`` and define a ``build-docs`` job:
.. code-block:: yaml
build-docs:
extends: .build-docs
The job expects the project to have the following documentation structure.
.. code-block::
└── docs
├── index.rst
└── Makefile
The ``Makefile`` should build the documentation into the ``build`` directory,
relative to the ``docs`` directory. A sample, re-usable example is is provided below:
.. code-block:: makefile
# SPDX-FileCopyrightText: Huawei Inc.
#
# SPDX-License-Identifier: Apache-2.0
.PHONY: all
all:
sphinx-build -W -C \
-D html_theme=sphinx_rtd_theme \
-D project='Name of the project' \
-D copyright='Copyright Holder' \
. build
clean:
rm -rf ./build
docs/ci/hidden-jobs/build-freertos.rst 0 → 100644
+ 17
− 0
View file @ b26828a2
.. SPDX-FileCopyrightText: Huawei Inc.
..
.. SPDX-License-Identifier: CC-BY-4.0
===============
.build-freertos
===============
The ``.build-freertos`` job extends the :doc:`bitbake-workspace` job. It sets
``CI_ONIRO_BUILD_FLAVOUR`` to ``freertos`` and builds the ``freertos-demo``
test image.
Usage Guide
===========
This job is not intended for direct use. Instead it serves as a base for all
the FreeRTOS-specific :doc:`../shared-jobs`.
docs/ci/hidden-jobs/build-image.rst 0 → 100644
+ 24
− 0
View file @ b26828a2
.. SPDX-FileCopyrightText: Huawei Inc.
..
.. SPDX-License-Identifier: CC-BY-4.0
============
.build-image
============
The ``.build-image`` job extends the :doc:`build-recipe` job to build a single
recipe, which should describe an image, and to collect the resulting image and
licenses as job artifacts.
Usage Guide
===========
This job is configured exactly the same as :doc:`build-recipe`.
Implementation Details
======================
The job handles differences between the name of the temporary build directory
between various All Scenarios OS flavours. Internally BitBake is interrogated for
the value of ``TMPDIR`` and the image is copied back to a subdirectory
of ``$CI_PROJECT_DIR`` for delivery to the GitLab runner.
docs/ci/hidden-jobs/build-linux-matrix.rst 0 → 100644
+ 19
− 0
View file @ b26828a2
.. SPDX-FileCopyrightText: Huawei Inc.
..
.. SPDX-License-Identifier: CC-BY-4.0
===================
.build-linux-matrix
===================
The ``.build-linux-matrix`` job extends the :doc:`build-linux` job and
otherwise behaves exactly the same but builds each of the
``CI_ONIRO_BITBAKE_TARGET`` as a separate job. This creates a wider pipeline
which unlocks additional parallelism.
Usage Guide
===========
Note that due to the way parallel matrix jobs are defined, to change the set of
bitbake recipes to build you must re-define the ``parallel/matrix`` element
entirely. Changing ``CI_ONIRO_BITBAKE_TARGETS`` is not effective.
docs/ci/hidden-jobs/build-linux.rst 0 → 100644
+ 41
− 0
View file @ b26828a2
.. SPDX-FileCopyrightText: Huawei Inc.
..
.. SPDX-License-Identifier: CC-BY-4.0
============
.build-linux
============
The ``.build-linux`` job extends the :doc:`bitbake-workspace` job. It sets
``CI_ONIRO_BUILD_FLAVOUR`` to ``linux`` and builds the bitbake targets (e.g.
images) as defined by ``CI_ONIRO_BITBAKE_TARGETS`` (defaults included).
The images are built one after another, allowing CI to fail quickly in case of
any problems with the more fundamental image. The set of built images may
change over time, as additional reference images are defined.
Usage Guide
===========
This job is not intended for direct use. Instead it serves as a base for all
the Linux-specific :doc:`../shared-jobs`. It may be re-defined in a pipeline to
alter ``rules`` or ``variables`` in a way that fits a particular purpose.
If used directly, it is recommended pick the desired ``MACHINE`` and to
override the entire script section and refer to the base ``.bitbake-workspace``
job as illustrated below. The set of build operations can then be tailored to
the purpose of the desired job.
.. code-block:: yaml
build-something-linux-specific:
extends: .build-linux
variables:
MACHINE: "..."
script:
- !reference [.bitbake-workspace, script]
- true # put your code here
.. note::
``CI_ONIRO_BITBAKE_TARGETS`` can be overwritten if defaults are not desired.
docs/ci/hidden-jobs/build-recipe.rst 0 → 100644
+ 43
− 0
View file @ b26828a2
.. SPDX-FileCopyrightText: Huawei Inc.
..
.. SPDX-License-Identifier: CC-BY-4.0
=============
.build-recipe
=============
The ``.build-recipe`` job extends the :doc:`bitbake-workspace` job to build a
single BitBake recipe. The recipe is built and all the results are discarded.
Variables
=========
CI_ONIRO_RECIPE_NAME
--------------------
The name of the recipe to build.
There is no default value, this must be set by the extending job.
Usage Guide
===========
To use this job in your pipeline include the generic build definition file,
``build-generic.yaml`` and define a job with the minimal configuration, as
illustrated below:
.. code-block:: yaml
build-something:
extends: .build-recipe
variables:
CI_ONIRO_BUILD_FLAVOUR: "..."
CI_ONIRO_RECIPE_NAME: "..."
MACHINE: "..."
Pick the desired ``CI_ONIRO_BUILD_FLAVOUR``, ``CI_ONIRO_RECIPE_NAME`` and
``MACHINE``. For discussion of ``CI_ONIRO_BUILD_FLAVOUR`` refer to the
:doc:`bitbake-workspace` job. ``CI_ONIRO_RECIPE_NAME`` is any BitBake recipe
available in the selected flavour. ``MACHINE`` is desired machine name.
Supported values are documented alongside each flavour in the *meta-ohos*
repository.
docs/ci/hidden-jobs/build-wic-image.rst 0 → 100644
+ 19
− 0
View file @ b26828a2
.. SPDX-FileCopyrightText: Huawei Inc.
..
.. SPDX-License-Identifier: CC-BY-4.0
================
.build-wic-image
================
The ``.build-wic-image`` job extends the :doc:`build-image` job to collect only
the `*.wic.*` and `*.bmap` files and remove all the other files that would
normally be collected by the artifact system. It is recommended for Linux
builds which produce wic images, as the size of subset of collected artifacts
is considerably smaller than what `.build-image` provides.
Usage Guide
===========
This job is configured exactly the same as :doc:`build-image` and
:doc:`build-recipe`.
docs/ci/hidden-jobs/build-zephyr.rst 0 → 100644
+ 17
− 0
View file @ b26828a2
.. SPDX-FileCopyrightText: Huawei Inc.
..
.. SPDX-License-Identifier: CC-BY-4.0
=============
.build-zephyr
=============
The ``.build-zephyr`` job extends the :doc:`bitbake-workspace` job. It sets
``CI_ONIRO_BUILD_FLAVOUR`` to ``zephyr`` and builds the bitbake targets (e.g.
images) as defined by ``CI_ONIRO_BITBAKE_TARGETS`` (defaults included).
Usage Guide
===========
This job is not intended for direct use. Instead it serves as a base for all
the Zephyr-specific :doc:`../shared-jobs`.
docs/ci/hidden-jobs/index.rst 0 → 100644
+ 29
− 0
View file @ b26828a2
.. SPDX-FileCopyrightText: Huawei Inc.
..
.. SPDX-License-Identifier: CC-BY-4.0
Hidden Jobs
===========
There's a number of *hidden jobs*, which start with the dot character, that are
used as foundation for the set of :doc:`shared jobs <../shared-jobs>`. Hidden
jobs do not participate in any pipeline directly. They can only be used as
templates, using the ``extends: ...`` mechanism, to share and reuse
implementation details.
.. toctree::
:maxdepth: 1
workspace
bitbake-workspace
build-linux
build-linux-matrix
build-zephyr
build-freertos
build-recipe
build-image
build-wic-image
build-docs
lava-test
lava-report
aggregate-docs
docs/ci/hidden-jobs/lava-report.rst 0 → 100644
+ 19
− 0
View file @ b26828a2
.. SPDX-FileCopyrightText: Huawei Inc.
..
.. SPDX-License-Identifier: CC-BY-4.0
============
.lava-report
============
The ``.lava-report`` job iterates through the submitted jobs from
:doc:`lava-test` and collects the artifacts from these jobs via LAVA REST API.
Job Variables
=============
CI_LAVA_INSTANCE
----------------
The base url of the LAVA server.
docs/ci/hidden-jobs/lava-test.rst 0 → 100644
+ 51
− 0
View file @ b26828a2
.. SPDX-FileCopyrightText: Huawei Inc.
..
.. SPDX-License-Identifier: CC-BY-4.0
==========
.lava-test
==========
The ``.lava-test`` job creates the LAVA test job definition based on the
pipeline information and submits the job to LAVA.
The artifact of this job is the list of LAVA job id(s) submitted by this job.
This will be consumed by the :doc:`lava-report`
Job Variables
=============
The ``.lava-test`` job defines several variables as a way to customize
the LAVA job definition before submiting the job to LAVA.
CI_LAVA_JOB_DEFINITION
----------------------
This is the url to the job definition template. The template needs to have
variables inside which will be replaced by this job. The list of variables is
the following:
- $ci_pipeline_id - pipeline ID of the CI job ($CI_PIPELINE_ID in GitLab)
- $ci_job_id - CI job ID ($CI_JOB_ID in GitLab)
- $ci_project_id - CI project ID ($CI_PROJECT_ID in GitLab)
- $build_job_id - job ID of the **build** job within the same pipeline
- $callback_url - url which triggers the execution or the manual job which
collects the results back from LAVA to GitLab. See :doc:`lava-report`
CI_BUILD_JOB_NAME
-----------------
The name of the job that builds the artifacts which will be used by LAVA to
boot up the DUT for this particular LAVA job. If we test multiple builds in one
CI job, each will have different build job name.
CI_REPORT_JOB_NAME
------------------
The name of the ``report`` job which will be triggered manually when the LAVA
job(s) are finished with executution. This job will collect the results from
LAVA and import them to GitLab.
CI_LAVA_INSTANCE
----------------
The base url of the LAVA server.
docs/ci/hidden-jobs/workspace.rst 0 → 100644
+ 118
− 0
View file @ b26828a2
.. SPDX-FileCopyrightText: Huawei Inc.
..
.. SPDX-License-Identifier: CC-BY-4.0
==========
.workspace
==========
The ``.workspace`` job assembles a *git-repo* workspace, as described by
*manifest* file. This job is a foundation for other jobs.
This workspace is **not** constructed in ``$CI_PROJECT_DIR``, which is by
GitLab runner. To avoid clashes with any files that may be present there, it is
constructed in a temporary directory. Additional logic stores and restores the
path of that directory between the code in ``before_script``, ``script`` and
``after_script`` as those execute in separate shell processes.
Constraints
===========
This job uses the ``ostc-builder`` container image, mainly for convenience, as
it is often extended to perform other tasks, such as bitbake builds. Other
containers can be used, as long as they have the ``repo`` program
pre-installed.
In addition this job uses the ``large-disk`` tag to be scheduled on a machine
with access to a shared NFS volume with git mirrors, that greatly speed up the
process of constructing the workspace from scratch.
Variables
=========
CI_ONIRO_RUNNER_TAG
-------------------
An arbitrary GitLab Runner tag selecting the runner which processes the
.workspace job, or its derivative. This defaults to an empty string but can be
used to pick a specific GitLab runner, as long as that runner has the matching
tag set.
This can be used to perform scheduled builds in a non-default location, as long
as the pipeline schedule defines this variable.
CI_ONIRO_MANIFEST_URL
---------------------
The URL consumed by git-repo. You only want to change this if you forked the
entire infrastructure and want to use it in private.
The default value is ``https://git.ostc-eu.org/OSTC/OHOS/manifest``.
If you change the default value, please set ``CI_ONIRO_MANIFEST_MIRROR``
as well.
CI_ONIRO_MANIFEST_BRANCH
------------------------
The name of the git branch of the manifest repository. Unless special
circumstances apply this does not need to be customized.
For testing changes coming into the *manifest* repository itself, use
``$CI_COMMIT_REF_NAME``, which will check out the manifest as described by the
specific commit being tested.
The default value is ``develop``.
CI_ONIRO_MANIFEST_NAME
----------------------
The name of the manifest file from the repository mentioned above.
The default value is ``default.xml``.
CI_ONIRO_MANIFEST_MIRROR
------------------------
Name of the ``git-repo`` mirror matching to use, expressed as a directory name
in the shared disk volume. Check the section about git-repo mirror below, for
details.
The default value is ``ostc-develop``.
CI_ONIRO_GIT_REPO_PATH
----------------------
The path of the git repository to deviate from what the git-repo manifest
describes. This variable should be used for constructing CI jobs for
repositories directly described by the manifest.
When set to a non-empty value, the specified git repository will be first
checked out by the git-repo according to what is described in the manifest, and
then changed again, to point to ``$CI_COMMIT_SHA``. The logic in the script
supports the forked repository workflow, by reusing the merge setup GitLab does
on pipelines for merge results. See GitLab documentation on `Pipelines for
Merge Results`_ for more information.
.. _Pipelines for Merge Results: https://docs.gitlab.com/ee/ci/merge_request_pipelines/pipelines_for_merged_results/index.html
The default value is the empty string.
Local git-repo Mirror
=====================
The ``.workspace`` job relies on a git-repo mirror that is mounted into the
execution environment provided by the GitLab worker. The mirror is created and
kept up-to-date by the scheduled run of the pipeline in `ostc-manifest-mirror
repository`_
.. _ostc-manifest-mirror repository: https://git.ostc-eu.org/OSTC/infrastructure/ostc-manifest-mirror
When the mirror is out-of-date additional git revisions needed to construct the
workspace are fetched form the respective upstream repositories. Some of the
git repositories described by the manifest are rather large and are hosted on
infrastructure outside of the OSTC cloud provider, making this an important
optimization.
The mirror is automatically published in `<https://cache.ostc-eu.org/git-repo-mirrors/>`_,
with two specific mirrors being available: *ostc-dev* and *gitee-dev*.
docs/ci/index.rst
+ 2
− 0
View file @ b26828a2
......@@ -9,3 +9,5 @@ Continuous Integration
:maxdepth: 1
machines-and-flavours
special-jobs
hidden-jobs/index
docs/ci/special-jobs.rst 0 → 100644
+ 16
− 0
View file @ b26828a2
.. SPDX-FileCopyrightText: Huawei Inc.
..
.. SPDX-License-Identifier: CC-BY-4.0
Special Jobs
------------
linux-glibc-qemu-x86_64
.......................
This job extends ``linux-qemu-x86_64`` and differs in the following way.
This job performs a build with the libc switched to ``glibc``. It only runs on a
schedule that is defined in the GitLab project settings for the `manifest`
repository. In practice it runs daily to check if the Linux favour could be
switched back to ``glibc``, from the default ``musl`` that is used right now.
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Copyright © Eclipse Foundation, Inc. All Rights Reserved.     Privacy Policy | Terms of Use | Copyright Agent