Skip to content
Snippets Groups Projects

Draft: Test CI

Closed Draft: Test CI
test_ci into kirkstone
Closed Kristijonas Čiulčinskas requested to merge test_ci into kirkstone
Compare
and
1 file
+ 3
− 476
Compare changes
  • Side-by-side
  • Inline
.gitlab-ci.yml
+ 3
− 476
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Huawei Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Note: While main CI operations are done on Eclipse Foundation infrastructure,
# a few extra operations are run on Huawei OSTC infrastructure, providing:
#
# - hosting Docker images on registry.ostc-eu.org
# - automatically updated mirror of source files in China
#
# The Docker images will move to Eclipse Foundation infrastructure when Docker
# registry becomes available on gitlab.eclipse.org.
#
# The mirror in China will need to be investigated further. Do we actually need
# it? Can it be supported by Eclipse Foundation infrastructure?
#
# switch between branch pipelines and merge request pipelines
workflow:
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
when: never
- if: $CI_COMMIT_BRANCH
include:
- local: '.gitlab-ci/container.yml'
variables:
# output upload and download progress every 10 seconds
TRANSFER_METER_FREQUENCY: "10s"
# Use no compression for artifacts
ARTIFACT_COMPRESSION_LEVEL: "fastest"
# Use no compression for caches
CACHE_COMPRESSION_LEVEL: "fastest"
# The bitbake-builder Docker image registry path.
#
# As we don't have Docker registry support on gitlab.eclipse.org, use this
# setup for now. When updating the bitbake-builder image, you therefore need
# to push to git.ostc-eu.org first, and wait for the bitbake-builder image to
# be pushed to the registry before starting a pipeline on gitlab.eclipse.org.
BITBAKE_BUILDER: "registry.ostc-eu.org/ostc/pre-integration/meta-openharmony/bitbake-builder"
# And same for openharmony-builder
OPENHARMONY_BUILDER: "registry.ostc-eu.org/ostc/pre-integration/meta-openharmony/openharmony-builder"
stages:
- container
- maintenance
- fetch
- build
- test
# Common job definition for bitbake driven jobs, such as fetch and build
.bitbake:
image:
name: $BITBAKE_BUILDER:$BITBAKE_CONTAINER_VERSION
variables:
MANIFEST_FILE: manifests/ci.xml
BB_ENV_PASSTHROUGH_ADDITIONS: DISTRO MACHINE
SOURCE_MIRROR_URL_CH: http://114.116.235.68/source-mirror
before_script:
# Tell git this repository is safe even if cloned with a different uid
- git config --global --add safe.directory $CI_PROJECT_DIR
# The repo tool seems to insist on having a branch checked out or
# something like that... Without this we get errors like
# fatal: couldn't find remote ref refs/heads/master
- git checkout -b master
# The repo command does not like shallow repos
- if [ "$(git rev-parse --is-shallow-repository)" = "true" ] ; then
git fetch --unshallow ;
fi
# Create/maintain a mirror of all manifest repositories in runner
# persistent storage, and use that as reference when initializing the
# build repo, effectively reducing the time spent fetching git repos to
# almost nothing.
#
# If for some reason, the repo mirror becomes corrupted, a pipeline with
# $REBUILD_REPO_MIRROR set to a non-empty value will remove the entire
# mirror, and rebuild it from scratch.
#
# Note the use of fd 9 to hold a lock while accessing the $REPO_MIRROR
# path, so we can handle parallel jobs. First we open the lock file on fd
# 9, then we lock it, and close/unlock it after repo sync.
- if [ -n "$CI_ONIRO_RUNNER_PERSISTENT_STORAGE" ] ; then
REPO_MIRROR="$CI_ONIRO_RUNNER_PERSISTENT_STORAGE/meta-openharmony/repo-mirror" ;
REPO_MIRROR_ARG="--reference=$REPO_MIRROR" ;
mkdir -p $(dirname $REPO_MIRROR) ;
exec 9>$REPO_MIRROR.lock ; flock 9 ;
if [ -n "$REBUILD_REPO_MIRROR" ] ; then
rm -rf "$REPO_MIRROR" ;
fi ;
if [ ! -e "$REPO_MIRROR" ] ; then
echo "Creating new repo mirror @ $REPO_MIRROR" ;
mkdir -p "$REPO_MIRROR" ;
pushd "$REPO_MIRROR" ;
repo init -u "$CI_PROJECT_DIR" -m "$MANIFEST_FILE" --mirror ;
else
echo "Reusing repo mirror @ $REPO_MIRROR" ;
pushd "$REPO_MIRROR" ;
repo init -u "$CI_PROJECT_DIR" -m "$MANIFEST_FILE" ;
fi ;
repo sync --no-clone-bundle ;
popd ;
fi
# Create the build environment in a repo subdir, which links back to this
# directory with a meta-openharmony symlink
- mkdir repo && cd repo
- repo init -u "$CI_PROJECT_DIR" -m "$MANIFEST_FILE" $REPO_MIRROR_ARG
- repo sync --no-clone-bundle
- ln -s "$CI_PROJECT_DIR" meta-openharmony
# Release repo mirror lock if held
- if [ -n "$REPO_MIRROR_ARG" ] ; then exec 9>&- ; fi
# Set mtime for all checked out files to commit time, allowing bitbake
# parser cache to be re-used
- repo forall -c git restore-mtime -q
# Initialize bitbake build configuration
- TEMPLATECONF=../meta-openharmony/conf source oe-core/oe-init-build-env meta-openharmony/build
# Try to reduce disk usage while building
- echo 'INHERIT += "rm_work"' >> conf/auto.conf
# Setup to use mirror of downloads
- if [ -n "$SOURCE_MIRROR_URL" ] ; then
echo 'INHERIT += "own-mirrors"' ;
echo "SOURCE_MIRROR_URL = \"$SOURCE_MIRROR_URL\"" ;
else
if [ -n "$CI_ONIRO_RUNNER_PERSISTENT_STORAGE" ] ; then
SOURCE_MIRROR_PATH="$CI_ONIRO_RUNNER_PERSISTENT_STORAGE/meta-openharmony/source" ;
else
SOURCE_MIRROR_PATH="$CI_PROJECT_DIR/build/mirror" ;
fi ;
mkdir -p "$SOURCE_MIRROR_PATH" ;
echo 'INHERIT += "own-mirrors"' ;
echo "SOURCE_MIRROR_URL = \"file://$SOURCE_MIRROR_PATH\"" ;
fi >> conf/auto.conf
- if [ -n "$CI_ONIRO_RUNNER_PERSISTENT_STORAGE" ] ; then
SSTATE_MIRROR_PATH="$CI_ONIRO_RUNNER_PERSISTENT_STORAGE/meta-openharmony/sstate" ;
mkdir -p "$SSTATE_MIRROR_PATH" ;
echo "SSTATE_MIRRORS = \"file://.* file://$SSTATE_MIRROR_PATH/PATH\""
>> conf/auto.conf ;
fi
- MAX_JOBS="${CI_ONIRO_PARALLEL_BUILD_MAX_JOBS:-$(nproc)}"
- echo "BB_NUMBER_PARSE_THREADS = \"$MAX_JOBS\"" >> conf/auto.conf
- echo "BB_NUMBER_THREADS = \"$MAX_JOBS\"" >> conf/auto.conf
- echo "PARALLEL_MAKE = \"-j$MAX_JOBS\"" >> conf/auto.conf
- echo "NINJA_ARGS:append = \" -j$MAX_JOBS\"" >> conf/auto.conf
- MAX_LOADAVG="${CI_ONIRO_PARALLEL_BUILD_MAX_LOADAVG}"
- if [ -n "$MAX_LOADAVG" ] ; then
echo "PARALLEL_MAKE += \"-l$MAX_LOADAVG\"" ;
echo "NINJA_ARGS:append = \" -l$MAX_LOADAVG\"" ;
echo "MAXLOAD_NINJA = \"$MAX_LOADAVG\"" ;
fi >> conf/auto.conf
- echo "RM_WORK_EXCLUDE += \"${RECIPE}\"" >> conf/auto.conf
- echo "OPENHARMONY_VERSION = \"${OPENHARMONY_VERSION}\"" >> conf/auto.conf
script:
- bitbake "${RECIPE}"
after_script:
# Update sstate-cache mirror
- if [ -n "$CI_ONIRO_RUNNER_PERSISTENT_STORAGE" -a -d build/sstate-cache ] ; then
SSTATE_MIRROR_PATH="$CI_ONIRO_RUNNER_PERSISTENT_STORAGE/meta-openharmony/sstate" ;
cp -urn build/sstate-cache/* "$SSTATE_MIRROR_PATH/" ;
fi
cache:
key: bitbake-cache
paths:
- build/downloads/uninative
- build/mirror
- build/cache
- build/tmp*/cache
policy: pull
# Pre-fetch all download files to mirror maintained in GitLab CI cache, and
# provide artifacts for updating an external mirror.
#
# To overcome (reasonable) limits to artifacts size, we skip the huge
# OpenHarmony tarballs, which must be uploaded to external mirrors manually if
# needed.
fetch:
stage: fetch
extends: .bitbake
needs:
- job: bitbake-builder
artifacts: false
optional: true
script:
# Configure build to create mirror tarballs of VCS repositories
- echo 'BB_GENERATE_MIRROR_TARBALLS = "1"' >> conf/auto.conf
# Run fetch tasks for all builds
# For this fetch job, the OPENHARMONY_VERSION variable is set just here
# and was not set in the before_script section of the .bitbake job.
# We modify it here in auto.conf with a sed substitution as it was added
# there by .bitbake before_script but with an empty string.
- export DISTRO MACHINE
- for DISTRO in oniro-openharmony-linux ; do
for OPENHARMONY_VERSION in 3.1 3.0 ; do
for MACHINE in qemuarma7 ; do
RECIPES="oniro-openharmony-bundle openharmony-standard-image openharmony-standard-image-tests" ;
echo Fetching DISTRO=$DISTRO OPENHARMONY_VERSION=$OPENHARMONY_VERSION MACHINE=$MACHINE RECIPE=$RECIPE ;
sed -r "s/^OPENHARMONY_VERSION.*/OPENHARMONY_VERSION = \"${OPENHARMONY_VERSION}\"/" -i conf/auto.conf ;
time bitbake $RECIPES --runall=fetch ;
done ; done ; done
after_script:
# Copy all regular files from downloads dir (except *.done files) to
# mirror dirs. Note, file is first copied to a temporary file (in same
# filesystem), and then atomically moved, so no concurrent CI job will see
# partial mirror files.
- if [ -n "$CI_ONIRO_RUNNER_PERSISTENT_STORAGE" ] ; then
SOURCE_MIRROR_PATH="$CI_ONIRO_RUNNER_PERSISTENT_STORAGE/meta-openharmony/source" ;
else
SOURCE_MIRROR_PATH="$CI_PROJECT_DIR/build/mirror" ;
fi
- echo "Syncing mirror files to $SOURCE_MIRROR_PATH"
- mkdir -pv build/mirror-updates-ch
- rm -f build/downloads/*.done build/downloads/npm2/*.{done,resolved}
- for f in build/downloads/* build/downloads/npm2/* ; do if test -f "$f" -a ! -L "$f" ; then
fname=$(basename "$f") ;
if [ -n "$SOURCE_MIRROR_PATH" -a ! -e "$SOURCE_MIRROR_PATH/$fname" ] ; then
cp "$f" "$SOURCE_MIRROR_PATH/$fname~tmp" ;
mv "$SOURCE_MIRROR_PATH/$fname~tmp" "$SOURCE_MIRROR_PATH/$fname" ;
echo "Persisted mirror file $SOURCE_MIRROR_PATH/$fname" ;
fi ;
if ! curl --output /dev/null --silent --head --fail "$SOURCE_MIRROR_URL_CH/$fname" ; then
ln -v "$f" build/mirror-updates-ch/ ;
fi ;
fi ; done
cache:
policy: pull-push
when: always
artifacts:
paths:
- build/mirror-updates-ch/
exclude:
- build/mirror-updates-ch/code-*.tar.gz
expire_in: 1 day
# Update source mirror in China
mirror_ch:
stage: fetch
needs:
- job: fetch
artifacts: true
rules:
- if: $CI_SERVER_HOST == "git.ostc-eu.org"
# We don't want to block MRs due to issues with mirror server
allow_failure: true
tags:
- huawei-cloud-ch
image:
name: ubuntu:20.04
script:
- for f in build/mirror-updates-ch/* ; do
if test ! -e /var/www/source-mirror/$(basename "$f") ; then
cp -v "$f" /var/www/source-mirror/ ;
fi ;
done
- template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml'
# Build OpenHarmony images using normal CI runners
build:
build-job:
stage: build
needs:
- job: fetch
artifacts: false
rules:
- if: $CI_SERVER_HOST == "git.ostc-eu.org" && $CHINA_CI != null
# Don't use CI resources on normal builds when doing China builds
when: never
- when: on_success
extends: .bitbake
parallel:
matrix:
- DISTRO: [oniro-openharmony-linux]
OPENHARMONY_VERSION: ["3.1", "3.0"]
MACHINE: [qemuarma7]
RECIPE: [oniro-openharmony-toolchain, oniro-openharmony-bundle, openharmony-standard-image]
artifacts:
paths:
- build/tmp-*/deploy/images/${MACHINE}
- build/tmp-*/deploy/sdk
expire_in: 1 month
# Build and execute OpenHarmony test image.
#
# This will execute all included ptest test suites.
ptest:
stage: test
needs:
- job: build
artifacts: false
rules:
- if: $CI_SERVER_HOST == "git.ostc-eu.org" && $CHINA_CI != null
# Don't use CI resources on normal tests when doing China builds
when: never
- when: on_success
extends: .bitbake
parallel:
matrix:
- DISTRO: [oniro-openharmony-linux]
OPENHARMONY_VERSION: ["3.1", "3.0"]
MACHINE: [qemuarma7]
RECIPE: [openharmony-standard-image-tests]
artifacts:
paths:
- build/tmp-*/work/*/*/*/testimage
expire_in: 1 month
when: always
# Build OpenHarmony images using CI runner in China.
#
# Note, you might want to allow the mirror_ch job to complete before running
# this job, or you could see this job failing on problems fetching new
# downloads blocked by firewall rules. Take care.
build_ch:
extends: build
needs:
- job: mirror_ch
artifacts: false
rules:
- if: $CI_SERVER_HOST == "git.ostc-eu.org" && $CHINA_CI != null
tags:
- region.china
- cpu.4
- mem.16
- large-disk
variables:
MANIFEST_FILE: manifests/gitee/ci.xml
SOURCE_MIRROR_URL: $SOURCE_MIRROR_URL_CH
# URL and filename for tarball containing entire OpenHarmony OS codebase
.openharmony_tarball:
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- if: $CI_COMMIT_BRANCH
when: manual
# We don't really want these jobs to fail, but we don't want to block
# pipeline and dependent jobs either
allow_failure: true
before_script:
- if [ "${OPENHARMONY_VERSION}" = "3.0" ]; then
TARBALL_URL="https://repo.huaweicloud.com/harmonyos/os/3.0.1/code-v3.0.1-LTS.tar.gz";
elif [ "${OPENHARMONY_VERSION}" = "3.1" ]; then
TARBALL_URL="https://repo.huaweicloud.com/harmonyos/os/3.1.1/code-v3.1.1-Release.tar.gz";
else
echo "OPENHARMONY_VERSION=${OPENHARMONY_VERSION} is not supported";
exit 1;
fi
- TARBALL_FILE=$(basename $TARBALL_URL)
- mkdir -p downloads
# Use runner persistent storage if available
- if [ -n "$CI_ONIRO_RUNNER_PERSISTENT_STORAGE" ] ; then
TARBALL_FILE="$CI_ONIRO_RUNNER_PERSISTENT_STORAGE/$TARBALL_FILE" ;
touch downloads/.dummy_file ;
else
TARBALL_FILE="downloads/$TARBALL_FILE" ;
fi ;
echo "Using $TARBALL_FILE"
# Fetch OpenHarmony OS codebase.
#
# This is for use in ohos test jobs.
ohos_fetch:
stage: fetch
image:
name: $OPENHARMONY_BUILDER:$OPENHARMONY_CONTAINER_VERSION
extends: .openharmony_tarball
parallel:
matrix:
- OPENHARMONY_VERSION: ["3.0", "3.1"]
script:
# Integrity check of cached tarball
- if [ -f "${TARBALL_FILE}.sha256" -a -f "${TARBALL_FILE}" ] ; then
read tarball_sha256 _ <<<$(sha256sum "${TARBALL_FILE}") ;
read expected_sha256 _ < "${TARBALL_FILE}.sha256" ;
if [ "$tarball_sha256" = "$expected_sha256" ] ; then
echo "Using cached ${TARBALL_FILE}" ;
exit 0 ;
else
echo "Deleting bad ${TARBALL_FILE}" ;
rm -f "${TARBALL_FILE}" "${TARBALL_FILE}.sha256" ;
fi ;
elif [ -f "${TARBALL_FILE}.sha256" ] ; then
echo "Deleting ${TARBALL_FILE}.sha256" ;
rm -f "${TARBALL_FILE}.sha256" ;
fi
- pushd $(dirname $TARBALL_FILE)
# Download tarball (should continue if partial download is found
- wget --progress=dot:giga "${TARBALL_URL}" --timestamping --no-if-modified-since
# Download sha256 signature
- wget "${TARBALL_URL}.sha256" --timestamping --no-if-modified-since
- popd
- read tarball_sha256 _ <<<$(sha256sum "${TARBALL_FILE}") ;
- read expected_sha256 _ < "${TARBALL_FILE}.sha256" ;
- if [ "$tarball_sha256" != "$expected_sha256" ] ; then
echo "Deleting bad ${TARBALL_FILE}" ;
rm -f "${TARBALL_FILE}" "${TARBALL_FILE}.sha256" ;
exit 1 ;
fi
artifacts:
paths:
- downloads
expire_in: 1 month
# Test prebuilts images against OpenHarmony OS codebase
ohos:
stage: test
needs:
- job: ohos_fetch
artifacts: true
- job: build
artifacts: true
parallel:
matrix:
- OPENHARMONY_VERSION: ["3.0", "3.1"]
PREBUILT_IMAGE: [oniro-openharmony-toolchain, oniro-openharmony-bundle]
image:
name: $OPENHARMONY_BUILDER:$OPENHARMONY_CONTAINER_VERSION
extends: .openharmony_tarball
script:
- tar xfz $TARBALL_FILE
- cd $(basename $TARBALL_FILE .tar.gz)/OpenHarmony
- ../../build/tmp-*/deploy/sdk/${PREBUILT_IMAGE}-${OPENHARMONY_VERSION}-*.sh -y -d oniro
- ./oniro/setup.sh
- MAX_JOBS="${CI_ONIRO_PARALLEL_BUILD_MAX_JOBS:-$(nproc)}"
- NINJA_ARGS="--ninja-args -j$MAX_JOBS"
- MAX_LOADAVG="${CI_ONIRO_PARALLEL_BUILD_MAX_LOADAVG}"
- if [ -n "$MAX_LOADAVG" ] ; then
NINJA_ARGS+=" --ninja-args -l$MAX_LOADAVG" ;
fi
- ./build.sh --product-name Hi3516DV300 $NINJA_ARGS
# Job for triggering a prune of the sstate-cache mirror in runner persistent
# storage. This should be done periodically, but as I believe it removes a
# bit too much, not too often. And a complete rebuild is done also, to refill
# the cache.
#
# To run it, use "CI/CD" -> "Pipelines" -> "Run pipeline", and then define
# PRUNE_SSTATE_CACHE variable (any value will do).
prune_sstate_cache:
stage: maintenance
rules:
- if: $PRUNE_SSTATE_CACHE != null && $CLEAR_PERSISTENT_STORAGE == null
extends: .bitbake
script:
- cd "$CI_PROJECT_DIR"
- if [ -n "$CI_ONIRO_RUNNER_PERSISTENT_STORAGE" ] ; then
SSTATE_MIRROR_PATH="$CI_ONIRO_RUNNER_PERSISTENT_STORAGE/meta-openharmony/sstate" ;
SSTATE_MIRROR_ARCHS=$(
find "$SSTATE_MIRROR_PATH" \( -name \*.tgz -o -name \*.tar.zst \) -exec basename {} \; |
awk -F ":" '{print $3}' | grep . | sort -u |
tr '\n' ',' | sed 's/,$//');
./repo/oe-core/scripts/sstate-cache-management.sh
--cache-dir="$SSTATE_MIRROR_PATH"
--extra-archs="$SSTATE_MIRROR_ARCHS"
--remove-duplicated -y ;
fi
# Manual job for completely clearing source and sstate-cache mirror
# directories in CI runner persistent storage. This should normally not be
# necessary, and is therefore not added as a proper manual job. To run it,
# use "CI/CD" -> "Pipelines" -> "Run pipeline", and then define
# CLEAR_PERSISTENT_STORAGE variable (any value will do).
clear_persistent_storage:
stage: maintenance
rules:
- if: $CLEAR_PERSISTENT_STORAGE != null
image:
name: ubuntu:20.04
script:
- if [ -n "$CI_ONIRO_RUNNER_PERSISTENT_STORAGE" ] ; then
rm -rf "$CI_ONIRO_RUNNER_PERSISTENT_STORAGE/meta-openharmony" ;
fi
- echo "Hello, $GITLAB_USER_LOGIN!"
\ No newline at end of file

Copyright © Eclipse Foundation, Inc. All Rights Reserved.     Privacy Policy | Terms of Use | Copyright Agent