From e4fcf3e8da5d62f55a76b6a3208264d661eb163d Mon Sep 17 00:00:00 2001
From: Chase Qi <chase.qi@linaro.org>
Date: Tue, 31 May 2022 12:12:45 +0800
Subject: [PATCH] ci: add oe-selftest gcc test jobs
gcc test suite has been integrated in oe-selftest framework. It allows
to run gcc test against qemu vm and within the build env using qemu
linux-user mode.
These tests can be run in gitlab CI with qemu friendly runners. This
commit add three oe-selftest jobs to run the tests using qemu
linux-user, qemu x86-64 vm and qemu arm64 vm.
Test log and sum files are collected as job artifacts.
When all test jobs finish, report job will be triggered to parse the
sum files to generate results in the format that supported by squad. The
report job will upload results to squad for reporting.
Closes issues #8 and #9.
Signed-off-by: Chase Qi <chase.qi@linaro.org>
---
.gitlab-ci.yml | 89 +++++++++++++++++++
.oniro-ci/build-generic.yaml | 38 ++++++++
.../containers/ubuntu-checkbox/.gitconfig | 6 ++
.../containers/ubuntu-checkbox/Dockerfile | 8 ++
docs/ci/hidden-jobs/index.rst | 1 +
docs/ci/hidden-jobs/oe-selftest.rst | 30 +++++++
6 files changed, 172 insertions(+)
create mode 100644 .oniro-ci/containers/ubuntu-checkbox/.gitconfig
create mode 100644 .oniro-ci/containers/ubuntu-checkbox/Dockerfile
create mode 100644 docs/ci/hidden-jobs/oe-selftest.rst
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 182e3136..9db233df 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -529,3 +529,92 @@ build-npm-cspell:
extends: .build-with-kaniko
variables:
CONTAINER_PATH: npm-cspell
+
+.oe-selftest-rules:
+ rules:
+ - if: $CI_PIPELINE_SOURCE == "schedule" && $OE_SELFTEST == "gcc"
+
+oe-selftest-gcc-qemu-linux-user:
+ extends: [.oe-selftest, .oe-selftest-rules]
+ variables:
+ # Set the variable to 'gcc' will run all tests defined in the gcc.py file.
+ # https://github.com/openembedded/openembedded-core/blob/master/meta/lib/oeqa/selftest/cases/gcc.py#L79
+ CI_ONIRO_OE_SELFTESTS:
+ gcc.GccCrossSelfTest
+ gcc.GccLibAtomicSelfTest
+ gcc.GccLibGompSelfTest
+ gcc.GccLibItmSelfTest
+ gcc.GccLibSspSelfTest
+ gcc.GccLibStdCxxSelfTest
+ gcc.GxxCrossSelfTest
+
+.oe-selftest-gcc-system-emulated:
+ timeout: 8h
+ extends: [.oe-selftest, .oe-selftest-rules]
+ variables:
+ MACHINE: qemux86-64
+ CI_ONIRO_OE_SELFTESTS:
+ gcc.GccCrossSelfTestSystemEmulated
+ gcc.GccLibAtomicSelfTestSystemEmulated
+ gcc.GccLibGompSelfTestSystemEmulated
+ gcc.GccLibItmSelfTestSystemEmulated
+ gcc.GccLibStdCxxSelfTestSystemEmulated
+ gcc.GccLibSspSelfTestSystemEmulated
+ gcc.GxxCrossSelfTestSystemEmulated
+
+oe-selftest-gcc-system-emulated-qemux86-64:
+ extends: .oe-selftest-gcc-system-emulated
+ variables:
+ MACHINE: qemux86-64
+
+oe-selftest-gcc-system-emulated-qemuarm64:
+ extends: .oe-selftest-gcc-system-emulated
+ variables:
+ MACHINE: qemuarm64
+
+oe-selftest-report:
+ timeout: 6h
+ extends: .oe-selftest-rules
+ image:
+ name: registry.ostc-eu.org/ostc/oniro/bitbake-builder:latest
+ stage: report
+ dependencies:
+ - oe-selftest-gcc-qemu-linux-user
+ - oe-selftest-gcc-system-emulated-qemux86-64
+ - oe-selftest-gcc-system-emulated-qemuarm64
+ needs:
+ - oe-selftest-gcc-qemu-linux-user
+ - oe-selftest-gcc-system-emulated-qemux86-64
+ - oe-selftest-gcc-system-emulated-qemuarm64
+ script:
+ - |
+ set -x
+ curl https://git.ostc-eu.org/OSTC/infrastructure/lava/lava-test-definitions/-/raw/gcc-test/automated/utils/gcc-result-parser.py \
+ > gcc-result-parser.py
+ for job_path in $(find artifacts/ -type d -name oe-selftest-gcc*); do
+ (
+ job=$(basename $job_path)
+ test_env="$(echo $job | cut -d'-' -f4-)"
+ cd $job_path
+ xz -d *.xz
+ # Parse sum file.
+ for sum_file in $(ls *.sum); do
+ python3 $CI_PROJECT_DIR/gcc-result-parser.py $sum_file
+ done
+ # Upload result to squad.
+ for result_file in $(ls result_*.json); do
+ component="$(basename $result_file .json | cut -d'_' -f2-)"
+ job_id="$job-$component"
+ time curl \
+ --header "Auth-Token: $CI_SQUAD_TOKEN" \
+ https://squadp.svc.ostc-eu.dev/api/submit/oniro-core/gcc-test/$CI_PIPELINE_ID/$test_env \
+ --form tests=@$result_file \
+ --form metadata='{"job_id": "'$job_id'"}'
+ done
+ )
+ done
+ set +x
+ artifacts:
+ paths:
+ - artifacts/oe-selftest-gcc*/*.json
+ - artifacts/oe-selftest-gcc*/*.txt
diff --git a/.oniro-ci/build-generic.yaml b/.oniro-ci/build-generic.yaml
index cf04b0e9..9b490352 100644
--- a/.oniro-ci/build-generic.yaml
+++ b/.oniro-ci/build-generic.yaml
@@ -468,3 +468,41 @@
# Run this job in case the pipeline changes.
- .oniro-ci/*.yml
- .gitlab-ci.yml
+
+# This job is documented in docs/ci/hidden-jobs/oe-selftest.rst
+.oe-selftest:
+ image:
+ name: registry.ostc-eu.org/ostc/oniro/bitbake-builder:latest
+ tags: [qemu-friendly, large-disk, $CI_ONIRO_RUNNER_TAG, $CI_ONIRO_INSTANCE_SIZE]
+ stage: test
+ dependencies: []
+ extends: .bitbake-workspace
+ variables:
+ MACHINE: qemux86-64
+ CI_ONIRO_BUILD_FLAVOUR: linux
+ CI_ONIRO_BB_LOCAL_CONF_SANITY_TESTED_DISTROS: ""
+ CI_ONIRO_OE_SELFTESTS: ""
+ CI_ONIRO_OE_SEFLTEST_SKIPS: ""
+ # The USER variable is required by oe-selftest but missing in env.
+ USER: "builder"
+ ARTIFACTS_DIR: "$CI_PROJECT_DIR/artifacts/$CI_JOB_NAME"
+ script:
+ - test -n "$CI_ONIRO_OE_SELFTESTS" || (
+ echo "nothing to test - CI_ONIRO_OE_SELFTESTS is empty"
+ && exit 1 )
+ - !reference [.bitbake-workspace, script]
+ # oe-selftest inherits the current 'build/conf/local.conf' to create
+ # '../build-st/conf/local.conf' for qemu image building. Adding the
+ # MACHINE variable to the current local conf allows test job to customize
+ # the qemu image arch for testing.
+ - echo "MACHINE = \"$MACHINE\"" >> conf/local.conf
+ - echo "Running specific tests $CI_ONIRO_OE_SELFTESTS ..."
+ - time oe-selftest -vvv -r $CI_ONIRO_OE_SELFTESTS -K
+ - rm -rf "$ARTIFACTS_DIR" & mkdir -p "$ARTIFACTS_DIR"
+ - find ../build-st \( -name "*.sum" -o -name "*.log" \) \( -path "*/gcc-runtime/*testsuite*" \)
+ -exec cp --verbose {} "$ARTIFACTS_DIR" \;
+ - for file in $(find "$ARTIFACTS_DIR" -type f); do xz $file; done
+ - rm -rf ../build-st
+ artifacts:
+ paths:
+ - $ARTIFACTS_DIR
diff --git a/.oniro-ci/containers/ubuntu-checkbox/.gitconfig b/.oniro-ci/containers/ubuntu-checkbox/.gitconfig
new file mode 100644
index 00000000..bd53c98f
--- /dev/null
+++ b/.oniro-ci/containers/ubuntu-checkbox/.gitconfig
@@ -0,0 +1,6 @@
+# SPDX-FileCopyrightText: 2021 Huawei Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+[user]
+ name = Oniro Project CI
+ email = oniro-dev@eclipse.org
diff --git a/.oniro-ci/containers/ubuntu-checkbox/Dockerfile b/.oniro-ci/containers/ubuntu-checkbox/Dockerfile
new file mode 100644
index 00000000..43ac2be3
--- /dev/null
+++ b/.oniro-ci/containers/ubuntu-checkbox/Dockerfile
@@ -0,0 +1,8 @@
+FROM ubuntu:20.04
+
+ARG DEBIAN_FRONTEND="noninteractive"
+RUN sed -i 's/archive.ubuntu.com/mirrors.ustc.edu.cn/g' /etc/apt/sources.list
+RUN apt-get update -qq \
+ && apt-get install -y software-properties-common \
+ && add-apt-repository -y ppa:hardware-certification/public \
+ && apt-get install -y checkbox-ng plainbox-provider-resource-generic plainbox-provider-certification-client plainbox-provider-checkbox canonical-certification-client jq
diff --git a/docs/ci/hidden-jobs/index.rst b/docs/ci/hidden-jobs/index.rst
index 39ba17d6..2d582a84 100644
--- a/docs/ci/hidden-jobs/index.rst
+++ b/docs/ci/hidden-jobs/index.rst
@@ -29,3 +29,4 @@ implementation details.
lava-report
aggregate-docs
publish-rauc-bundle-to-hawkbit
+ oe-selftest
diff --git a/docs/ci/hidden-jobs/oe-selftest.rst b/docs/ci/hidden-jobs/oe-selftest.rst
new file mode 100644
index 00000000..c297e0bb
--- /dev/null
+++ b/docs/ci/hidden-jobs/oe-selftest.rst
@@ -0,0 +1,30 @@
+.. SPDX-FileCopyrightText: Huawei Inc.
+..
+.. SPDX-License-Identifier: CC-BY-4.0
+
+============
+.oe-selftest
+============
+
+The ``.oe-selftest`` job extends the :doc:`bitbake-workspace` job to configure
+OpenEmbedded Self test environment. The job does not run any tests by itself,
+actual tests should be run in the downstream test jobs.
+
+Job Variables
+=============
+
+Oe-selftest is based on Python unitest. Tests are organized by module, class
+and method. The ``.oe-selftest`` job defines two variables as a way to
+customize what tests to run.
+
+CI_ONIRO_OE_SELFTESTS
+---------------------
+
+The specific tests to run. The default value is empty. The order the tests are
+running is alphabetical.
+
+
+CI_ONIRO_OE_SEFLTEST_SKIPS
+--------------------------
+
+The specific tests to skip. The default value is empty.
--
GitLab