Skip to content
Snippets Groups Projects
Commit 66c95aec authored by Esben Haabendal's avatar Esben Haabendal
Browse files

CI: Test toolchain and bundle images against OHOS codebase


Fetch OpenHarmony OS codebase release tarballs and test
oniro-openharmony-toolchain and oniro-openharmony-bundle against these.

In order to save CI resources, only MR pipelines run these new jobs
automatically.  For normal branch pipelines, the job are set to manual, so
developer can choose to run them as needed.

Signed-off-by: default avatarEsben Haabendal <esben.haabendal@huawei.com>
parent d020802c
No related branches found
No related tags found
1 merge request!20CI: Test toolchain and bundle images against OHOS codebase
...@@ -52,6 +52,8 @@ variables: ...@@ -52,6 +52,8 @@ variables:
# to push to git.ostc-eu.org first, and wait for the bitbake-builder image to # to push to git.ostc-eu.org first, and wait for the bitbake-builder image to
# be pushed to the registry before starting a pipeline on gitlab.eclipse.org. # be pushed to the registry before starting a pipeline on gitlab.eclipse.org.
BITBAKE_BUILDER: "registry.ostc-eu.org/ostc/pre-integration/meta-openharmony/bitbake-builder" BITBAKE_BUILDER: "registry.ostc-eu.org/ostc/pre-integration/meta-openharmony/bitbake-builder"
# And same for openharmony-builder
OPENHARMONY_BUILDER: "registry.ostc-eu.org/ostc/pre-integration/meta-openharmony/openharmony-builder"
stages: stages:
- container - container
...@@ -274,8 +276,10 @@ build: ...@@ -274,8 +276,10 @@ build:
- build/tmp-*/deploy/sdk - build/tmp-*/deploy/sdk
expire_in: 1 month expire_in: 1 month
# Build OpenHarmony images using normal CI runners # Build and execute OpenHarmony test image.
test: #
# This will execute all included ptest test suites.
ptest:
stage: test stage: test
needs: needs:
- job: build - job: build
...@@ -319,6 +323,103 @@ build_ch: ...@@ -319,6 +323,103 @@ build_ch:
MANIFEST_FILE: manifests/gitee/ci.xml MANIFEST_FILE: manifests/gitee/ci.xml
SOURCE_MIRROR_URL: $SOURCE_MIRROR_URL_CH SOURCE_MIRROR_URL: $SOURCE_MIRROR_URL_CH
# URL and filename for tarball containing entire OpenHarmony OS codebase
.openharmony_tarball:
before_script:
- if [ "${OPENHARMONY_VERSION}" = "3.0" ]; then
TARBALL_URL="https://repo.huaweicloud.com/harmonyos/os/3.0/code-v3.0-LTS.tar.gz";
elif [ "${OPENHARMONY_VERSION}" = "3.0.1" ]; then
TARBALL_URL="https://repo.huaweicloud.com/harmonyos/os/3.0.1/code-v3.0.1-LTS.tar.gz";
elif [ "${OPENHARMONY_VERSION}" = "3.0.2" ]; then
TARBALL_URL="https://repo.huaweicloud.com/harmonyos/os/3.0.2/code-v3.0.2-LTS.tar.gz";
elif [ "${OPENHARMONY_VERSION}" = "3.0.3" ]; then
TARBALL_URL="https://repo.huaweicloud.com/harmonyos/os/3.0.3/code-v3.0.3-LTS.tar.gz";
elif [ "${OPENHARMONY_VERSION}" = "3.1" ]; then
TARBALL_URL="https://repo.huaweicloud.com/harmonyos/os/3.1-Release/code-v3.1-Release.tar.gz";
elif [ "${OPENHARMONY_VERSION}" = "3.1.1" ]; then
TARBALL_URL="https://repo.huaweicloud.com/harmonyos/os/3.1.1/code-v3.1.1-Release.tar.gz";
else
echo "OPENHARMONY_VERSION=${OPENHARMONY_VERSION} is not supported";
exit 1;
fi
- TARBALL_FILE=$(basename $TARBALL_URL)
- mkdir -p downloads
# Use runner persistent storage if available
- if [ -n "$CI_ONIRO_RUNNER_PERSISTENT_STORAGE" ] ; then
TARBALL_FILE="$CI_ONIRO_RUNNER_PERSISTENT_STORAGE/$TARBALL_FILE" ;
touch downloads/.dummy_file ;
else
TARBALL_FILE="downloads/$TARBALL_FILE" ;
fi ;
echo "Using $TARBALL_FILE"
# Fetch OpenHarmony OS codebase.
#
# This is for use in ohos test jobs.
ohos_fetch:
stage: fetch
image:
name: $OPENHARMONY_BUILDER:$OPENHARMONY_CONTAINER_VERSION
extends: .openharmony_tarball
parallel:
matrix:
- OPENHARMONY_VERSION: ["3.0", "3.1"]
script:
# Integrity check of cached tarball
- if [ -f "${TARBALL_FILE}.sha256" -a -f "${TARBALL_FILE}" ] ; then
read tarball_sha256 _ <<<$(sha256sum "${TARBALL_FILE}") ;
read expected_sha256 _ < "${TARBALL_FILE}.sha256" ;
if [ "$tarball_sha256" = "$expected_sha256" ] ; then
echo "Using cached ${TARBALL_FILE}" ;
exit 0 ;
else
echo "Deleting bad ${TARBALL_FILE}" ;
rm -f "${TARBALL_FILE}" "${TARBALL_FILE}.sha256" ;
fi ;
elif [ -f "${TARBALL_FILE}.sha256" ] ; then
echo "Deleting ${TARBALL_FILE}.sha256" ;
rm -f "${TARBALL_FILE}.sha256" ;
fi
- pushd $(dirname $TARBALL_FILE)
# Download tarball (should continue if partial download is found
- wget --progress=dot:giga "${TARBALL_URL}" --timestamping --no-if-modified-since
# Download sha256 signature
- wget "${TARBALL_URL}.sha256" --timestamping --no-if-modified-since
- popd
- read tarball_sha256 _ <<<$(sha256sum "${TARBALL_FILE}") ;
- read expected_sha256 _ < "${TARBALL_FILE}.sha256" ;
- if [ "$tarball_sha256" != "$expected_sha256" ] ; then
echo "Deleting bad ${TARBALL_FILE}" ;
rm -f "${TARBALL_FILE}" "${TARBALL_FILE}.sha256" ;
exit 1 ;
fi
artifacts:
paths:
- downloads
expire_in: 1 month
# Test prebuilts images against OpenHarmony OS codebase
ohos:
stage: test
needs:
- job: ohos_fetch
artifacts: true
- job: build
artifacts: true
parallel:
matrix:
- OPENHARMONY_VERSION: ["3.0", "3.1"]
PREBUILT_IMAGE: [oniro-openharmony-toolchain, oniro-openharmony-bundle]
image:
name: $OPENHARMONY_BUILDER:$OPENHARMONY_CONTAINER_VERSION
extends: .openharmony_tarball
script:
- tar xfz $TARBALL_FILE
- cd $(basename $TARBALL_FILE .tar.gz)/OpenHarmony
- ../../build/tmp-*/deploy/sdk/${PREBUILT_IMAGE}-${OPENHARMONY_VERSION}-*.sh -y -d oniro
- ./oniro/setup.sh
- ./build.sh --product-name Hi3516DV300
# Job for triggering a prune of the sstate-cache mirror in runner persistent # Job for triggering a prune of the sstate-cache mirror in runner persistent
# storage. This should be done periodically, but as I believe it removes a # storage. This should be done periodically, but as I believe it removes a
# bit too much, not too often. And a complete rebuild is done also, to refill # bit too much, not too often. And a complete rebuild is done also, to refill
......
...@@ -15,6 +15,7 @@ ...@@ -15,6 +15,7 @@
variables: variables:
BITBAKE_CONTAINER_VERSION: "0.6" BITBAKE_CONTAINER_VERSION: "0.6"
OPENHARMONY_CONTAINER_VERSION: "0.1"
.kaniko: .kaniko:
stage: container stage: container
...@@ -83,3 +84,21 @@ bitbake-toolbox: ...@@ -83,3 +84,21 @@ bitbake-toolbox:
needs: needs:
- job: bitbake-builder - job: bitbake-builder
artifacts: false artifacts: false
openharmony-builder:
extends: .kaniko
variables:
CONTAINER_NAME: openharmony-builder
CONTAINER_VERSION: $OPENHARMONY_CONTAINER_VERSION
openharmony-toolbox:
extends: .kaniko
variables:
CONTAINER_NAME: openharmony-toolbox
CONTAINER_VERSION: $OPENHARMONY_CONTAINER_VERSION
CONTAINER_PATH: openharmony-builder
CONTAINER_DOCKERFILE: Dockerfile.toolbox
CONTAINER_FROM: openharmony-builder
needs:
- job: openharmony-builder
artifacts: false
# SPDX-FileCopyrightText: 2021 Huawei Inc.
# SPDX-License-Identifier: Apache-2.0
[user]
name = OSTC Builder
email = ostc-builder@example.org
# Copyright 2021 Huawei Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# SPDX-License-Identifier: Apache-2.0
# This builds a container image for use in building OpenHarmony images,
# following the build system specification in
# https://repo.huaweicloud.com/harmonyos/os/2.0/tool_chain/Standard_System.md
FROM ubuntu:18.04
ARG DEBIAN_FRONTEND="noninteractive"
RUN apt-get update -qq \
&& apt-get install -qq -y \
bash locales \
apt-utils vim software-properties-common openssh-server iputils-ping \
curl net-tools bsdmainutils kmod bc rsync gawk ssh ccache zip python-dev \
make m4 gcc-multilib ca-certificates-java unzip python3-yaml perl \
openssl libssl1.1 gnupg xsltproc x11proto-core-dev tcl python3-crypto \
python-crypto libxml2-utils libxml2-dev libx11-dev libssl-dev \
libgl1-mesa-dev lib32z1-dev lib32ncurses5-dev g++-multilib flex bison \
doxygen git subversion tofrodos pigz expect python3-xlrd python-xlrd \
git-core gperf build-essential zlib1g-dev libc6-dev-i386 lib32z-dev \
openjdk-8-jdk \
&& apt-get clean && rm -rf /var/lib/apt/lists/*
RUN locale-gen
# Let's just have /bin/sh as bash
RUN echo "dash dash/sh boolean false" | debconf-set-selections \
&& dpkg-reconfigure dash
# For some reason for containers built using kaniko ping errors with:
# `permission denied` for user other than root. Even running container in
# privileged mode doesn't help for non-root user. Most probably it is related
# to this issue: https://github.com/GoogleContainerTools/kaniko/issues/1851
# ping is required by Yocto testing machinery and therefore temporarily setting
# suid to get it working
RUN chmod u+s $(command -v ping)
RUN apt-get update -qq \
&& apt-get install -qq -y python3.8 python3-pip \
&& apt-get clean && rm -rf /var/lib/apt/lists/*
RUN update-alternatives --install /usr/bin/python python /usr/bin/python2.7 2 \
&& update-alternatives --install /usr/bin/python python /usr/bin/python3.6 3 \
&& update-alternatives --install /usr/bin/python python /usr/bin/python3.8 4 \
&& update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 2 \
&& update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 3
RUN pip3 install --trusted-host mirrors.aliyun.com -i https://mirrors.aliyun.com/pypi/simple setuptools \
&& pip3 install --trusted-host mirrors.aliyun.com -i https://mirrors.aliyun.com/pypi/simple kconfiglib \
&& pip3 install --trusted-host mirrors.aliyun.com -i http://mirrors.aliyun.com/pypi/simple pycryptodome \
&& pip3 install --trusted-host mirrors.aliyun.com -i https://mirrors.aliyun.com/pypi/simple ecdsa \
&& python3 -m pip install ohos-build==0.4.6
RUN mkdir -p /opt/harmonyos
RUN wget -q -P /opt/harmonyos https://repo.huaweicloud.com/harmonyos/compiler/clang/10.0.1-62608/linux/llvm.tar.gz \
&& tar xf /opt/harmonyos/llvm.tar.gz -C /opt/harmonyos \
&& rm /opt/harmonyos/llvm.tar.gz
RUN wget -q -P /opt/harmonyos https://repo.huaweicloud.com/harmonyos/compiler/hc-gen/0.65/linux/hc-gen-0.65-linux.tar \
&& tar xf /opt/harmonyos/hc-gen-0.65-linux.tar -C /opt/harmonyos \
&& rm /opt/harmonyos/hc-gen-0.65-linux.tar
RUN wget -q -P /opt/harmonyos https://repo.huaweicloud.com/harmonyos/compiler/gcc_riscv32/7.3.0/linux/gcc_riscv32-linux-7.3.0.tar.gz \
&& tar xf /opt/harmonyos/gcc_riscv32-linux-7.3.0.tar.gz -C /opt/harmonyos \
&& rm /opt/harmonyos/gcc_riscv32-linux-7.3.0.tar.gz
RUN wget -q -P /opt/harmonyos https://repo.huaweicloud.com/harmonyos/compiler/ninja/1.9.0/linux/ninja.1.9.0.tar \
&& tar xf /opt/harmonyos/ninja.1.9.0.tar -C /opt/harmonyos \
&& rm /opt/harmonyos/ninja.1.9.0.tar
RUN wget -q -P /opt/harmonyos https://repo.huaweicloud.com/harmonyos/compiler/gn/1717/linux/gn-linux-x86-1717.tar.gz \
&& tar xf /opt/harmonyos/gn-linux-x86-1717.tar.gz -C /opt/harmonyos \
&& rm /opt/harmonyos/gn-linux-x86-1717.tar.gz
RUN wget -q -P /opt/harmonyos https://mirrors.huaweicloud.com/nodejs/v12.20.0/node-v12.20.0-linux-x64.tar.gz \
&& tar xf /opt/harmonyos/node-v12.20.0-linux-x64.tar.gz -C /opt/harmonyos \
&& rm /opt/harmonyos/node-v12.20.0-linux-x64.tar.gz
RUN wget -q -P /opt/harmonyos https://repo.huaweicloud.com/harmonyos/develop_tools/hmos_app_packing_tool.jar
RUN wget -q -P /opt/harmonyos https://repo.huaweicloud.com/harmonyos/develop_tools/hapsigntoolv2.jar
RUN sed -i '$aexport PATH=/opt/harmonyos/llvm/bin:$PATH' /root/.bashrc \
&& sed -i '$aexport PATH=/opt/harmonyos/hc-gen:$PATH' /root/.bashrc \
&& sed -i '$aexport PATH=/opt/harmonyos/gcc_riscv32/bin:$PATH' /root/.bashrc \
&& sed -i '$aexport PATH=/opt/harmonyos/ninja:$PATH' /root/.bashrc \
&& sed -i '$aexport PATH=/opt/harmonyos/node-v12.20.0-linux-x64/bin:$PATH' /root/.bashrc \
&& sed -i '$aexport PATH=/opt/harmonyos:$PATH' /root/.bashrc \
&& sed -i '$aexport PATH=/root/.local/bin:$PATH' /root/.bashrc \
&& export PATH=/opt/harmonyos/llvm/bin:$PATH \
&& export PATH=/opt/harmonyos/hc-gen:$PATH \
&& export PATH=/opt/harmonyos/gcc_riscv32/bin:$PATH \
&& export PATH=/opt/harmonyos/ninja:$PATH \
&& export PATH=/opt/harmonyos/node-v12.20.0-linux-x64/bin:$PATH \
&& export PATH=/opt/harmonyos:$PATH \
&& export PATH=/root/.local/bin:$PATH \
&& npm install -g @ohos/hpm-cli --registry https://mirrors.huaweicloud.com/repository/npm/
RUN useradd --create-home --uid 1000 --shell /usr/bin/bash builder
COPY --chown=builder:builder .gitconfig /home/builder/.gitconfig
USER builder
WORKDIR /home/builder
# Copyright 2020 Huawei Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# SPDX-License-Identifier: Apache-2.0
FROM openharmony-builder
# Revert setup with builder user account
USER root
WORKDIR /
RUN userdel -r builder
# Install packages needed by Toolbox
ARG DEBIAN_FRONTEND="noninteractive"
RUN apt-get update -qq \
&& apt-get install -qq -y \
bash sudo libcap2-bin locales \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# Use UTF-8 locale
RUN echo "locales locales/locales_to_be_generated multiselect en_US.UTF-8 UTF-8" | debconf-set-selections \
&& echo "locales locales/default_environment_locale select en_US.UTF-8" | debconf-set-selections \
&& sed -i 's/^# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \
&& dpkg-reconfigure locales
ENV LANG=en_US.UTF-8
# Allow password-less sudo
RUN echo '%sudo ALL=(ALL:ALL) NOPASSWD:ALL' > /etc/sudoers.d/toolbox
LABEL com.github.containers.toolbox="true" \
com.github.debarshiray.toolbox="true"
<!--
SPDX-FileCopyrightText: 2021 Huawei Inc.
SPDX-License-Identifier: Apache-2.0
-->
# OpenHarmony build container
This container is intended for use in building OpenHarmony images,
following the build system specification in
https://repo.huaweicloud.com/harmonyos/os/2.0/tool_chain/Standard_System.md
The default user is called builder, making usage of bitbake easier.
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment