CI: Test toolchain and bundle images against OHOS codebase

Fetch OpenHarmony OS codebase release tarballs and test
oniro-openharmony-toolchain and oniro-openharmony-bundle against these.

In order to save CI resources, only MR pipelines run these new jobs
automatically.  For normal branch pipelines, the job are set to manual, so
developer can choose to run them as needed.

Signed-off-by: Esben Haabendal <esben.haabendal@huawei.com>

.gitlab-ci.yml

@@ -52,6 +52,8 @@ variables:
   # to push to git.ostc-eu.org first, and wait for the bitbake-builder image to
   # be pushed to the registry before starting a pipeline on gitlab.eclipse.org.
   BITBAKE_BUILDER: "registry.ostc-eu.org/ostc/pre-integration/meta-openharmony/bitbake-builder"
+  # And same for openharmony-builder
+  OPENHARMONY_BUILDER: "registry.ostc-eu.org/ostc/pre-integration/meta-openharmony/openharmony-builder"
 
 stages:
   - container
@@ -274,8 +276,10 @@ build:
       - build/tmp-*/deploy/sdk
     expire_in: 1 month
 
-# Build OpenHarmony images using normal CI runners
-test:
+# Build and execute OpenHarmony test image.
+#
+# This will execute all included ptest test suites.
+ptest:
   stage: test
   needs:
     - job: build
@@ -319,6 +323,103 @@ build_ch:
     MANIFEST_FILE: manifests/gitee/ci.xml
     SOURCE_MIRROR_URL: $SOURCE_MIRROR_URL_CH
 
+# URL and filename for tarball containing entire OpenHarmony OS codebase
+.openharmony_tarball:
+  before_script:
+    - if [ "${OPENHARMONY_VERSION}" = "3.0" ]; then
+        TARBALL_URL="https://repo.huaweicloud.com/harmonyos/os/3.0/code-v3.0-LTS.tar.gz";
+      elif [ "${OPENHARMONY_VERSION}" = "3.0.1" ]; then
+        TARBALL_URL="https://repo.huaweicloud.com/harmonyos/os/3.0.1/code-v3.0.1-LTS.tar.gz";
+      elif [ "${OPENHARMONY_VERSION}" = "3.0.2" ]; then
+        TARBALL_URL="https://repo.huaweicloud.com/harmonyos/os/3.0.2/code-v3.0.2-LTS.tar.gz";
+      elif [ "${OPENHARMONY_VERSION}" = "3.0.3" ]; then
+        TARBALL_URL="https://repo.huaweicloud.com/harmonyos/os/3.0.3/code-v3.0.3-LTS.tar.gz";
+      elif [ "${OPENHARMONY_VERSION}" = "3.1" ]; then
+        TARBALL_URL="https://repo.huaweicloud.com/harmonyos/os/3.1-Release/code-v3.1-Release.tar.gz";
+      elif [ "${OPENHARMONY_VERSION}" = "3.1.1" ]; then
+        TARBALL_URL="https://repo.huaweicloud.com/harmonyos/os/3.1.1/code-v3.1.1-Release.tar.gz";
+      else
+        echo "OPENHARMONY_VERSION=${OPENHARMONY_VERSION} is not supported";
+        exit 1;
+      fi
+    - TARBALL_FILE=$(basename $TARBALL_URL)
+    - mkdir -p downloads
+    # Use runner persistent storage if available
+    - if [ -n "$CI_ONIRO_RUNNER_PERSISTENT_STORAGE" ] ; then
+        TARBALL_FILE="$CI_ONIRO_RUNNER_PERSISTENT_STORAGE/$TARBALL_FILE" ;
+        touch downloads/.dummy_file ;
+      else
+        TARBALL_FILE="downloads/$TARBALL_FILE" ;
+      fi ;
+      echo "Using $TARBALL_FILE"
+
+# Fetch OpenHarmony OS codebase.
+#
+# This is for use in ohos test jobs.
+ohos_fetch:
+  stage: fetch
+  image:
+    name: $OPENHARMONY_BUILDER:$OPENHARMONY_CONTAINER_VERSION
+  extends: .openharmony_tarball
+  parallel:
+    matrix:
+      - OPENHARMONY_VERSION: ["3.0", "3.1"]
+  script:
+    # Integrity check of cached tarball
+    - if [ -f "${TARBALL_FILE}.sha256" -a -f "${TARBALL_FILE}" ] ; then
+        read tarball_sha256 _ <<<$(sha256sum "${TARBALL_FILE}") ;
+        read expected_sha256 _ < "${TARBALL_FILE}.sha256" ;
+        if [ "$tarball_sha256" = "$expected_sha256" ] ; then
+          echo "Using cached ${TARBALL_FILE}" ;
+          exit 0 ;
+        else
+          echo "Deleting bad ${TARBALL_FILE}" ;
+          rm -f "${TARBALL_FILE}" "${TARBALL_FILE}.sha256" ;
+        fi ;
+      elif [ -f "${TARBALL_FILE}.sha256" ] ; then
+          echo "Deleting ${TARBALL_FILE}.sha256" ;
+          rm -f "${TARBALL_FILE}.sha256" ;
+      fi
+    - pushd $(dirname $TARBALL_FILE)
+    # Download tarball (should continue if partial download is found
+    - wget --progress=dot:giga "${TARBALL_URL}" --timestamping --no-if-modified-since
+    # Download sha256 signature
+    - wget "${TARBALL_URL}.sha256" --timestamping --no-if-modified-since
+    - popd
+    - read tarball_sha256 _ <<<$(sha256sum "${TARBALL_FILE}") ;
+    - read expected_sha256 _ < "${TARBALL_FILE}.sha256" ;
+    - if [ "$tarball_sha256" != "$expected_sha256" ] ; then
+        echo "Deleting bad ${TARBALL_FILE}" ;
+        rm -f "${TARBALL_FILE}" "${TARBALL_FILE}.sha256" ;
+        exit 1 ;
+      fi
+  artifacts:
+    paths:
+      - downloads
+    expire_in: 1 month
+
+# Test prebuilts images against OpenHarmony OS codebase
+ohos:
+  stage: test
+  needs:
+    - job: ohos_fetch
+      artifacts: true
+    - job: build
+      artifacts: true
+  parallel:
+    matrix:
+      - OPENHARMONY_VERSION: ["3.0", "3.1"]
+        PREBUILT_IMAGE: [oniro-openharmony-toolchain, oniro-openharmony-bundle]
+  image:
+    name: $OPENHARMONY_BUILDER:$OPENHARMONY_CONTAINER_VERSION
+  extends: .openharmony_tarball
+  script:
+    - tar xfz $TARBALL_FILE
+    - cd $(basename $TARBALL_FILE .tar.gz)/OpenHarmony
+    - ../../build/tmp-*/deploy/sdk/${PREBUILT_IMAGE}-${OPENHARMONY_VERSION}-*.sh -y -d oniro
+    - ./oniro/setup.sh
+    - ./build.sh --product-name Hi3516DV300
+
 # Job for triggering a prune of the sstate-cache mirror in runner persistent
 # storage.  This should be done periodically, but as I believe it removes a
 # bit too much, not too often.  And a complete rebuild is done also, to refill

.gitlab-ci/container.yml

@@ -15,6 +15,7 @@
 
 variables:
   BITBAKE_CONTAINER_VERSION: "0.6"
+  OPENHARMONY_CONTAINER_VERSION: "0.1"
 
 .kaniko:
   stage: container
@@ -83,3 +84,21 @@ bitbake-toolbox:
   needs:
     - job: bitbake-builder
       artifacts: false
+
+openharmony-builder:
+  extends: .kaniko
+  variables:
+    CONTAINER_NAME: openharmony-builder
+    CONTAINER_VERSION: $OPENHARMONY_CONTAINER_VERSION
+
+openharmony-toolbox:
+  extends: .kaniko
+  variables:
+    CONTAINER_NAME: openharmony-toolbox
+    CONTAINER_VERSION: $OPENHARMONY_CONTAINER_VERSION
+    CONTAINER_PATH: openharmony-builder
+    CONTAINER_DOCKERFILE: Dockerfile.toolbox
+    CONTAINER_FROM: openharmony-builder
+  needs:
+    - job: openharmony-builder
+      artifacts: false

.gitlab-ci/container/openharmony-builder/.gitconfig

+# SPDX-FileCopyrightText: 2021 Huawei Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+[user]
+	name = OSTC Builder
+	email = ostc-builder@example.org

.gitlab-ci/container/openharmony-builder/Dockerfile

+# Copyright 2021 Huawei Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# This builds a container image for use in building OpenHarmony images,
+# following the build system specification in
+# https://repo.huaweicloud.com/harmonyos/os/2.0/tool_chain/Standard_System.md
+
+FROM ubuntu:18.04
+
+ARG DEBIAN_FRONTEND="noninteractive"
+
+RUN apt-get update -qq \
+ && apt-get install -qq -y \
+      bash locales \
+      apt-utils vim software-properties-common openssh-server iputils-ping \
+      curl net-tools bsdmainutils kmod bc rsync gawk ssh ccache zip python-dev \
+      make m4 gcc-multilib ca-certificates-java unzip python3-yaml perl \
+      openssl libssl1.1 gnupg xsltproc x11proto-core-dev tcl python3-crypto \
+      python-crypto libxml2-utils libxml2-dev libx11-dev libssl-dev \
+      libgl1-mesa-dev lib32z1-dev lib32ncurses5-dev g++-multilib flex bison \
+      doxygen git subversion tofrodos pigz expect python3-xlrd python-xlrd \
+      git-core gperf build-essential zlib1g-dev libc6-dev-i386 lib32z-dev \
+      openjdk-8-jdk \
+ && apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN locale-gen
+
+# Let's just have /bin/sh as bash
+RUN echo "dash dash/sh boolean false" | debconf-set-selections \
+ && dpkg-reconfigure dash
+
+# For some reason for containers built using kaniko ping errors with:
+# `permission denied` for user other than root. Even running container in
+# privileged mode doesn't help for non-root user. Most probably it is related
+# to this issue: https://github.com/GoogleContainerTools/kaniko/issues/1851
+# ping is required by Yocto testing machinery and therefore temporarily setting
+# suid to get it working
+RUN chmod u+s $(command -v ping)
+
+RUN apt-get update -qq \
+ && apt-get install -qq -y python3.8 python3-pip \
+ && apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python2.7 2 \
+ && update-alternatives --install /usr/bin/python python /usr/bin/python3.6 3 \
+ && update-alternatives --install /usr/bin/python python /usr/bin/python3.8 4 \
+ && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 2 \
+ && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 3
+RUN pip3 install --trusted-host mirrors.aliyun.com -i https://mirrors.aliyun.com/pypi/simple setuptools \
+ && pip3 install --trusted-host mirrors.aliyun.com -i https://mirrors.aliyun.com/pypi/simple kconfiglib \
+ && pip3 install --trusted-host mirrors.aliyun.com -i http://mirrors.aliyun.com/pypi/simple pycryptodome \
+ && pip3 install --trusted-host mirrors.aliyun.com -i https://mirrors.aliyun.com/pypi/simple ecdsa \
+ && python3 -m pip install ohos-build==0.4.6
+
+RUN mkdir -p /opt/harmonyos
+RUN wget -q -P /opt/harmonyos https://repo.huaweicloud.com/harmonyos/compiler/clang/10.0.1-62608/linux/llvm.tar.gz \
+ && tar xf /opt/harmonyos/llvm.tar.gz -C /opt/harmonyos \
+ && rm /opt/harmonyos/llvm.tar.gz
+RUN wget -q -P /opt/harmonyos https://repo.huaweicloud.com/harmonyos/compiler/hc-gen/0.65/linux/hc-gen-0.65-linux.tar \
+ && tar xf /opt/harmonyos/hc-gen-0.65-linux.tar -C /opt/harmonyos \
+ && rm /opt/harmonyos/hc-gen-0.65-linux.tar
+RUN wget -q -P /opt/harmonyos https://repo.huaweicloud.com/harmonyos/compiler/gcc_riscv32/7.3.0/linux/gcc_riscv32-linux-7.3.0.tar.gz \
+ && tar xf /opt/harmonyos/gcc_riscv32-linux-7.3.0.tar.gz -C /opt/harmonyos \
+ && rm /opt/harmonyos/gcc_riscv32-linux-7.3.0.tar.gz
+RUN wget -q -P /opt/harmonyos https://repo.huaweicloud.com/harmonyos/compiler/ninja/1.9.0/linux/ninja.1.9.0.tar \
+ && tar xf /opt/harmonyos/ninja.1.9.0.tar -C /opt/harmonyos \
+ && rm /opt/harmonyos/ninja.1.9.0.tar
+RUN wget -q -P /opt/harmonyos https://repo.huaweicloud.com/harmonyos/compiler/gn/1717/linux/gn-linux-x86-1717.tar.gz \
+ && tar xf /opt/harmonyos/gn-linux-x86-1717.tar.gz -C /opt/harmonyos \
+ && rm /opt/harmonyos/gn-linux-x86-1717.tar.gz
+RUN wget -q -P /opt/harmonyos https://mirrors.huaweicloud.com/nodejs/v12.20.0/node-v12.20.0-linux-x64.tar.gz \
+ && tar xf /opt/harmonyos/node-v12.20.0-linux-x64.tar.gz -C /opt/harmonyos \
+ && rm /opt/harmonyos/node-v12.20.0-linux-x64.tar.gz
+RUN wget -q -P /opt/harmonyos https://repo.huaweicloud.com/harmonyos/develop_tools/hmos_app_packing_tool.jar
+RUN wget -q -P /opt/harmonyos https://repo.huaweicloud.com/harmonyos/develop_tools/hapsigntoolv2.jar
+
+RUN sed -i '$aexport PATH=/opt/harmonyos/llvm/bin:$PATH' /root/.bashrc \
+ && sed -i '$aexport PATH=/opt/harmonyos/hc-gen:$PATH' /root/.bashrc \
+ && sed -i '$aexport PATH=/opt/harmonyos/gcc_riscv32/bin:$PATH' /root/.bashrc \
+ && sed -i '$aexport PATH=/opt/harmonyos/ninja:$PATH' /root/.bashrc \
+ && sed -i '$aexport PATH=/opt/harmonyos/node-v12.20.0-linux-x64/bin:$PATH' /root/.bashrc \
+ && sed -i '$aexport PATH=/opt/harmonyos:$PATH' /root/.bashrc \
+ && sed -i '$aexport PATH=/root/.local/bin:$PATH' /root/.bashrc \
+ && export PATH=/opt/harmonyos/llvm/bin:$PATH \
+ && export PATH=/opt/harmonyos/hc-gen:$PATH \
+ && export PATH=/opt/harmonyos/gcc_riscv32/bin:$PATH \
+ && export PATH=/opt/harmonyos/ninja:$PATH \
+ && export PATH=/opt/harmonyos/node-v12.20.0-linux-x64/bin:$PATH \
+ && export PATH=/opt/harmonyos:$PATH \
+ && export PATH=/root/.local/bin:$PATH \
+ && npm install -g @ohos/hpm-cli --registry https://mirrors.huaweicloud.com/repository/npm/
+
+RUN useradd --create-home --uid 1000 --shell /usr/bin/bash builder
+COPY --chown=builder:builder .gitconfig /home/builder/.gitconfig
+USER builder
+WORKDIR /home/builder

.gitlab-ci/container/openharmony-builder/Dockerfile.toolbox

+# Copyright 2020 Huawei Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+FROM openharmony-builder
+
+# Revert setup with builder user account
+USER root
+WORKDIR /
+RUN userdel -r builder
+
+# Install packages needed by Toolbox
+ARG DEBIAN_FRONTEND="noninteractive"
+RUN apt-get update -qq \
+ && apt-get install -qq -y \
+        bash sudo libcap2-bin locales \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+# Use UTF-8 locale
+RUN echo "locales locales/locales_to_be_generated multiselect en_US.UTF-8 UTF-8" | debconf-set-selections \
+ && echo "locales locales/default_environment_locale select en_US.UTF-8" | debconf-set-selections \
+ && sed -i 's/^# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \
+ && dpkg-reconfigure locales
+ENV LANG=en_US.UTF-8
+
+# Allow password-less sudo
+RUN echo '%sudo ALL=(ALL:ALL) NOPASSWD:ALL' > /etc/sudoers.d/toolbox
+
+LABEL com.github.containers.toolbox="true" \
+      com.github.debarshiray.toolbox="true"

.gitlab-ci/container/openharmony-builder/README.md

+<!--
+SPDX-FileCopyrightText: 2021 Huawei Inc.
+SPDX-License-Identifier: Apache-2.0
+-->
+
+# OpenHarmony build container
+
+This container is intended for use in building OpenHarmony images,
+following the build system specification in
+https://repo.huaweicloud.com/harmonyos/os/2.0/tool_chain/Standard_System.md
+
+The default user is called builder, making usage of bitbake easier.