Merge branch 'feat/ef-runner' into 'kirkstone'

Migration EF runner

See merge request eclipse/oniro-core/meta-openharmony!86

.gitlab-ci.yml

@@ -45,7 +45,7 @@ workflow:
     # To use these pipelines, you must either enable Docker registry or set
     # CI_REGISTRY_IMAGE variable to an external Docker registry which hosts the
     # needed container images
-    - if: $CI_REGISTRY_IMAGE == null
+    - if: $CI_EF_REGISTRY_IMAGE == null
       when: never
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
     - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
@@ -55,6 +55,11 @@ workflow:
 include:
  - local: '.gitlab-ci/container.yml'
 
+default:
+  tags:
+    - origin:eclipse
+    - ctx:oniro-group
+
 variables:
   # Force the use of git clone to avoid odd git and repo problems
   GIT_STRATEGY: "clone"
@@ -77,9 +82,9 @@ variables:
   # But on the other hand, forking meta-openharmony to a GitLab instance with
   # Docker registry support should simply work, automatically building and using
   # the registry of the project/fork.
-  BITBAKE_BUILDER: "$CI_REGISTRY_IMAGE/bitbake-builder"
+  BITBAKE_BUILDER: "$CI_EF_REGISTRY_IMAGE/bitbake-builder"
   # And same for openharmony-builder
-  OPENHARMONY_BUILDER: "$CI_REGISTRY_IMAGE/openharmony-builder"
+  OPENHARMONY_BUILDER: "$CI_EF_REGISTRY_IMAGE/openharmony-builder"
 
 stages:
   - container
@@ -90,15 +95,24 @@ stages:
 
 # Common job definition for bitbake driven jobs, such as fetch and build
 .bitbake:
+  tags:
+    - origin:eclipse
+    - tool:bitbake
+  retry:
+    max: 2
+    when: scheduler_failure
   image:
     name: $BITBAKE_BUILDER:$BITBAKE_CONTAINER_VERSION
   variables:
     MANIFEST_FILE: manifests/ci.xml
     BB_ENV_PASSTHROUGH_ADDITIONS: DISTRO MACHINE
-    SOURCE_MIRROR_URL_CH: http://114.116.235.68/source-mirror
+    CI_ONIRO_BB_LOCAL_CONF_FETCHCMD_wget: "/usr/bin/env wget -L -t 6 -T 5 --no-check-certificate --passive-ftp"
   before_script:
     # Tell git this repository is safe even if cloned with a different uid
-    - git config --global --add safe.directory $CI_PROJECT_DIR
+    - |
+      git config --global --add safe.directory $CI_PROJECT_DIR
+      git config --global user.email "$GITLAB_USER_EMAIL"
+      git config --global user.name "$GITLAB_USER_NAME"
     # The repo tool seems to insist on having a branch checked out or
     # something like that... Without this we get errors like
     #     fatal: couldn't find remote ref refs/heads/master
@@ -119,7 +133,11 @@ stages:
     # Note the use of fd 9 to hold a lock while accessing the $REPO_MIRROR
     # path, so we can handle parallel jobs.  First we open the lock file on fd
     # 9, then we lock it, and close/unlock it after repo sync.
-    - if [ -n "$CI_ONIRO_RUNNER_PERSISTENT_STORAGE" ] ; then
+    - |
+      set +eo pipefail
+      set +o errexit
+    - |
+      if [ -n "$CI_ONIRO_RUNNER_PERSISTENT_STORAGE" ] ; then
         REPO_MIRROR="$CI_ONIRO_RUNNER_PERSISTENT_STORAGE/meta-openharmony/repo-mirror" ;
         REPO_MIRROR_ARG="--reference=$REPO_MIRROR" ;
         mkdir -p $(dirname $REPO_MIRROR) ;
@@ -137,7 +155,16 @@ stages:
           pushd "$REPO_MIRROR" ;
           repo init -u "$CI_PROJECT_DIR" -m "$MANIFEST_FILE" ;
         fi ;
-        repo sync --no-clone-bundle ;
+        repo sync --no-clone-bundle;
+         if [ $? -ne 0 ] ; then
+          echo "Repo sync failed! try to recreate repo mirror @ $REPO_MIRROR" ;
+          rm -rf "$REPO_MIRROR" ;
+          echo "Creating new repo mirror @ $REPO_MIRROR" ;
+          mkdir -p "$REPO_MIRROR" ;
+          pushd "$REPO_MIRROR" ;
+          repo init -u "$CI_PROJECT_DIR" -m "$MANIFEST_FILE" --mirror ;
+          repo sync --no-clone-bundle;
+        fi;
         popd ;
       fi
     # Create the build environment in a repo subdir, which links back to this
@@ -175,17 +202,15 @@ stages:
         echo "SSTATE_MIRRORS = \"file://.* file://$SSTATE_MIRROR_PATH/PATH\""
           >> conf/auto.conf ;
       fi
-    - MAX_JOBS="${CI_ONIRO_PARALLEL_BUILD_MAX_JOBS:-$(nproc)}"
-    - echo "BB_NUMBER_PARSE_THREADS = \"$MAX_JOBS\"" >> conf/auto.conf
-    - echo "BB_NUMBER_THREADS = \"$MAX_JOBS\"" >> conf/auto.conf
-    - echo "PARALLEL_MAKE = \"-j$MAX_JOBS\"" >> conf/auto.conf
-    - echo "NINJA_ARGS:append = \" -j$MAX_JOBS\"" >> conf/auto.conf
-    - MAX_LOADAVG="${CI_ONIRO_PARALLEL_BUILD_MAX_LOADAVG}"
-    - if [ -n "$MAX_LOADAVG" ] ; then
-        echo "PARALLEL_MAKE += \"-l$MAX_LOADAVG\"" ;
-        echo "NINJA_ARGS:append = \" -l$MAX_LOADAVG\"" ;
-        echo "MAXLOAD_NINJA = \"$MAX_LOADAVG\"" ;
-      fi >> conf/auto.conf
+    - |
+      ( set +o pipefail;
+        env \
+          | grep -v "_plus_equals_" \
+          | grep -E '^CI_ONIRO_BB_LOCAL_CONF_[a-zA-Z_0-9]+=' \
+          | sed -e 's/^CI_ONIRO_BB_LOCAL_CONF_//g' -e 's/_colon_/:/g' -e 's/"/\\"/g' -e 's/=/ = "/g' -e 's/$/"/g' \
+          | sort \
+          | tee -a conf/auto.conf )
+    
     - echo "RM_WORK_EXCLUDE += \"${RECIPE}\"" >> conf/auto.conf
     - echo "OPENHARMONY_VERSION = \"${OPENHARMONY_VERSION}\"" >> conf/auto.conf
   script:
@@ -218,13 +243,7 @@ fetch:
     - job: bitbake-builder
       artifacts: false
       optional: true
-  tags:
-    - cpu.light
   script:
-    # Fetching is io-bound, not cpu-bound, so we should be able to ignore the
-    # runner configured max on build jobs without eating up too much cpu time.
-    - echo "PARALLEL_MAKE = \"-j$(nproc)\"" >> conf/auto.conf
-    # Configure build to create mirror tarballs of VCS repositories
     - echo 'BB_GENERATE_MIRROR_TARBALLS = "1"' >> conf/auto.conf
     # Run fetch tasks for all builds
     # For this fetch job, the OPENHARMONY_VERSION variable is set just here
@@ -250,26 +269,18 @@ fetch:
       else
         SOURCE_MIRROR_PATH="$CI_PROJECT_DIR/build/mirror" ;
       fi
-    - echo "Syncing mirror files to $SOURCE_MIRROR_PATH"
-    - if [ "$CI_SERVER_HOST" = "gitlab.eclipse.org" -a
-           "$CI_PROJECT_PATH" = "eclipse/oniro-core/meta-openharmony" -a
-           "$CI_COMMIT_BRANCH" = "kirkstone" ] ; then
-        DO_MIRROR_UPDATES_CH=1 ;
-      fi
     - mkdir -pv build/mirror-updates-ch
     - rm -f build/downloads/*.{done,resolved} build/downloads/npm2/*.{done,resolved} build/downloads/uninative/*/*.{done,resolved}
-    - for f in build/downloads/* build/downloads/npm2/* build/downloads/uninative/*/*; do if test -f "$f" -a ! -L "$f" ; then
+    - |
+      for f in build/downloads/* build/downloads/npm2/* build/downloads/uninative/*/*; do if test -f "$f" -a ! -L "$f" ; then
         fname=$(basename "$f") ;
         if [ -n "$SOURCE_MIRROR_PATH" -a ! -e "$SOURCE_MIRROR_PATH/$fname" ] ; then
           cp "$f" "$SOURCE_MIRROR_PATH/$fname~tmp" ;
           mv "$SOURCE_MIRROR_PATH/$fname~tmp" "$SOURCE_MIRROR_PATH/$fname" ;
           echo "Persisted mirror file $SOURCE_MIRROR_PATH/$fname" ;
         fi ;
-        if [ "$DO_MIRROR_UPDATES_CH" = 1 ] ; then
-          if ! curl --output /dev/null --silent --head --fail "$SOURCE_MIRROR_URL_CH/$fname" ; then
-            ln -v "$f" build/mirror-updates-ch/ ;
-          fi ;
-        fi ;
+        # [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]] && ln -v -f "$f" build/mirror-updates-ch/
+        ln -v "$f" build/mirror-updates-ch/
       fi ; done
   cache:
     policy: pull-push
@@ -288,7 +299,7 @@ mirror_ch:
     - job: fetch
       artifacts: true
   rules:
-    - if: $CI_SERVER_HOST == "gitlab.eclipse.org" && $CI_PROJECT_PATH == "eclipse/oniro-core/meta-openharmony" && $CI_COMMIT_BRANCH == "kirkstone"
+    - if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"
   # No need to mark builds broken due to infrastructure problems
   allow_failure: true
   tags:
@@ -318,8 +329,6 @@ build:
   after_script:
     - !reference [.bitbake, after_script]
     - time repo/oe-core/scripts/pybootchartgui/pybootchartgui.py build/tmp-*/buildstats/* -o build/tmp-*/buildstats
-  tags:
-    - cpu.heavy
   parallel:
     matrix:
       - DISTRO: [oniro-openharmony-linux]
@@ -347,8 +356,6 @@ ptest:
       when: never
     - when: on_success
   extends: .bitbake
-  tags:
-    - cpu.heavy
   parallel:
     matrix:
       - DISTRO: [oniro-openharmony-linux]
@@ -373,11 +380,9 @@ build_ch:
       artifacts: false
   rules:
     - if: $CHINA_CI != null
-  tags:
-    - region.china
   variables:
     MANIFEST_FILE: manifests/gitee/ci.xml
-    SOURCE_MIRROR_URL: $SOURCE_MIRROR_URL_CH
+    SOURCE_MIRROR_URL: http://114.116.235.68/source-mirror
 
 # URL and filename for tarball containing entire OpenHarmony OS codebase
 .openharmony_tarball:
@@ -417,8 +422,6 @@ ohos_fetch:
   image:
     name: $OPENHARMONY_BUILDER:$OPENHARMONY_CONTAINER_VERSION
   extends: .openharmony_tarball
-  tags:
-    - cpu.light
   parallel:
     matrix:
       - OPENHARMONY_VERSION: ["3.0"]
@@ -475,7 +478,11 @@ ohos:
     name: $OPENHARMONY_BUILDER:$OPENHARMONY_CONTAINER_VERSION
   extends: .openharmony_tarball
   tags:
-    - cpu.heavy
+    - origin:eclipse
+    - tool:bitbake
+  retry:
+    max: 2
+    when: scheduler_failure
   script:
     - tar xfz $TARBALL_FILE
     - cd $(basename $TARBALL_FILE .tar.gz)/OpenHarmony

.gitlab-ci/container.yml

@@ -17,59 +17,80 @@ variables:
   BITBAKE_CONTAINER_VERSION: "0.8"
   OPENHARMONY_CONTAINER_VERSION: "0.1"
 
-.kaniko:
+.buildkit:
   stage: container
-  rules:
-    - if: $CI_REGISTRY && $CI_REGISTRY_USER && $CI_REGISTRY_PASSWORD && $CI_REGISTRY_IMAGE
-      changes:
-        - .gitlab-ci/container.yml
-        - .gitlab-ci/container/$CONTAINER_PATH/**
   image:
-    name: gcr.io/kaniko-project/executor:debug
+    name: eclipsefdn/containertools
     entrypoint: [""]
   variables:
     CONTAINER_DOCKERFILE: Dockerfile
-  tags:
-    - cpu.light
+    SERVICE_ADDRESS: "tcp://buildkitd.foundation-internal-infra-buildkitd:1234"
   script:
+    - test -n "$CI_EF_REGISTRY" || (
+          echo "CI_EF_REGISTRY not provided. Docker registry deployment can not continue."
+          && exit 1 )
+    - test -n "$CI_EF_REGISTRY_IMAGE" || (
+          echo "CI_EF_REGISTRY_IMAGE not provided. Docker registry deployment can not continue."
+          && exit 1 )
+    - test -n "$CI_EF_REGISTRY_USER" || (
+          echo "CI_EF_REGISTRY_USER not provided. Docker registry deployment can not continue."
+          && exit 1 )
+    - test -n "$CI_EF_REGISTRY_PASSWORD" || (
+          echo "CI_EF_REGISTRY_PASSWORD not provided. Docker registry deployment can not continue."
+          && exit 1 )
+    - set -xe
     - CONTEXT="$CI_PROJECT_DIR/.gitlab-ci/container/$CONTAINER_PATH"
     - DOCKERFILE="$CONTEXT/$CONTAINER_DOCKERFILE"
     - if [ -n "$CONTAINER_FROM" ] ; then
-        sed -e "s|^FROM .*|FROM $CI_REGISTRY_IMAGE/$CONTAINER_FROM:$CONTAINER_VERSION|"
+        sed -e "s|^FROM .*|FROM $CI_EF_REGISTRY_IMAGE/$CONTAINER_FROM:$CONTAINER_VERSION|"
           -i "$DOCKERFILE" ;
       fi
-    - mkdir -p /kaniko/.docker
-    - printf '{"auths":{"%s":{"auth":"%s"}}}\n' "$CI_REGISTRY" "$(printf '%s:%s' "$CI_REGISTRY_USER" "$CI_REGISTRY_PASSWORD" | base64)" > /kaniko/.docker/config.json
-    - BUILD_DATE="$(date '+%FT%T%z' | sed -E -n 's/(\+[0-9]{2})([0-9]{2})$/\1:\2/p')" #rfc 3339 date
-    - BUILD_TITLE=$(echo "$CI_PROJECT_TITLE" | tr " " "_")
-    - IMAGE_LABELS="--label build-date=$BUILD_DATE
-          --label com.gitlab.ci.cijoburl=$CI_JOB_URL
-          --label com.gitlab.ci.commiturl=$CI_PROJECT_URL/commit/$CI_COMMIT_SHA
-          --label com.gitlab.ci.email=$GITLAB_USER_EMAIL
-          --label com.gitlab.ci.mrurl=$CI_PROJECT_URL/-/merge_requests/$CI_MERGE_REQUEST_ID
-          --label com.gitlab.ci.pipelineurl=$CI_PIPELINE_URL
-          --label com.gitlab.ci.tagorbranch=$CI_COMMIT_REF_NAME
-          --label com.gitlab.ci.user=$CI_SERVER_URL/$GITLAB_USER_LOGIN
-          --label org.opencontainers.image.authors=$CI_SERVER_URL/$GITLAB_USER_LOGIN
-          --label org.opencontainers.image.created=$BUILD_DATE
-          --label org.opencontainers.image.description=$BUILD_TITLE
-          --label org.opencontainers.image.documentation=$CI_PROJECT_URL
-          --label org.opencontainers.image.licenses=$CI_PROJECT_URL
-          --label org.opencontainers.image.ref.name=$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
-          --label org.opencontainers.image.revision=$CI_COMMIT_SHA
-          --label org.opencontainers.image.source=$CI_PROJECT_URL
-          --label org.opencontainers.image.title=$BUILD_TITLE
-          --label org.opencontainers.image.url=$CI_PROJECT_URL
-          --label org.opencontainers.image.vendor=$CI_SERVER_URL/$GITLAB_USER_LOGIN
-          --label org.opencontainers.image.version=$CI_COMMIT_TAG
-          --label vcs-url=$CI_PROJECT_URL"
-    - echo "Building and shipping image to $CI_REGISTRY_IMAGE"
-    - exec /kaniko/executor --cache=true
-        --context "$CONTEXT" --dockerfile "$DOCKERFILE"
-        --destination "$CI_REGISTRY_IMAGE/$CONTAINER_NAME:$CONTAINER_VERSION"
+    - |
+      export HOME=$CI_PROJECT_DIR
+      crane auth login "$CI_EF_REGISTRY" --username "$CI_EF_REGISTRY_USER" --password "$CI_EF_REGISTRY_PASSWORD"
+      BUILD_DATE="$(date '+%FT%T%z' | sed -E -n 's/(\+[0-9]{2})([0-9]{2})$/\1:\2/p')" #rfc 3339 date
+      BUILD_TITLE=$(echo "$CI_PROJECT_TITLE" | tr " " "_")
+      IMAGE_ANNOTATIONS="annotation.build-date=$BUILD_DATE,"
+      IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.cijoburl=$CI_JOB_URL,"
+      IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.commiturl=$CI_PROJECT_URL/commit/$CI_COMMIT_SHA,"
+      IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.email=$GITLAB_USER_EMAIL,"
+      IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.mrurl=$CI_PROJECT_URL/-/merge_requests/$CI_MERGE_REQUEST_ID,"
+      IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.pipelineurl=$CI_PIPELINE_URL,"
+      IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.tagorbranch=$CI_COMMIT_REF_NAME,"
+      IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.user=$CI_SERVER_URL/$GITLAB_USER_LOGIN,"
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.authors=$CI_SERVER_URL/$GITLAB_USER_LOGIN,"
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.created=$BUILD_DATE,"
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.description=$BUILD_TITLE,"
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.documentation=$CI_PROJECT_URL,"
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.licenses=$CI_PROJECT_URL,"
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.ref.name=$CI_EF_REGISTRY:$CI_COMMIT_REF_NAME,"
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.revision=$CI_COMMIT_SHA,"
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.source=$CI_PROJECT_URL,"
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.title=$BUILD_TITLE,"
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.url=$CI_PROJECT_URL,"
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.vendor=$CI_SERVER_URL/$GITLAB_USER_LOGIN,"
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.version=$CI_COMMIT_TAG,"
+      IMAGE_ANNOTATIONS+="annotation.vcs-url=$CI_PROJECT_URL"
+
+      echo "Building and shipping image to $CI_EF_REGISTRY"
+      OUTPUT="$CONTAINER_NAME.tar"
+      buildctl --addr="$SERVICE_ADDRESS" \
+        build \
+          --progress=plain \
+          --frontend=dockerfile.v0 \
+          --local context="$CONTEXT" \
+          --local dockerfile="$CONTEXT" \
+          --output "type=oci,$IMAGE_ANNOTATIONS,dest=$OUTPUT"
+      crane push "$OUTPUT" "$CI_EF_REGISTRY_IMAGE/$CONTAINER_NAME"
+      crane tag "$CI_EF_REGISTRY_IMAGE/$CONTAINER_NAME" "$CONTAINER_VERSION"    
+  rules:
+    - if: $CI_EF_REGISTRY && $CI_EF_REGISTRY_USER && $CI_EF_REGISTRY_PASSWORD && $CI_EF_REGISTRY_IMAGE
+      changes:
+        - .gitlab-ci/container.yml
+        - .gitlab-ci/container/$CONTAINER_PATH/**
 
 bitbake-builder:
-  extends: .kaniko
+  extends: .buildkit
   variables:
     CONTAINER_NAME: bitbake-builder
     CONTAINER_VERSION: $BITBAKE_CONTAINER_VERSION
@@ -86,7 +107,7 @@ bitbake-toolbox:
       artifacts: false
 
 openharmony-builder:
-  extends: .kaniko
+  extends: .buildkit
   variables:
     CONTAINER_NAME: openharmony-builder
     CONTAINER_VERSION: $OPENHARMONY_CONTAINER_VERSION

.gitlab-ci/container/bitbake-builder/Dockerfile

@@ -33,6 +33,7 @@ RUN eatmydata apt-get update -qq \
         lz4 zstd git-restore-mtime \
         python3-cairo \
  && eatmydata apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN setcap cap_net_raw,cap_net_admin+p /usr/bin/ping
 RUN locale-gen
 
 # Let's just have /bin/sh as bash