diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5d78a8821bfc4941834e23722cf62fde0c4ee7ca..447a0425e98c92b6540f7c980f9d421a4adb1dae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -45,7 +45,7 @@ workflow: # To use these pipelines, you must either enable Docker registry or set # CI_REGISTRY_IMAGE variable to an external Docker registry which hosts the # needed container images - - if: $CI_REGISTRY_IMAGE == null + - if: $CI_EF_REGISTRY_IMAGE == null when: never - if: $CI_PIPELINE_SOURCE == "merge_request_event" - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS @@ -55,6 +55,11 @@ workflow: include: - local: '.gitlab-ci/container.yml' +default: + tags: + - origin:eclipse + - ctx:oniro-group + variables: # Force the use of git clone to avoid odd git and repo problems GIT_STRATEGY: "clone" @@ -77,9 +82,9 @@ variables: # But on the other hand, forking meta-openharmony to a GitLab instance with # Docker registry support should simply work, automatically building and using # the registry of the project/fork. - BITBAKE_BUILDER: "$CI_REGISTRY_IMAGE/bitbake-builder" + BITBAKE_BUILDER: "$CI_EF_REGISTRY_IMAGE/bitbake-builder" # And same for openharmony-builder - OPENHARMONY_BUILDER: "$CI_REGISTRY_IMAGE/openharmony-builder" + OPENHARMONY_BUILDER: "$CI_EF_REGISTRY_IMAGE/openharmony-builder" stages: - container @@ -90,15 +95,24 @@ stages: # Common job definition for bitbake driven jobs, such as fetch and build .bitbake: + tags: + - origin:eclipse + - tool:bitbake + retry: + max: 2 + when: scheduler_failure image: name: $BITBAKE_BUILDER:$BITBAKE_CONTAINER_VERSION variables: MANIFEST_FILE: manifests/ci.xml BB_ENV_PASSTHROUGH_ADDITIONS: DISTRO MACHINE - SOURCE_MIRROR_URL_CH: http://114.116.235.68/source-mirror + CI_ONIRO_BB_LOCAL_CONF_FETCHCMD_wget: "/usr/bin/env wget -L -t 6 -T 5 --no-check-certificate --passive-ftp" before_script: # Tell git this repository is safe even if cloned with a different uid - - git config --global --add safe.directory $CI_PROJECT_DIR + - | + git config --global --add safe.directory $CI_PROJECT_DIR + git config --global user.email "$GITLAB_USER_EMAIL" + git config --global user.name "$GITLAB_USER_NAME" # The repo tool seems to insist on having a branch checked out or # something like that... Without this we get errors like # fatal: couldn't find remote ref refs/heads/master @@ -119,7 +133,11 @@ stages: # Note the use of fd 9 to hold a lock while accessing the $REPO_MIRROR # path, so we can handle parallel jobs. First we open the lock file on fd # 9, then we lock it, and close/unlock it after repo sync. - - if [ -n "$CI_ONIRO_RUNNER_PERSISTENT_STORAGE" ] ; then + - | + set +eo pipefail + set +o errexit + - | + if [ -n "$CI_ONIRO_RUNNER_PERSISTENT_STORAGE" ] ; then REPO_MIRROR="$CI_ONIRO_RUNNER_PERSISTENT_STORAGE/meta-openharmony/repo-mirror" ; REPO_MIRROR_ARG="--reference=$REPO_MIRROR" ; mkdir -p $(dirname $REPO_MIRROR) ; @@ -137,7 +155,16 @@ stages: pushd "$REPO_MIRROR" ; repo init -u "$CI_PROJECT_DIR" -m "$MANIFEST_FILE" ; fi ; - repo sync --no-clone-bundle ; + repo sync --no-clone-bundle; + if [ $? -ne 0 ] ; then + echo "Repo sync failed! try to recreate repo mirror @ $REPO_MIRROR" ; + rm -rf "$REPO_MIRROR" ; + echo "Creating new repo mirror @ $REPO_MIRROR" ; + mkdir -p "$REPO_MIRROR" ; + pushd "$REPO_MIRROR" ; + repo init -u "$CI_PROJECT_DIR" -m "$MANIFEST_FILE" --mirror ; + repo sync --no-clone-bundle; + fi; popd ; fi # Create the build environment in a repo subdir, which links back to this @@ -175,17 +202,15 @@ stages: echo "SSTATE_MIRRORS = \"file://.* file://$SSTATE_MIRROR_PATH/PATH\"" >> conf/auto.conf ; fi - - MAX_JOBS="${CI_ONIRO_PARALLEL_BUILD_MAX_JOBS:-$(nproc)}" - - echo "BB_NUMBER_PARSE_THREADS = \"$MAX_JOBS\"" >> conf/auto.conf - - echo "BB_NUMBER_THREADS = \"$MAX_JOBS\"" >> conf/auto.conf - - echo "PARALLEL_MAKE = \"-j$MAX_JOBS\"" >> conf/auto.conf - - echo "NINJA_ARGS:append = \" -j$MAX_JOBS\"" >> conf/auto.conf - - MAX_LOADAVG="${CI_ONIRO_PARALLEL_BUILD_MAX_LOADAVG}" - - if [ -n "$MAX_LOADAVG" ] ; then - echo "PARALLEL_MAKE += \"-l$MAX_LOADAVG\"" ; - echo "NINJA_ARGS:append = \" -l$MAX_LOADAVG\"" ; - echo "MAXLOAD_NINJA = \"$MAX_LOADAVG\"" ; - fi >> conf/auto.conf + - | + ( set +o pipefail; + env \ + | grep -v "_plus_equals_" \ + | grep -E '^CI_ONIRO_BB_LOCAL_CONF_[a-zA-Z_0-9]+=' \ + | sed -e 's/^CI_ONIRO_BB_LOCAL_CONF_//g' -e 's/_colon_/:/g' -e 's/"/\\"/g' -e 's/=/ = "/g' -e 's/$/"/g' \ + | sort \ + | tee -a conf/auto.conf ) + - echo "RM_WORK_EXCLUDE += \"${RECIPE}\"" >> conf/auto.conf - echo "OPENHARMONY_VERSION = \"${OPENHARMONY_VERSION}\"" >> conf/auto.conf script: @@ -218,13 +243,7 @@ fetch: - job: bitbake-builder artifacts: false optional: true - tags: - - cpu.light script: - # Fetching is io-bound, not cpu-bound, so we should be able to ignore the - # runner configured max on build jobs without eating up too much cpu time. - - echo "PARALLEL_MAKE = \"-j$(nproc)\"" >> conf/auto.conf - # Configure build to create mirror tarballs of VCS repositories - echo 'BB_GENERATE_MIRROR_TARBALLS = "1"' >> conf/auto.conf # Run fetch tasks for all builds # For this fetch job, the OPENHARMONY_VERSION variable is set just here @@ -250,26 +269,18 @@ fetch: else SOURCE_MIRROR_PATH="$CI_PROJECT_DIR/build/mirror" ; fi - - echo "Syncing mirror files to $SOURCE_MIRROR_PATH" - - if [ "$CI_SERVER_HOST" = "gitlab.eclipse.org" -a - "$CI_PROJECT_PATH" = "eclipse/oniro-core/meta-openharmony" -a - "$CI_COMMIT_BRANCH" = "kirkstone" ] ; then - DO_MIRROR_UPDATES_CH=1 ; - fi - mkdir -pv build/mirror-updates-ch - rm -f build/downloads/*.{done,resolved} build/downloads/npm2/*.{done,resolved} build/downloads/uninative/*/*.{done,resolved} - - for f in build/downloads/* build/downloads/npm2/* build/downloads/uninative/*/*; do if test -f "$f" -a ! -L "$f" ; then + - | + for f in build/downloads/* build/downloads/npm2/* build/downloads/uninative/*/*; do if test -f "$f" -a ! -L "$f" ; then fname=$(basename "$f") ; if [ -n "$SOURCE_MIRROR_PATH" -a ! -e "$SOURCE_MIRROR_PATH/$fname" ] ; then cp "$f" "$SOURCE_MIRROR_PATH/$fname~tmp" ; mv "$SOURCE_MIRROR_PATH/$fname~tmp" "$SOURCE_MIRROR_PATH/$fname" ; echo "Persisted mirror file $SOURCE_MIRROR_PATH/$fname" ; fi ; - if [ "$DO_MIRROR_UPDATES_CH" = 1 ] ; then - if ! curl --output /dev/null --silent --head --fail "$SOURCE_MIRROR_URL_CH/$fname" ; then - ln -v "$f" build/mirror-updates-ch/ ; - fi ; - fi ; + # [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]] && ln -v -f "$f" build/mirror-updates-ch/ + ln -v "$f" build/mirror-updates-ch/ fi ; done cache: policy: pull-push @@ -288,7 +299,7 @@ mirror_ch: - job: fetch artifacts: true rules: - - if: $CI_SERVER_HOST == "gitlab.eclipse.org" && $CI_PROJECT_PATH == "eclipse/oniro-core/meta-openharmony" && $CI_COMMIT_BRANCH == "kirkstone" + - if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH" # No need to mark builds broken due to infrastructure problems allow_failure: true tags: @@ -318,8 +329,6 @@ build: after_script: - !reference [.bitbake, after_script] - time repo/oe-core/scripts/pybootchartgui/pybootchartgui.py build/tmp-*/buildstats/* -o build/tmp-*/buildstats - tags: - - cpu.heavy parallel: matrix: - DISTRO: [oniro-openharmony-linux] @@ -347,8 +356,6 @@ ptest: when: never - when: on_success extends: .bitbake - tags: - - cpu.heavy parallel: matrix: - DISTRO: [oniro-openharmony-linux] @@ -373,11 +380,9 @@ build_ch: artifacts: false rules: - if: $CHINA_CI != null - tags: - - region.china variables: MANIFEST_FILE: manifests/gitee/ci.xml - SOURCE_MIRROR_URL: $SOURCE_MIRROR_URL_CH + SOURCE_MIRROR_URL: http://114.116.235.68/source-mirror # URL and filename for tarball containing entire OpenHarmony OS codebase .openharmony_tarball: @@ -417,8 +422,6 @@ ohos_fetch: image: name: $OPENHARMONY_BUILDER:$OPENHARMONY_CONTAINER_VERSION extends: .openharmony_tarball - tags: - - cpu.light parallel: matrix: - OPENHARMONY_VERSION: ["3.0"] @@ -475,7 +478,11 @@ ohos: name: $OPENHARMONY_BUILDER:$OPENHARMONY_CONTAINER_VERSION extends: .openharmony_tarball tags: - - cpu.heavy + - origin:eclipse + - tool:bitbake + retry: + max: 2 + when: scheduler_failure script: - tar xfz $TARBALL_FILE - cd $(basename $TARBALL_FILE .tar.gz)/OpenHarmony diff --git a/.gitlab-ci/container.yml b/.gitlab-ci/container.yml index ef36eb5dd83ec6e57355ae66476212730983f1ff..7a7e62593fe6c2cdb77ecd1b51ee81cf7e189ccc 100644 --- a/.gitlab-ci/container.yml +++ b/.gitlab-ci/container.yml @@ -17,59 +17,80 @@ variables: BITBAKE_CONTAINER_VERSION: "0.8" OPENHARMONY_CONTAINER_VERSION: "0.1" -.kaniko: +.buildkit: stage: container - rules: - - if: $CI_REGISTRY && $CI_REGISTRY_USER && $CI_REGISTRY_PASSWORD && $CI_REGISTRY_IMAGE - changes: - - .gitlab-ci/container.yml - - .gitlab-ci/container/$CONTAINER_PATH/** image: - name: gcr.io/kaniko-project/executor:debug + name: eclipsefdn/containertools entrypoint: [""] variables: CONTAINER_DOCKERFILE: Dockerfile - tags: - - cpu.light + SERVICE_ADDRESS: "tcp://buildkitd.foundation-internal-infra-buildkitd:1234" script: + - test -n "$CI_EF_REGISTRY" || ( + echo "CI_EF_REGISTRY not provided. Docker registry deployment can not continue." + && exit 1 ) + - test -n "$CI_EF_REGISTRY_IMAGE" || ( + echo "CI_EF_REGISTRY_IMAGE not provided. Docker registry deployment can not continue." + && exit 1 ) + - test -n "$CI_EF_REGISTRY_USER" || ( + echo "CI_EF_REGISTRY_USER not provided. Docker registry deployment can not continue." + && exit 1 ) + - test -n "$CI_EF_REGISTRY_PASSWORD" || ( + echo "CI_EF_REGISTRY_PASSWORD not provided. Docker registry deployment can not continue." + && exit 1 ) + - set -xe - CONTEXT="$CI_PROJECT_DIR/.gitlab-ci/container/$CONTAINER_PATH" - DOCKERFILE="$CONTEXT/$CONTAINER_DOCKERFILE" - if [ -n "$CONTAINER_FROM" ] ; then - sed -e "s|^FROM .*|FROM $CI_REGISTRY_IMAGE/$CONTAINER_FROM:$CONTAINER_VERSION|" + sed -e "s|^FROM .*|FROM $CI_EF_REGISTRY_IMAGE/$CONTAINER_FROM:$CONTAINER_VERSION|" -i "$DOCKERFILE" ; fi - - mkdir -p /kaniko/.docker - - printf '{"auths":{"%s":{"auth":"%s"}}}\n' "$CI_REGISTRY" "$(printf '%s:%s' "$CI_REGISTRY_USER" "$CI_REGISTRY_PASSWORD" | base64)" > /kaniko/.docker/config.json - - BUILD_DATE="$(date '+%FT%T%z' | sed -E -n 's/(\+[0-9]{2})([0-9]{2})$/\1:\2/p')" #rfc 3339 date - - BUILD_TITLE=$(echo "$CI_PROJECT_TITLE" | tr " " "_") - - IMAGE_LABELS="--label build-date=$BUILD_DATE - --label com.gitlab.ci.cijoburl=$CI_JOB_URL - --label com.gitlab.ci.commiturl=$CI_PROJECT_URL/commit/$CI_COMMIT_SHA - --label com.gitlab.ci.email=$GITLAB_USER_EMAIL - --label com.gitlab.ci.mrurl=$CI_PROJECT_URL/-/merge_requests/$CI_MERGE_REQUEST_ID - --label com.gitlab.ci.pipelineurl=$CI_PIPELINE_URL - --label com.gitlab.ci.tagorbranch=$CI_COMMIT_REF_NAME - --label com.gitlab.ci.user=$CI_SERVER_URL/$GITLAB_USER_LOGIN - --label org.opencontainers.image.authors=$CI_SERVER_URL/$GITLAB_USER_LOGIN - --label org.opencontainers.image.created=$BUILD_DATE - --label org.opencontainers.image.description=$BUILD_TITLE - --label org.opencontainers.image.documentation=$CI_PROJECT_URL - --label org.opencontainers.image.licenses=$CI_PROJECT_URL - --label org.opencontainers.image.ref.name=$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME - --label org.opencontainers.image.revision=$CI_COMMIT_SHA - --label org.opencontainers.image.source=$CI_PROJECT_URL - --label org.opencontainers.image.title=$BUILD_TITLE - --label org.opencontainers.image.url=$CI_PROJECT_URL - --label org.opencontainers.image.vendor=$CI_SERVER_URL/$GITLAB_USER_LOGIN - --label org.opencontainers.image.version=$CI_COMMIT_TAG - --label vcs-url=$CI_PROJECT_URL" - - echo "Building and shipping image to $CI_REGISTRY_IMAGE" - - exec /kaniko/executor --cache=true - --context "$CONTEXT" --dockerfile "$DOCKERFILE" - --destination "$CI_REGISTRY_IMAGE/$CONTAINER_NAME:$CONTAINER_VERSION" + - | + export HOME=$CI_PROJECT_DIR + crane auth login "$CI_EF_REGISTRY" --username "$CI_EF_REGISTRY_USER" --password "$CI_EF_REGISTRY_PASSWORD" + BUILD_DATE="$(date '+%FT%T%z' | sed -E -n 's/(\+[0-9]{2})([0-9]{2})$/\1:\2/p')" #rfc 3339 date + BUILD_TITLE=$(echo "$CI_PROJECT_TITLE" | tr " " "_") + IMAGE_ANNOTATIONS="annotation.build-date=$BUILD_DATE," + IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.cijoburl=$CI_JOB_URL," + IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.commiturl=$CI_PROJECT_URL/commit/$CI_COMMIT_SHA," + IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.email=$GITLAB_USER_EMAIL," + IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.mrurl=$CI_PROJECT_URL/-/merge_requests/$CI_MERGE_REQUEST_ID," + IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.pipelineurl=$CI_PIPELINE_URL," + IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.tagorbranch=$CI_COMMIT_REF_NAME," + IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.user=$CI_SERVER_URL/$GITLAB_USER_LOGIN," + IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.authors=$CI_SERVER_URL/$GITLAB_USER_LOGIN," + IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.created=$BUILD_DATE," + IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.description=$BUILD_TITLE," + IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.documentation=$CI_PROJECT_URL," + IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.licenses=$CI_PROJECT_URL," + IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.ref.name=$CI_EF_REGISTRY:$CI_COMMIT_REF_NAME," + IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.revision=$CI_COMMIT_SHA," + IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.source=$CI_PROJECT_URL," + IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.title=$BUILD_TITLE," + IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.url=$CI_PROJECT_URL," + IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.vendor=$CI_SERVER_URL/$GITLAB_USER_LOGIN," + IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.version=$CI_COMMIT_TAG," + IMAGE_ANNOTATIONS+="annotation.vcs-url=$CI_PROJECT_URL" + + echo "Building and shipping image to $CI_EF_REGISTRY" + OUTPUT="$CONTAINER_NAME.tar" + buildctl --addr="$SERVICE_ADDRESS" \ + build \ + --progress=plain \ + --frontend=dockerfile.v0 \ + --local context="$CONTEXT" \ + --local dockerfile="$CONTEXT" \ + --output "type=oci,$IMAGE_ANNOTATIONS,dest=$OUTPUT" + crane push "$OUTPUT" "$CI_EF_REGISTRY_IMAGE/$CONTAINER_NAME" + crane tag "$CI_EF_REGISTRY_IMAGE/$CONTAINER_NAME" "$CONTAINER_VERSION" + rules: + - if: $CI_EF_REGISTRY && $CI_EF_REGISTRY_USER && $CI_EF_REGISTRY_PASSWORD && $CI_EF_REGISTRY_IMAGE + changes: + - .gitlab-ci/container.yml + - .gitlab-ci/container/$CONTAINER_PATH/** bitbake-builder: - extends: .kaniko + extends: .buildkit variables: CONTAINER_NAME: bitbake-builder CONTAINER_VERSION: $BITBAKE_CONTAINER_VERSION @@ -86,7 +107,7 @@ bitbake-toolbox: artifacts: false openharmony-builder: - extends: .kaniko + extends: .buildkit variables: CONTAINER_NAME: openharmony-builder CONTAINER_VERSION: $OPENHARMONY_CONTAINER_VERSION diff --git a/.gitlab-ci/container/bitbake-builder/Dockerfile b/.gitlab-ci/container/bitbake-builder/Dockerfile index 383207cecbb31b790c0a96e7e0334bb7281ed45c..a088191879cd299981b2372cd7c7fd4c0f871d25 100644 --- a/.gitlab-ci/container/bitbake-builder/Dockerfile +++ b/.gitlab-ci/container/bitbake-builder/Dockerfile @@ -33,6 +33,7 @@ RUN eatmydata apt-get update -qq \ lz4 zstd git-restore-mtime \ python3-cairo \ && eatmydata apt-get clean && rm -rf /var/lib/apt/lists/* +RUN setcap cap_net_raw,cap_net_admin+p /usr/bin/ping RUN locale-gen # Let's just have /bin/sh as bash