build-generic.yaml

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Huawei Inc.

# This job is documented in docs/ci/hidden-jobs/workspace.rst
.workspace:
  interruptible: true
  image:
    name: registry.ostc-eu.org/ostc/oniro/bitbake-builder:latest
  tags: [large-disk, $CI_ONIRO_RUNNER_TAG, $CI_ONIRO_INSTANCE_SIZE]
  variables:
    CI_ONIRO_RUNNER_TAG : ""
    CI_ONIRO_MANIFEST_URL: https://gitlab.eclipse.org/eclipse/oniro-core/oniro
    CI_ONIRO_MANIFEST_BRANCH: kirkstone
    CI_ONIRO_MANIFEST_NAME: default.xml
    CI_ONIRO_MANIFEST_MIRROR: oniro-develop
    CI_ONIRO_GIT_REPO_PATH: ""
    CI_ONIRO_INSTANCE_SIZE: s3.large.8
  before_script:
    - test ! -e "$CI_PROJECT_DIR"/.scratch-dir-name || (
        echo "precondition failed - concurrent modification of $CI_PROJECT_DIR"
        && env | grep CI_ | sort
        && ls -l
        && exit 1 )

    # Create scratch space, being careful not to pollute the working directory.
    # This is done so that we are not attached to the contents of
    # $CI_PROJECT_DIR which contains something that GitLab CI prepared for us.
    - SCRATCH_DIR="$(mktemp -p /tmp -d workspace.XXXXXXXXXX)"
    - echo "$SCRATCH_DIR" > "$CI_PROJECT_DIR"/.scratch-dir-name
    - cd "$SCRATCH_DIR"

    - |
        set -x
        echo "Building repo workspace with the following properties:"
        echo "CI_ONIRO_MANIFEST_URL: $CI_ONIRO_MANIFEST_URL"
        echo "CI_ONIRO_MANIFEST_NAME: $CI_ONIRO_MANIFEST_NAME"
        echo "CI_ONIRO_MANIFEST_BRANCH: $CI_ONIRO_MANIFEST_BRANCH"
        repo init --reference \
           "$CI_ONIRO_RUNNER_PERSISTENT_STORAGE"/pub/git-repo-mirrors/"$CI_ONIRO_MANIFEST_MIRROR" \
           --manifest-url "$CI_ONIRO_MANIFEST_URL" \
           --manifest-name "$CI_ONIRO_MANIFEST_NAME" \
           --manifest-branch "$CI_ONIRO_MANIFEST_BRANCH"
        set +x
    - time repo sync --no-clone-bundle

    # See the documentation of CI_ONIRO_GIT_REPO_PATH
    - |
      set -x
      if [ -n "$CI_ONIRO_GIT_REPO_PATH" ]; then
        if [ -n "$CI_MERGE_REQUEST_SOURCE_PROJECT_URL" ]; then
            echo "CI: Bootstrapping '$CI_PROJECT_DIR' as 'incoming-merged' remote in '$CI_ONIRO_GIT_REPO_PATH'";
            ( cd "$CI_ONIRO_GIT_REPO_PATH" \
                && git remote add incoming-merged "$CI_MERGE_REQUEST_SOURCE_PROJECT_URL" \
                && git fetch incoming-merged );
        fi;
        echo "CI: Switching $CI_ONIRO_GIT_REPO_PATH to $CI_COMMIT_SHA";
        ( cd "$CI_ONIRO_GIT_REPO_PATH" && git checkout "$CI_COMMIT_SHA" );
      fi
      set +x

  script:
    # Reload the value of SCRATCH_DIR set in the before_script phase. Those run
    # in separate shell processes and do not share environment variables.
    - SCRATCH_DIR="$(cat "$CI_PROJECT_DIR"/.scratch-dir-name)"
    - cd "$SCRATCH_DIR"

  after_script:
    # If the primary script failed early enough, the scratch dir may not have
    # been created yet. Check for that to avoid confusing errors.
    - test -f "$CI_PROJECT_DIR"/.scratch-dir-name || exit 0
    # Reload the value of SCRATCH_DIR set in the before_script phase.
    - SCRATCH_DIR="$(cat "$CI_PROJECT_DIR"/.scratch-dir-name)"
    # Clean up after ourselves.
    - rm -f "$CI_PROJECT_DIR"/.scratch-dir-name
    - rm -rf "$SCRATCH_DIR"

# This job is documented in docs/ci/hidden-jobs/bitbake-workspace.rst
.bitbake-workspace:
  extends: .workspace
  stage: build
  timeout: 6h
  variables:
    CI_ONIRO_BUILD_FLAVOUR: ""
    CI_ONIRO_BUILD_CACHE: "private"
    CI_ONIRO_BB_LOCAL_CONF_CONNECTIVITY_CHECK_URIS: "https://example.net/"
    CI_ONIRO_BB_LOCAL_CONF_DL_DIR: $CI_ONIRO_RUNNER_PERSISTENT_STORAGE/$CI_ONIRO_BUILD_CACHE/bitbake/downloads
    CI_ONIRO_BB_LOCAL_CONF_SSTATE_DIR: $CI_ONIRO_RUNNER_PERSISTENT_STORAGE/$CI_ONIRO_BUILD_CACHE/bitbake/sstate-cache
    CI_ONIRO_BB_LOCAL_CONF_IMAGE_VERSION_SUFFIX: ""
    CI_ONIRO_BB_LOCAL_CONF_CVE_CHECK_DB_DIR: "$${TMPDIR}/CVE_CHECK/"
    CI_ONIRO_BB_LOCAL_CONF_plus_equals_INHERIT: cve-check
    CI_ONIRO_BB_LOCAL_CONF_plus_equals_USER_CLASSES: "buildstats buildstats-summary"
    CI_ONIRO_DEVTOOL_RECIPE_NAME: ""
    CI_ONIRO_DEVTOOL_LAYER_PATH: ""

  before_script:
    # Bitbake requires a non-root user to operate.
    # The container should have a non-root user by default.
    - test "$(id -u)" -ne 0 || (
        echo "precondition failed - this job cannot run as root"
        && exit 1 )
    # Check if the job is configured properly.
    - test -n "$CI_ONIRO_BUILD_FLAVOUR" || (
        echo "precondition failed - set CI_ONIRO_BUILD_FLAVOUR to \"flavour\" of the build to use (e.g. linux)"
        && exit 1 )
    - test -n "$MACHINE" || (
        echo "precondition failed - set MACHINE to supported machines (e.g. qemux86-64)"
        && exit 1 )
    # Check devtool operation is enabled and configured properly.
    - |
      if { [ -n "$CI_ONIRO_DEVTOOL_RECIPE_NAME" ] && [ -z "$CI_ONIRO_DEVTOOL_LAYER_PATH" ]; } || { [ -z "$CI_ONIRO_DEVTOOL_RECIPE_NAME" ] && [ -n "$CI_ONIRO_DEVTOOL_LAYER_PATH" ]; } then
        echo "precondition failed - cannot define just one of CI_ONIRO_DEVTOOL_RECIPE_NAME and CI_ONIRO_DEVTOOL_LAYER_PATH"
        exit 1
      fi
    # Bitbake is configured to use $CI_ONIRO_RUNNER_PERSISTENT_STORAGE/$CI_ONIRO_BUILD_CACHE/bitbake
    # directory for both the download directory and the sstate-cache.
    - test -n "$CI_ONIRO_RUNNER_PERSISTENT_STORAGE" || (
        echo "precondition failed - CI_ONIRO_RUNNER_PERSISTENT_STORAGE is not set"
        && exit 1 )
    - test -w "$CI_ONIRO_RUNNER_PERSISTENT_STORAGE/$CI_ONIRO_BUILD_CACHE/bitbake" || (
        echo "precondition failed - expected $CI_ONIRO_RUNNER_PERSISTENT_STORAGE/$CI_ONIRO_BUILD_CACHE/bitbake to be writable"
        && exit 1 )
    # Log available disk space on the persistent shared disk.
    - df -h "$CI_ONIRO_RUNNER_PERSISTENT_STORAGE/$CI_ONIRO_BUILD_CACHE/bitbake"
    - !reference [.workspace, before_script]

  script:
    - !reference [.workspace, script]
    # Initialize bitbake build environment by sourcing the oe-init-build-env
    # into the running bash process. This has the side-effect of changing the
    # current working directory and populating the $SCRATCH_DIR/build
    # sub-directory with default configuration.
    - TEMPLATECONF=../oniro/flavours/"$CI_ONIRO_BUILD_FLAVOUR"
        . ./oe-core/oe-init-build-env build
    # xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
    # NOTE: From now on, we are running inside "$SCRATCH_DIR"/build
    # with bash modified by oe-init-build-env. We now have access to bitbake,
    # devtool and other related tools.
    # xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx

    # Generic local.conf customization point. All environment variables with a
    # given prefix are converted to local.conf variables, with the prefix
    # dropped. Some extra sed processing is used to quote the variables.
    # Everything runs in a sub-shell with bash pipefail disabled, as otherwise
    # lack of variables matching the grep pattern causes the execution to stop.
    # Variables in the form of 'CI_ONIRO_BB_LOCAL_CONF_attr' are converted to
    # 'attr = "value"'. It can include '_colon_' substrings which are converted
    # to ':'.
    - |
      ( set +o pipefail;
        env \
          | grep -v "_plus_equals_" \
          | grep -E '^CI_ONIRO_BB_LOCAL_CONF_[a-zA-Z_0-9]+=' \
          | sed -e 's/^CI_ONIRO_BB_LOCAL_CONF_//g' -e 's/_colon_/:/g' -e 's/"/\\"/g' -e 's/=/ = "/g' -e 's/$/"/g' \
          | sort \
          | tee -a conf/local.conf )
    # Variables in the form of 'CI_ONIRO_BB_LOCAL_CONF_plus_equals_attr' are
    # converted to 'attr += "value"'
    - |
      ( set +o pipefail;
        env \
          | grep -E '^CI_ONIRO_BB_LOCAL_CONF_plus_equals_[A-Z_0-9]+=' \
          | sed -e 's/^CI_ONIRO_BB_LOCAL_CONF_plus_equals_//g' -e 's/"/\\"/g' -e 's/=/ += "/g' -e 's/$/"/g' \
          | sort \
          | tee -a conf/local.conf )

    # Sanity check: disallow using public build cache with a specific setting
    # in local.conf. The list of settings may grow over time.
    - |
      if [ "$CI_ONIRO_BUILD_CACHE" != private ]; then
        if grep -qF 'ACCEPT_FSL_EULA = "1"' conf/local.conf; then
          echo "cannot use CI_ONIRO_BUILD_CACHE=$CI_ONIRO_BUILD_CACHE with ACCEPT_FSL_EULA = \"1\""
          exit 1
        fi
      fi

    # Update a specific bitbake recipe, which must use git, to point to the
    # commit that is being tested here.
    - |
      if [ -n "$CI_ONIRO_DEVTOOL_RECIPE_NAME" ] && [ -n "$CI_ONIRO_DEVTOOL_LAYER_PATH" ]; then
        echo "CI: Switching $CI_ONIRO_DEVTOOL_RECIPE_NAME from $CI_ONIRO_DEVTOOL_LAYER_PATH to $CI_COMMIT_SHA";
        # Upgrade the recipe to point to the sources in the project directory.
        # Pass --no-patch to skip applying patches that are present in the
        # recipe.
        #
        # NOTE: This requires the git repository specified in the recipe to
        # have access to the SHA presented here.
        time devtool upgrade --no-patch --srcrev "$CI_COMMIT_SHA" --srcbranch "$CI_COMMIT_REF_NAME" "$CI_ONIRO_DEVTOOL_RECIPE_NAME";
        # Switch the detached head created by devtool upgrade above, to a
        # scratch branch. This is required for the devtool finish to work.
        ( cd workspace/"$CI_ONIRO_DEVTOOL_RECIPE_NAME" && git checkout -b scratch );
        # Finish the upgrade process and commit the change back to the layer.
        # Because our recipe may have had patches, this step is required to
        # correctly discard them ("devtool upgrade --no-patch" simply skipped
        # them).
        time devtool finish --remove-work --force "$CI_ONIRO_DEVTOOL_RECIPE_NAME" "$(basename "$CI_ONIRO_DEVTOOL_LAYER_PATH")";
      fi
  rules:
    # Run the job when a merge request is created.
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    # Run the job when a tag is placed.
    - if: '$CI_COMMIT_TAG'
    # Run the build for scheduled pipelines.
    - if: '$CI_PIPELINE_SOURCE == "schedule"'

# This job is documented in docs/ci/hidden-jobs/build-linux.rst
.build-linux:
  extends: .bitbake-workspace
  variables:
    CI_ONIRO_BUILD_FLAVOUR: linux
    CI_ONIRO_BITBAKE_TARGETS: "oniro-image-base-tests oniro-image-extra-tests"
    # Linux builds can be extremely large. Use rm_work to remove the temporary
    # build data as soon as possible to reclaim space.
    CI_ONIRO_BB_LOCAL_CONF_plus_equals_INHERIT: "rm_work"
  script:
    - test -n "$CI_ONIRO_BITBAKE_TARGETS" || (
        echo "nothing to build - CI_ONIRO_BITBAKE_TARGETS is empty"
        && exit 1 )
    - !reference [.bitbake-workspace, script]
    - |
      for target in $CI_ONIRO_BITBAKE_TARGETS; do
        time bitbake "$target"
        du -sh tmp
      done

# This job is documented in docs/ci/hidden-jobs/build-linux-marix.rst
.build-linux-matrix:
  extends: .build-linux
  parallel:
    matrix:
      - CI_ONIRO_BITBAKE_TARGETS: ["oniro-image-base-tests", "oniro-image-extra-tests"]

# This job is documented in docs/ci/hidden-jobs/build-zephyr.rst
.build-zephyr:
  extends: .bitbake-workspace
  variables:
    CI_ONIRO_BUILD_FLAVOUR: zephyr
    CI_ONIRO_BITBAKE_TARGETS: "zephyr-philosophers"
    CI_ONIRO_INSTANCE_SIZE: s3.large.2
  script:
    - test -n "$CI_ONIRO_BITBAKE_TARGETS" || (
        echo "nothing to build - CI_ONIRO_BITBAKE_TARGETS is empty"
        && exit 1 )
    - !reference [.bitbake-workspace, script]
    - |
      for target in $CI_ONIRO_BITBAKE_TARGETS; do
        time bitbake "$target"
      done

# This job is documented in docs/ci/hidden-jobs/build-freertos.rst
.build-freertos:
  extends: .bitbake-workspace
  variables:
    CI_ONIRO_BUILD_FLAVOUR: freertos
    CI_ONIRO_INSTANCE_SIZE: s3.large.2
  script:
    - !reference [.bitbake-workspace, script]
    - time bitbake freertos-demo

# This job is documented in docs/ci/hidden-jobs/build-recipe.rst
.build-recipe:
  extends: .bitbake-workspace
  variables:
    CI_ONIRO_RECIPE_NAME: ""
  script:
    - !reference [.bitbake-workspace, script]
    - time bitbake "$CI_ONIRO_RECIPE_NAME"

# This job is documented in docs/ci/hidden-jobs/build-image.rst
.build-image:
  extends: .build-recipe
  variables:
    CI_ONIRO_JOB_ARTIFACTS: ""
  script:
    - !reference [.build-recipe, script]
    # Move artifacts for recovery, which only considers $CI_PROJECT_DIR and
    # subdirectories.
    - mkdir -p "$CI_PROJECT_DIR"/artifacts
    - rm -rf "$CI_PROJECT_DIR"/artifacts/*
    # The name of the build-specific tmp directory may vary. Ask bitbake
    # instead of hard-coding it. This is currently only relevant for Zephyr,
    # which uses tmp-newlib.
    - eval "$(bitbake -e | grep ^TMPDIR= | sed -e 's/^TMPDIR/BITBAKE_TMPDIR/g')"
    # Uncompressed wic image is not needed in CI. We cannot remove it from
    # meta-oniro-core/classes/oniro-image.bbclass as runqemu needs it for local
    # build and boot.
    - find "$BITBAKE_TMPDIR"/deploy/images/ -name *.wic -exec rm -rf {} \;
    - cp -a "$BITBAKE_TMPDIR"/deploy/licenses/ "$CI_PROJECT_DIR"/artifacts || true
    # Only copy the files defined in the CI_ONIRO_JOB_ARTIFACTS variable. Most
    # of the remaining files are redundant and quite large.
    - |
      set -x
      if test -n "$CI_ONIRO_JOB_ARTIFACTS" && test -d "$BITBAKE_TMPDIR"/deploy/; then
        (
          cd "$BITBAKE_TMPDIR"/deploy/
          for artifact in $CI_ONIRO_JOB_ARTIFACTS; do
            find ./ \( -type f -o -type l \) -name "$artifact" -exec cp --verbose --archive --parents {} "$CI_PROJECT_DIR"/artifacts \;
          done
        )
      fi
      set +x
  artifacts:
    paths:
      - artifacts/

.build-wic-image:
  extends: .build-image
  variables:
    CI_ONIRO_JOB_ARTIFACTS: "*.wic.* *.bmap"
  script:
    - !reference [.build-image, script]

.build-rauc-bundle:
  extends: .build-image
  variables:
    # Put timestamp into bundle name.
    CI_ONIRO_BB_LOCAL_CONF_IMAGE_VERSION_SUFFIX: "-$${DATETIME}"
    # Collect rauc bundles as artifacts.
    CI_ONIRO_JOB_ARTIFACTS: "*.raucb"
  # Avoid pulling in any artifacts from the previous stage.
  dependencies: []

.publish-rauc-bundle-to-hawkbit:
  image: zyga/hawkbitctl:latest-ubuntu
  # Use the deploy stage so that we only publish bundles to HawkBit if and only
  # if all the bundles built in their stage completed successfully. This delays
  # publishing to ensure that we have a consistent set and not some partial set
  # when something failed to build.
  stage: deploy
  variables:
    # Those variables have to be provided by specialized jobs.
    CI_ONIRO_HAWKBIT_SWMOD_NAME: ""
    CI_ONIRO_HAWKBIT_DS_NAME: ""
    CI_ONIRO_RAUC_BUNDLE_NAME: ""
    MACHINE: ""
  script:
    - set -e
    - test -n "${CI_ONIRO_HAWKBIT_SWMOD_NAME:-}" || (
        echo "precondition failed - CI_ONIRO_HAWKBIT_SWMOD_NAME is not set."
        && echo "It should be set to the name of the HawkBit software module"
        && exit 1 )
    - test -n "${CI_ONIRO_HAWKBIT_DS_NAME:-}" || (
        echo "precondition failed - CI_ONIRO_HAWKBIT_DS_NAME is not set."
        && echo "It should be set to the name of the HawkBit distribution set."
        && exit 1 )
    - test -n "${CI_ONIRO_RAUC_BUNDLE_NAME:-}" || (
        echo "precondition failed - CI_ONIRO_RAUC_BUNDLE_NAME is not set."
        && echo "It should be set to the name of the Yocto recipe constructing the RAUC bundle."
        && exit 1 )
    - test -n "${MACHINE:-}" || (
        echo "precondition failed - MACHINE is not set."
        && echo "It should be set to the name of the Yocto machine name."
        && exit 1 )
    - test -n "${HAWKBIT_URL:-}" || (
        echo "precondition failed - HAWKBIT_URL is not set."
        && exit 1 )
    - test -n "${HAWKBIT_USERNAME:-}" || (
        echo "precondition failed - HAWKBIT_USERNAME is not set."
        && exit 1 )
    - test -n "${HAWKBIT_PASSWORD:-}" || (
        echo "precondition failed - HAWKBIT_PASSWORD is not set."
        && echo "It should be set with a protected and masked variable."
        && exit 1 )
    - test -L "${CI_PROJECT_DIR}/artifacts/images/${MACHINE}/${CI_ONIRO_RAUC_BUNDLE_NAME}-${MACHINE}.raucb" || (
        echo "precondition failed - bundle file is either missing or is not a symbolic link."
        && find "${CI_PROJECT_DIR}/artifacts"
        && exit 1 )
    - BUNDLE_FILE="$(readlink "${CI_PROJECT_DIR}/artifacts/images/${MACHINE}/${CI_ONIRO_RAUC_BUNDLE_NAME}-${MACHINE}.raucb")"
    - test -n "$BUNDLE_FILE" || (
        echo "Cannot compute BUNDLE_FILE"
        && ls -l "${CI_PROJECT_DIR}/artifacts/images/${MACHINE}/${CI_ONIRO_RAUC_BUNDLE_NAME}-${MACHINE}.raucb"
        && exit 1 )
    - BUNDLE_VERSION="$(echo "$BUNDLE_FILE" | awk 'BEGIN { RS = "-" } /.*\.raucb/ { sub(".raucb", "", $NF); print $NF }')"
    - test -n "$BUNDLE_VERSION" || (
        echo "Cannot compute BUNDLE_VERSION"
        && echo "BUNDLE_FILE=$BUNDLE_FILE"
        && exit 1 )
    # TODO: create or find sw mod type for SysOTA + RAUC bundle
    - hawkbitctl create software-module
        -name "$CI_ONIRO_HAWKBIT_SWMOD_NAME"
        -version "$BUNDLE_VERSION"
        -type os
        -vendor "Onrio"
        -description "Automatic build from Oniro CI"
    # TODO: create or find ds type for MACHINE and OS
    - hawkbitctl create distribution-set
        -name "$CI_ONIRO_HAWKBIT_DS_NAME"
        -version "$BUNDLE_VERSION"
        -type os
        -modules "$CI_ONIRO_HAWKBIT_SWMOD_NAME:$BUNDLE_VERSION"
        -description "Automatic build from Oniro CI"
    - hawkbitctl upload
        -m "$CI_ONIRO_HAWKBIT_SWMOD_NAME:$BUNDLE_VERSION"
        -a "${CI_PROJECT_DIR}/artifacts/images/${MACHINE}/$(readlink "${CI_PROJECT_DIR}/artifacts/images/${MACHINE}/${CI_ONIRO_RAUC_BUNDLE_NAME}-${MACHINE}.raucb")"
  rules:
    # Publishing is done only for scheduled builds
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    # For merge requests, the publishing is optional and can be triggered manually.
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: manual
      allow_failure: true

.build-zephyr-image:
  extends: .build-image
  variables:
    CI_ONIRO_JOB_ARTIFACTS: "*.elf *.bin"
  script:
    - !reference [.build-image, script]

# This job is currently used to customize the behavior in oniro and xts-acts.
# It will be removed when that is safe to do so. It is not documented.
.build:
 extends: .build-recipe

# This job is documented in docs/ci/hidden-jobs/build-docs.rst
.build-docs:
  interruptible: true
  image:
    name: registry.ostc-eu.org/ostc/oniro/docs-builder:latest
  stage: build
  script:
    - make -C docs
  artifacts:
    paths:
      - docs/build
  rules:
    # Build the docs when a merge request is created.
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      changes:
        # React to changes to the docs directory.
        - docs/**/*
        # Run this job in case the pipeline changes.
        - .oniro-ci/*.yml
        - .gitlab-ci.yml
    # Or when things land.
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
      changes:
        # React to changes to the docs directory.
        - docs/**/*
        # Run this job in case the pipeline changes.
        - .oniro-ci/*.yml
        - .gitlab-ci.yml

# This job is documented in docs/ci/hidden-jobs/aggregate-docs.rst
.aggregate-docs:
  stage: deploy
  trigger: eclipse/oniro-core/docs
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
      when: never
    # Update the documentation when things land in the default branch.
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
      changes:
        # React to changes to the docs directory.
        - docs/**/*
        # Run this job in case the pipeline changes.
        - .oniro-ci/*.yml
        - .gitlab-ci.yml