.gitlab-ci.yml

# SPDX-License-Identifier: Apache-2.0
#
# Copyright 2020-2021 Huawei Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

stages:
  - compliance
  - build
  - update
  - test
  - report
  - deploy

include:
 - local: '/.oniro-ci/build-generic.yaml'
 - local: '/.oniro-ci/test-generic.yaml'
 - local: '/.oniro-ci/machines-and-flavours.yaml'
 - local: '/.oniro-ci/dco.yaml'
 - template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml'

dco:
  extends: .dco

build-docs:
  extends: .build-docs

aggregate-docs:
  extends: .aggregate-docs
  needs: [build-docs]

# Customize the .workspace job to set the path of the git repository to deviate
# from what the git-repo prepares. This effectively allows testing incoming
# changes that match the repository holding this CI pipeline.
.workspace:
  variables:
    CI_ONIRO_GIT_REPO_PATH: oniro

# Customize the .bitbake-workspace job to set the rules governing when a build
# is attempted to: modifications (changes) to the pipeline, meta-layers,
# flavours and assets OR to placement of a tag on a commit OR when a job is
# scheduled. In addition, draft merge requests will no longer start the heavy
# build jobs automatically, giving an option to the developer, to start the
# desired jobs manually.
.bitbake-workspace:
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
      variables:
        CI_ONIRO_MANIFEST_URL: "$CI_PROJECT_URL"
        CI_ONIRO_MANIFEST_BRANCH: "$CI_COMMIT_REF_NAME"
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TITLE =~ /^(wip|draft):.*/i'
      when: manual
      variables:
        # See below for rationale.
        CI_ONIRO_MANIFEST_URL: "$CI_MERGE_REQUEST_SOURCE_PROJECT_URL"
        CI_ONIRO_MANIFEST_BRANCH: "$CI_COMMIT_REF_NAME"
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      changes:
        - assets/**/*
        - flavours/**/*
        - manifests/*.xml
        - meta-*/**/*
        - .oniro-ci/*
        - .gitlab-ci.yml
      # When building a merge request substitute the CI_ONIRO_MANIFEST_URL to point
      # to the incoming repository, and CI_ONIRO_MANIFEST_BRANCH, to the right
      # branch name. This allows testing the changes coming into project. This
      # is done here, so that it does not clobber scheduled pipelines.
      variables:
        CI_ONIRO_MANIFEST_URL: "$CI_MERGE_REQUEST_SOURCE_PROJECT_URL"
        CI_ONIRO_MANIFEST_BRANCH: "$CI_COMMIT_REF_NAME"
    - if: '$CI_COMMIT_TAG'
      # XXX: This needs CI_ONIRO_MANIFEST_BRANCH as well, most likely.

# Build a RAUC update bundle for Raspberry Pi 4
bundle-raspberrypi4-64:
  extends: .build-rauc-bundle
  stage: update
  # Depend on the build job to prevent repeating build failures.
  needs: [linux-raspberrypi4-64]
  variables:
    MACHINE: raspberrypi4-64
    CI_ONIRO_BUILD_FLAVOUR: linux
    CI_ONIRO_RECIPE_NAME: oniro-bundle-base

# Build a RAUC update bundle for SECO B68
bundle-seco-intel-b68:
  extends: .build-rauc-bundle
  stage: update
  # Depend on the build job to prevent repeating build failures.
  needs: [linux-seco-intel-b68]
  variables:
    MACHINE: seco-intel-b68
    CI_ONIRO_BUILD_FLAVOUR: linux
    CI_ONIRO_RECIPE_NAME: oniro-bundle-base

publish-raspberrypi4-64:
  extends: .publish-rauc-bundle-to-hawkbit
  dependencies: [bundle-raspberrypi4-64]
  needs: [bundle-raspberrypi4-64]
  variables:
    CI_ONIRO_HAWKBIT_SWMOD_NAME: oniro-bundle-base-raspberrypi4-64
    CI_ONIRO_HAWKBIT_DS_NAME: oniro-image-base-raspberrypi4-64
    CI_ONIRO_RAUC_BUNDLE_NAME: oniro-bundle-base
    MACHINE: raspberrypi4-64

publish-seco-intel-b68:
  extends: .publish-rauc-bundle-to-hawkbit
  dependencies: [bundle-seco-intel-b68]
  needs: [bundle-seco-intel-b68]
  variables:
    CI_ONIRO_HAWKBIT_SWMOD_NAME: oniro-bundle-base-seco-intel-b68
    CI_ONIRO_HAWKBIT_DS_NAME: oniro-image-base-seco-intel-b68
    CI_ONIRO_RAUC_BUNDLE_NAME: oniro-bundle-base
    MACHINE: seco-intel-b68

.build-wic-image-daily:
  extends: .build-wic-image
  variables:
    # Add packages needed by daily testing.
    CI_ONIRO_BB_LOCAL_CONF_IMAGE_INSTALL_colon_append: " ltp"
  rules:
    # Run the build for scheduled pipelines.
    - if: '$CI_PIPELINE_SOURCE == "schedule"'

# Mimic the updated rules for lava-test from the bitbake-workspace.
.lava-test:
  rules:
    - if: '$CI_LAVA_TOKEN == null'
      when: never
    - if: '$CI_PIPELINE_SOURCE == "schedule"'

# Mimic the updated rules for lava-report from the bitbake-workspace.
.lava-report:
  rules:
    - if: '$CI_LAVA_TOKEN == null'
      when: never
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
      when: manual

##
## Submit jobs to LAVA
##
lava-qemu-x86:
  needs: [linux-qemu-x86]
  stage: test
  extends: .lava-test
  variables:
    MACHINE: qemux86
    CI_BUILD_JOB_NAME: linux-qemu-x86
    CI_LAVA_JOB_DEFINITION: "https://git.ostc-eu.org/OSTC/infrastructure/lava/lava-config/-/raw/master/lava.ostc-eu.org/job-definitions/ci/qemu-x86.yaml"
    CI_REPORT_JOB_NAME: lava-report

lava-qemu-x86_64:
  needs: [linux-qemu-x86_64]
  stage: test
  extends: .lava-test
  variables:
    MACHINE: qemux86-64
    CI_BUILD_JOB_NAME: linux-qemu-x86_64
    CI_LAVA_JOB_DEFINITION: "https://git.ostc-eu.org/OSTC/infrastructure/lava/lava-config/-/raw/master/lava.ostc-eu.org/job-definitions/ci/qemu-x86_64.yaml"
    CI_REPORT_JOB_NAME: lava-report

lava-raspberrypi4-64:
  needs: [linux-raspberrypi4-64]
  stage: test
  extends: .lava-test
  variables:
    MACHINE: raspberrypi4-64
    CI_BUILD_JOB_NAME: linux-raspberrypi4-64
    CI_LAVA_JOB_DEFINITION: "https://git.ostc-eu.org/OSTC/infrastructure/lava/lava-config/-/raw/master/lava.ostc-eu.org/job-definitions/ci/raspberrypi4-64.yaml"
    CI_REPORT_JOB_NAME: lava-report

lava-seco-intel-b68:
  needs: [linux-seco-intel-b68]
  stage: test
  extends: .lava-test
  variables:
    MACHINE: seco-intel-b68
    CI_BUILD_JOB_NAME: linux-seco-intel-b68
    CI_LAVA_JOB_DEFINITION: "https://git.ostc-eu.org/OSTC/infrastructure/lava/lava-config/-/raw/master/lava.ostc-eu.org/job-definitions/ci/seco-intel-b68.yaml"
    CI_REPORT_JOB_NAME: lava-report

lava-seco-c61:
  needs: [linux-seco-imx8mm-c61-4gb]
  stage: test
  extends: .lava-test
  variables:
    MACHINE: seco-imx8mm-c61-4gb
    CI_BUILD_JOB_NAME: linux-seco-imx8mm-c61-4gb
    CI_LAVA_JOB_DEFINITION: "https://git.ostc-eu.org/OSTC/infrastructure/lava/lava-config/-/raw/master/lava.ostc-eu.org/job-definitions/ci/seco-c61.yaml"
    CI_REPORT_JOB_NAME: lava-report

lava-qemu-x86-ltp:
  needs: [linux-qemu-x86]
  stage: test
  extends: .lava-test
  variables:
    MACHINE: qemux86
    CI_BUILD_JOB_NAME: linux-qemu-x86
    CI_LAVA_JOB_DEFINITION: "https://git.ostc-eu.org/OSTC/infrastructure/lava/lava-config/-/raw/master/lava.ostc-eu.org/job-definitions/ci/qemu-x86-ltp.yaml"
    CI_REPORT_JOB_NAME: lava-report

lava-qemu-x86_64-ltp:
  needs: [linux-qemu-x86_64]
  stage: test
  extends: .lava-test
  variables:
    MACHINE: qemux86-64
    CI_BUILD_JOB_NAME: linux-qemu-x86_64
    CI_LAVA_JOB_DEFINITION: "https://git.ostc-eu.org/OSTC/infrastructure/lava/lava-config/-/raw/master/lava.ostc-eu.org/job-definitions/ci/qemu-x86_64-ltp.yaml"
    CI_REPORT_JOB_NAME: lava-report

lava-raspberrypi4-64-ltp:
  needs: [linux-raspberrypi4-64]
  stage: test
  extends: .lava-test
  variables:
    MACHINE: raspberrypi4-64
    CI_BUILD_JOB_NAME: linux-raspberrypi4-64
    CI_LAVA_JOB_DEFINITION: "https://git.ostc-eu.org/OSTC/infrastructure/lava/lava-config/-/raw/master/lava.ostc-eu.org/job-definitions/ci/raspberrypi4-64-ltp.yaml"
    CI_REPORT_JOB_NAME: lava-report

lava-seco-intel-b68-ltp:
  needs: [linux-seco-intel-b68]
  stage: test
  extends: .lava-test
  variables:
    MACHINE: seco-intel-b68
    CI_BUILD_JOB_NAME: linux-seco-intel-b68
    CI_LAVA_JOB_DEFINITION: "https://git.ostc-eu.org/OSTC/infrastructure/lava/lava-config/-/raw/master/lava.ostc-eu.org/job-definitions/ci/seco-intel-b68-ltp.yaml"
    CI_REPORT_JOB_NAME: lava-report

lava-seco-c61-ltp:
  needs: [linux-seco-imx8mm-c61-4gb]
  stage: test
  extends: .lava-test
  variables:
    MACHINE: seco-imx8mm-c61-4gb
    CI_BUILD_JOB_NAME: linux-seco-imx8mm-c61-4gb
    CI_LAVA_JOB_DEFINITION: "https://git.ostc-eu.org/OSTC/infrastructure/lava/lava-config/-/raw/master/lava.ostc-eu.org/job-definitions/ci/seco-c61-ltp.yaml"
    CI_REPORT_JOB_NAME: lava-report

lava-qemu-x86_64-kselftest:
  needs: [linux-qemu-x86_64]
  stage: test
  extends: .lava-test
  variables:
    MACHINE: qemux86-64
    CI_BUILD_JOB_NAME: linux-qemu-x86_64
    CI_LAVA_JOB_DEFINITION: "https://git.ostc-eu.org/OSTC/infrastructure/lava/lava-config/-/raw/master/lava.ostc-eu.org/job-definitions/ci/qemu-x86_64-kselftest.yaml"
    CI_REPORT_JOB_NAME: lava-report

lava-raspberrypi4-64-kselftest:
  needs: [linux-raspberrypi4-64]
  stage: test
  extends: .lava-test
  variables:
    MACHINE: raspberrypi4-64
    CI_BUILD_JOB_NAME: linux-raspberrypi4-64
    CI_LAVA_JOB_DEFINITION: "https://git.ostc-eu.org/OSTC/infrastructure/lava/lava-config/-/raw/master/lava.ostc-eu.org/job-definitions/ci/raspberrypi4-64-kselftest.yaml"
    CI_REPORT_JOB_NAME: lava-report

lava-seco-c61-kselftest:
  needs: [linux-seco-imx8mm-c61-4gb]
  stage: test
  extends: .lava-test
  variables:
    MACHINE: seco-imx8mm-c61-4gb
    CI_BUILD_JOB_NAME: linux-seco-imx8mm-c61-4gb
    CI_LAVA_JOB_DEFINITION: "https://git.ostc-eu.org/OSTC/infrastructure/lava/lava-config/-/raw/master/lava.ostc-eu.org/job-definitions/ci/seco-c61-kselftest.yaml"
    CI_REPORT_JOB_NAME: lava-report

lava-zephyr-96b-nitrogen-twister:
  needs: [zephyr-96b-nitrogen-twister]
  stage: test
  extends: .lava-test
  variables:
    MACHINE: 96b-nitrogen
    CI_BUILD_JOB_NAME: zephyr-96b-nitrogen-twister
    CI_LAVA_JOB_DEFINITION: "https://git.ostc-eu.org/OSTC/infrastructure/lava/lava-config/-/raw/master/lava.ostc-eu.org/job-definitions/ci/nitrogen-twister.yaml"
    CI_REPORT_JOB_NAME: lava-report

lava-qemu-x86-perf:
  needs: [linux-qemu-x86]
  stage: test
  extends: .lava-test
  variables:
    MACHINE: qemux86
    CI_BUILD_JOB_NAME: linux-qemu-x86
    CI_LAVA_JOB_DEFINITION: "https://git.ostc-eu.org/OSTC/infrastructure/lava/lava-config/-/raw/master/lava.ostc-eu.org/job-definitions/ci/qemu-x86-perf.yaml"
    CI_REPORT_JOB_NAME: lava-report

lava-qemu-x86_64-perf:
  needs: [linux-qemu-x86_64]
  stage: test
  extends: .lava-test
  variables:
    MACHINE: qemux86-64
    CI_BUILD_JOB_NAME: linux-qemu-x86_64
    CI_LAVA_JOB_DEFINITION: "https://git.ostc-eu.org/OSTC/infrastructure/lava/lava-config/-/raw/master/lava.ostc-eu.org/job-definitions/ci/qemu-x86_64-perf.yaml"
    CI_REPORT_JOB_NAME: lava-report

lava-raspberrypi4-64-perf:
  needs: [linux-raspberrypi4-64]
  stage: test
  extends: .lava-test
  variables:
    MACHINE: raspberrypi4-64
    CI_BUILD_JOB_NAME: linux-raspberrypi4-64
    CI_LAVA_JOB_DEFINITION: "https://git.ostc-eu.org/OSTC/infrastructure/lava/lava-config/-/raw/master/lava.ostc-eu.org/job-definitions/ci/raspberrypi4-64-perf.yaml"
    CI_REPORT_JOB_NAME: lava-report

lava-seco-intel-b68-perf:
  needs: [linux-seco-intel-b68]
  stage: test
  extends: .lava-test
  variables:
    MACHINE: seco-intel-b68
    CI_BUILD_JOB_NAME: linux-seco-intel-b68
    CI_LAVA_JOB_DEFINITION: "https://git.ostc-eu.org/OSTC/infrastructure/lava/lava-config/-/raw/master/lava.ostc-eu.org/job-definitions/ci/seco-intel-b68-perf.yaml"
    CI_REPORT_JOB_NAME: lava-report

lava-seco-c61-perf:
  needs: [linux-seco-imx8mm-c61-4gb]
  stage: test
  extends: .lava-test
  variables:
    MACHINE: seco-imx8mm-c61-4gb
    CI_BUILD_JOB_NAME: linux-seco-imx8mm-c61-4gb
    CI_LAVA_JOB_DEFINITION: "https://git.ostc-eu.org/OSTC/infrastructure/lava/lava-config/-/raw/master/lava.ostc-eu.org/job-definitions/ci/seco-c61-perf.yaml"
    CI_REPORT_JOB_NAME: lava-report

lava-zephyr-qemu-cortex-m3:
  needs: [zephyr-qemu-cortex-m3]
  stage: test
  extends: .lava-test
  variables:
    MACHINE: qemu-cortex-m3
    CI_BUILD_JOB_NAME: zephyr-qemu-cortex-m3
    CI_LAVA_JOB_DEFINITION: "https://git.ostc-eu.org/OSTC/infrastructure/lava/lava-config/-/raw/master/lava.ostc-eu.org/job-definitions/ci/qemu-zephyr-cortex-m3.yaml"
    CI_REPORT_JOB_NAME: lava-report

lava-zephyr-qemu-x86:
  needs: [zephyr-qemu-x86]
  stage: test
  extends: .lava-test
  variables:
    MACHINE: qemu-x86
    CI_BUILD_JOB_NAME: zephyr-qemu-x86
    CI_LAVA_JOB_DEFINITION: "https://git.ostc-eu.org/OSTC/infrastructure/lava/lava-config/-/raw/master/lava.ostc-eu.org/job-definitions/ci/qemu-zephyr-x86.yaml"
    CI_REPORT_JOB_NAME: lava-report

lava-report:
  needs:
    - lava-qemu-x86
    - lava-qemu-x86_64
    - lava-raspberrypi4-64
    - lava-seco-intel-b68
    - lava-seco-c61
    - lava-qemu-x86-ltp
    - lava-qemu-x86_64-ltp
    - lava-raspberrypi4-64-ltp
    - lava-seco-intel-b68-ltp
    - lava-seco-c61-ltp
    - lava-qemu-x86-perf
    - lava-qemu-x86_64-perf
    - lava-raspberrypi4-64-perf
    - lava-seco-intel-b68-perf
    - lava-seco-c61-perf
    - lava-zephyr-96b-nitrogen-twister
    - lava-zephyr-qemu-cortex-m3
    - lava-zephyr-qemu-x86
    - lava-qemu-x86_64-kselftest
    - lava-raspberrypi4-64-kselftest
    - lava-seco-c61-kselftest
  extends: .lava-report

lava-badge:
  needs: [lava-report]
  extends: .lava-badge

.build-with-kaniko:
  stage: build
  image:
    name: gcr.io/kaniko-project/executor:debug
    entrypoint: [""]
  script:
    - test -n "$CI_ONIRO_REGISTRY" || (
          echo "CI_ONIRO_REGISTRY not provided. Docker registry deployment can not continue."
          && exit 1 )
    - test -n "$CI_ONIRO_REGISTRY_IMAGE" || (
          echo "CI_ONIRO_REGISTRY_IMAGE not provided. Docker registry deployment can not continue."
          && exit 1 )
    - test -n "$CI_ONIRO_REGISTRY_USER" || (
          echo "CI_ONIRO_REGISTRY_USER not provided. Docker registry deployment can not continue."
          && exit 1 )
    - test -n "$CI_ONIRO_REGISTRY_TOKEN" || (
          echo "CI_ONIRO_REGISTRY_TOKEN not provided. Docker registry deployment can not continue."
          && exit 1 )
    - |
      set -xe
      mkdir -p /kaniko/.docker
      printf '{"auths":{"%s":{"auth":"%s"}}}\n' "$CI_ONIRO_REGISTRY" "$(printf '%s:%s' "$CI_ONIRO_REGISTRY_USER" "$CI_ONIRO_REGISTRY_TOKEN" | base64)" > /kaniko/.docker/config.json
      BUILD_DATE="$(date '+%FT%T%z' | sed -E -n 's/(\+[0-9]{2})([0-9]{2})$/\1:\2/p')" #rfc 3339 date
      BUILD_TITLE=$(echo "$CI_PROJECT_TITLE" | tr " " "_")
      IMAGE_LABELS="$(cat <<EOM
          --label build-date=$BUILD_DATE
          --label com.gitlab.ci.cijoburl=$CI_JOB_URL
          --label com.gitlab.ci.commiturl=$CI_PROJECT_URL/commit/$CI_COMMIT_SHA
          --label com.gitlab.ci.email=$GITLAB_USER_EMAIL
          --label com.gitlab.ci.mrurl=$CI_PROJECT_URL/-/merge_requests/$CI_MERGE_REQUEST_ID
          --label com.gitlab.ci.pipelineurl=$CI_PIPELINE_URL
          --label com.gitlab.ci.tagorbranch=$CI_COMMIT_REF_NAME
          --label com.gitlab.ci.user=$CI_SERVER_URL/$GITLAB_USER_LOGIN
          --label org.opencontainers.image.authors=$CI_SERVER_URL/$GITLAB_USER_LOGIN
          --label org.opencontainers.image.created=$BUILD_DATE
          --label org.opencontainers.image.description=$BUILD_TITLE
          --label org.opencontainers.image.documentation=$CI_PROJECT_URL
          --label org.opencontainers.image.licenses=$CI_PROJECT_URL
          --label org.opencontainers.image.ref.name=$CI_ONIRO_REGISTRY:$CI_COMMIT_REF_NAME
          --label org.opencontainers.image.revision=$CI_COMMIT_SHA
          --label org.opencontainers.image.source=$CI_PROJECT_URL
          --label org.opencontainers.image.title=$BUILD_TITLE
          --label org.opencontainers.image.url=$CI_PROJECT_URL
          --label org.opencontainers.image.vendor=$CI_SERVER_URL/$GITLAB_USER_LOGIN
          --label org.opencontainers.image.version=$CI_COMMIT_TAG
          --label vcs-url=$CI_PROJECT_URL
      EOM
      )"

      ADDITIONAL_TAG_LIST="$CI_COMMIT_REF_NAME $CI_COMMIT_SHORT_SHA"
      if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then
          ADDITIONAL_TAG_LIST="$ADDITIONAL_TAG_LIST latest";
      fi

      if [ -n "$ADDITIONAL_TAG_LIST" ]; then
          for TAG in $ADDITIONAL_TAG_LIST; do
              FORMATTED_TAG_LIST="$FORMATTED_TAG_LIST --tag $CI_REGISTRY_IMAGE:$TAG "
          done
      fi
      FORMATTED_TAG_LIST="$(echo "$FORMATTED_TAG_LIST" | sed -e 's/--tag/--destination/g')"

      echo "Building and shipping image to $CI_ONIRO_REGISTRY"
      exec /kaniko/executor --context "$CI_PROJECT_DIR/.oniro-ci/containers/$CONTAINER_PATH" --dockerfile "$CI_PROJECT_DIR/.oniro-ci/containers/$CONTAINER_PATH/Dockerfile" --destination $CI_ONIRO_REGISTRY_IMAGE/$CONTAINER_PATH $IMAGE_LABELS
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
      when: never
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      changes:
        - .oniro-ci/containers/$CONTAINER_PATH/*
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
      changes:
        - .oniro-ci/containers/$CONTAINER_PATH/*

build-bitbake-builder:
  extends: .build-with-kaniko
  variables:
    CONTAINER_PATH: bitbake-builder

build-docs-builder:
  extends: .build-with-kaniko
  variables:
    CONTAINER_PATH: docs-builder

build-dco-check:
  extends: .build-with-kaniko
  variables:
    CONTAINER_PATH: dco-check

build-reuse:
  extends: .build-with-kaniko
  variables:
    CONTAINER_PATH: reuse

build-npm-cspell:
  extends: .build-with-kaniko
  variables:
    CONTAINER_PATH: npm-cspell