Merge branch 'feat/ef-runner' into 'main'

Draft: feat: EF pipeline migration See merge request !13

Merge branch 'feat/ef-runner' into 'main'
Draft: feat: EF pipeline migration See merge request !13
8eda15e7 · Sébastien Heurtematte · 4fc590e1 · e2ec1690 · 8eda15e7 · 8eda15e7
Commit 8eda15e7 authored 2 years ago by Sébastien Heurtematte
--- a/.gitignore
+++ b/.gitignore
@@ -16,3 +16,4 @@ man/*.7
 man/*.5
 sysota-*.tar.gz
 sysota-*.tar.gz.asc
+coverage.txt
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Huawei Inc.
+default:
+  tags:
+    - origin:eclipse
+    - ctx:oniro-group
 stages:
  - compliance
  - test
@@ -16,12 +21,13 @@ include:
    # The project requires Go 1.16 compatibility.
    # For Yocto Kirkstone the baseline will move to Go 1.18.
    CI_GO_VERSION: "1.16"
+    GOPATH: $CI_PROJECT_DIR.tmp/go
+  needs: []
 dco:
  interruptible: true
-  tags: [kvm] # Technically we want x86_64 or aarch64 but there's no OR syntax for tags, this is a way to cheat.
  stage: compliance
-  image: christophebedard/dco-check:latest
+  image: docker.io/onirocore/dco-check:latest
  rules:
    - if: $CI_MERGE_REQUEST_ID
    - if: $CI_EXTERNAL_PULL_REQUEST_IID
@@ -42,36 +48,25 @@ dco:
 reuse:
  interruptible: true
  stage: compliance
-  tags: [x86_64] # The aarch64 binary is actually an x86_64 binary.
  image:
-    name: fsfe/reuse:latest
+    name: docker.io/onirocore/reuse:latest
    entrypoint: [""]
  script:
    - reuse lint
 go-test:
-  tags: [$RUNNER_OS, $RUNNER_ARCH]
+  image: docker.io/onirocore/go-test:$CI_GO_VERSION
+  stage: test
+  needs: 
+    - job: docker-go-test
+      optional: true
  parallel:
    matrix:
-      - RUNNER_OS: linux
+      - CI_GO_VERSION: ["1.16", latest]
-        RUNNER_ARCH: [x86_64, aarch64, armv7l]
-        CI_GO_VERSION: ["1.16", latest]
-  before_script:
-    # Set up caching.
-    - !reference [.go, before_script]
-    - echo -e "\e[0Ksection_start:$(date +%s):install_debian[collapsed=true]\r\e[0KInstalling Debian dependencies..."
-    # Install dbus-daemon and squashfs-tools which are used by dbus and
-    # squashfs-tools tests. The golang:latest image is based on Debian so we
-    # can just apt-get install it.
-    - apt-get update && apt-get install -y dbus squashfs-tools
-    - echo -e "\e[0Ksection_end:$(date +%s):install_debian\r\e[0K"
-    # Install go package for emitting GitLab friendly coverage information.
-    - echo -e "\e[0Ksection_start:$(date +%s):install_cover\r\e[0KInstalling Go coverage dependencies..."
-    - go install github.com/boumenot/gocover-cobertura@latest
-    - echo -e "\e[0Ksection_end:$(date +%s):install_cover\r\e[0K"
  script:
    - echo -e "\e[0Ksection_start:$(date +%s):test\r\e[0KTesting..."
-    - go test $(case $(uname -m) in x86_64|aarch64) echo -race ;; esac) -coverprofile=coverage.txt -covermode atomic ./...
+    - export TMPDIR=/tmp
+    - go test $(case $(uname -m) in x86_64) echo -race ;; esac) -coverprofile=coverage.txt -covermode atomic ./...
    - echo -e "\e[0Ksection_end:$(date +%s):test\r\e[0K"
    - echo -e "\e[0Ksection_start:$(date +%s):test_coverage[collapsed=true]\r\e[0KGenerating coverage report..."
    - go tool cover -func=coverage.txt
@@ -81,76 +76,86 @@ go-test:
    paths:
      - coverage.xml
    reports:
-      cobertura: coverage.xml
+      coverage_report:
+        coverage_format: cobertura
+        path: coverage.xml
-.build-with-kaniko:
+.build-with-buildkit:
-  stage: build
+  stage: image
-  # This is artificially limited because we don't handle build tags correctly
-  # and the other side of this job requires x86_64 (for spread).
-  tags: [x86_64]
  image:
-    name: gcr.io/kaniko-project/executor:debug
+    name: eclipsefdn/containertools
    entrypoint: [""]
+  variables:
+    SERVICE_ADDRESS: "tcp://buildkitd.foundation-internal-infra-buildkitd:1234"
+    IMAGE_VERSION: latest
+    BUILD_ARG: ""
  script:
+    - test -n "$CI_REGISTRY" || (
+          echo "CI_REGISTRY not provided. Docker registry deployment can not continue."
+          && exit 1 )
+    - test -n "$CI_REGISTRY_IMAGE" || (
+          echo "CI_REGISTRY_IMAGE not provided. Docker registry deployment can not continue."
+          && exit 1 )
+    - test -n "$CI_REGISTRY_USER" || (
+          echo "CI_REGISTRY_USER not provided. Docker registry deployment can not continue."
+          && exit 1 )
+    - test -n "$CI_REGISTRY_PASSWORD" || (
+          echo "CI_REGISTRY_PASSWORD not provided. Docker registry deployment can not continue."
+          && exit 1 )
    - |
      set -xe
-      mkdir -p /kaniko/.docker
+      export HOME=$CI_PROJECT_DIR
-      printf '{"auths":{"%s":{"auth":"%s"}}}\n' "$CI_REGISTRY" "$(printf '%s:%s' "$CI_REGISTRY_USER" "$CI_REGISTRY_PASSWORD" | base64)" > /kaniko/.docker/config.json
+      crane auth login "$CI_REGISTRY" --username "$CI_REGISTRY_USER" --password "$CI_REGISTRY_PASSWORD"
      BUILD_DATE="$(date '+%FT%T%z' | sed -E -n 's/(\+[0-9]{2})([0-9]{2})$/\1:\2/p')" #rfc 3339 date
      BUILD_TITLE=$(echo "$CI_PROJECT_TITLE" | tr " " "_")
-      IMAGE_LABELS="$(cat <<EOM
+      IMAGE_ANNOTATIONS="annotation.build-date=$BUILD_DATE,"
-          --label build-date=$BUILD_DATE
+      IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.cijoburl=$CI_JOB_URL,"
-          --label com.gitlab.ci.cijoburl=$CI_JOB_URL
+      IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.commiturl=$CI_PROJECT_URL/commit/$CI_COMMIT_SHA,"
-          --label com.gitlab.ci.commiturl=$CI_PROJECT_URL/commit/$CI_COMMIT_SHA
+      IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.email=$GITLAB_USER_EMAIL,"
-          --label com.gitlab.ci.email=$GITLAB_USER_EMAIL
+      IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.mrurl=$CI_PROJECT_URL/-/merge_requests/$CI_MERGE_REQUEST_ID,"
-          --label com.gitlab.ci.mrurl=$CI_PROJECT_URL/-/merge_requests/$CI_MERGE_REQUEST_ID
+      IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.pipelineurl=$CI_PIPELINE_URL,"
-          --label com.gitlab.ci.pipelineurl=$CI_PIPELINE_URL
+      IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.tagorbranch=$CI_COMMIT_REF_NAME,"
-          --label com.gitlab.ci.tagorbranch=$CI_COMMIT_REF_NAME
+      IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.user=$CI_SERVER_URL/$GITLAB_USER_LOGIN,"
-          --label com.gitlab.ci.user=$CI_SERVER_URL/$GITLAB_USER_LOGIN
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.authors=$CI_SERVER_URL/$GITLAB_USER_LOGIN,"
-          --label org.opencontainers.image.authors=$CI_SERVER_URL/$GITLAB_USER_LOGIN
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.created=$BUILD_DATE,"
-          --label org.opencontainers.image.created=$BUILD_DATE
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.description=$BUILD_TITLE,"
-          --label org.opencontainers.image.description=$BUILD_TITLE
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.documentation=$CI_PROJECT_URL,"
-          --label org.opencontainers.image.documentation=$CI_PROJECT_URL
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.licenses=$CI_PROJECT_URL,"
-          --label org.opencontainers.image.licenses=$CI_PROJECT_URL
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.ref.name=$CI_REGISTRY:$CI_COMMIT_REF_NAME,"
-          --label org.opencontainers.image.ref.name=$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.revision=$CI_COMMIT_SHA,"
-          --label org.opencontainers.image.revision=$CI_COMMIT_SHA
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.source=$CI_PROJECT_URL,"
-          --label org.opencontainers.image.source=$CI_PROJECT_URL
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.title=$BUILD_TITLE,"
-          --label org.opencontainers.image.title=$BUILD_TITLE
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.url=$CI_PROJECT_URL,"
-          --label org.opencontainers.image.url=$CI_PROJECT_URL
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.vendor=$CI_SERVER_URL/$GITLAB_USER_LOGIN,"
-          --label org.opencontainers.image.vendor=$CI_SERVER_URL/$GITLAB_USER_LOGIN
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.version=$CI_COMMIT_TAG,"
-          --label org.opencontainers.image.version=$CI_COMMIT_TAG
+      IMAGE_ANNOTATIONS+="annotation.vcs-url=$CI_PROJECT_URL"
-          --label vcs-url=$CI_PROJECT_URL
-      EOM
-      )"
-      ADDITIONAL_TAG_LIST="$CI_COMMIT_REF_NAME $CI_COMMIT_SHORT_SHA"
+      echo "Building and shipping image to $CI_REGISTRY"
-      if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then
+      OUTPUT="$CONTAINER_PATH.tar"
-          ADDITIONAL_TAG_LIST="$ADDITIONAL_TAG_LIST latest";
+      buildctl --addr="$SERVICE_ADDRESS" \
-      fi
+        build \
+          --progress=plain \
-      if [ -n "$ADDITIONAL_TAG_LIST" ]; then
+          --frontend=dockerfile.v0 \
-          for TAG in $ADDITIONAL_TAG_LIST; do
+          --local context="$CI_PROJECT_DIR/docker/$CONTAINER_PATH" \
-              FORMATTED_TAG_LIST="$FORMATTED_TAG_LIST --tag $CI_REGISTRY_IMAGE:$TAG "
+          --local dockerfile="$CI_PROJECT_DIR/docker/$CONTAINER_PATH" \
-          done
+          --output "type=oci,$IMAGE_ANNOTATIONS,dest=$OUTPUT" \
-      fi
+          $BUILD_ARG
-      FORMATTED_TAG_LIST="$(echo "$FORMATTED_TAG_LIST" | sed -e 's/--tag/--destination/g')"
+      crane push "$OUTPUT" "$CI_REGISTRY_IMAGE/$CONTAINER_PATH"
+      crane tag "$CI_REGISTRY_IMAGE/$CONTAINER_PATH" "$IMAGE_VERSION"
-      echo "Building and shipping image to $CI_REGISTRY_IMAGE"
-      exec /kaniko/executor --context "$CI_PROJECT_DIR/$CONTAINER_PATH" --dockerfile "$CI_PROJECT_DIR/$CONTAINER_PATH/Dockerfile" --destination $CI_REGISTRY_IMAGE/$CONTAINER_PATH $IMAGE_LABELS
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      changes:
-        - $CONTAINER_PATH/*
+        - docker/$CONTAINER_PATH/*
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
      changes:
-        - $CONTAINER_PATH/*
+        - docker/$CONTAINER_PATH/*
 spell:
  interruptible: true
  stage: test
  image:
-    name: registry.booting.oniroproject.org/distro/oniro/npm-cspell:latest
+    name: docker.io/onirocore/npm-cspell:latest
-  tags: [x86_64]  # TODO(zyga): publish a container ourselves or switch to node containers.
+  needs: []
  script:
    # Note that the particular options are duplicated with what's defined in
    # Makefile. To avoid this we'd have to get zmk as an installable dependency
@@ -159,31 +164,40 @@ spell:
    - "cspell lint '**/*.go' '**/*.md' '**/*.yaml' '**/*.json' '**/*.txt' '**/*.log' '**/*.mk' Makefile '.tours/*.tour' 'man/*.in'"
 docker-spread:
-  extends: .build-with-kaniko
+  extends: .build-with-buildkit
+  variables:
+    CONTAINER_PATH: spread
+docker-go-test:
+  extends: .build-with-buildkit
+  parallel:
+    matrix:
+      - IMAGE_VERSION: "1.16"
+        BUILD_ARG: "--opt build-arg:BASE_IMAGE_VERSION=$IMAGE_VERSION"
+      - IMAGE_VERSION: "latest"
+        BUILD_ARG: "--opt build-arg:BASE_IMAGE_VERSION=$IMAGE_VERSION"
  variables:
-    CONTAINER_PATH: docker/spread
+    CONTAINER_PATH: go-test
 spread-qemu:
  interruptible: true
  # TODO(zyga): depend on amd64/linux cross build explicitly when
  # GitLab supports this: https://gitlab.com/gitlab-org/gitlab/-/issues/254821
  stage: integration
+  tags:
+    - origin:eclipse
+    - tools:qemu
  needs:
    - job: go-test
      artifacts: false
  image: registry.booting.oniroproject.org/distro/components/sysota/docker/spread:latest
-  tags: [kvm, x86_64]  # TODO(zyga): build this container locally, add aarch64 support
  variables:
-    DEBIAN_FRONTEND: noninteractive
    SPREAD_QEMU_KVM: "1"
    TRANSFER_METER_FREQUENCY: "2s"
    ARTIFACT_COMPRESSION_LEVEL: "fast"
    CACHE_COMPRESSION_LEVEL: "fastest"
  parallel:
    matrix:
-      - VERSION_CODENAME: [impish]
-        ID: [ubuntu]
-        ID_VERSION: ["21.10"]
      - VERSION_CODENAME: [jammy]
        ID: [ubuntu]
        ID_VERSION: ["22.04"]

--- a/boot/piboot/piboot.go
+++ b/boot/piboot/piboot.go
@@ -14,9 +14,9 @@
 // configuration.
 //
 // References:
-//  - https://github.com/raspberrypi/linux/commit/757666748ebf69dc161a262faa3717a14d68e5aa
+//   - https://github.com/raspberrypi/linux/commit/757666748ebf69dc161a262faa3717a14d68e5aa
-//  - https://github.com/raspberrypi/rpi-eeprom/blob/master/firmware/release-notes.md
+//   - https://github.com/raspberrypi/rpi-eeprom/blob/master/firmware/release-notes.md
-//  - https://www.raspberrypi.org/documentation/hardware/raspberrypi/bootmodes/README.md
+//   - https://www.raspberrypi.org/documentation/hardware/raspberrypi/bootmodes/README.md
 package piboot
 import (

--- a/docker/go-test/Dockerfile
+++ b/docker/go-test/Dockerfile
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Huawei Inc.
+ARG BASE_IMAGE=golang
+ARG BASE_IMAGE_VERSION=latest
+FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION}
+ARG DEBIAN_FRONTEND="noninteractive"
+# hadolint ignore=DL3008
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends eatmydata && \
+    eatmydata apt-get install -y --no-install-recommends \
+      dbus squashfs-tools \
+      \
+    && eatmydata apt-get clean && rm -rf /var/lib/apt/lists/*
--- a/docker/spread/Dockerfile
+++ b/docker/spread/Dockerfile
@@ -10,6 +10,7 @@ ARG DEBIAN_FRONTEND="noninteractive"
 RUN apt-get update -qq \
 && apt-get install -y eatmydata \
 && eatmydata apt-get install -qq -y 'spread=0~20201124-1+ohos3' \
- && eatmydata apt-get install -qq -y 'ca-certificates=20210119~20.04.2' \
+ && eatmydata apt-get install -qq -y ca-certificates \
 && eatmydata apt-get install -qq -y autopkgtest qemu-system-x86 python3-distro-info genisoimage \
- && eatmydata apt-get clean && rm -rf /var/lib/apt/lists/*
+ && eatmydata apt-get install -qq -y net-tools iputils-ping \
+ && eatmydata apt-get clean && rm -rf /var/lib/apt/lists/* 
--- a/pkg/dbusutil/watcher.go
+++ b/pkg/dbusutil/watcher.go
@@ -36,32 +36,32 @@ type WatchedObject struct {
 //
 // Callers should adopt the following structure:
 //
-//     conn, err := dbus.SessionBusPrivate(dbus.WithSignalHandler(dbus.NewSequentialSignalHandler()))
+//	conn, err := dbus.SessionBusPrivate(dbus.WithSignalHandler(dbus.NewSequentialSignalHandler()))
-//     if err != nil {
+//	if err != nil {
-//         /* error handling */
+//	    /* error handling */
-//     }
+//	}
 //
-//     ctx, cancel := context.WithCancel(ctx)
+//	ctx, cancel := context.WithCancel(ctx)
-//     /* make sure to cancel the context at some point to remove the D-Bus watch */
+//	/* make sure to cancel the context at some point to remove the D-Bus watch */
-//     defer cancel()
+//	defer cancel()
 //
-//     w, err := NewWatchedObject(ctx, conn, busName, path, iface1, iface2)
+//	w, err := NewWatchedObject(ctx, conn, busName, path, iface1, iface2)
-//     if err != nil {
+//	if err != nil {
-//         /* error handling */
+//	    /* error handling */
-//     }
+//	}
 //
-//     for /* loop termination condition (e.g. something is marked as done) */ {
+//	for /* loop termination condition (e.g. something is marked as done) */ {
-//         select {
+//	    select {
-//             case iface <- w.Changes(): /* changes are not buffered */
+//	        case iface <- w.Changes(): /* changes are not buffered */
-//                 fmt.Printf("properties on interface %s have changed\n", iface)
+//	            fmt.Printf("properties on interface %s have changed\n", iface)
-//             case <-w.ChangeLost(): /* change lost has a buffer dept of one */
+//	        case <-w.ChangeLost(): /* change lost has a buffer dept of one */
-//                 fmt.Printf("properties have changed while we were busy\n")
+//	            fmt.Printf("properties have changed while we were busy\n")
-//         }
+//	    }
-//     }
+//	}
 //
-//     <- w.Done() /* Wait for monitoring shut-down. */
+//	<- w.Done() /* Wait for monitoring shut-down. */
 //
-//     /* Updates have ceased and final state is available for inspection. */
+//	/* Updates have ceased and final state is available for inspection. */
 //
 // This guarantees that the terminating condition is never lost. It is
 // important to check before falling asleep for changes, as the changes may

--- a/spread.yaml
+++ b/spread.yaml
@@ -4,15 +4,9 @@
 project: sysota
 backends:
-    lxd:
-        systems:
-            - ubuntu-21.04:
    qemu:
        memory: 1G
        systems:
-            - ubuntu-21.10-64:
-                username: ubuntu
-                password: ubuntu
            - ubuntu-22.04-64:
                username: ubuntu
                password: ubuntu