Merge branch 'feat/ef-runner' into 'main'

Draft: feat: EF pipeline migration

See merge request !13

.gitignore

@@ -16,3 +16,4 @@ man/*.7
 man/*.5
 sysota-*.tar.gz
 sysota-*.tar.gz.asc
+coverage.txt

.gitlab-ci.yml

 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Huawei Inc.
 
+default:
+  tags:
+    - origin:eclipse
+    - ctx:oniro-group
+    
 stages:
   - compliance
+  - image
   - test
   - build
   - integration
@@ -16,12 +22,13 @@ include:
     # The project requires Go 1.16 compatibility.
     # For Yocto Kirkstone the baseline will move to Go 1.18.
     CI_GO_VERSION: "1.16"
+    GOPATH: $CI_PROJECT_DIR.tmp/go
+  needs: []
 
 dco:
   interruptible: true
-  tags: [kvm] # Technically we want x86_64 or aarch64 but there's no OR syntax for tags, this is a way to cheat.
   stage: compliance
-  image: christophebedard/dco-check:latest
+  image: docker.io/onirocore/dco-check:latest
   rules:
     - if: $CI_MERGE_REQUEST_ID
     - if: $CI_EXTERNAL_PULL_REQUEST_IID
@@ -42,36 +49,25 @@ dco:
 reuse:
   interruptible: true
   stage: compliance
-  tags: [x86_64] # The aarch64 binary is actually an x86_64 binary.
   image:
-    name: fsfe/reuse:latest
+    name: docker.io/onirocore/reuse:latest
     entrypoint: [""]
   script:
     - reuse lint
 
 go-test:
-  tags: [$RUNNER_OS, $RUNNER_ARCH]
+  image: docker.io/onirocore/go-test:$CI_GO_VERSION
+  stage: test
+  needs: 
+    - job: docker-go-test
+      optional: true
   parallel:
     matrix:
-      - RUNNER_OS: linux
-        RUNNER_ARCH: [x86_64, aarch64, armv7l]
-        CI_GO_VERSION: ["1.16", latest]
-  before_script:
-    # Set up caching.
-    - !reference [.go, before_script]
-    - echo -e "\e[0Ksection_start:$(date +%s):install_debian[collapsed=true]\r\e[0KInstalling Debian dependencies..."
-    # Install dbus-daemon and squashfs-tools which are used by dbus and
-    # squashfs-tools tests. The golang:latest image is based on Debian so we
-    # can just apt-get install it.
-    - apt-get update && apt-get install -y dbus squashfs-tools
-    - echo -e "\e[0Ksection_end:$(date +%s):install_debian\r\e[0K"
-    # Install go package for emitting GitLab friendly coverage information.
-    - echo -e "\e[0Ksection_start:$(date +%s):install_cover\r\e[0KInstalling Go coverage dependencies..."
-    - go install github.com/boumenot/gocover-cobertura@latest
-    - echo -e "\e[0Ksection_end:$(date +%s):install_cover\r\e[0K"
+      - CI_GO_VERSION: ["1.16", latest]
   script:
     - echo -e "\e[0Ksection_start:$(date +%s):test\r\e[0KTesting..."
-    - go test $(case $(uname -m) in x86_64|aarch64) echo -race ;; esac) -coverprofile=coverage.txt -covermode atomic ./...
+    - export TMPDIR=/tmp
+    - go test $(case $(uname -m) in x86_64) echo -race ;; esac) -coverprofile=coverage.txt -covermode atomic ./...
     - echo -e "\e[0Ksection_end:$(date +%s):test\r\e[0K"
     - echo -e "\e[0Ksection_start:$(date +%s):test_coverage[collapsed=true]\r\e[0KGenerating coverage report..."
     - go tool cover -func=coverage.txt
@@ -81,76 +77,86 @@ go-test:
     paths:
       - coverage.xml
     reports:
-      cobertura: coverage.xml
+      coverage_report:
+        coverage_format: cobertura
+        path: coverage.xml
 
-.build-with-kaniko:
-  stage: build
-  # This is artificially limited because we don't handle build tags correctly
-  # and the other side of this job requires x86_64 (for spread).
-  tags: [x86_64]
+.build-with-buildkit:
+  stage: image
   image:
-    name: gcr.io/kaniko-project/executor:debug
+    name: eclipsefdn/containertools
     entrypoint: [""]
+  variables:
+    SERVICE_ADDRESS: "tcp://buildkitd.foundation-internal-infra-buildkitd:1234"
+    IMAGE_VERSION: latest
+    BUILD_ARG: ""
   script:
+    - test -n "$CI_REGISTRY" || (
+          echo "CI_REGISTRY not provided. Docker registry deployment can not continue."
+          && exit 1 )
+    - test -n "$CI_REGISTRY_IMAGE" || (
+          echo "CI_REGISTRY_IMAGE not provided. Docker registry deployment can not continue."
+          && exit 1 )
+    - test -n "$CI_REGISTRY_USER" || (
+          echo "CI_REGISTRY_USER not provided. Docker registry deployment can not continue."
+          && exit 1 )
+    - test -n "$CI_REGISTRY_PASSWORD" || (
+          echo "CI_REGISTRY_PASSWORD not provided. Docker registry deployment can not continue."
+          && exit 1 )
     - |
       set -xe
-      mkdir -p /kaniko/.docker
-      printf '{"auths":{"%s":{"auth":"%s"}}}\n' "$CI_REGISTRY" "$(printf '%s:%s' "$CI_REGISTRY_USER" "$CI_REGISTRY_PASSWORD" | base64)" > /kaniko/.docker/config.json
+      export HOME=$CI_PROJECT_DIR
+      crane auth login "$CI_REGISTRY" --username "$CI_REGISTRY_USER" --password "$CI_REGISTRY_PASSWORD"
       BUILD_DATE="$(date '+%FT%T%z' | sed -E -n 's/(\+[0-9]{2})([0-9]{2})$/\1:\2/p')" #rfc 3339 date
       BUILD_TITLE=$(echo "$CI_PROJECT_TITLE" | tr " " "_")
-      IMAGE_LABELS="$(cat <<EOM
-          --label build-date=$BUILD_DATE
-          --label com.gitlab.ci.cijoburl=$CI_JOB_URL
-          --label com.gitlab.ci.commiturl=$CI_PROJECT_URL/commit/$CI_COMMIT_SHA
-          --label com.gitlab.ci.email=$GITLAB_USER_EMAIL
-          --label com.gitlab.ci.mrurl=$CI_PROJECT_URL/-/merge_requests/$CI_MERGE_REQUEST_ID
-          --label com.gitlab.ci.pipelineurl=$CI_PIPELINE_URL
-          --label com.gitlab.ci.tagorbranch=$CI_COMMIT_REF_NAME
-          --label com.gitlab.ci.user=$CI_SERVER_URL/$GITLAB_USER_LOGIN
-          --label org.opencontainers.image.authors=$CI_SERVER_URL/$GITLAB_USER_LOGIN
-          --label org.opencontainers.image.created=$BUILD_DATE
-          --label org.opencontainers.image.description=$BUILD_TITLE
-          --label org.opencontainers.image.documentation=$CI_PROJECT_URL
-          --label org.opencontainers.image.licenses=$CI_PROJECT_URL
-          --label org.opencontainers.image.ref.name=$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
-          --label org.opencontainers.image.revision=$CI_COMMIT_SHA
-          --label org.opencontainers.image.source=$CI_PROJECT_URL
-          --label org.opencontainers.image.title=$BUILD_TITLE
-          --label org.opencontainers.image.url=$CI_PROJECT_URL
-          --label org.opencontainers.image.vendor=$CI_SERVER_URL/$GITLAB_USER_LOGIN
-          --label org.opencontainers.image.version=$CI_COMMIT_TAG
-          --label vcs-url=$CI_PROJECT_URL
-      EOM
-      )"
-
-      ADDITIONAL_TAG_LIST="$CI_COMMIT_REF_NAME $CI_COMMIT_SHORT_SHA"
-      if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then
-          ADDITIONAL_TAG_LIST="$ADDITIONAL_TAG_LIST latest";
-      fi
-
-      if [ -n "$ADDITIONAL_TAG_LIST" ]; then
-          for TAG in $ADDITIONAL_TAG_LIST; do
-              FORMATTED_TAG_LIST="$FORMATTED_TAG_LIST --tag $CI_REGISTRY_IMAGE:$TAG "
-          done
-      fi
-      FORMATTED_TAG_LIST="$(echo "$FORMATTED_TAG_LIST" | sed -e 's/--tag/--destination/g')"
+      IMAGE_ANNOTATIONS="annotation.build-date=$BUILD_DATE,"
+      IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.cijoburl=$CI_JOB_URL,"
+      IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.commiturl=$CI_PROJECT_URL/commit/$CI_COMMIT_SHA,"
+      IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.email=$GITLAB_USER_EMAIL,"
+      IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.mrurl=$CI_PROJECT_URL/-/merge_requests/$CI_MERGE_REQUEST_ID,"
+      IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.pipelineurl=$CI_PIPELINE_URL,"
+      IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.tagorbranch=$CI_COMMIT_REF_NAME,"
+      IMAGE_ANNOTATIONS+="annotation.com.gitlab.ci.user=$CI_SERVER_URL/$GITLAB_USER_LOGIN,"
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.authors=$CI_SERVER_URL/$GITLAB_USER_LOGIN,"
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.created=$BUILD_DATE,"
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.description=$BUILD_TITLE,"
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.documentation=$CI_PROJECT_URL,"
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.licenses=$CI_PROJECT_URL,"
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.ref.name=$CI_REGISTRY:$CI_COMMIT_REF_NAME,"
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.revision=$CI_COMMIT_SHA,"
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.source=$CI_PROJECT_URL,"
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.title=$BUILD_TITLE,"
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.url=$CI_PROJECT_URL,"
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.vendor=$CI_SERVER_URL/$GITLAB_USER_LOGIN,"
+      IMAGE_ANNOTATIONS+="annotation.org.opencontainers.image.version=$CI_COMMIT_TAG,"
+      IMAGE_ANNOTATIONS+="annotation.vcs-url=$CI_PROJECT_URL"
 
-      echo "Building and shipping image to $CI_REGISTRY_IMAGE"
-      exec /kaniko/executor --context "$CI_PROJECT_DIR/$CONTAINER_PATH" --dockerfile "$CI_PROJECT_DIR/$CONTAINER_PATH/Dockerfile" --destination $CI_REGISTRY_IMAGE/$CONTAINER_PATH $IMAGE_LABELS
+      echo "Building and shipping image to $CI_REGISTRY"
+      OUTPUT="$CONTAINER_PATH.tar"
+      buildctl --addr="$SERVICE_ADDRESS" \
+        build \
+          --progress=plain \
+          --frontend=dockerfile.v0 \
+          --local context="$CI_PROJECT_DIR/docker/$CONTAINER_PATH" \
+          --local dockerfile="$CI_PROJECT_DIR/docker/$CONTAINER_PATH" \
+          --output "type=oci,$IMAGE_ANNOTATIONS,dest=$OUTPUT" \
+          $BUILD_ARG
+      crane push "$OUTPUT" "$CI_REGISTRY_IMAGE/$CONTAINER_PATH"
+      crane tag "$CI_REGISTRY_IMAGE/$CONTAINER_PATH" "$IMAGE_VERSION"
   rules:
     - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
       changes:
-        - $CONTAINER_PATH/*
+        - docker/$CONTAINER_PATH/*
     - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
       changes:
-        - $CONTAINER_PATH/*
+        - docker/$CONTAINER_PATH/*
 
 spell:
   interruptible: true
   stage: test
   image:
-    name: registry.booting.oniroproject.org/distro/oniro/npm-cspell:latest
-  tags: [x86_64]  # TODO(zyga): publish a container ourselves or switch to node containers.
+    name: docker.io/onirocore/npm-cspell:latest
+  needs: []
   script:
     # Note that the particular options are duplicated with what's defined in
     # Makefile. To avoid this we'd have to get zmk as an installable dependency
@@ -159,20 +165,33 @@ spell:
     - "cspell lint '**/*.go' '**/*.md' '**/*.yaml' '**/*.json' '**/*.txt' '**/*.log' '**/*.mk' Makefile '.tours/*.tour' 'man/*.in'"
 
 docker-spread:
-  extends: .build-with-kaniko
+  extends: .build-with-buildkit
+  variables:
+    CONTAINER_PATH: spread
+
+docker-go-test:
+  extends: .build-with-buildkit
+  parallel:
+    matrix:
+      - IMAGE_VERSION: "1.16"
+        BUILD_ARG: "--opt build-arg:BASE_IMAGE_VERSION=$IMAGE_VERSION"
+      - IMAGE_VERSION: "latest"
+        BUILD_ARG: "--opt build-arg:BASE_IMAGE_VERSION=$IMAGE_VERSION"
   variables:
-    CONTAINER_PATH: docker/spread
+    CONTAINER_PATH: go-test
 
 spread-qemu:
   interruptible: true
   # TODO(zyga): depend on amd64/linux cross build explicitly when
   # GitLab supports this: https://gitlab.com/gitlab-org/gitlab/-/issues/254821
   stage: integration
+  tags:
+    - origin:eclipse
+    - tools:qemu
   needs:
     - job: go-test
       artifacts: false
-  image: registry.booting.oniroproject.org/distro/components/sysota/docker/spread:latest
-  tags: [kvm, x86_64]  # TODO(zyga): build this container locally, add aarch64 support
+  image: docker.io/onirocore/spread:latest
   variables:
     DEBIAN_FRONTEND: noninteractive
     SPREAD_QEMU_KVM: "1"
@@ -181,9 +200,6 @@ spread-qemu:
     CACHE_COMPRESSION_LEVEL: "fastest"
   parallel:
     matrix:
-      - VERSION_CODENAME: [impish]
-        ID: [ubuntu]
-        ID_VERSION: ["21.10"]
       - VERSION_CODENAME: [jammy]
         ID: [ubuntu]
         ID_VERSION: ["22.04"]
@@ -200,7 +216,7 @@ spread-qemu:
         time autopkgtest-buildvm-ubuntu-cloud -r $VERSION_CODENAME --post-command "apt-get install ssh golang-go zmk rauc-service squashfs-tools jq git && apt-get install -d postgresql"
         cd ..
       fi
-    - ln -s $CI_PROJECT_DIR/.cache/autopkgtest-$VERSION_CODENAME-amd64.img ~/.spread/qemu/$ID-$ID_VERSION-64.img
+    - ln -s -f $CI_PROJECT_DIR/.cache/autopkgtest-$VERSION_CODENAME-amd64.img ~/.spread/qemu/$ID-$ID_VERSION-64.img
   script:
     - 'time spread -v qemu:$ID-$ID_VERSION-64:'
   cache:

boot/piboot/piboot.go

@@ -14,9 +14,9 @@
 // configuration.
 //
 // References:
-//  - https://github.com/raspberrypi/linux/commit/757666748ebf69dc161a262faa3717a14d68e5aa
-//  - https://github.com/raspberrypi/rpi-eeprom/blob/master/firmware/release-notes.md
-//  - https://www.raspberrypi.org/documentation/hardware/raspberrypi/bootmodes/README.md
+//   - https://github.com/raspberrypi/linux/commit/757666748ebf69dc161a262faa3717a14d68e5aa
+//   - https://github.com/raspberrypi/rpi-eeprom/blob/master/firmware/release-notes.md
+//   - https://www.raspberrypi.org/documentation/hardware/raspberrypi/bootmodes/README.md
 package piboot
 
 import (

cspell.json

@@ -15,6 +15,7 @@
         "BOOTNAME", // a RAUC variable name
         "bootup", // boot-up, manual page name
         "boumenot", // a GitHub profile name
+        "buildvcs", // a go options for GOFLAGS
         "buildvm", // a particular abbreviation of build virtual machine
         "busctl", // bus control tool, part of systemd
         "busybox", // collection of small command line programs
@@ -65,6 +66,7 @@
         "goconst", // duplicate constant tool for Go
         "gocover", // code test coverage tool for Go
         "godbus", // a Go library for D-Bus
+        "GOFLAGS", // a Go options
         "GOMAXPROCS", // Go Maximum Processes, concurrency control variable for Go
         "gopath", // lower case spelling of GOPATH
         "gopkg", // https://gopkg.in Go package service

docker/go-test/Dockerfile

+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Huawei Inc.
+
+ARG BASE_IMAGE=golang
+ARG BASE_IMAGE_VERSION=latest
+FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION}
+
+ARG DEBIAN_FRONTEND="noninteractive"
+
+# hadolint ignore=DL3008
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends eatmydata && \
+    eatmydata apt-get install -y --no-install-recommends \
+      dbus squashfs-tools \
+      \
+    && eatmydata apt-get clean && rm -rf /var/lib/apt/lists/*
+

docker/spread/Dockerfile

@@ -10,6 +10,6 @@ ARG DEBIAN_FRONTEND="noninteractive"
 RUN apt-get update -qq \
  && apt-get install -y eatmydata \
  && eatmydata apt-get install -qq -y 'spread=0~20201124-1+ohos3' \
- && eatmydata apt-get install -qq -y 'ca-certificates=20210119~20.04.2' \
+ && eatmydata apt-get install -qq -y ca-certificates \
  && eatmydata apt-get install -qq -y autopkgtest qemu-system-x86 python3-distro-info genisoimage \
  && eatmydata apt-get clean && rm -rf /var/lib/apt/lists/*

pkg/dbusutil/watcher.go

@@ -36,32 +36,32 @@ type WatchedObject struct {
 //
 // Callers should adopt the following structure:
 //
-//     conn, err := dbus.SessionBusPrivate(dbus.WithSignalHandler(dbus.NewSequentialSignalHandler()))
-//     if err != nil {
-//         /* error handling */
-//     }
+//	conn, err := dbus.SessionBusPrivate(dbus.WithSignalHandler(dbus.NewSequentialSignalHandler()))
+//	if err != nil {
+//	    /* error handling */
+//	}
 //
-//     ctx, cancel := context.WithCancel(ctx)
-//     /* make sure to cancel the context at some point to remove the D-Bus watch */
-//     defer cancel()
+//	ctx, cancel := context.WithCancel(ctx)
+//	/* make sure to cancel the context at some point to remove the D-Bus watch */
+//	defer cancel()
 //
-//     w, err := NewWatchedObject(ctx, conn, busName, path, iface1, iface2)
-//     if err != nil {
-//         /* error handling */
-//     }
+//	w, err := NewWatchedObject(ctx, conn, busName, path, iface1, iface2)
+//	if err != nil {
+//	    /* error handling */
+//	}
 //
-//     for /* loop termination condition (e.g. something is marked as done) */ {
-//         select {
-//             case iface <- w.Changes(): /* changes are not buffered */
-//                 fmt.Printf("properties on interface %s have changed\n", iface)
-//             case <-w.ChangeLost(): /* change lost has a buffer dept of one */
-//                 fmt.Printf("properties have changed while we were busy\n")
-//         }
-//     }
+//	for /* loop termination condition (e.g. something is marked as done) */ {
+//	    select {
+//	        case iface <- w.Changes(): /* changes are not buffered */
+//	            fmt.Printf("properties on interface %s have changed\n", iface)
+//	        case <-w.ChangeLost(): /* change lost has a buffer dept of one */
+//	            fmt.Printf("properties have changed while we were busy\n")
+//	    }
+//	}
 //
-//     <- w.Done() /* Wait for monitoring shut-down. */
+//	<- w.Done() /* Wait for monitoring shut-down. */
 //
-//     /* Updates have ceased and final state is available for inspection. */
+//	/* Updates have ceased and final state is available for inspection. */
 //
 // This guarantees that the terminating condition is never lost. It is
 // important to check before falling asleep for changes, as the changes may

spread.yaml

@@ -4,15 +4,9 @@
 project: sysota
 
 backends:
-    lxd:
-        systems:
-            - ubuntu-21.04:
     qemu:
         memory: 1G
         systems:
-            - ubuntu-21.10-64:
-                username: ubuntu
-                password: ubuntu
             - ubuntu-22.04-64:
                 username: ubuntu
                 password: ubuntu
@@ -47,7 +41,8 @@ prepare: |
         type "$fn" | tail -n +2 >> $SPREAD_PATH/tests/bin/"$fn"
         echo "$fn \"\$@\"" >> $SPREAD_PATH/tests/bin/"$fn"
     done
-
+    export GOFLAGS="-buildvcs=false"
+    go version
     # Build and install locally
     make configure
     ./configure --prefix=/usr