diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c656595e7c0a74757802715ba2b6242674fdd262..2f7e8b5bfb92f80bf1972ea9bc51f862eb47e4e3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,23 +26,16 @@ include: file: - 'dco.yaml' - 'reuse.yaml' - - project: 'OSTC/OHOS/manifest' - file: '.oniro-ci/build-generic.yaml' - - project: 'OSTC/OHOS/manifest' - file: '.oniro-ci/test-generic.yaml' -# By default, the CI pipeline is invoked using an explicit git ref (branch -# name) in the manifest. This include is a HACK to allow the CI pipeline to get -# changes to the machine-and-flavours.yaml picked up directly from the incoming -# branch in the merge request instead of the explicit branch name in the -# manifest. - local: '/.oniro-ci/machines-and-flavours.yaml' + - local: '/.oniro-ci/build-generic.yaml' + - local: '/.oniro-ci/test-generic.yaml' dco: extends: .dco reuse: extends: .reuse - # FIXME: https://git.ostc-eu.org/OSTC/OHOS/meta-ohos/-/issues/19 + # FIXME: https://git.ostc-eu.org/distro/oniro/-/issues/19 allow_failure: true build-docs: @@ -53,11 +46,11 @@ aggregate-docs: needs: [build-docs] # Customize the .workspace job to set the path of the git repository to deviate -# from what the git-repo manifest prepares. This effectively allows testing -# incoming changes that match the repository holding this CI pipeline. +# from what the git-repo prepares. This effectively allows testing incoming +# changes that match the repository holding this CI pipeline. .workspace: variables: - CI_ONIRO_GIT_REPO_PATH: sources/meta-ohos + CI_ONIRO_GIT_REPO_PATH: oniro # Customize the .bitbake-workspace job to set the rules governing when a build # is attempted to: modifications (changes) to the pipeline, meta-layers, @@ -75,7 +68,7 @@ aggregate-docs: - assets/**/* - flavours/**/* - meta-*/**/* - - .ostc-ci/* + - .oniro-ci/* - if: '$CI_COMMIT_TAG' # Mimic the updated rules for lava-test from the bitbake-workspace. @@ -89,7 +82,7 @@ aggregate-docs: - assets/**/* - flavours/**/* - meta-*/**/* - - .ostc-ci/* + - .oniro-ci/* - if: '$CI_COMMIT_TAG' # Mimic the updated rules for lava-report from the bitbake-workspace. @@ -103,7 +96,7 @@ aggregate-docs: - assets/**/* - flavours/**/* - meta-*/**/* - - .ostc-ci/* + - .oniro-ci/* - if: '$CI_COMMIT_TAG' ## diff --git a/.oniro-ci/build-generic.yaml b/.oniro-ci/build-generic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ef2778a427f0a6fc9e48ea4a105e95523159e03 --- /dev/null +++ b/.oniro-ci/build-generic.yaml @@ -0,0 +1,329 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Huawei Inc. + +# This job is documented in docs/ci/hidden-jobs/workspace.rst +.workspace: + interruptible: true + image: + name: registry.ostc-eu.org/ostc/containers/ostc-builder:latest + tags: [large-disk, $CI_ONIRO_RUNNER_TAG] + variables: + CI_ONIRO_RUNNER_TAG : "" + CI_ONIRO_MANIFEST_URL: https://git.ostc-eu.org/distro/oniro + CI_ONIRO_MANIFEST_BRANCH: develop + CI_ONIRO_MANIFEST_NAME: default.xml + CI_ONIRO_MANIFEST_MIRROR: ostc-develop + CI_ONIRO_GIT_REPO_PATH: "" + before_script: + - test ! -e "$CI_PROJECT_DIR"/.scratch-dir-name || ( + echo "precondition failed - concurrent modification of $CI_PROJECT_DIR" + && env | grep CI_ | sort + && ls -l + && exit 1 ) + + # Create scratch space, being careful not to pollute the working directory. + # This is done so that we are not attached to the contents of + # $CI_PROJECT_DIR which contains something that GitLab CI prepared for us. + - SCRATCH_DIR="$(mktemp -p /tmp -d workspace.XXXXXXXXXX)" + - echo "$SCRATCH_DIR" > "$CI_PROJECT_DIR"/.scratch-dir-name + - cd "$SCRATCH_DIR" + + - | + echo "Building repo workspace with the following properties:" + echo "CI_ONIRO_MANIFEST_URL: $CI_ONIRO_MANIFEST_URL" + echo "CI_ONIRO_MANIFEST_NAME: $CI_ONIRO_MANIFEST_NAME" + echo "CI_ONIRO_MANIFEST_BRANCH: $CI_ONIRO_MANIFEST_BRANCH" + repo init --reference \ + /var/shared/pub/git-repo-mirrors/"$CI_ONIRO_MANIFEST_MIRROR" \ + --manifest-url "$CI_ONIRO_MANIFEST_URL" \ + --manifest-name "$CI_ONIRO_MANIFEST_NAME" \ + --manifest-branch "$CI_ONIRO_MANIFEST_BRANCH" + - time repo sync --no-clone-bundle + + # See the documentation of CI_ONIRO_GIT_REPO_PATH + - | + if [ -n "$CI_ONIRO_GIT_REPO_PATH" ]; then + if [ -n "$CI_MERGE_REQUEST_SOURCE_PROJECT_URL" ]; then + echo "CI: Bootstrapping '$CI_PROJECT_DIR' as 'incoming-merged' remote in '$CI_ONIRO_GIT_REPO_PATH'"; + ( cd "$CI_ONIRO_GIT_REPO_PATH" \ + && git remote add incoming-merged "$CI_PROJECT_DIR" \ + && git fetch incoming-merged HEAD ); + fi; + echo "CI: Switching $CI_ONIRO_GIT_REPO_PATH to $CI_COMMIT_SHA"; + ( cd "$CI_ONIRO_GIT_REPO_PATH" && git checkout "$CI_COMMIT_SHA" ); + fi + + script: + # Reload the value of SCRATCH_DIR set in the before_script phase. Those run + # in separate shell processes and do not share environment variables. + - SCRATCH_DIR="$(cat "$CI_PROJECT_DIR"/.scratch-dir-name)" + - cd "$SCRATCH_DIR" + + after_script: + # If the primary script failed early enough, the scratch dir may not have + # been created yet. Check for that to avoid confusing errors. + - test -f "$CI_PROJECT_DIR"/.scratch-dir-name || exit 0 + # Reload the value of SCRATCH_DIR set in the before_script phase. + - SCRATCH_DIR="$(cat "$CI_PROJECT_DIR"/.scratch-dir-name)" + # Clean up after ourselves. + - rm -f "$CI_PROJECT_DIR"/.scratch-dir-name + - rm -rf "$SCRATCH_DIR" + +# This job is documented in docs/ci/hidden-jobs/bitbake-workspace.rst +.bitbake-workspace: + extends: .workspace + stage: build + timeout: 6h + variables: + CI_ONIRO_BUILD_FLAVOUR: "" + CI_ONIRO_BUILD_CACHE: "private" + CI_ONIRO_BB_LOCAL_CONF_CONNECTIVITY_CHECK_URIS: "https://example.net/" + CI_ONIRO_BB_LOCAL_CONF_DL_DIR: /var/shared/$CI_ONIRO_BUILD_CACHE/bitbake/downloads + CI_ONIRO_BB_LOCAL_CONF_SSTATE_DIR: /var/shared/$CI_ONIRO_BUILD_CACHE/bitbake/sstate-cache + CI_ONIRO_BB_LOCAL_CONF_IMAGE_VERSION_SUFFIX: "" + CI_ONIRO_BB_LOCAL_CONF_CVE_CHECK_DB_DIR: "$${TMPDIR}/CVE_CHECK/" + CI_ONIRO_BB_LOCAL_CONF_plus_equals_INHERIT: cve-check + CI_ONIRO_BB_LOCAL_CONF_plus_equals_USER_CLASSES: "buildstats buildstats-summary" + CI_ONIRO_DEVTOOL_RECIPE_NAME: "" + CI_ONIRO_DEVTOOL_LAYER_PATH: "" + + before_script: + # Bitbake requires a non-root user to operate. + # The container should have a non-root user by default. + - test "$(id -u)" -ne 0 || ( + echo "precondition failed - this job cannot run as root" + && exit 1 ) + # Check if the job is configured properly. + - test -n "$CI_ONIRO_BUILD_FLAVOUR" || ( + echo "precondition failed - set CI_ONIRO_BUILD_FLAVOUR to \"flavour\" of the build to use (e.g. linux)" + && exit 1 ) + # Check devtool operation is enabled and configured properly. + - | + if { [ -n "$CI_ONIRO_DEVTOOL_RECIPE_NAME" ] && [ -z "$CI_ONIRO_DEVTOOL_LAYER_PATH" ]; } || { [ -z "$CI_ONIRO_DEVTOOL_RECIPE_NAME" ] && [ -n "$CI_ONIRO_DEVTOOL_LAYER_PATH" ]; } then + echo "precondition failed - cannot define just one of CI_ONIRO_DEVTOOL_RECIPE_NAME and CI_ONIRO_DEVTOOL_LAYER_PATH" + exit 1 + fi + # Bitbake is configured to use /var/shared/$CI_ONIRO_BUILD_CACHE/bitbake + # directory for both the download directory and the sstate-cache. + - test -w /var/shared/$CI_ONIRO_BUILD_CACHE/bitbake || ( + echo "precondition failed - expected /var/shared/$CI_ONIRO_BUILD_CACHE/bitbake to be writable" + && exit 1 ) + # Log available disk space on the persistent shared disk. + - df -h /var/shared/$CI_ONIRO_BUILD_CACHE/bitbake + - !reference [.workspace, before_script] + + script: + - !reference [.workspace, script] + # Initialize bitbake build environment by sourcing the oe-init-build-env + # into the running bash process. This has the side-effect of changing the + # current working directory and populating the $SCRATCH_DIR/build + # sub-directory with default configuration. + - TEMPLATECONF=../oniro/flavours/"$CI_ONIRO_BUILD_FLAVOUR" + . ./oe-core/oe-init-build-env build + # xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + # NOTE: From now on, we are running inside "$SCRATCH_DIR"/build + # with bash modified by oe-init-build-env. We now have access to bitbake, + # devtool and other related tools. + # xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + + # Generic local.conf customization point. All environment variables with a + # given prefix are converted to local.conf variables, with the prefix + # dropped. Some extra sed processing is used to quote the variables. + # Everything runs in a sub-shell with bash pipefail disabled, as otherwise + # lack of variables matching the grep pattern causes the execution to stop. + - | + ( set +o pipefail; + env \ + | grep -E '^CI_ONIRO_BB_LOCAL_CONF_[A-Z_0-9]+=' \ + | sed -e 's/^CI_ONIRO_BB_LOCAL_CONF_//g' -e 's/"/\\"/g' -e 's/=/ = "/g' -e 's/$/"/g' \ + | sort \ + | tee -a conf/local.conf ) + ( set +o pipefail; + env \ + | grep -E '^CI_ONIRO_BB_LOCAL_CONF_plus_equals_[A-Z_0-9]+=' \ + | sed -e 's/^CI_ONIRO_BB_LOCAL_CONF_plus_equals_//g' -e 's/"/\\"/g' -e 's/=/ += "/g' -e 's/$/"/g' \ + | sort \ + | tee -a conf/local.conf ) + + + # Sanity check: disallow using public build cache with a specific setting + # in local.conf. The list of settings may grow over time. + - | + if [ "$CI_ONIRO_BUILD_CACHE" != private ]; then + if grep -qF 'ACCEPT_FSL_EULA = "1"' conf/local.conf; then + echo "cannot use CI_ONIRO_BUILD_CACHE=$CI_ONIRO_BUILD_CACHE with ACCEPT_FSL_EULA = \"1\"" + exit 1 + fi + fi + + # Update a specific bitbake recipe, which must use git, to point to the + # commit that is being tested here. + - | + if [ -n "$CI_ONIRO_DEVTOOL_RECIPE_NAME" ] && [ -n "$CI_ONIRO_DEVTOOL_LAYER_PATH" ]; then + echo "CI: Switching $CI_ONIRO_DEVTOOL_RECIPE_NAME from $CI_ONIRO_DEVTOOL_LAYER_PATH to $CI_COMMIT_SHA"; + # Upgrade the recipe to point to the sources in the project directory. + # Pass --no-patch to skip applying patches that are present in the + # recipe. + # + # NOTE: This requires the git repository specified in the recipe to + # have access to the SHA presented here. + time devtool upgrade --no-patch --srcrev "$CI_COMMIT_SHA" --srcbranch "$CI_COMMIT_REF_NAME" "$CI_ONIRO_DEVTOOL_RECIPE_NAME"; + # Switch the detached head created by devtool upgrade above, to a + # scratch branch. This is required for the devtool finish to work. + ( cd workspace/"$CI_ONIRO_DEVTOOL_RECIPE_NAME" && git checkout -b scratch ); + # Finish the upgrade process and commit the change back to the layer. + # Because our recipe may have had patches, this step is required to + # correctly discard them ("devtool upgrade --no-patch" simply skipped + # them). + time devtool finish --remove-work --force "$CI_ONIRO_DEVTOOL_RECIPE_NAME" "$(basename "$CI_ONIRO_DEVTOOL_LAYER_PATH")"; + fi + rules: + # Run the job when a merge request is created. + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + # Run the job when a tag is placed. + - if: '$CI_COMMIT_TAG' + # Run the build for scheduled pipelines. + - if: '$CI_PIPELINE_SOURCE == "schedule"' + +# This job is documented in docs/ci/hidden-jobs/build-linux.rst +.build-linux: + extends: .bitbake-workspace + variables: + CI_ONIRO_BUILD_FLAVOUR: linux + CI_ONIRO_BITBAKE_TARGETS: "oniro-image-base-tests oniro-image-extra-tests" + # Linux builds can be extremely large. Use rm_work to remove the temporary + # build data as soon as possible to reclaim space. + CI_ONIRO_BB_LOCAL_CONF_plus_equals_INHERIT: "rm_work" + script: + - test -n "$CI_ONIRO_BITBAKE_TARGETS" || ( + echo "nothing to build - CI_ONIRO_BITBAKE_TARGETS is empty" + && exit 1 ) + - !reference [.bitbake-workspace, script] + - | + for target in $CI_ONIRO_BITBAKE_TARGETS; do + time bitbake "$target" + du -sh tmp + done + +# This job is documented in docs/ci/hidden-jobs/build-linux-marix.rst +.build-linux-matrix: + extends: .build-linux + parallel: + matrix: + - CI_ONIRO_BITBAKE_TARGETS: ["oniro-image-base-tests", "oniro-image-extra-tests"] + +# This job is documented in docs/ci/hidden-jobs/build-zephyr.rst +.build-zephyr: + extends: .bitbake-workspace + variables: + CI_ONIRO_BUILD_FLAVOUR: zephyr + CI_ONIRO_BITBAKE_TARGETS: "zephyr-philosophers" + script: + - test -n "$CI_ONIRO_BITBAKE_TARGETS" || ( + echo "nothing to build - CI_ONIRO_BITBAKE_TARGETS is empty" + && exit 1 ) + - !reference [.bitbake-workspace, script] + - | + for target in $CI_ONIRO_BITBAKE_TARGETS; do + time bitbake "$target" + done + +# This job is documented in docs/ci/hidden-jobs/build-freertos.rst +.build-freertos: + extends: .bitbake-workspace + variables: + CI_ONIRO_BUILD_FLAVOUR: freertos + script: + - !reference [.bitbake-workspace, script] + - time bitbake freertos-demo + +# This job is documented in docs/ci/hidden-jobs/build-recipe.rst +.build-recipe: + extends: .bitbake-workspace + variables: + CI_ONIRO_RECIPE_NAME: "" + script: + - !reference [.bitbake-workspace, script] + - time bitbake "$CI_ONIRO_RECIPE_NAME" + +# This job is documented in docs/ci/hidden-jobs/build-image.rst +.build-image: + extends: .build-recipe + script: + - !reference [.build-recipe, script] + # Move artifacts for recovery, which only considers $CI_PROJECT_DIR and + # subdirectories. + - mkdir -p "$CI_PROJECT_DIR"/artifacts + # The name of the build-specific tmp directory may vary. Ask bitbake + # instead of hard-coding it. This is currently only relevant for Zephyr, + # which uses tmp-newlib. + - eval "$(bitbake -e | grep ^TMPDIR= | sed -e 's/^TMPDIR/BITBAKE_TMPDIR/g')" + # Uncompressed wic image is not needed in CI. We cannot remove it from + # meta-oniro-core/classes/ohos-image.bbclass as runqemu needs it for local + # build and boot. + - find "$BITBAKE_TMPDIR"/deploy/images/ -name *.wic -exec rm -rf {} \; + - mv "$BITBAKE_TMPDIR"/deploy/images/ "$CI_PROJECT_DIR"/artifacts || true + - mv "$BITBAKE_TMPDIR"/deploy/licenses/ "$CI_PROJECT_DIR"/artifacts || true + artifacts: + paths: + - artifacts/ + +.build-wic-image: + extends: .build-image + script: + - !reference [.build-image, script] + # Remove everything _except_ for the .wic.* and *.bmap files. Most of the + # remaining files are redundant and quite large. Our Linux builds really + # only care about the wic files. + - echo "Removing non-wic/bmap image files" + - find "$CI_PROJECT_DIR"/artifacts/images/ \( -type f -o -type l \) -a ! -name "*.wic.*" -print -delete + - echo "Pruning empty directories" + - find "$CI_PROJECT_DIR"/artifacts/ -type d -exec rmdir --verbose --ignore-fail-on-non-empty {} \; 2>/dev/null || true + +# This job is currently used to customize the behavior in oniro and xts-acts. +# It will be removed when that is safe to do so. It is not documented. +.build: + extends: .build-recipe + +# This job is documented in docs/ci/hidden-jobs/build-docs.rst +.build-docs: + interruptible: true + image: + name: registry.ostc-eu.org/ostc/containers/ostc-docs-builder + stage: build + script: + - make -C docs + artifacts: + paths: + - docs/build + rules: + # Build the docs when a merge request is created. + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + changes: + # React to changes to the docs directory. + - docs/**/* + # Run this job in case the pipeline changes. + - .oniro-ci/*.yml + - .gitlab-ci.yml + # Or when things land. + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + changes: + # React to changes to the docs directory. + - docs/**/* + # Run this job in case the pipeline changes. + - .oniro-ci/*.yml + - .gitlab-ci.yml + +# This job is documented in docs/ci/hidden-jobs/aggregate-docs.rst +.aggregate-docs: + stage: deploy + trigger: OSTC/OHOS/docs + rules: + # Update the documentation when things land in the default branch. + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + changes: + # React to changes to the docs directory. + - docs/**/* + # Run this job in case the pipeline changes. + - .oniro-ci/*.yml + - .gitlab-ci.yml diff --git a/.oniro-ci/machines-and-flavours.yaml b/.oniro-ci/machines-and-flavours.yaml index f64a764d69d70965fa4d320c12a7345af8f12920..c2ffc7621c97ee0305eaf22097890e5f60fed4f7 100644 --- a/.oniro-ci/machines-and-flavours.yaml +++ b/.oniro-ci/machines-and-flavours.yaml @@ -1,14 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Huawei Inc. -# NOTE: This file is included from the manifest repository. It is not -# self-sufficient. It is here only because the meta-ohos repository also -# contains all the flavour and machine definitions and having both in one place -# allows one to add a new MACHINE and the associated build jobs in one go. -# -# Please do not include this file directly. - -# The following jobs are documented in docs/ci/shared-jobs.rst +# Please do not include this file directly. It should be included through the +# build-generic.yaml only. + +# The following jobs are documented in docs/ci/machines-and-flavours.rst linux-qemu-x86: extends: .build-wic-image diff --git a/.oniro-ci/test-generic.yaml b/.oniro-ci/test-generic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36d8308b638764fcd0fb957d88eb65f6bb85ff3e --- /dev/null +++ b/.oniro-ci/test-generic.yaml @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Huawei Inc. + +## +## Submits a job to LAVA with CI variables +## +.lava-test: + interruptible: true + image: + name: registry.ostc-eu.org/ostc/containers/ostc-builder:latest + before_script: + # Check if the job is configured properly. + - test -n "$CI_LAVA_JOB_DEFINITION" || ( + echo "precondition failed - set CI_LAVA_JOB_DEFINITION to the URL of the LAVA test job definition" + && exit 1 ) + - test -n "$CI_BUILD_JOB_NAME" || ( + echo "precondition failed - set CI_BUILD_JOB_NAME to the appropriate job name from which LAVA will pick up build artifact" + && exit 1 ) + - test -n "$CI_REPORT_JOB_NAME" || ( + echo "precondition failed - set CI_REPORT_JOB_NAME to the CI job name which will gather results back from LAVA" + && exit 1 ) + script: + # Build callback URL for the "report" job + - curl --silent "https://git.ostc-eu.org/api/v4/projects/$CI_PROJECT_ID/pipelines/$CI_PIPELINE_ID/jobs?per_page=100" > jobs-manual.json + - job_id="$(jq -r ".[] | select(.name == \"$CI_REPORT_JOB_NAME\") | .id" jobs-manual.json)" + - build_job_id="$(jq -r ".[] | select(.name == \"$CI_BUILD_JOB_NAME\") | .id" jobs-manual.json)" + - CALLBACK_URL="https://git.ostc-eu.org/api/v4/projects/$CI_PROJECT_ID/jobs/${job_id}/play" + # Get the job definition from remote source. + - curl --silent "$CI_LAVA_JOB_DEFINITION" > job_def.yaml + # Update the job definition with CI data. + - sed -i -e 's/$ci_job_id/'"$CI_JOB_ID"'/' + -e 's/$ci_project_id/'"$CI_PROJECT_ID"'/' + -e 's/$ci_pipeline_id/'"$CI_PIPELINE_ID"'/' + -e 's,$ci_pipeline_url,'"$CI_PIPELINE_URL"',' + -e 's/$build_job_id/'"$build_job_id"'/' + -e 's,$callback_url,'"$CALLBACK_URL"',' job_def.yaml + # Generate job definitions for zephyr build + - | + mkdir lava_jobs + if echo "${CI_BUILD_JOB_NAME}" | grep -i "zephyr"; then + for image in $(find artifacts/ -name "*.elf" -exec basename {} \;); do + job_name=$(basename "${image}" ".elf") + echo "--- Generating lava job definition ${job_name}.yaml ---" + sed "s/\$elf_file/$image/" job_def.yaml | tee lava_jobs/"${job_name}".yaml + done + else + mv job_def.yaml lava_jobs/ + fi + # Submit the job to LAVA. + - | + for job_def in $(find lava_jobs/ -name "*.yaml"); do + lava_job_id=$(curl -X POST -H "Authorization: Token $CI_LAVA_TOKEN" -F "definition=$(<${job_def})" "$CI_LAVA_INSTANCE/api/v0.2/jobs/" | jq ".job_ids" | tr -d "[\n ]") + if [ -n "${lava_job_id}" ]; then + echo "$CI_LAVA_INSTANCE/scheduler/job/$lava_job_id" + echo "$lava_job_id" >> job_ids_"${CI_JOB_NAME}_${lava_job_id}".txt + else + echo "Failed to submit LAVA job" + exit 1 + fi + done + artifacts: + paths: + - job_ids_*.txt + rules: + # Run the build when it is scheduled. + - if: $CI_PIPELINE_SOURCE == "schedule" + # Do not run pipelines for draft merge requests unless manually triggered. + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TITLE =~ /^(wip|draft):.*/i' + when: manual + # Run the build when a merge request is created. + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + # Run the build when a tag is placed. + - if: '$CI_COMMIT_TAG' + +.lava-report: + interruptible: true + image: + name: registry.ostc-eu.org/ostc/containers/ostc-builder:latest + stage: report + script: + - | + for file in $(find ./ -name "job_ids_*.txt"); do + echo "Job file: $file" + while read -r p; do + echo "Job ID: $p" + mkdir "$p" + # Echo LAVA job health and link. + link="$CI_LAVA_INSTANCE/scheduler/job/$p" + health="$(curl --silent "$CI_LAVA_INSTANCE/api/v0.2/jobs/$p/" | jq '.health' | tr -d '"')" + echo "[$health] $link" + # Get the JUnit export from LAVA. + curl --silent -o job_$p.xml "$CI_LAVA_INSTANCE/api/v0.2/jobs/$p/junit/" + echo "Report file: $(ls job_$p.xml)" + done < "$file" + done + artifacts: + when: always + paths: + - job_*.xml + reports: + junit: + - job_*.xml + rules: + # Run the build when it is scheduled. + - if: $CI_PIPELINE_SOURCE == "schedule" + when: manual + # Do not run pipelines for draft merge requests unless manually triggered. + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TITLE =~ /^(wip|draft):.*/i' + when: manual + # Run the build when a merge request is created. + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: manual + # Run the build when a tag is placed. + - if: '$CI_COMMIT_TAG' + when: manual diff --git a/docs/ci/hidden-jobs/aggregate-docs.rst b/docs/ci/hidden-jobs/aggregate-docs.rst new file mode 100644 index 0000000000000000000000000000000000000000..ba7f00c20dd5a8335d7c9a6adae0f40e45c83cb1 --- /dev/null +++ b/docs/ci/hidden-jobs/aggregate-docs.rst @@ -0,0 +1,43 @@ +.. SPDX-FileCopyrightText: Huawei Inc. +.. +.. SPDX-License-Identifier: CC-BY-4.0 + +=============== +.aggregate-docs +=============== + +The ``.aggregate-docs`` job triggers a build of the aggregated documentation of +All Scenarios OS. + +All Scenarios OS documentation is maintained in an unusual way. Individual +repositories contain dedicated documentation that can be built with +:doc`build-docs` job. A special *centralized* documentation project aggregates +documentation from multiple git repositories, as checked out by *git-repo* based +on the *manifest* file, to build a standalone document. + +This job encapsulates knowledge on how to trigger the aggregated build upon +changes to documentation specific to a given project. + +Usage Guide +=========== + + +.. warning:: + If a project is pinned in the *git-repo manifest* then using + this job makes no sense, as the resulting aggregate will not + change without a prior modification of the manifest. + +To use this job in your pipeline include the generic build definition file, +``build-generic.yaml`` and define the ``aggregate-docs`` job in addition to the +:doc:`build-docs` job. + +.. code-block:: yaml + + # Build documentation present in the repository. + build-docs: + extends: .build-docs + + # Trigger aggregation of All Scenarios OS documentation. + aggregate-docs: + extends: .aggregate-docs + needs: [build-docs] diff --git a/docs/ci/hidden-jobs/bitbake-workspace.rst b/docs/ci/hidden-jobs/bitbake-workspace.rst new file mode 100644 index 0000000000000000000000000000000000000000..a31a504127ed73833ea28c7614669c09f9e60bf7 --- /dev/null +++ b/docs/ci/hidden-jobs/bitbake-workspace.rst @@ -0,0 +1,176 @@ +.. SPDX-FileCopyrightText: Huawei Inc. +.. +.. SPDX-License-Identifier: CC-BY-4.0 + +================== +.bitbake-workspace +================== + +The ``.bitbake-workspace`` job extends the :doc:`workspace` job to configure +and initialize BitBake for the desired build operations. The job does not build +anything by itself, that functionality is available in the ``.build-recipe`` +job. + +Job Variables +============= + +The ``.bitbake-workspace`` job defines several variables as a way to customize +the way BitBake is configured. + +CI_ONIRO_BUILD_FLAVOUR +---------------------- + +The name of the *flavour* of All Scenarios OS, which effectively picks the kernel +type. This is used to select the initial BitBake configuration template. +Templates are stored in the **meta-ohos** repository. + +Available values are ``linux``, ``zephyr`` and ``freertos``. There is no +default value. This variable must be set by a derivative job, it is usually set +by the three ``.build-linux``, ``.build-zephyr`` and ``.build-freertos`` jobs. +Specific build jobs, in turn, extend those. + +CI_ONIRO_BUILD_CACHE +-------------------- + +The set of build caches to use. + +Currently there are only two sets ``private`` or ``pub``. The names directly +correspond to file system path where the cache is stored. + +By default all builds are assumed to be tainted, and use the private cache. If +a given build configuration is not legally problematic, in the sense of pulling +in code or blobs that are non-redistributable, this attribute can be set to +``pub``. + +Public build cache is exposed as `<https://cache.ostc-eu.org/>`_ and can be +used by third parties to speed up local builds. + +The default value is ``private``. + +CI_ONIRO_DEVTOOL_RECIPE_NAME +---------------------------- + +Name of the BitBake recipe to upgrade. + +This may be set by jobs which extend or re-define the ``.bitbake-workspace`` +job. It must be used in tandem with ``CI_ONIRO_DEVTOOL_LAYER_PATH``. The side +effects are discussed below. + +CI_ONIRO_DEVTOOL_LAYER_PATH +--------------------------- + +Path of the layer containing the recipe to upgrade. + +This may be set by jobs which extend or re-define the ``.bitbake-workspace`` +job. It must be used in tandem with ``CI_ONIRO_DEVTOOL_RECIPE_NAME``. + +Before the build is attempted, the recipe is updated to ignore any existing +patches and point to the code corresponding to the repository and commit being +tested. + +.. code-block:: bash + + devtool upgrade \ + --no-patch \ + --srcrev "$CI_COMMIT_SHA" \ + --srcbranch "$CI_COMMIT_REF_NAME" \ + "$CI_ONIRO_DEVTOOL_RECIPE_NAME" + devtool finish \ + --remove-work \ + --force \ + "$CI_ONIRO_DEVTOOL_RECIPE_NAME" \ + "$(basename "$CI_ONIRO_DEVTOOL_LAYER_PATH")" + +This functionality is useful for testing incoming changes to repositories that +contain source code that is already packaged one of the layers. + +Configuring BitBake +=================== + +The ``local.conf`` file can define numerous variables that influence the +BitBake build process. This job offers a declarative method of doing that. Job +variables with have the prefix ``CI_ONIRO_BB_LOCAL_CONF_`` are converted to +``attr = "value"`` and those with prefix +``CI_ONIRO_BB_LOCAL_CONF_plus_equals_`` are converted to ``attr += "value"``. + +This method is friendly to job inheritance and re-definition. Derivative jobs +can add or re-define variables without having to duplicate any imperative logic +or maintaining synchronized settings across distinct jobs. + +This mechanism is used to set the following BitBake variables + +CONNECTIVITY_CHECK_URIS +----------------------- + +BitBake contains a connectivity check system. That system relies on access to +``https://example.com/``. OSTC cloud provider has a DNS configuration problem, +where that specific domain is not resolved correctly. For a compatible +workaround the connectivity check URL is set to ``https://example.net/``. + +This is implemented using the ``CI_ONIRO_BB_LOCAL_CONF_`` system. + +DL_DIR +------ + +BitBake downloads source archives and git repositories in the through the +``fetch`` task of many recipes. Those are all stored in the local file system +and can be shared between separate builds for a great speed up, as the files +are obtained from multiple third party servers and their connectivity varies. + +To optimize the build process, the download directory is set to point at the +shared NFS volume that persists between job execution, and is more efficient +than the artifact system, that copies all the data regardless of the need to +actually use that data in practice. + +The default location is changed to +``/var/shared/$CI_ONIRO_BUILD_CACHE/bitbake/downloads``, but this should +be treated as an implementation detail. The location may change in the future. +The download cache is not automatically purged yet. In the future it may be +purged periodically, if space becomes an issue. + +Note that the location relies on the value of ``$CI_ONIRO_BUILD_CACHE`` +discussed above. + +SSTATE_CACHE +------------ + +BitBake relies on an elaborate cache system, that can be used to avoid +duplicating work at the level of a specific recipe. The dependencies and +side-effects of each recipe are recorded in the cache, and are reused whenever +possible. + +Having access to a persistent cache has a dramatic effect on the performance of +the CI system as, in the fast-path, it can avoid virtually all compilation +tasks and simply assemble the desired system image out of intermediate files +present in the cache. + +The default location is changed to +``/var/shared/$CI_ONIRO_BUILD_CACHE/bitbake/sstate-cache``, but this +should be treated as an implementation detail. The location may change in the +future. The sstate cache is not automatically purged yet. It can be purged +periodically with the only caveat, that initial builds will be much slower. + +Cache Considerations +==================== + +The ``.bitbake-workspace`` job configures BitBake to use a persistent directory +that is shared between CI jobs, for the location of the ``download`` directory +as well as the ``sstate-cache`` directory. + +The job is using GitLab runner tags to schedule jobs in the environment where +that shared storage is available. When a new dependency is added or when the +layers and recipes are changed or updated, the download is automatically +populated with the necessary source archives. Similarly ``sstate-cache`` is +populated by all the build jobs present throughout the CI system. + +Due to legal restrictions, the caches are split into two pairs, public and private. +The public cache is automatically published in https://cache.ostc-eu.org/bitbake/ +The private cache, which is used by default, is available on the same volume but it is +not shared anywhere. + +In case the cache is fed with a software package that is, in retrospective +somehow problematic, for example, by not being freely redistributable, the cache +can be purged at will. + +For details on how cache selection and BitBake configuration looks like, please +refer to the pipeline source code. diff --git a/docs/ci/hidden-jobs/build-docs.rst b/docs/ci/hidden-jobs/build-docs.rst new file mode 100644 index 0000000000000000000000000000000000000000..00bc54a6309c5df76868680d20931cf3fe45ee5e --- /dev/null +++ b/docs/ci/hidden-jobs/build-docs.rst @@ -0,0 +1,53 @@ +.. SPDX-FileCopyrightText: Huawei Inc. +.. +.. SPDX-License-Identifier: CC-BY-4.0 + +=========== +.build-docs +=========== + +The ``.build-docs`` job builds reStructuredText documentation. + +This job offers no configuration and works by convention. This is done to both +simplify its use and improve uniformity of developer experience. The job +automatically reacts to changes in the ``docs/`` directory as well as the +definition of the pipeline. Built documentation is collected as job artifacts. + +Usage Guide +=========== + +To use this job in your pipeline include the generic build definition file, +``build-generic.yaml`` and define a ``build-docs`` job: + +.. code-block:: yaml + + build-docs: + extends: .build-docs + +The job expects the project to have the following documentation structure. + +.. code-block:: + + └── docs +   ├── index.rst +   └── Makefile + +The ``Makefile`` should build the documentation into the ``build`` directory, +relative to the ``docs`` directory. A sample, re-usable example is is provided below: + +.. code-block:: makefile + + # SPDX-FileCopyrightText: Huawei Inc. + # + # SPDX-License-Identifier: Apache-2.0 + + .PHONY: all + all: + sphinx-build -W -C \ + -D html_theme=sphinx_rtd_theme \ + -D project='Name of the project' \ + -D copyright='Copyright Holder' \ + . build + + clean: + rm -rf ./build diff --git a/docs/ci/hidden-jobs/build-freertos.rst b/docs/ci/hidden-jobs/build-freertos.rst new file mode 100644 index 0000000000000000000000000000000000000000..6159bbf27ec95fd97d72f3a3ed2adfb889e6866b --- /dev/null +++ b/docs/ci/hidden-jobs/build-freertos.rst @@ -0,0 +1,17 @@ +.. SPDX-FileCopyrightText: Huawei Inc. +.. +.. SPDX-License-Identifier: CC-BY-4.0 + +=============== +.build-freertos +=============== + +The ``.build-freertos`` job extends the :doc:`bitbake-workspace` job. It sets +``CI_ONIRO_BUILD_FLAVOUR`` to ``freertos`` and builds the ``freertos-demo`` +test image. + +Usage Guide +=========== + +This job is not intended for direct use. Instead it serves as a base for all +the FreeRTOS-specific :doc:`../shared-jobs`. diff --git a/docs/ci/hidden-jobs/build-image.rst b/docs/ci/hidden-jobs/build-image.rst new file mode 100644 index 0000000000000000000000000000000000000000..f7b0ae21a96afd5890d1401e310169580a30bc76 --- /dev/null +++ b/docs/ci/hidden-jobs/build-image.rst @@ -0,0 +1,24 @@ +.. SPDX-FileCopyrightText: Huawei Inc. +.. +.. SPDX-License-Identifier: CC-BY-4.0 + +============ +.build-image +============ + +The ``.build-image`` job extends the :doc:`build-recipe` job to build a single +recipe, which should describe an image, and to collect the resulting image and +licenses as job artifacts. + +Usage Guide +=========== + +This job is configured exactly the same as :doc:`build-recipe`. + +Implementation Details +====================== + +The job handles differences between the name of the temporary build directory +between various All Scenarios OS flavours. Internally BitBake is interrogated for +the value of ``TMPDIR`` and the image is copied back to a subdirectory +of ``$CI_PROJECT_DIR`` for delivery to the GitLab runner. diff --git a/docs/ci/hidden-jobs/build-linux-matrix.rst b/docs/ci/hidden-jobs/build-linux-matrix.rst new file mode 100644 index 0000000000000000000000000000000000000000..070243bf8a524d814cc53bb4a6be8058c1b2bac6 --- /dev/null +++ b/docs/ci/hidden-jobs/build-linux-matrix.rst @@ -0,0 +1,19 @@ +.. SPDX-FileCopyrightText: Huawei Inc. +.. +.. SPDX-License-Identifier: CC-BY-4.0 + +=================== +.build-linux-matrix +=================== + +The ``.build-linux-matrix`` job extends the :doc:`build-linux` job and +otherwise behaves exactly the same but builds each of the +``CI_ONIRO_BITBAKE_TARGET`` as a separate job. This creates a wider pipeline +which unlocks additional parallelism. + +Usage Guide +=========== + +Note that due to the way parallel matrix jobs are defined, to change the set of +bitbake recipes to build you must re-define the ``parallel/matrix`` element +entirely. Changing ``CI_ONIRO_BITBAKE_TARGETS`` is not effective. diff --git a/docs/ci/hidden-jobs/build-linux.rst b/docs/ci/hidden-jobs/build-linux.rst new file mode 100644 index 0000000000000000000000000000000000000000..21b515004bae4f62bdc002767dec30576d350f2c --- /dev/null +++ b/docs/ci/hidden-jobs/build-linux.rst @@ -0,0 +1,41 @@ +.. SPDX-FileCopyrightText: Huawei Inc. +.. +.. SPDX-License-Identifier: CC-BY-4.0 + +============ +.build-linux +============ + +The ``.build-linux`` job extends the :doc:`bitbake-workspace` job. It sets +``CI_ONIRO_BUILD_FLAVOUR`` to ``linux`` and builds the bitbake targets (e.g. +images) as defined by ``CI_ONIRO_BITBAKE_TARGETS`` (defaults included). + +The images are built one after another, allowing CI to fail quickly in case of +any problems with the more fundamental image. The set of built images may +change over time, as additional reference images are defined. + +Usage Guide +=========== + +This job is not intended for direct use. Instead it serves as a base for all +the Linux-specific :doc:`../shared-jobs`. It may be re-defined in a pipeline to +alter ``rules`` or ``variables`` in a way that fits a particular purpose. + +If used directly, it is recommended pick the desired ``MACHINE`` and to +override the entire script section and refer to the base ``.bitbake-workspace`` +job as illustrated below. The set of build operations can then be tailored to +the purpose of the desired job. + +.. code-block:: yaml + + build-something-linux-specific: + extends: .build-linux + variables: + MACHINE: "..." + script: + - !reference [.bitbake-workspace, script] + - true # put your code here + +.. note:: + + ``CI_ONIRO_BITBAKE_TARGETS`` can be overwritten if defaults are not desired. diff --git a/docs/ci/hidden-jobs/build-recipe.rst b/docs/ci/hidden-jobs/build-recipe.rst new file mode 100644 index 0000000000000000000000000000000000000000..a4827566aa3716473539e05085fd09d617e244cc --- /dev/null +++ b/docs/ci/hidden-jobs/build-recipe.rst @@ -0,0 +1,43 @@ +.. SPDX-FileCopyrightText: Huawei Inc. +.. +.. SPDX-License-Identifier: CC-BY-4.0 + +============= +.build-recipe +============= + +The ``.build-recipe`` job extends the :doc:`bitbake-workspace` job to build a +single BitBake recipe. The recipe is built and all the results are discarded. + +Variables +========= + +CI_ONIRO_RECIPE_NAME +-------------------- + +The name of the recipe to build. + +There is no default value, this must be set by the extending job. + +Usage Guide +=========== + +To use this job in your pipeline include the generic build definition file, +``build-generic.yaml`` and define a job with the minimal configuration, as +illustrated below: + +.. code-block:: yaml + + build-something: + extends: .build-recipe + variables: + CI_ONIRO_BUILD_FLAVOUR: "..." + CI_ONIRO_RECIPE_NAME: "..." + MACHINE: "..." + +Pick the desired ``CI_ONIRO_BUILD_FLAVOUR``, ``CI_ONIRO_RECIPE_NAME`` and +``MACHINE``. For discussion of ``CI_ONIRO_BUILD_FLAVOUR`` refer to the +:doc:`bitbake-workspace` job. ``CI_ONIRO_RECIPE_NAME`` is any BitBake recipe +available in the selected flavour. ``MACHINE`` is desired machine name. +Supported values are documented alongside each flavour in the *meta-ohos* +repository. diff --git a/docs/ci/hidden-jobs/build-wic-image.rst b/docs/ci/hidden-jobs/build-wic-image.rst new file mode 100644 index 0000000000000000000000000000000000000000..a566c20579e1053938f1335a21db5ce29cf12e91 --- /dev/null +++ b/docs/ci/hidden-jobs/build-wic-image.rst @@ -0,0 +1,19 @@ +.. SPDX-FileCopyrightText: Huawei Inc. +.. +.. SPDX-License-Identifier: CC-BY-4.0 + +================ +.build-wic-image +================ + +The ``.build-wic-image`` job extends the :doc:`build-image` job to collect only +the `*.wic.*` and `*.bmap` files and remove all the other files that would +normally be collected by the artifact system. It is recommended for Linux +builds which produce wic images, as the size of subset of collected artifacts +is considerably smaller than what `.build-image` provides. + +Usage Guide +=========== + +This job is configured exactly the same as :doc:`build-image` and +:doc:`build-recipe`. diff --git a/docs/ci/hidden-jobs/build-zephyr.rst b/docs/ci/hidden-jobs/build-zephyr.rst new file mode 100644 index 0000000000000000000000000000000000000000..e966cfc9717f0494aa207972bde248a210ee0463 --- /dev/null +++ b/docs/ci/hidden-jobs/build-zephyr.rst @@ -0,0 +1,17 @@ +.. SPDX-FileCopyrightText: Huawei Inc. +.. +.. SPDX-License-Identifier: CC-BY-4.0 + +============= +.build-zephyr +============= + +The ``.build-zephyr`` job extends the :doc:`bitbake-workspace` job. It sets +``CI_ONIRO_BUILD_FLAVOUR`` to ``zephyr`` and builds the bitbake targets (e.g. +images) as defined by ``CI_ONIRO_BITBAKE_TARGETS`` (defaults included). + +Usage Guide +=========== + +This job is not intended for direct use. Instead it serves as a base for all +the Zephyr-specific :doc:`../shared-jobs`. diff --git a/docs/ci/hidden-jobs/index.rst b/docs/ci/hidden-jobs/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..e1f9cd7aa2cb32701dec78bab208085921bb18b4 --- /dev/null +++ b/docs/ci/hidden-jobs/index.rst @@ -0,0 +1,29 @@ +.. SPDX-FileCopyrightText: Huawei Inc. +.. +.. SPDX-License-Identifier: CC-BY-4.0 + +Hidden Jobs +=========== + +There's a number of *hidden jobs*, which start with the dot character, that are +used as foundation for the set of :doc:`shared jobs <../shared-jobs>`. Hidden +jobs do not participate in any pipeline directly. They can only be used as +templates, using the ``extends: ...`` mechanism, to share and reuse +implementation details. + +.. toctree:: + :maxdepth: 1 + + workspace + bitbake-workspace + build-linux + build-linux-matrix + build-zephyr + build-freertos + build-recipe + build-image + build-wic-image + build-docs + lava-test + lava-report + aggregate-docs diff --git a/docs/ci/hidden-jobs/lava-report.rst b/docs/ci/hidden-jobs/lava-report.rst new file mode 100644 index 0000000000000000000000000000000000000000..1dbc0154e282ff4850c0c4048584a5c384c67f1b --- /dev/null +++ b/docs/ci/hidden-jobs/lava-report.rst @@ -0,0 +1,19 @@ +.. SPDX-FileCopyrightText: Huawei Inc. +.. +.. SPDX-License-Identifier: CC-BY-4.0 + +============ +.lava-report +============ + +The ``.lava-report`` job iterates through the submitted jobs from +:doc:`lava-test` and collects the artifacts from these jobs via LAVA REST API. + + +Job Variables +============= + +CI_LAVA_INSTANCE +---------------- + +The base url of the LAVA server. diff --git a/docs/ci/hidden-jobs/lava-test.rst b/docs/ci/hidden-jobs/lava-test.rst new file mode 100644 index 0000000000000000000000000000000000000000..bd8fce7efedc021656f34a18f5ec1c80eda43dfd --- /dev/null +++ b/docs/ci/hidden-jobs/lava-test.rst @@ -0,0 +1,51 @@ +.. SPDX-FileCopyrightText: Huawei Inc. +.. +.. SPDX-License-Identifier: CC-BY-4.0 + +========== +.lava-test +========== + +The ``.lava-test`` job creates the LAVA test job definition based on the +pipeline information and submits the job to LAVA. + +The artifact of this job is the list of LAVA job id(s) submitted by this job. +This will be consumed by the :doc:`lava-report` + +Job Variables +============= + +The ``.lava-test`` job defines several variables as a way to customize +the LAVA job definition before submiting the job to LAVA. + +CI_LAVA_JOB_DEFINITION +---------------------- + +This is the url to the job definition template. The template needs to have +variables inside which will be replaced by this job. The list of variables is +the following: +- $ci_pipeline_id - pipeline ID of the CI job ($CI_PIPELINE_ID in GitLab) +- $ci_job_id - CI job ID ($CI_JOB_ID in GitLab) +- $ci_project_id - CI project ID ($CI_PROJECT_ID in GitLab) +- $build_job_id - job ID of the **build** job within the same pipeline +- $callback_url - url which triggers the execution or the manual job which +collects the results back from LAVA to GitLab. See :doc:`lava-report` + +CI_BUILD_JOB_NAME +----------------- + +The name of the job that builds the artifacts which will be used by LAVA to +boot up the DUT for this particular LAVA job. If we test multiple builds in one +CI job, each will have different build job name. + +CI_REPORT_JOB_NAME +------------------ + +The name of the ``report`` job which will be triggered manually when the LAVA +job(s) are finished with executution. This job will collect the results from +LAVA and import them to GitLab. + +CI_LAVA_INSTANCE +---------------- + +The base url of the LAVA server. diff --git a/docs/ci/hidden-jobs/workspace.rst b/docs/ci/hidden-jobs/workspace.rst new file mode 100644 index 0000000000000000000000000000000000000000..6855524403b266ebe4f291f5bedf39d52eef5e60 --- /dev/null +++ b/docs/ci/hidden-jobs/workspace.rst @@ -0,0 +1,118 @@ +.. SPDX-FileCopyrightText: Huawei Inc. +.. +.. SPDX-License-Identifier: CC-BY-4.0 + +========== +.workspace +========== + +The ``.workspace`` job assembles a *git-repo* workspace, as described by +*manifest* file. This job is a foundation for other jobs. + +This workspace is **not** constructed in ``$CI_PROJECT_DIR``, which is by +GitLab runner. To avoid clashes with any files that may be present there, it is +constructed in a temporary directory. Additional logic stores and restores the +path of that directory between the code in ``before_script``, ``script`` and +``after_script`` as those execute in separate shell processes. + +Constraints +=========== + +This job uses the ``ostc-builder`` container image, mainly for convenience, as +it is often extended to perform other tasks, such as bitbake builds. Other +containers can be used, as long as they have the ``repo`` program +pre-installed. + +In addition this job uses the ``large-disk`` tag to be scheduled on a machine +with access to a shared NFS volume with git mirrors, that greatly speed up the +process of constructing the workspace from scratch. + +Variables +========= + +CI_ONIRO_RUNNER_TAG +------------------- + +An arbitrary GitLab Runner tag selecting the runner which processes the +.workspace job, or its derivative. This defaults to an empty string but can be +used to pick a specific GitLab runner, as long as that runner has the matching +tag set. + +This can be used to perform scheduled builds in a non-default location, as long +as the pipeline schedule defines this variable. + +CI_ONIRO_MANIFEST_URL +--------------------- + +The URL consumed by git-repo. You only want to change this if you forked the +entire infrastructure and want to use it in private. + +The default value is ``https://git.ostc-eu.org/OSTC/OHOS/manifest``. + +If you change the default value, please set ``CI_ONIRO_MANIFEST_MIRROR`` +as well. + +CI_ONIRO_MANIFEST_BRANCH +------------------------ + +The name of the git branch of the manifest repository. Unless special +circumstances apply this does not need to be customized. + +For testing changes coming into the *manifest* repository itself, use +``$CI_COMMIT_REF_NAME``, which will check out the manifest as described by the +specific commit being tested. + +The default value is ``develop``. + +CI_ONIRO_MANIFEST_NAME +---------------------- + +The name of the manifest file from the repository mentioned above. + +The default value is ``default.xml``. + +CI_ONIRO_MANIFEST_MIRROR +------------------------ + +Name of the ``git-repo`` mirror matching to use, expressed as a directory name +in the shared disk volume. Check the section about git-repo mirror below, for +details. + +The default value is ``ostc-develop``. + +CI_ONIRO_GIT_REPO_PATH +---------------------- + +The path of the git repository to deviate from what the git-repo manifest +describes. This variable should be used for constructing CI jobs for +repositories directly described by the manifest. + +When set to a non-empty value, the specified git repository will be first +checked out by the git-repo according to what is described in the manifest, and +then changed again, to point to ``$CI_COMMIT_SHA``. The logic in the script +supports the forked repository workflow, by reusing the merge setup GitLab does +on pipelines for merge results. See GitLab documentation on `Pipelines for +Merge Results`_ for more information. + +.. _Pipelines for Merge Results: https://docs.gitlab.com/ee/ci/merge_request_pipelines/pipelines_for_merged_results/index.html + +The default value is the empty string. + +Local git-repo Mirror +===================== + +The ``.workspace`` job relies on a git-repo mirror that is mounted into the +execution environment provided by the GitLab worker. The mirror is created and +kept up-to-date by the scheduled run of the pipeline in `ostc-manifest-mirror +repository`_ + +.. _ostc-manifest-mirror repository: https://git.ostc-eu.org/OSTC/infrastructure/ostc-manifest-mirror + +When the mirror is out-of-date additional git revisions needed to construct the +workspace are fetched form the respective upstream repositories. Some of the +git repositories described by the manifest are rather large and are hosted on +infrastructure outside of the OSTC cloud provider, making this an important +optimization. + +The mirror is automatically published in `<https://cache.ostc-eu.org/git-repo-mirrors/>`_, +with two specific mirrors being available: *ostc-dev* and *gitee-dev*. diff --git a/docs/ci/index.rst b/docs/ci/index.rst index 5a81e12a5a2a016138fd3f3421a96be8abc1135f..71196aefb316567765ec267703d244de0af92286 100644 --- a/docs/ci/index.rst +++ b/docs/ci/index.rst @@ -9,3 +9,5 @@ Continuous Integration :maxdepth: 1 machines-and-flavours + special-jobs + hidden-jobs/index diff --git a/docs/ci/special-jobs.rst b/docs/ci/special-jobs.rst new file mode 100644 index 0000000000000000000000000000000000000000..94cc6e9a248661a65b82291b6bb68b325fad1d65 --- /dev/null +++ b/docs/ci/special-jobs.rst @@ -0,0 +1,16 @@ +.. SPDX-FileCopyrightText: Huawei Inc. +.. +.. SPDX-License-Identifier: CC-BY-4.0 + +Special Jobs +------------ + +linux-glibc-qemu-x86_64 +....................... + +This job extends ``linux-qemu-x86_64`` and differs in the following way. + +This job performs a build with the libc switched to ``glibc``. It only runs on a +schedule that is defined in the GitLab project settings for the `manifest` +repository. In practice it runs daily to check if the Linux favour could be +switched back to ``glibc``, from the default ``musl`` that is used right now.