Merge branch 'ci-tuning' into 'kirkstone'

CI: Attempt to throttle load caused by builds better

Closes #33

See merge request eclipse/oniro-core/meta-openharmony!38

.gitlab-ci.yml

@@ -77,6 +77,10 @@ stages:
     # something like that... Without this we get errors like
     #     fatal: couldn't find remote ref refs/heads/master
     - git checkout -b master
+    # The repo command does not like shallow repos
+    - if [ "$(git rev-parse --is-shallow-repository)" = "true" ] ; then
+        git fetch --unshallow ;
+      fi
     # Create/maintain a mirror of all manifest repositories in runner
     # persistent storage, and use that as reference when initializing the
     # build repo, effectively reducing the time spent fetching git repos to
@@ -145,9 +149,16 @@ stages:
         echo "SSTATE_MIRRORS = \"file://.* file://$SSTATE_MIRROR_PATH/PATH\""
           >> conf/auto.conf ;
       fi
-    - if [ -n "$CI_ONIRO_NUM_CPUS" ] ; then
-        echo "BB_NUMBER_THREADS = \"$CI_ONIRO_NUM_CPUS\"" ;
-        echo "BB_NUMBER_PARSE_THREADS = \"$CI_ONIRO_NUM_CPUS\"" ;
+    - MAX_JOBS="${CI_ONIRO_PARALLEL_BUILD_MAX_JOBS:-$(nproc)}"
+    - echo "BB_NUMBER_PARSE_THREADS = \"$MAX_JOBS\"" >> conf/auto.conf
+    - echo "BB_NUMBER_THREADS = \"$MAX_JOBS\"" >> conf/auto.conf
+    - echo "PARALLEL_MAKE = \"-j$MAX_JOBS\"" >> conf/auto.conf
+    - echo "NINJA_ARGS:append = \" -j$MAX_JOBS\"" >> conf/auto.conf
+    - MAX_LOADAVG="${CI_ONIRO_PARALLEL_BUILD_MAX_LOADAVG}"
+    - if [ -n "$MAX_LOADAVG" ] ; then
+        echo "PARALLEL_MAKE += \"-l$MAX_LOADAVG\"" ;
+        echo "NINJA_ARGS:append = \" -l$MAX_LOADAVG\"" ;
+        echo "MAXLOAD_NINJA = \"$MAX_LOADAVG\"" ;
       fi >> conf/auto.conf
     - echo "RM_WORK_EXCLUDE += \"${RECIPE}\"" >> conf/auto.conf
     - echo "OPENHARMONY_VERSION = \"${OPENHARMONY_VERSION}\"" >> conf/auto.conf
@@ -209,7 +220,7 @@ fetch:
         SOURCE_MIRROR_PATH="$CI_PROJECT_DIR/build/mirror" ;
       fi
     - echo "Syncing mirror files to $SOURCE_MIRROR_PATH"
-    - mkdir -pv mirror-updates-ch
+    - mkdir -pv build/mirror-updates-ch
     - rm -f build/downloads/*.done build/downloads/npm2/*.{done,resolved}
     - for f in build/downloads/* build/downloads/npm2/* ; do if test -f "$f" -a ! -L "$f" ; then
         fname=$(basename "$f") ;
@@ -426,7 +437,13 @@ ohos:
     - cd $(basename $TARBALL_FILE .tar.gz)/OpenHarmony
     - ../../build/tmp-*/deploy/sdk/${PREBUILT_IMAGE}-${OPENHARMONY_VERSION}-*.sh -y -d oniro
     - ./oniro/setup.sh
-    - ./build.sh --product-name Hi3516DV300
+    - MAX_JOBS="${CI_ONIRO_PARALLEL_BUILD_MAX_JOBS:-$(nproc)}"
+    - NINJA_ARGS="--ninja-args -j$MAX_JOBS"
+    - MAX_LOADAVG="${CI_ONIRO_PARALLEL_BUILD_MAX_LOADAVG}"
+    - if [ -n "$MAX_LOADAVG" ] ; then
+        NINJA_ARGS+=" --ninja-args -l$MAX_LOADAVG" ;
+      fi
+    - ./build.sh --product-name Hi3516DV300 $NINJA_ARGS
 
 # Job for triggering a prune of the sstate-cache mirror in runner persistent
 # storage.  This should be done periodically, but as I believe it removes a

manifests/pin.xml

@@ -6,10 +6,10 @@ SPDX-FileCopyrightText: Huawei Inc.
 <manifest>
   <include name="manifests/branch.xml" />
 
-  <extend-project name="oniro" revision="3258864533369a3a01d65934d0aefe582cb56c47" />
+  <extend-project name="oniro" revision="e196cd07ee6cbb25158693b33ea49271bebbbbd5" />
 
   <extend-project name="bitbake" revision="b8fd6f5d9959d27176ea016c249cf6d35ac8ba03" />
-  <extend-project name="openembedded-core" revision="eea52e0c3d24c79464f4afdbc3c397e1cb982231" />
+  <extend-project name="openembedded-core" revision="4667abcc925ae0c430cccb480ec530506f6201ae" />
   <extend-project name="meta-openembedded" revision="fcc7d7eae82be4c180f2e8fa3db90a8ab3be07b7" />
   <extend-project name="meta-clang" revision="d669d873edf68dc7440bb07096737203bb7ec505" />
   <extend-project name="meta-java" revision="1a8059f6b257ebe6fcae6416e499784d976afd24" />

recipes-openharmony/prebuilts/oniro-openharmony-toolchain-integration-3.0.inc

@@ -2,5 +2,4 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-# This file is intentionally left empty.
-# It is needed so the 'require' statement in oniro-openharmony-toolchain-integration.bb doesn't fail
+SRC_URI += "file://patches/build-ninja_args.patch;apply=no;subdir=src"

recipes-openharmony/prebuilts/openharmony-3.0/patches/build-ninja_args.patch

+# SPDX-FileCopyrightText: Huawei Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+Patch for //build git repository of OpenHarmony 3.0 codebase.
+
+This fixes support for using --ninja-args argument multiple times with build.sh
+script.
+
+Signed-off-by: Esben Haabendal <esben.haabendal@huawei.com>
+Upstream-Status: Pending
+
+diff --git a/build/core/build_scripts/parse_cmdline.sh b/build/core/build_scripts/parse_cmdline.sh
+index d8e351ede111..b7a49a2d04ca 100755
+--- a/build/core/build_scripts/parse_cmdline.sh
++++ b/build/core/build_scripts/parse_cmdline.sh
+@@ -33,7 +33,7 @@ parse_cmdline()
+         variant)          BUILD_VARIANT="$PARAM" ;;
+         out_dir)          OUT_DIR="$PARAM" ;;
+         gn_args)          GN_ARGS="$GN_ARGS $PARAM" ;;
+-        ninja_args)       NINJA_ARGS="$PARAM" ;;
++        ninja_args)       NINJA_ARGS="$NINJA_ARGS $PARAM" ;;
+         versionmode)      TARGET_VERSION_MODE="$PARAM" ;;
+         coverage)         COVERAGE="$PARAM" ;;
+         custom_clang)     CUSTOM_CLANG="$PARAM" ;;