Reduce disk usage

See also eclipsefdn/helpdesk#5250 (closed)

utils/ci/Jenkinsfile

-/*******************************************************************************
+/********************************************************************************
  * Copyright (c) 2021 in-tech GmbH
+ *               2022-2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
  *
- * This program and the accompanying materials are made
- * available under the terms of the Eclipse Public License 2.0
- * which is available at https://www.eclipse.org/legal/epl-2.0/
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
  *
  * SPDX-License-Identifier: EPL-2.0
- *******************************************************************************/
+ ********************************************************************************/
 
 pipeline {
   agent none
+  parameters {
+    booleanParam(name: 'BUILD_DOCKER_IMAGE', defaultValue: false, description: 'Force docker image (re-)build')
+  }
   options {
     checkoutToSubdirectory('repo')
+    timeout(time: 10, unit: 'HOURS')
+    timestamps()
+    disableConcurrentBuilds(abortPrevious: true)
+    buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '2'))
+  }
+  environment {
+    IMAGE_NAME = "eclipseopenpass/opsim-ci"
+    REPO_URL = "https://gitlab.eclipse.org/eclipse/openpass/opSimulation.git"
+    DOCKERFILE_PATH = "utils/Dockerfile"
   }
   stages {
+    stage('Build docker image or set image tag') {
+      agent {
+            kubernetes {
+              inheritFrom 'openpass-agent-pod-' + env.BUILD_NUMBER
+              yaml """
+apiVersion: v1
+kind: Pod
+spec:
+  containers:
+  - name: ose-build
+    image: eclipseopenpass/ubuntu:base
+    tty: true
+    resources:
+      limits:
+        memory: "16Gi"
+        cpu: "4"
+      requests:
+        memory: "16Gi"
+        cpu: "4"
+  - name: jnlp
+    volumeMounts:
+    - name: volume-known-hosts
+      mountPath: /home/jenkins/.ssh
+  volumes:
+  - name: volume-known-hosts
+    configMap:
+      name: known-hosts
+"""
+            }
+          }
+      stages {
+        stage('Build docker image') {
+          when {
+            expression {
+              return params.BUILD_DOCKER_IMAGE
+            }
+          }
+          steps {
+            script {
+              env.IMAGE_TAG = "v1.2.0"
+            }
+            build job: 'Docker-build', parameters: [string(name: 'IMAGE_NAME', value: "${env.IMAGE_NAME}"),
+                                                    string(name: 'IMAGE_TAG', value:"${env.IMAGE_TAG}"),
+                                                    string(name: 'REPO_URL', value: "${env.REPO_URL}"),
+                                                    string(name: 'BRANCH_NAME', value: "${env.GIT_BRANCH}"),
+                                                    string(name: 'DOCKERFILE_PATH', value: "${env.DOCKERFILE_PATH}")],
+                                       propagate: true
+          }
+        }
+        stage('Set image tag') {
+          when {
+            expression {
+              return !params.BUILD_DOCKER_IMAGE
+            }
+          }
+          steps {
+            script {
+              env.IMAGE_TAG = "v1.2.0"
+            }
+          }
+        }
+      }
+    }
     stage('Linux and Windows build') {
       parallel {
         stage('Linux') {
           agent {
             kubernetes {
-              label 'openpass-agent-pod-' + env.BUILD_NUMBER
+              inheritFrom 'openpass-agent-pod-' + env.BUILD_NUMBER
               yaml """
 apiVersion: v1
 kind: Pod
 spec:
   containers:
   - name: openpass-build
-    image: rbiegel/openpass-build:latest
+    image: "${env.IMAGE_NAME}:${env.IMAGE_TAG}"
     tty: true
     resources:
       limits:
         memory: "16Gi"
         cpu: "4"
-      requests:
-        memory: "16Gi"
-        cpu: "4"
+    volumeMounts:
+    - name: openpass-cache-storage
+      mountPath: "/home/jenkins/cache"
   - name: jnlp
     volumeMounts:
     - name: volume-known-hosts
@@ -43,11 +119,23 @@ spec:
   - name: volume-known-hosts
     configMap:
       name: known-hosts
+  - name: openpass-cache-storage
+    persistentVolumeClaim:
+      claimName: openpass-cache-storage
+  securityContext:
+      seLinuxOptions:
+        level: s0:c4,c54
+        type: spc_t
 """
             }
           }
           environment {
-            CONAN_USER_HOME = '/home/jenkins/agent/workspace'
+            CONAN_HOME = '/opsimulation/conan'
+            CCACHE_REMOTE_STORAGE = 'file:///home/jenkins/cache/opSimulation/ccache/linux'
+            CCACHE_REMOTE_ONLY = 'true'
+            CCACHE_BASE_DIR = "${env.WORKSPACE}"
+            CTCACHE_BASE_DIR = '/home/jenkins/cache/opSimulation/ctcache'
+            PYTHON_EXECUTABLE = 'python3'
           }
           stages {
             stage('Linux: Cleanup and prepare dependencies') {
@@ -57,6 +145,14 @@ spec:
                 }
               }
             }
+            stage('Linux: Code compliance') {
+              steps {
+                container('openpass-build') {
+                  sh 'bash repo/utils/ci/scripts/61_check_eol.sh'
+                  sh 'bash repo/utils/ci/scripts/62_static_analysis.sh'
+                }
+              }
+            }
             stage('Linux: Build core and run end-to-end tests') {
               steps {
                 container('openpass-build') {
@@ -71,20 +167,35 @@ spec:
                 }
               }
             }
+            stage('Linux: Cleanup') {
+              steps {
+                container('openpass-build') {
+                  sh 'bash repo/utils/ci/scripts/cleanup.sh'
+                }
+              }
+            }
             stage('Linux: Deploy') {
               when {
                 anyOf {
-                  tag pattern: "openPASS_.*", comparator: "REGEXP"
-                  triggeredBy 'TimerTrigger'
+                  tag pattern: "v.*", comparator: "REGEXP"
+                  branch "develop"
                 }
               }
               steps {
                 container('jnlp') {
                   sshagent ( ['projects-storage.eclipse.org-bot-ssh']) {
-                    sh '''
-                      ssh -o BatchMode=yes genie.simopenpass@projects-storage.eclipse.org mkdir -p /home/data/httpd/download.eclipse.org/simopenpass/snapshots
-                      [ -f artifacts/openPASS_SIM.tar.gz ] && scp -o BatchMode=yes artifacts/openPASS_SIM.tar.gz "genie.simopenpass@projects-storage.eclipse.org:/home/data/httpd/download.eclipse.org/simopenpass/snapshots/openPASS_SIM_$(date -u +%Y%m%d_%H%M%S).tar.gz"
-                    '''
+                    sh 'bash repo/utils/ci/scripts/publish_artifacts.sh'
+                  }
+                }
+              }
+            }
+            stage('Linux: Deploy Docs') {
+              steps {
+                container('openpass-build')
+                {
+                  withCredentials([string(credentialsId: 'gitlab-api-token', variable: 'GITLAB_BOT_TOKEN')])
+                  {
+                    sh 'bash repo/utils/ci/scripts/auto_deploy_docu.sh'
                   }
                 }
               }
@@ -92,50 +203,63 @@ spec:
           }
           post {
             always {
-              archiveArtifacts allowEmptyArchive: true, artifacts: 'artifacts/**,dist/opSim/testreport/**', followSymlinks: false
+              archiveArtifacts allowEmptyArchive: true, artifacts: 'artifacts/**', followSymlinks: false
               junit allowEmptyResults: true, testResults: 'build/**/*Tests.xml,repo/sim/tests/endToEndTests/pyOpenPASS/result_*.xml'
             }
           }
         }
         stage('Windows') {
           agent {
-            label 'windows'
+            label 'windows10'
           }
           environment {
             MSYSTEM = 'MINGW64'
             CHERE_INVOKING = 'yes'
-            PYTHON_WINDOWS_EXE = 'C:/Program Files/Python39/python.exe'
+            PYTHON_EXECUTABLE = 'C:/op/python/python.exe'
+            OP_BASE_DIR = '/w'
+            OP_REPO_DIR = '/w/repo'
+            OP_DEPS_DIR = '/w/deps'
           }
           stages {
             stage('Windows: Cleanup and prepare dependencies') {
               steps {
                 bat 'subst W: %WORKSPACE%'
                 dir('W:/') {
-                  bat 'C:\\msys64\\usr\\bin\\bash -lc repo/utils/ci/scripts/build_prepare.sh'
+                  bat 'C:\\op\\msys64\\usr\\bin\\bash -lc repo/utils/ci/scripts/build_prepare.sh'
                 }
               }
             }
             stage('Windows: Build core and run end-to-end tests') {
               steps {
                 dir('W:/') {
-                  bat 'C:\\msys64\\usr\\bin\\bash -lc repo/utils/ci/scripts/build_core_and_e2e.sh'
+                  bat 'C:\\op\\msys64\\usr\\bin\\bash -lc repo/utils/ci/scripts/build_core_and_e2e.sh'
                 }
               }
             }
             stage('Windows: Build and run unit tests') {
               steps {
                 dir('W:/') {
-                  bat 'C:\\msys64\\usr\\bin\\bash -lc repo/utils/ci/scripts/build_unittests.sh'
+                  bat 'C:\\op\\msys64\\usr\\bin\\bash -lc repo/utils/ci/scripts/build_unittests.sh'
+                }
+              }
+            }
+            stage('Windows: Cleanup') {
+              steps {
+                dir('W:/') {
+                  bat 'C:\\op\\msys64\\usr\\bin\\bash -lc repo/utils/ci/scripts/cleanup.sh'
                 }
               }
             }
             stage('Windows: Deploy') {
+              when {
+                anyOf {
+                  tag pattern: "v.*", comparator: "REGEXP"
+                  branch "develop"
+                }
+              }
               steps {
                 sshagent ( ['projects-storage.eclipse.org-bot-ssh']) {
-                  bat '''
-                    C:\\msys64\\usr\\bin\\bash -lc 'ssh -o BatchMode=yes genie.simopenpass@projects-storage.eclipse.org mkdir -p /home/data/httpd/download.eclipse.org/simopenpass/snapshots'
-                    C:\\msys64\\usr\\bin\\bash -lc '[ -f artifacts/openPASS_SIM.zip ] && scp -o BatchMode=yes artifacts/openPASS_SIM.zip "genie.simopenpass@projects-storage.eclipse.org:/home/data/httpd/download.eclipse.org/simopenpass/snapshots/openPASS_SIM_$(date -u +%%Y%%m%%d_%%H%%M%%S).zip"'
-                    '''
+                  bat 'C:\\op\\msys64\\usr\\bin\\bash -lc repo/utils/ci/scripts/publish_artifacts.sh'
                 }
               }
             }
@@ -143,7 +267,7 @@ spec:
           post {
             always {
               bat 'subst W: /d'
-              archiveArtifacts allowEmptyArchive: true, artifacts: 'artifacts/**,dist/opSim/testreport/**', followSymlinks: false
+              archiveArtifacts allowEmptyArchive: true, artifacts: 'artifacts/**', followSymlinks: false
               junit allowEmptyResults: true, testResults: 'build/**/*Tests.xml,repo/sim/tests/endToEndTests/pyOpenPASS/result_*.xml'
             }
           }