Merge branch 'use-python-venvs' into 'develop'

Use python venvs

See merge request !238

sim/tests/endToEndTests/CMakeLists.txt

@@ -89,7 +89,7 @@ foreach(i RANGE ${LEN_CONFIGS})
   add_custom_target(
     pyOpenPASS_${CURRENT_TESTCASE}
     COMMAND ${CMAKE_COMMAND} -E copy ${CURRENT_CONFIG} ${CMAKE_CURRENT_LIST_DIR}/pyOpenPASS/${CURRENT_TESTCASE}.json
-    COMMAND ${Python3_EXECUTABLE} ${PYOPENPASS_ARGS}
+    COMMAND python ${PYOPENPASS_ARGS}
     WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/pyOpenPASS
     )
 

utils/Dockerfile

@@ -2,7 +2,7 @@
 # Copyright (c) 2020-2021 in-tech GmbH
 #               2021 ITK-Engineering GmbH
 #               2023 Deutsches Zentrum für Luft- und Raumfahrt e. V. (DLR)
-#               2022-2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
+#               2022-2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
 #
 # This program and the accompanying materials are made available under the
 # terms of the Eclipse Public License 2.0 which is available at
@@ -46,6 +46,7 @@ RUN apt-get -qq update && apt-get -qq dist-upgrade && apt-get install -qq -y --n
     python3 \
     python3-distutils \
     python3-pip \
+    python3-venv \
     qtbase5-dev \
     uuid-dev \
     && apt-get -qq clean
@@ -105,6 +106,8 @@ RUN mkdir -p /opsimulation/conan
 # set the folder as a conan home
 ENV CONAN_HOME='/opsimulation/conan'
 
+ENV PYTHON_EXECUTABLE=python3
+
 # copy the required contents from the repo to build conan packages
 COPY repo/utils/ci/scripts/15_prepare_thirdParty.sh   repo/utils/ci/scripts/15_prepare_thirdParty.sh
 COPY repo/utils/ci/conan                              repo/utils/ci/conan

utils/ci/Jenkinsfile

@@ -64,7 +64,7 @@ spec:
           }
           steps {
             script {
-              env.IMAGE_TAG = "v1.1.9"
+              env.IMAGE_TAG = "v1.2.0"
             }
             build job: 'Docker-build', parameters: [string(name: 'IMAGE_NAME', value: "${env.IMAGE_NAME}"),
                                                     string(name: 'IMAGE_TAG', value:"${env.IMAGE_TAG}"),
@@ -82,7 +82,7 @@ spec:
           }
           steps {
             script {
-              env.IMAGE_TAG = "v1.1.9"
+              env.IMAGE_TAG = "v1.2.0"
             }
           }
         }
@@ -137,6 +137,7 @@ spec:
             CCACHE_REMOTE_ONLY = 'true'
             CCACHE_BASE_DIR = "${env.WORKSPACE}"
             CTCACHE_BASE_DIR = '/home/jenkins/cache/opSimulation/ctcache'
+            PYTHON_EXECUTABLE = 'python3'
           }
           stages {
             stage('Linux: Cleanup and prepare dependencies') {
@@ -216,7 +217,7 @@ spec:
           environment {
             MSYSTEM = 'MINGW64'
             CHERE_INVOKING = 'yes'
-            PYTHON_WINDOWS_EXE = 'C:/op/python/python.exe'
+            PYTHON_EXECUTABLE = 'C:/op/python/python.exe'
             OP_BASE_DIR = '/w'
             OP_REPO_DIR = '/w/repo'
             OP_DEPS_DIR = '/w/deps'

utils/ci/Jenkinsfile.CrossCompilation

@@ -58,6 +58,7 @@ spec:
         environment {
           CONAN_HOME = '/opsimulation/cross-compilation'
           CROSS_COMPILE = 'true'
+          PYTHON_EXECUTABLE = 'python3'
         }
         stages {
           stage('Linux: Cleanup and prepare dependencies') {
@@ -172,4 +173,4 @@ spec:
         }
       }
   }
-}
\ No newline at end of file
+}

utils/ci/Jenkinsfile.nightly

@@ -55,13 +55,14 @@ spec:
           }
           environment {
             CONAN_HOME = '/opsimulation/conan'
+            PYTHON_EXECUTABLE = 'python3'
           }
           stages {
             stage('Linux: Create and Run Docker image') {
               steps {
                 container('openpass-build') {
                   sh 'bash repo/utils/ci/scripts/nightly/create_docker_image.sh'
-                }              
+                }
               }
             }
           }
@@ -78,7 +79,7 @@ spec:
             environment {
               MSYSTEM = 'MINGW64'
               CHERE_INVOKING = 'yes'
-              PYTHON_WINDOWS_EXE = 'C:/Program Files/Python39/python.exe'
+              PYTHON_EXECUTABLE = 'C:/Program Files/Python39/python.exe'
               CONAN_HOME = 'C:/_dev/workspace/nightly-conan'
               bash_executable='C:\\msys-nightly\\msys2\\bin\\msys64\\usr\\bin\\bash'
             }
@@ -121,8 +122,8 @@ spec:
               }
             }
           }
-        } 
-      } 
-    } 
-  } 
-}
\ No newline at end of file
+        }
+      }
+    }
+  }
+}

utils/ci/scripts/10_prepare.sh

@@ -34,6 +34,9 @@ else
   rm -f repo/sim/tests/endToEndTests/pyOpenPASS/result_*.xml
 fi
 
+# wipe old python env if exists
+rm -rf "repo/.env"
+
 printenv
 
 if [[ "${OSTYPE}" = "msys" ]]; then

utils/ci/scripts/14_prepare_python_env.sh

+#!/bin/bash
+
+################################################################################
+# Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
+#
+# This program and the accompanying materials are made available under the
+# terms of the Eclipse Public License 2.0 which is available at
+# http://www.eclipse.org/legal/epl-2.0.
+#
+# SPDX-License-Identifier: EPL-2.0
+################################################################################
+
+################################################################################
+# This script creates a python virtual environments
+# and install python packages mentioned in the requirements.txt
+################################################################################
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+OP_REPO_DIR="${OP_REPO_DIR:=$SCRIPT_DIR/../../..}"
+
+echo "Python virtual environment (.env) is being created at ${OP_REPO_DIR}"
+
+if [ ! -f "${OP_REPO_DIR}/requirements.txt" ]; then
+    echo "Error: requirements.txt not found at ${OP_REPO_DIR}!"
+    exit 1
+fi
+
+"${PYTHON_EXECUTABLE}" -m venv "${OP_REPO_DIR}/.env"
+
+if [[ "${OSTYPE}" = "msys" ]]; then
+    venv_python_executable="${OP_REPO_DIR}/.env/Scripts/python.exe"
+else
+    venv_python_executable="${OP_REPO_DIR}/.env/bin/python3"
+fi
+
+"${venv_python_executable}" -m pip install --upgrade pip
+
+"${venv_python_executable}" -m pip install -r "${OP_REPO_DIR}/requirements.txt"  || { echo "Failed to install dependencies"; exit 1; }
+
+echo "Following packages are installed successfully in the virtual environment at ${OP_REPO_DIR}/.env."
+
+"${venv_python_executable}" -m pip list -v

utils/ci/scripts/15_prepare_thirdParty.sh

@@ -36,12 +36,10 @@ build_strategy="--build=missing"
 
 # Set python command depending on OS
 if [[ "${OSTYPE}" = "msys" ]]; then
-  PYTHON_COMMAND="${PYTHON_WINDOWS_EXE}"
   export CONAN_CMAKE_GENERATOR="MSYS Makefiles"
   conanprofile="$OP_REPO_DIR/utils/ci/conan/conanprofile_windows"
   conanprofilehost="$OP_REPO_DIR/utils/ci/conan/conanprofile_windows"
 else
-  PYTHON_COMMAND=python3
   conanprofilehost="$OP_REPO_DIR/utils/ci/conan/conanprofile_linux"
   conanprofile="$OP_REPO_DIR/utils/ci/conan/conanprofile_linux"
 fi
@@ -74,8 +72,8 @@ while [[ "$#" -gt 0 ]]; do
     shift
 done
 
-"$PYTHON_COMMAND" -m conans.conan remote enable conancenter
-"$PYTHON_COMMAND" -m conans.conan remote add eclipse-conan-local-repo ${OP_REPO_DIR}/utils/ci/conan --force
+"$PYTHON_EXECUTABLE" -m conans.conan remote enable conancenter
+"$PYTHON_EXECUTABLE" -m conans.conan remote add eclipse-conan-local-repo ${OP_REPO_DIR}/utils/ci/conan --force
 
 # Note: If there is a change in recipe of the existing package and would want to
 # reinstall the existing package. Then execute conan remove <package> command
@@ -86,7 +84,7 @@ rm -rf "$OP_DEPS_DIR"
 
 # Command to install all the packages into the required folder.
 # --build=missing argument is necessary as if the package is not available it builds from my-local-repo (utils/ci/conan/recipes)
-"$PYTHON_COMMAND" -m conans.conan install $conanfile $CONAN_ARGS $build_strategy --deployer=direct_deploy -of="$OP_DEPS_DIR" -s:a $build_type -r eclipse-conan-local-repo -r conancenter || { "$PYTHON_COMMAND" -m conans.conan remote remove -c eclipse-conan-local-repo; "$PYTHON_COMMAND" -m conans.conan cache clean --source --build "*/*"; exit 1;}
+"$PYTHON_EXECUTABLE" -m conans.conan install $conanfile $CONAN_ARGS $build_strategy --deployer=direct_deploy -of="$OP_DEPS_DIR" -s:a $build_type -r eclipse-conan-local-repo -r conancenter || { "$PYTHON_EXECUTABLE" -m conans.conan remote remove -c eclipse-conan-local-repo; "$PYTHON_EXECUTABLE" -m conans.conan cache clean --source --build "*/*"; exit 1;}
 
-"$PYTHON_COMMAND" -m conans.conan remote remove -c eclipse-conan-local-repo
-"$PYTHON_COMMAND" -m conans.conan cache clean --source --build "*/*"
+"$PYTHON_EXECUTABLE" -m conans.conan remote remove -c eclipse-conan-local-repo
+"$PYTHON_EXECUTABLE" -m conans.conan cache clean --source --build "*/*"

utils/ci/scripts/20_configure.sh

@@ -25,6 +25,7 @@ function join_paths()
 MYDIR="$(dirname "$(readlink -f $0)")"
 cd "$MYDIR/../../../../build" || exit 1
 
+OP_REPO_DIR="${OP_REPO_DIR:=$MYDIR/../../..}"
 # dependencies built previously
 DEPS=(
   "$PWD/../deps/direct_deploy/fmilibrary"
@@ -59,11 +60,6 @@ if [[ "${OSTYPE}" = "msys" ]]; then
   # set the correct CMake generator
   CMAKE_GENERATOR_ARG="-GMSYS Makefiles"
 
-  # set python command
-  if [[ -n "${PYTHON_WINDOWS_EXE}" ]]; then
-    CMAKE_PYTHON_COMMAND_ARG="-DPython3_EXECUTABLE=${PYTHON_WINDOWS_EXE}"
-  fi
-
   # prepare dependency paths
   # it seems cmake doesn't like MSYS paths starting with drive letters (e.g. /c/thirdParty/...)
   # cygpath is used here to format the paths in "mixed format" (e.g. c:/thirdparty/...)
@@ -86,9 +82,13 @@ elif [[ -n "${GIT_BRANCH}" || -n "${GIT_COMMIT}" ]]; then
   CMAKE_VERSION_ARG="-DSIMCORE_VERSION_TAG=${GIT_BRANCH:-no-branch}_${GIT_COMMIT}"
 fi
 
+case "${OSTYPE}" in
+    msys*) source "${OP_REPO_DIR}/.env/Scripts/activate" ;;
+    *)     source "${OP_REPO_DIR}/.env/bin/activate" ;;
+esac
+
 cmake \
   "$CMAKE_GENERATOR_ARG" \
-  "$CMAKE_PYTHON_COMMAND_ARG" \
   $CMAKE_VERSION_ARG \
   "$CMAKE_C_COMPILER" \
   "$CMAKE_CXX_COMPILER" \

utils/ci/scripts/30_build.sh

@@ -2,6 +2,7 @@
 
 ################################################################################
 # Copyright (c) 2021 in-tech GmbH
+#               2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
 #
 # This program and the accompanying materials are made available under the
 # terms of the Eclipse Public License 2.0 which is available at
@@ -15,6 +16,7 @@
 ################################################################################
 
 MYDIR="$(dirname "$(readlink -f $0)")"
+OP_REPO_DIR="${OP_REPO_DIR:=$MYDIR/../../..}"
 cd "$MYDIR/../../../../build" || exit 1
 
 if hash nproc 2>/dev/null; then
@@ -25,11 +27,16 @@ else
   MAKE_JOB_COUNT=1
 fi
 
+case "${OSTYPE}" in
+    msys*) source "${OP_REPO_DIR}/.env/Scripts/activate" ;;
+    *)     source "${OP_REPO_DIR}/.env/bin/activate" ;;
+esac
+
 make -j$MAKE_JOB_COUNT install
 
 if [[ "${CROSS_COMPILE}" = true ]]; then
   cd ../dist/opSimulation
-  cp $MYDIR/../../../../deps/direct_deploy/yase/bin/libagnostic_behavior_tree.dll $MYDIR/../../../../dist/opSimulation 
-  cp $MYDIR/../../../../deps/direct_deploy/openscenario_engine/bin/libOpenScenarioEngine.dll $MYDIR/../../../../dist/opSimulation 
+  cp $MYDIR/../../../../deps/direct_deploy/yase/bin/libagnostic_behavior_tree.dll $MYDIR/../../../../dist/opSimulation
+  cp $MYDIR/../../../../deps/direct_deploy/openscenario_engine/bin/libOpenScenarioEngine.dll $MYDIR/../../../../dist/opSimulation
   cp $MYDIR/../../../../deps/direct_deploy/openscenario_api/lib/*.dll $MYDIR/../../../../dist/opSimulation
-fi
\ No newline at end of file
+fi

utils/ci/scripts/55_endtoend.sh

@@ -2,7 +2,7 @@
 
 ################################################################################
 # Copyright (c) 2021 in-tech GmbH
-#               2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
+#               2023-2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
 #
 # This program and the accompanying materials are made available under the
 # terms of the Eclipse Public License 2.0 which is available at
@@ -16,6 +16,7 @@
 ################################################################################
 
 MYDIR="$(dirname "$(readlink -f $0)")"
+OP_REPO_DIR="${OP_REPO_DIR:=$MYDIR/../../..}"
 
 if [[ "${CROSS_COMPILE}" = true ]]; then
   echo "When cross compiling, end-to-end test is not executed on build system"
@@ -28,5 +29,10 @@ if [[ "${OSTYPE}" = "msys" ]]; then
 fi
 # END OF HOTFIX
 
+case "${OSTYPE}" in
+    msys*) source "${OP_REPO_DIR}/.env/Scripts/activate" ;;
+    *)     source "${OP_REPO_DIR}/.env/bin/activate" ;;
+esac
+
 cd "$MYDIR/../../../../build"
 make pyOpenPASS

utils/ci/scripts/60_check_clang_format.sh

 #!/usr/bin/env bash
 
 ################################################################################
-# Copyright (c) 2022-2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
+# Copyright (c) 2022-2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
 #
 # This program and the accompanying materials are made available under the
 # terms of the Eclipse Public License 2.0 which is available at
@@ -26,7 +26,7 @@ fi
 
 clang-format --version
 
-for file in $(find . -type f \( -name "*.h" -o -name "*.cpp" \) ! -path "./sim/deps/*" ! -path "./build/*" ! -path "./gui/*" ! -path "./.git/*"); do
+for file in $(find . -type f \( -name "*.h" -o -name "*.cpp" \) ! -path "./sim/deps/*" ! -path "./build/*" ! -path "./gui/*" ! -path "./.git/*" ! -path "./.env/*"); do
     if $(clang-format -output-replacements-xml $file | grep -c "<replacement " > /dev/null); then
         if [[ ! "$file" =~ version\.h$ ]]; then # filtering version.h as it is autogenerated
             echo "ERROR [clang-format]: $file is not formatted properly"

utils/ci/scripts/62_static_analysis.sh

@@ -36,6 +36,7 @@ CLANG_TIDY_CHECK_EXCLUDE_DIRS=(
   "contrib"
   "deps"
   "doc"
+  ".env"
 )
 
 CLANG_TIDY_CHECK_EXCLUDE_FILES=(

utils/ci/scripts/65_check_include_style.sh

 #!/usr/bin/env bash
 
 ################################################################################
-# Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
+# Copyright (c) 2023-2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
 #
 # This program and the accompanying materials are made available under the
 # terms of the Eclipse Public License 2.0 which is available at
@@ -31,7 +31,7 @@ while IFS= read -r file; do
         filename=$(basename "$include" | sed 's/["><]//g')
 
         # Use find to search for the filename in all subdirectories
-        file_count=$(find sim ! -path "sim/deps/*" -name "$filename" | wc -l)
+        file_count=$(find sim ! -path "sim/deps/*" ! -path ".env/*" -name "$filename" | wc -l)
 
         if [[ $file_count -gt 0 ]]; then
             if [[ "$include" != "\""*"\""  ]]; then
@@ -45,7 +45,7 @@ while IFS= read -r file; do
             fi
         fi
     done
-done < <(find ./sim -type f \( -name "*.h" -o -name "*.cpp" \) ! -path "./sim/deps/*" ! -path "./sim/contrib/*" ! -path "./sim/doc/*" ! -path "./build/*" ! -path "./gui/*" ! -path "./.git/*")
+done < <(find ./sim -type f \( -name "*.h" -o -name "*.cpp" \) ! -path "./sim/deps/*" ! -path "./.env/*" ! -path "./sim/contrib/*" ! -path "./sim/doc/*" ! -path "./build/*" ! -path "./gui/*" ! -path "./.git/*")
 
 if [ $exit_code -eq 0 ]; then
     echo "SUCCESS [header-include-style]: Header files are enclosed correctly"

utils/ci/scripts/build_prepare.sh

@@ -2,6 +2,7 @@
 
 ################################################################################
 # Copyright (c) 2021 in-tech GmbH
+#               2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
 #
 # This program and the accompanying materials are made available under the
 # terms of the Eclipse Public License 2.0 which is available at
@@ -19,7 +20,7 @@ cd "$MYDIR" || exit 1
 
 export CCACHE_DIR=$MYDIR/../../../../ccache
 
-for SCRIPT in 10_prepare.sh 15_prepare_thirdParty.sh 20_configure.sh; do
+for SCRIPT in 10_prepare.sh 14_prepare_python_env.sh 15_prepare_thirdParty.sh 20_configure.sh; do
   echo
   echo "======================================================================="
   echo "Executing ${SCRIPT}..."
@@ -28,4 +29,3 @@ for SCRIPT in 10_prepare.sh 15_prepare_thirdParty.sh 20_configure.sh; do
 
   ./$SCRIPT || exit 1
 done
-