Merge branch 'CI_INTEGRATION_28062024' into 'main'

Ci integration 28062024

See merge request eclipse-research-labs/codeco-project/privacy-preserving-decentralised-learning-and-context-awareness-pdlc/pdlc-integration!4

.gitlab-ci.yml

+stages:
+  - build
+
+image: ubuntu:24.04
+
+variables:
+  KIND_VERSION: v0.23.0
+  SDA_REPO: https://gitlab.eclipse.org/eclipse-research-labs/codeco-project/experimentation-framework-and-demonstrations/data-generators-and-datasets/synthetic-data-generator.git  # Synthetic Data Generator URI
+
+build_environment:
+  stage: build
+  script:
+    # Installing sudo & kind & yq (for file processing)
+    - apt-get update && apt-get install -y curl
+    - apt install sudo -y
+    - '[ $(uname -m) = x86_64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64'
+    - chmod +x ./kind
+    - sudo mv ./kind /usr/local/bin/kind
+    - kind version
+    - sudo apt-get install -y yq
+
+    # Installing Docker
+    - for pkg in docker.io docker-doc docker-compose docker-compose-v2 podman-docker containerd runc; do sudo apt-get remove $pkg; done
+    - sudo apt-get update
+    - sudo apt-get install ca-certificates curl
+    - sudo install -m 0755 -d /etc/apt/keyrings
+    - sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+    - sudo chmod a+r /etc/apt/keyrings/docker.asc
+    - 'echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null'
+    - sudo apt-get update
+    - sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y
+    - sudo docker run hello-world
+    - sudo docker --version
+
+    # Creating kind cluster
+    - 'echo -e "kind: Cluster\napiVersion: kind.x-k8s.io/v1alpha4\nnodes:\n- role: control-plane\n- role: worker\n- role: worker" > config.yaml'
+    - kind delete cluster --name sonem    # Delete Cluster to Recreate it from the Start
+    - kind create cluster --config config.yaml --name sonem
+    - sleep 10    # Wait for 10 seconds for the cluster to startup
+
+    # Test Cluster
+    - docker exec sonem-control-plane kubectl cluster-info
+    - docker exec sonem-control-plane kubectl get pods --all-namespaces
+
+    # Downloading Repos
+    - git clone $CI_REPOSITORY_URL  # Clone current Repo
+    - cd pdlc-integration
+    - git checkout $CI_COMMIT_BRANCH # Checkout the commit that triggered the pipeline (the latest commit)
+    - cd ..
+    - git clone $SDA_REPO           # Clone Synthetic Data Generator Repo
+
+    # Configuring Synthetic Data Generator
+    - |
+      sed -i 's/value: "node1,node2,node3"/value: "sonem-control-plane,sonem-worker,sonem-worker2"/' synthetic-data-generator/netma-controller/netma-controller-deployment.yaml
+    - |
+      sed -i 's/value: "node1,node2,node3"/value: "sonem-control-plane,sonem-worker,sonem-worker2"/' synthetic-data-generator/acm-controller/acm-controller-deployment.yaml
+    - cat synthetic-data-generator/netma-controller/netma-controller-deployment.yaml
+    - cat synthetic-data-generator/acm-controller/acm-controller-deployment.yaml
+    - docker cp synthetic-data-generator sonem-control-plane:synthetic-data-generator
+    - docker exec sonem-control-plane ls synthetic-data-generator
+
+    # Configuring PDLC Integration
+    - docker cp pdlc-integration sonem-control-plane:pdlc-integration
+    - docker exec sonem-control-plane ls pdlc-integration
+
+    # Apply The Synthetic Data Generator (and Dummy Data)
+    - docker exec sonem-control-plane chmod +x synthetic-data-generator/apply-controllers.sh
+    - docker exec sonem-control-plane chmod +x synthetic-data-generator/apply-dummy.sh
+    - docker exec sonem-control-plane bash -c "cd ./synthetic-data-generator && ./apply-controllers.sh"
+    - docker exec sonem-control-plane bash -c "cd ./synthetic-data-generator && ./apply-dummy.sh"
+    - sleep 300 # Wait some seconds for the Synthetic Data Generator to Install (5 mins)
+
+    # Print Synthetic Data Generator Information
+    - docker exec sonem-control-plane kubectl get pods --all-namespaces
+    - docker exec sonem-control-plane kubectl describe deployment acm-controller -n he-codeco-acm
+    - docker exec sonem-control-plane kubectl describe codecoapp -n test-namespace  # Get ACM CR
+    - docker exec sonem-control-plane kubectl describe assignmentplan   # Get SWM Assignment Plan
+    - docker exec sonem-control-plane kubectl describe application -n test-namespace   # Get SWM Application
+
+    # Apply PDLC Integration
+    - docker exec sonem-control-plane chmod +x pdlc-integration/apply_yamls.sh
+    - docker exec sonem-control-plane bash -c "cd ./pdlc-integration && ./apply_yamls.sh"
+    - sleep 60 # Wait some seconds for the PDLC DP to Install and Extract Results (1 mins)
+
+    # Print PDLC DP Logs
+    - docker exec sonem-control-plane kubectl get pods --all-namespaces
+    - docker exec sonem-control-plane kubectl logs -n he-codeco-pdlc -l app=pdlc-dp
+    # Logs for CA output
+    - docker exec sonem-worker head -n 10 /data/CA/data_CA.csv
+    # Logs for GNNs
+    - docker exec sonem-worker head -n 10 /data/GNN/data_GNN.csv
+    # Logs gor RL
+    - docker exec sonem-worker head -n 10 /data/RL/test_integracio.csv
+    - docker exec sonem-worker head -n 10 /data/RL/test_integracio.csv
+
+    # Print PDLC Outputs from CA and GNNs
+    - docker exec sonem-worker ls /data/RL
+
+    # Log the Node Reccomendations
+    - docker exec sonem-control-plane kubectl describe application -n test-namespace   # Get SWM Application
+
+    # Delete Cluster
+    - kind delete cluster --name sonem

apply_yamls.sh

@@ -21,17 +21,21 @@ kubectl apply -f ./volume/shared-pv-ro.yaml
 kubectl apply -f ./data_preprocessing/pdlc-dp-role.yaml
 kubectl apply -f ./data_preprocessing/pdlc-dp-role-binding.yaml
 kubectl apply -f ./data_preprocessing/pdlc-dp-deployment.yaml
+sleep 300   # Wait for Image to Download and Start
 
 ## Install CA Subcomponent
 kubectl apply -f ./context_awareness/pdlc-ca-deployment.yaml
+sleep 180   # Wait for Image to Download and Start
 
 ## Install GNN Subcomponent
 kubectl apply -f ./gnn_model/gnn-role.yaml
 kubectl apply -f ./gnn_model/gnn-role-binding.yaml
 kubectl apply -f ./gnn_model/gnn_inference.yaml
 kubectl apply -f ./gnn_model/gnn_controller.yaml
+sleep 180   # Wait for Image to Download and Start
 
 ## Install RL Subcomponent
 kubectl apply -f ./rl_model/rl-role.yaml
 kubectl apply -f ./rl_model/rl-role-binding.yaml
-kubectl apply -f ./rl_model/rl-model-deployment.yaml
+kubectl apply -f ./rl_model/rl-model-deployment.yaml
\ No newline at end of file
+sleep 300   # Wait for Image to Download and Start
\ No newline at end of file

data_preprocessing/pdlc-dp-deployment.yaml

@@ -28,7 +28,7 @@ spec:
         - name: COPY_MODELS_PATH  # Will be removed when MLOOps are ready
           value: "/data/"
         - name: ACM_NAMESPACE
-          value: "he-codeco-acm"
+          value: "test-namespace"
         - name: MDM_NAMESPACE
           value: "he-codeco-mdm"
         - name: NETMA_NAMESPACE