From 0dce76075d626df10e5fb763e712b2a496b646a8 Mon Sep 17 00:00:00 2001
From: Alex ubuntu vm <alexdecb@yahoo.es>
Date: Wed, 28 Feb 2024 12:27:12 +0100
Subject: [PATCH] pods deployment: added ipam plugin
Following discussions with the SWM team, static ip address implementation was required. This is done now by specifying with the pod an optional ip address field. If no ip is defined, a default ipv6 will be used
---
.../custom-installation/deploySwitch.yaml | 26 +++---
.../interfaces_definitions/test/poda.yaml | 21 +++++
.../interfaces_definitions/test/podb.yaml | 21 +++++
.../interfaces_definitions/test/ptp1.yaml | 15 ++++
.../interfaces_definitions/vhost1.yaml | 5 +-
.../interfaces_definitions/vhost10.yaml | 5 +-
.../interfaces_definitions/vhost2.yaml | 5 +-
.../interfaces_definitions/vhost3.yaml | 5 +-
.../interfaces_definitions/vhost4.yaml | 5 +-
.../interfaces_definitions/vhost5.yaml | 5 +-
.../interfaces_definitions/vhost6.yaml | 5 +-
.../interfaces_definitions/vhost7.yaml | 5 +-
.../interfaces_definitions/vhost8.yaml | 5 +-
.../interfaces_definitions/vhost9.yaml | 5 +-
.../l2sm-network-crd.yaml | 0
deployments/l2sm-deployment.yaml | 50 ++++++++---
src/operator/l2sm-operator.py | 85 +++++++++----------
17 files changed, 190 insertions(+), 78 deletions(-)
create mode 100644 deployments/custom-installation/interfaces_definitions/test/poda.yaml
create mode 100644 deployments/custom-installation/interfaces_definitions/test/podb.yaml
create mode 100644 deployments/custom-installation/interfaces_definitions/test/ptp1.yaml
rename deployments/{custom-installation => inter-cluster}/l2sm-network-crd.yaml (100%)
diff --git a/deployments/custom-installation/deploySwitch.yaml b/deployments/custom-installation/deploySwitch.yaml
index e509e08..c93a73b 100644
--- a/deployments/custom-installation/deploySwitch.yaml
+++ b/deployments/custom-installation/deploySwitch.yaml
@@ -14,7 +14,7 @@ spec:
labels:
l2sm-component: l2sm-switch
annotations:
- k8s.v1.cni.cncf.io/networks: veth1, veth2, veth3, veth4, veth5, veth6, veth7, veth8, veth9, veth10
+ k8s.v1.cni.cncf.io/networks: '[{ "name": "veth1", "ips": ["fe80::58d0:b8ff:fe42:debf/64"]}, { "name": "veth2", "ips": ["fe80::58d0:b8ff:fe42:debe/64"]}, { "name": "veth3", "ips": ["fe80::58d0:b8ff:fe42:debd/64"]}, { "name": "veth4", "ips": ["fe80::58d0:b8ff:fe42:debc/64"]}, { "name": "veth5", "ips": ["fe80::58d0:b8ff:fe42:debb/64"]}, { "name": "veth6", "ips": ["fe80::58d0:b8ff:fe42:deba/64"]}, { "name": "veth7", "ips": ["fe80::58d0:b8ff:fe42:deb9/64"]}, { "name": "veth8", "ips": ["fe80::58d0:b8ff:fe42:deb8/64"]}, { "name": "veth9", "ips": ["fe80::58d0:b8ff:fe42:deb7/64"]}, { "name": "veth10", "ips": ["fe80::58d0:b8ff:fe42:deb6/64"]}]'
spec:
tolerations:
# this toleration is to have the daemonset runnable on master nodes
@@ -22,18 +22,18 @@ spec:
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
- initContainers:
- - name: wait-for-l2sm-operator
- image: curlimages/curl
- args:
- - /bin/sh
- - -c
- - >
- set -x;
- while [ $(curl -sw '%{http_code}' "http://l2sm-operator-service:8080/healthz" -o /dev/null) -ne 200 ]; do
- sleep 15;
- done;
- sleep 5;
+ # initContainers:
+ # - name: wait-for-l2sm-operator
+ # image: curlimages/curl
+ # args:
+ # - /bin/sh
+ # - -c
+ # - >
+ # set -x;
+ # while [ $(curl -sw '%{http_code}' "http://l2sm-operator-service:8080/healthz" -o /dev/null) -ne 200 ]; do
+ # sleep 15;
+ # done;
+ # sleep 5;
containers:
- name: l2sm-switch
image: alexdecb/l2sm-switch:2.2
diff --git a/deployments/custom-installation/interfaces_definitions/test/poda.yaml b/deployments/custom-installation/interfaces_definitions/test/poda.yaml
new file mode 100644
index 0000000..c22dd4d
--- /dev/null
+++ b/deployments/custom-installation/interfaces_definitions/test/poda.yaml
@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: poda
+ labels:
+ app: ping-pong
+ annotations:
+ k8s.v1.cni.cncf.io/networks: '[
+ { "name": "ptp1",
+ "ips": ["192.168.1.6/16"]
+ }]'
+spec:
+ containers:
+ - name: router
+ command: ["/bin/ash", "-c", "trap : TERM INT; sleep infinity & wait"]
+ image: alpine:latest
+ securityContext:
+ capabilities:
+ add: ["NET_ADMIN"]
+ # Use this parameter if you want to place the pod in a specific node
+ #nodeName: workerk8s
diff --git a/deployments/custom-installation/interfaces_definitions/test/podb.yaml b/deployments/custom-installation/interfaces_definitions/test/podb.yaml
new file mode 100644
index 0000000..e5c9084
--- /dev/null
+++ b/deployments/custom-installation/interfaces_definitions/test/podb.yaml
@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: podb
+ labels:
+ app: ping-pong
+ annotations:
+ k8s.v1.cni.cncf.io/networks: '[
+ { "name": "ptp1",
+ "ips": ["fe80::58d0:b8ff:fe42:debf/64"]
+ }]'
+spec:
+ containers:
+ - name: router
+ command: ["/bin/ash", "-c", "trap : TERM INT; sleep infinity & wait"]
+ image: alpine:latest
+ securityContext:
+ capabilities:
+ add: ["NET_ADMIN"]
+ # Use this parameter if you want to place the pod in a specific node
+ #nodeName: workerk8s
diff --git a/deployments/custom-installation/interfaces_definitions/test/ptp1.yaml b/deployments/custom-installation/interfaces_definitions/test/ptp1.yaml
new file mode 100644
index 0000000..c2b261a
--- /dev/null
+++ b/deployments/custom-installation/interfaces_definitions/test/ptp1.yaml
@@ -0,0 +1,15 @@
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+ name: veth1
+spec:
+ config: '{
+ "cniVersion": "0.3.0",
+ "type": "bridge",
+ "name":"veth1",
+ "mtu": 1400,
+ "device":"veth1",
+ "ipam": {
+ "type":"static"
+ }
+ }'
\ No newline at end of file
diff --git a/deployments/custom-installation/interfaces_definitions/vhost1.yaml b/deployments/custom-installation/interfaces_definitions/vhost1.yaml
index 5eed802..20e0002 100644
--- a/deployments/custom-installation/interfaces_definitions/vhost1.yaml
+++ b/deployments/custom-installation/interfaces_definitions/vhost1.yaml
@@ -9,5 +9,8 @@ spec:
"type": "bridge",
"bridge": "br1",
"mtu": 1400,
- "device": "veth1"
+ "device": "veth1",
+ "ipam": {
+ "type":"static"
+ }
}'
diff --git a/deployments/custom-installation/interfaces_definitions/vhost10.yaml b/deployments/custom-installation/interfaces_definitions/vhost10.yaml
index db850d1..86e6e19 100644
--- a/deployments/custom-installation/interfaces_definitions/vhost10.yaml
+++ b/deployments/custom-installation/interfaces_definitions/vhost10.yaml
@@ -9,5 +9,8 @@ spec:
"type": "bridge",
"bridge": "br10",
"mtu": 1400,
- "device": "veth10"
+ "device": "veth10",
+ "ipam": {
+ "type":"static"
+ }
}'
diff --git a/deployments/custom-installation/interfaces_definitions/vhost2.yaml b/deployments/custom-installation/interfaces_definitions/vhost2.yaml
index ebee425..d16f057 100644
--- a/deployments/custom-installation/interfaces_definitions/vhost2.yaml
+++ b/deployments/custom-installation/interfaces_definitions/vhost2.yaml
@@ -9,5 +9,8 @@ spec:
"type": "bridge",
"bridge": "br2",
"mtu": 1400,
- "device": "veth2"
+ "device": "veth2",
+ "ipam": {
+ "type":"static"
+ }
}'
diff --git a/deployments/custom-installation/interfaces_definitions/vhost3.yaml b/deployments/custom-installation/interfaces_definitions/vhost3.yaml
index 984a574..c91fad3 100644
--- a/deployments/custom-installation/interfaces_definitions/vhost3.yaml
+++ b/deployments/custom-installation/interfaces_definitions/vhost3.yaml
@@ -9,5 +9,8 @@ spec:
"type": "bridge",
"bridge": "br3",
"mtu": 1400,
- "device": "veth3"
+ "device": "veth3",
+ "ipam": {
+ "type":"static"
+ }
}'
diff --git a/deployments/custom-installation/interfaces_definitions/vhost4.yaml b/deployments/custom-installation/interfaces_definitions/vhost4.yaml
index be25ebf..f42ba14 100644
--- a/deployments/custom-installation/interfaces_definitions/vhost4.yaml
+++ b/deployments/custom-installation/interfaces_definitions/vhost4.yaml
@@ -9,5 +9,8 @@ spec:
"type": "bridge",
"bridge": "br4",
"mtu": 1400,
- "device": "veth4"
+ "device": "veth4",
+ "ipam": {
+ "type":"static"
+ }
}'
diff --git a/deployments/custom-installation/interfaces_definitions/vhost5.yaml b/deployments/custom-installation/interfaces_definitions/vhost5.yaml
index 9742122..a1cfdd3 100644
--- a/deployments/custom-installation/interfaces_definitions/vhost5.yaml
+++ b/deployments/custom-installation/interfaces_definitions/vhost5.yaml
@@ -9,5 +9,8 @@ spec:
"type": "bridge",
"bridge": "br5",
"mtu": 1400,
- "device": "veth5"
+ "device": "veth5",
+ "ipam": {
+ "type":"static"
+ }
}'
diff --git a/deployments/custom-installation/interfaces_definitions/vhost6.yaml b/deployments/custom-installation/interfaces_definitions/vhost6.yaml
index 8601245..a000d58 100644
--- a/deployments/custom-installation/interfaces_definitions/vhost6.yaml
+++ b/deployments/custom-installation/interfaces_definitions/vhost6.yaml
@@ -9,5 +9,8 @@ spec:
"type": "bridge",
"bridge": "br6",
"mtu": 1400,
- "device": "veth6"
+ "device": "veth6",
+ "ipam": {
+ "type":"static"
+ }
}'
diff --git a/deployments/custom-installation/interfaces_definitions/vhost7.yaml b/deployments/custom-installation/interfaces_definitions/vhost7.yaml
index c6cdd78..dfa6b2e 100644
--- a/deployments/custom-installation/interfaces_definitions/vhost7.yaml
+++ b/deployments/custom-installation/interfaces_definitions/vhost7.yaml
@@ -9,5 +9,8 @@ spec:
"type": "bridge",
"bridge": "br7",
"mtu": 1400,
- "device": "veth7"
+ "device": "veth7",
+ "ipam": {
+ "type":"static"
+ }
}'
diff --git a/deployments/custom-installation/interfaces_definitions/vhost8.yaml b/deployments/custom-installation/interfaces_definitions/vhost8.yaml
index e3b478d..f3fbafc 100644
--- a/deployments/custom-installation/interfaces_definitions/vhost8.yaml
+++ b/deployments/custom-installation/interfaces_definitions/vhost8.yaml
@@ -9,5 +9,8 @@ spec:
"type": "bridge",
"bridge": "br8",
"mtu": 1400,
- "device": "veth8"
+ "device": "veth8",
+ "ipam": {
+ "type":"static"
+ }
}'
diff --git a/deployments/custom-installation/interfaces_definitions/vhost9.yaml b/deployments/custom-installation/interfaces_definitions/vhost9.yaml
index 5b68835..cd1502e 100644
--- a/deployments/custom-installation/interfaces_definitions/vhost9.yaml
+++ b/deployments/custom-installation/interfaces_definitions/vhost9.yaml
@@ -9,5 +9,8 @@ spec:
"type": "bridge",
"bridge": "br9",
"mtu": 1400,
- "device": "veth9"
+ "device": "veth9",
+ "ipam": {
+ "type":"static"
+ }
}'
diff --git a/deployments/custom-installation/l2sm-network-crd.yaml b/deployments/inter-cluster/l2sm-network-crd.yaml
similarity index 100%
rename from deployments/custom-installation/l2sm-network-crd.yaml
rename to deployments/inter-cluster/l2sm-network-crd.yaml
diff --git a/deployments/l2sm-deployment.yaml b/deployments/l2sm-deployment.yaml
index 16f2655..5a9c1f1 100644
--- a/deployments/l2sm-deployment.yaml
+++ b/deployments/l2sm-deployment.yaml
@@ -9,7 +9,10 @@ spec:
"type": "bridge",
"bridge": "br1",
"mtu": 1400,
- "device": "veth1"
+ "device": "veth1",
+ "ipam": {
+ "type":"static"
+ }
}'
---
apiVersion: "k8s.cni.cncf.io/v1"
@@ -22,7 +25,10 @@ spec:
"type": "bridge",
"bridge": "br2",
"mtu": 1400,
- "device": "veth2"
+ "device": "veth2",
+ "ipam": {
+ "type":"static"
+ }
}'
---
apiVersion: "k8s.cni.cncf.io/v1"
@@ -35,7 +41,10 @@ spec:
"type": "bridge",
"bridge": "br3",
"mtu": 1400,
- "device": "veth3"
+ "device": "veth3",
+ "ipam": {
+ "type":"static"
+ }
}'
---
apiVersion: "k8s.cni.cncf.io/v1"
@@ -48,7 +57,10 @@ spec:
"type": "bridge",
"bridge": "br4",
"mtu": 1400,
- "device": "veth4"
+ "device": "veth4",
+ "ipam": {
+ "type":"static"
+ }
}'
---
apiVersion: "k8s.cni.cncf.io/v1"
@@ -61,7 +73,10 @@ spec:
"type": "bridge",
"bridge": "br5",
"mtu": 1400,
- "device": "veth5"
+ "device": "veth5",
+ "ipam": {
+ "type":"static"
+ }
}'
---
apiVersion: "k8s.cni.cncf.io/v1"
@@ -74,7 +89,10 @@ spec:
"type": "bridge",
"bridge": "br6",
"mtu": 1400,
- "device": "veth6"
+ "device": "veth6",
+ "ipam": {
+ "type":"static"
+ }
}'
---
@@ -88,7 +106,10 @@ spec:
"type": "bridge",
"bridge": "br7",
"mtu": 1400,
- "device": "veth7"
+ "device": "veth7",
+ "ipam": {
+ "type":"static"
+ }
}'
---
apiVersion: "k8s.cni.cncf.io/v1"
@@ -101,7 +122,10 @@ spec:
"type": "bridge",
"bridge": "br8",
"mtu": 1400,
- "device": "veth8"
+ "device": "veth8",
+ "ipam": {
+ "type":"static"
+ }
}'
---
apiVersion: "k8s.cni.cncf.io/v1"
@@ -114,7 +138,10 @@ spec:
"type": "bridge",
"bridge": "br9",
"mtu": 1400,
- "device": "veth9"
+ "device": "veth9",
+ "ipam": {
+ "type":"static"
+ }
}'
---
apiVersion: "k8s.cni.cncf.io/v1"
@@ -127,7 +154,10 @@ spec:
"type": "bridge",
"bridge": "br10",
"mtu": 1400,
- "device": "veth10"
+ "device": "veth10",
+ "ipam": {
+ "type":"static"
+ }
}'
---
apiVersion: v1
diff --git a/src/operator/l2sm-operator.py b/src/operator/l2sm-operator.py
index 9ecdb96..e537416 100644
--- a/src/operator/l2sm-operator.py
+++ b/src/operator/l2sm-operator.py
@@ -167,20 +167,7 @@ def create_vn(spec, name, namespace, logger, **kwargs):
sql = "INSERT INTO networks (name, type) VALUES (%s, %s) ON DUPLICATE KEY UPDATE name = VALUES(name), type = VALUES(type)"
cursor.execute(sql, (name.strip(), spec['type']))
connection.commit()
- # # Prepare data for the REST API call
- # data = {"networkId": name.strip()}
- # response = session.post(base_controller_url + '/l2sm/networks', json=data)
-
- # # Check the response status
- # if response.status_code == 204:
- # # Commit database changes only if the network is successfully created in the controller
- # connection.commit()
- # logger.info(f"Network '{name}' has been successfully created in both the database and the L2SM controller.")
- # else:
- # # Roll back the database transaction if the network creation in the controller fails
- # connection.rollback()
- # logger.error(f"Failed to create network '{name}' in the L2SM controller. Database transaction rolled back.")
-
+
except Exception as e:
# Roll back the database transaction in case of any error
connection.rollback()
@@ -191,11 +178,9 @@ def create_vn(spec, name, namespace, logger, **kwargs):
-#ASSIGN POD TO NETWORK (TRIGGERS ONLY IF ANNOTATION IS PRESENT)
@kopf.on.create('pods.v1', annotations={'l2sm/networks': kopf.PRESENT})
def pod_vn(body, name, namespace, logger, annotations, **kwargs):
"""Assign Pod to a network if a specific annotation is present."""
-
# Avoid database overlap by introducing a random sleep time
time.sleep(random.uniform(0, 0.8))
@@ -206,36 +191,32 @@ def pod_vn(body, name, namespace, logger, annotations, **kwargs):
return
existing_networks = get_existing_networks(namespace)
- target_networks = filter_target_networks(multus_networks, existing_networks)
- if not target_networks:
+ # Need to extract just the names from multus_networks for comparison
+ target_networks_info = filter_target_networks([net['name'] for net in multus_networks], existing_networks)
+ if not target_networks_info:
logger.info("No target networks found. Letting Multus handle the network assignment.")
return
+ # Update `target_networks` to include IP information if available
+ target_networks = [net for net in multus_networks if net['name'] in target_networks_info]
+
api = CustomObjectsApi()
- l2sm_network = api.get_namespaced_custom_object(
- group="l2sm.k8s.local",
- version="v1",
- namespace=namespace,
- plural="l2sm-networks",
- name="ping-network",
- )
- print(l2sm_network)
- for target_network in target_networks:
- update_network(target_network,name,namespace,api)
+ # Assign pods to each of the target networks, this part remains unchanged
+ for network in target_networks:
+ update_network(network['name'], name, namespace, api)
if 'spec' in body and 'nodeName' in body['spec']:
node_name = body['spec']['nodeName']
-
free_interfaces = get_free_interfaces(node_name)
if len(free_interfaces) < len(target_networks):
raise kopf.PermanentError(f"Node {node_name} has no free interfaces left")
openflow_id = get_openflow_id(node_name)
-
+ # Now pass the full network info including IPs if present
update_network_assignments(name, namespace, node_name, free_interfaces, target_networks, logger, openflow_id)
else:
raise kopf.TemporaryError("The Pod is not yet ready", delay=1)
-
+
def update_network(l2sm_network_name,pod_name,namespace,api):
l2sm_network = api.get_namespaced_custom_object(
group="l2sm.k8s.local",
@@ -288,8 +269,17 @@ def remove_from_network(l2sm_network_name,pod_name,namespace,api):
def extract_multus_networks(annotations):
- """Extract and return Multus networks from annotations."""
- return [network.strip() for network in annotations.get('l2sm/networks').split(",")]
+ """Extract and return Multus networks from annotations."""
+ annotation = annotations.get('l2sm/networks')
+ if annotation.startswith('['): # New JSON format
+ try:
+ networks = json.loads(annotation)
+ return [{ 'name': net['name'], 'ips': net.get('ips', []) } for net in networks]
+ except json.JSONDecodeError:
+ raise ValueError("Failed to decode JSON from annotations")
+ else: # Original string format
+ return [{'name': network.strip(), 'ips': []} for network in annotation.split(",")]
+
def get_existing_networks(namespace):
"""Return existing networks in the namespace."""
@@ -322,12 +312,24 @@ def get_free_interfaces(node_name):
connection.close()
return free_interfaces
-def update_pod_annotation(pod_name, namespace, interfaces):
- """Update the Pod's annotation with assigned interfaces."""
+def update_pod_annotation(pod_name, namespace, networks_info):
+ """Update the Pod's annotation with assigned networks and optional IPs."""
v1 = client.CoreV1Api()
pod = v1.read_namespaced_pod(pod_name, namespace)
pod_annotations = pod.metadata.annotations or {}
- pod_annotations['k8s.v1.cni.cncf.io/networks'] = ', '.join(interfaces)
+
+ # Format the annotations based on whether IPs are provided
+ formatted_networks = []
+ for network_info in networks_info:
+ if network_info['ips']: # Case B with IP addresses
+ formatted_networks.append(json.dumps({
+ "name": network_info['name'],
+ "ips": network_info['ips']
+ }))
+ else: # Case A without specific IP addresses
+ formatted_networks.append(network_info['name'])
+
+ pod_annotations['k8s.v1.cni.cncf.io/networks'] = '[' + ', '.join(formatted_networks) + ']'
v1.patch_namespaced_pod(pod_name, namespace, {'metadata': {'annotations': pod_annotations}})
@@ -434,15 +436,8 @@ def delete_vn(spec, name, logger, **kwargs):
- response = session.delete(base_controller_url + '/l2sm/networks/' + name)
-
- if response.status_code == 204:
- # Successful request
- logger.info(f"Network has been deleted in the SDN Controller")
- connection.commit()
- else:
- # Handle errors
- logger.info(f"Error: {response.status_code}")
+
+ connection.commit()
finally:
connection.close()
logger.info(f"Network {name} removed")
--
GitLab