diff --git a/src/kubernetes-api/.env b/src/kubernetes-api/.env index c37241b6021c0cbc7bd2e49fbdbd69f77bd2a764..7916204fe0fb3fd65e7289c8324681c160e33b69 100644 --- a/src/kubernetes-api/.env +++ b/src/kubernetes-api/.env @@ -1,3 +1,3 @@ -CONTROLLER_IP=10.152.183.42 +CONTROLLER_IP=10.152.183.223 CONTROLLER_PORT=8181 SWITCHES_NAMESPACE=he-codeco-netma \ No newline at end of file diff --git a/src/kubernetes-api/api/v1/overlay_types.go b/src/kubernetes-api/api/v1/overlay_types.go index 70a76e8680d3c5da34eb95131bde5026bde639f1..9af3a45f835b5cb6480ae5c06389bcc466a84cc8 100644 --- a/src/kubernetes-api/api/v1/overlay_types.go +++ b/src/kubernetes-api/api/v1/overlay_types.go @@ -20,9 +20,13 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +type Link struct { + EndpointA string `json:"endpointA"` + EndpointB string `json:"endpointB"` +} type TopologySpec struct { Nodes []string `json:"nodes"` - Links []string `json:"links"` + Links []Link `json:"links"` } // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! diff --git a/src/kubernetes-api/api/v1/zz_generated.deepcopy.go b/src/kubernetes-api/api/v1/zz_generated.deepcopy.go index cc21e3d022723107c953952db807792ffbf6412a..b1b38649a1a720981b8955dbc861f53280b87c7b 100644 --- a/src/kubernetes-api/api/v1/zz_generated.deepcopy.go +++ b/src/kubernetes-api/api/v1/zz_generated.deepcopy.go @@ -139,6 +139,21 @@ func (in *L2NetworkStatus) DeepCopy() *L2NetworkStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Link) DeepCopyInto(out *Link) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Link. +func (in *Link) DeepCopy() *Link { + if in == nil { + return nil + } + out := new(Link) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NeighborSpec) DeepCopyInto(out *NeighborSpec) { *out = *in @@ -475,7 +490,7 @@ func (in *TopologySpec) DeepCopyInto(out *TopologySpec) { } if in.Links != nil { in, out := &in.Links, &out.Links - *out = make([]string, len(*in)) + *out = make([]Link, len(*in)) copy(*out, *in) } } diff --git a/src/kubernetes-api/bin/controller-gen-v0.14.0 b/src/kubernetes-api/bin/controller-gen-v0.14.0 index d0dfee37bf48b49b968f9a7ed26358e506b505e6..fbc1cc38d886075f9cf82a642a7a2f9b67fb145d 100755 Binary files a/src/kubernetes-api/bin/controller-gen-v0.14.0 and b/src/kubernetes-api/bin/controller-gen-v0.14.0 differ diff --git a/src/kubernetes-api/bin/kustomize-v5.3.0 b/src/kubernetes-api/bin/kustomize-v5.3.0 index 60e62f5c2994eeddcf2e6bf697330f28ade1a186..12cd834ca5c254096e98574a7dd02872aee2f380 100755 Binary files a/src/kubernetes-api/bin/kustomize-v5.3.0 and b/src/kubernetes-api/bin/kustomize-v5.3.0 differ diff --git a/src/kubernetes-api/config/crd/bases/l2sm.l2sm.k8s.local_overlays.yaml b/src/kubernetes-api/config/crd/bases/l2sm.l2sm.k8s.local_overlays.yaml index 90d1d1e526d60048afa6264782990e3abde2a0f4..28e19475a8aba36e12f1c56e3fe6f93ee6e476f4 100644 --- a/src/kubernetes-api/config/crd/bases/l2sm.l2sm.k8s.local_overlays.yaml +++ b/src/kubernetes-api/config/crd/bases/l2sm.l2sm.k8s.local_overlays.yaml @@ -4539,7 +4539,15 @@ spec: properties: links: items: - type: string + properties: + endpointA: + type: string + endpointB: + type: string + required: + - endpointA + - endpointB + type: object type: array nodes: items: diff --git a/src/kubernetes-api/config/dev/webhookcainjection_patch.yaml b/src/kubernetes-api/config/dev/webhookcainjection_patch.yaml index 0319edaa439a0ad46711c1da6bfe4d4570e62a87..8a9f00ff6270225f9a4c89b3a2436e38bfe64a75 100644 --- a/src/kubernetes-api/config/dev/webhookcainjection_patch.yaml +++ b/src/kubernetes-api/config/dev/webhookcainjection_patch.yaml @@ -13,7 +13,7 @@ webhooks: - name: mpod.kb.io clientConfig: url: https://10.0.2.4:9443/mutate-v1-pod - caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURQRENDQWlTZ0F3SUJBZ0lVSkpMTEZLV2ZrWkthay9uRzNiZEVRSDdCNVdrd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0dERVdNQlFHQTFVRUF3d05iRzlqWVd3dGQyVmlhRzl2YXpBZUZ3MHlOREEyTVRRd09UQXpNVGxhRncweQpOVEEyTVRRd09UQXpNVGxhTUJneEZqQVVCZ05WQkFNTURXeHZZMkZzTFhkbFltaHZiMnN3Z2dFaU1BMEdDU3FHClNJYjNEUUVCQVFVQUE0SUJEd0F3Z2dFS0FvSUJBUUM3UER0S1dTVUp2MkZISDliTkdaeWpHNDF5NnRUandFZWkKRVZreDhyMTVsVDZ3ekhUSGF5RGRlYUJXTmhvaVV6YS9JZ0J0N1dCNjV1YTFsdDRyNlJGSTVzSDE4VndKRDZqNApCWlMycmtaQ2J2NDZyOVlxV3E5ZFlxNHJNZ2tzekliZkllWTlvbndvRkoyMFU3aHZLZGs5cjdJNzBJWi9LOHcrCjVha0w3eXI1YnpWRmp1enJSRWVlY0RwdWI0WjV0WXUwRWxXVnVwK1J0dytYTnZLOG1keHBUV2hWdVlHdUU1bFYKbW9UQ2NZZy91TXV3WlVmY1BNRC8zWCtZamo2TWNVQW9IVjdrQStDQnp6UGdTN21PblhZUktrelVERE0ydW5EVgpnZCtldzk5OUVSQWllejBoeTVqUFBxMjNQbE1zVU5wV2srRlM1bm0vemt3UTU4MEp3Z3dYQWdNQkFBR2pmakI4Ck1EWUdBMVVkRVFRdk1DMkhCSDhBQUFHSEJBb0FBZ1NDQ1d4dlkyRnNhRzl6ZElJVWQyVmlhRzl2YXkxelpYSjIKWlhJdWJHOWpZV3d3RGdZRFZSMFBBUUgvQkFRREFnV2dNQk1HQTFVZEpRUU1NQW9HQ0NzR0FRVUZCd01CTUIwRwpBMVVkRGdRV0JCUWlVUVRaNzJFYWdacXlKV1BCQ3BzVnYxOEtPREFOQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBClBBdWk2RktaS2pkWG1jbGVHUm4xc25DOTFMalh6S0VrYm1JRGpveXNoRC9UNE9yMmFPYmwvUVAzTFd5bXorNEgKV0NNUHNMUHlzUVdzbkNraVRXbEljTkxtaisrSUh4Wm5IckxRSnBYeEYyMVUrRkY1UGphT1R2UGpiT0VXdHI0RwowbHJuV0RrSURDSzdBcUxmdU5jSHZOVUY1elhBclYybm9hanU3d1k5a0Q2WTFraW12V2ZMRmZoOUxRblhaNGp0CkRwWUdERGlHZTNGa0dYTXVzY2VaQW44SjBtVHRhWUZOZXZNRFdpYllqV2E3OHJUSnRQRGN4Ti9DZmJkQ29EanIKV0swN1NFY1BjTnZjUjFWODZpYXlFM1A2WUJSdkhyWm9UY1dSU2VzcGVrSWgxRmljTUJkWkRSeDlHdmhpaE1qVApUdzRFVnJyQkVPUUROWWJGYUF6V0RRPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIVENDQWdXZ0F3SUJBZ0lVU1hpQm41YVR3TGgyWXZKQWZtWGNiZ25tNVlZd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0dERVdNQlFHQTFVRUF3d05iRzlqWVd3dGQyVmlhRzl2YXpBZUZ3MHlOREE0TURZd09EVTRNVE5hRncweQpOVEE0TURZd09EVTRNVE5hTUJneEZqQVVCZ05WQkFNTURXeHZZMkZzTFhkbFltaHZiMnN3Z2dFaU1BMEdDU3FHClNJYjNEUUVCQVFVQUE0SUJEd0F3Z2dFS0FvSUJBUUN0QmFjb05ISDBwanQ4dkJjS3NxUHVBaHVtV21VMnkzNloKdFhjMGtOVHpOL2Nqekhpc1JIZ2crQVdVaHJ6bllUOGZ2clNlQU9nODhuS2RrMG8wZGJ6aXR5d09vdmJOeG5jbApnU3crRlZKQlprN0tYWTRtN1RzTkxDbXh4RE0vZjN6Rm93NGRKU2dBbzBCOWdZUy9XQnc3WHF2VHdROEdBRUhxCm9WUkluaXoxdW5LVWg1YkluNzNsRk5aMTIzVDI1cERtRGxIWU1lcjZORTdhL3BGZE5LZi9IQUlHYVBCT3FWeTgKaWNTM2pKRjV6V2w2NllNWHNFM2xwa2VORm9ZN3JsYmRFbm5rYUhmL2d3Uk9aNmtRSzFSTG14RUN5akVuaXRiRgpyYkhOU1R6ZWIyYXg1bVkvM3hDWnJKSkd2SmdLdm9UU0JpbmVqWVNHQjJVVElHaE1ROUV4QWdNQkFBR2pYekJkCk1EWUdBMVVkRVFRdk1DMkhCSDhBQUFHSEJBb0FBZ1NDQ1d4dlkyRnNhRzl6ZElJVWQyVmlhRzl2YXkxelpYSjIKWlhJdWJHOWpZV3d3RGdZRFZSMFBBUUgvQkFRREFnV2dNQk1HQTFVZEpRUU1NQW9HQ0NzR0FRVUZCd01CTUEwRwpDU3FHU0liM0RRRUJDd1VBQTRJQkFRQ2lxNFpMQVVkQ1Z6ajBOUU1ncUkyZEs5YzNEc1IyejUxNjF5ZTMxakR1CkpBV0RNQjFnc2NnTmt0M2JnbGdGZVpQbjZxZHF4VHBVdk1ZUDRSQzR4S25paEQxcm1CMStMOU9oQTdmbGVuTUYKdllaMHVVdnhKdjdMYkdwOVhYMUxFZFFPUWZwd1l6cmVrU25Bci9iYXF4ME1UZDNGZkNpZ205WTJDYjFORVRPMwpUYTBaRS91amNVb2s5cHpJWVdybWxhVDN2SkE1OWZkbXZzcE9Ed3VnbUdQVkRGMzVCZ09MTDg0dnFPb2gvOUM1CjFyN1ZOcFFKcnBoMG8zVTNHcGxQbE9wa1JjRkZFR3Q4MHcrL2pzcUNNamJ3NkpiTWtEL3NmUlFrQ3lwTjhaa2wKTWdhbVRTeDVBcWgrOWZoU1Zudy9HN0RNdTZ4UFQ2M3J4U3R5RDVNenVqMVgKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= rules: - operations: [ "CREATE", "UPDATE" ] apiGroups: [""] diff --git a/src/kubernetes-api/config/samples/l2sm_v1_overlay.yaml b/src/kubernetes-api/config/samples/l2sm_v1_overlay.yaml index 63f0e1f067bfc4589dae186b91badf26e9c02185..7482d8a20d0c309b5c6bb3fe0dcdf5e4f500c62d 100644 --- a/src/kubernetes-api/config/samples/l2sm_v1_overlay.yaml +++ b/src/kubernetes-api/config/samples/l2sm_v1_overlay.yaml @@ -9,4 +9,21 @@ metadata: app.kubernetes.io/created-by: controllermanager name: overlay-sample spec: - # TODO(user): Add fields here + networkController: + name: example-network-controller + domain: controller.example.com + topology: + nodes: + - ant-machine + - mole + links: + - endpointA: ant-machine + endpointB: mole + switchTemplate: + spec: + containers: + - name: l2sm-switch + image: alexdecb/l2sm-switch:2.5 + resources: {} + ports: + - containerPort: 80 diff --git a/src/kubernetes-api/internal/controller/overlay_controller.go b/src/kubernetes-api/internal/controller/overlay_controller.go index ba8dbd4423390361b3159e291fefd991216cb5ad..c033dced83cf0319fe44f21e9d290fd8ea63213f 100644 --- a/src/kubernetes-api/internal/controller/overlay_controller.go +++ b/src/kubernetes-api/internal/controller/overlay_controller.go @@ -18,13 +18,20 @@ package controller import ( "context" + "encoding/json" + "fmt" + "time" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + l2smv1 "l2sm.k8s.local/controllermanager/api/v1" + "l2sm.k8s.local/controllermanager/internal/utils" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" - - l2smv1 "l2sm.k8s.local/controllermanager/api/v1" ) // OverlayReconciler reconciles a Overlay object @@ -33,6 +40,10 @@ type OverlayReconciler struct { Scheme *runtime.Scheme } +var replicaSetOwnerKeyOverlay = ".metadata.controller.overlay" + +// +kubebuilder:rbac:groups=l2sm.l2sm.k8s.local,resources=replicasets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps,resources=replicasets,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=l2sm.l2sm.k8s.local,resources=overlays,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=l2sm.l2sm.k8s.local,resources=overlays/status,verbs=get;update;patch //+kubebuilder:rbac:groups=l2sm.l2sm.k8s.local,resources=overlays/finalizers,verbs=update @@ -47,16 +58,302 @@ type OverlayReconciler struct { // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.17.0/pkg/reconcile func (r *OverlayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = log.FromContext(ctx) - // TODO(user): your logic here + log := log.FromContext(ctx) + + overlay := &l2smv1.Overlay{} + + if err := r.Get(ctx, req.NamespacedName, overlay); err != nil { + // we'll ignore not-found errors, since they can't be fixed by an immediate + // requeue (we'll need to wait for a new notification), and we can get them + // on deleted requests. + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // name of our custom finalizer + l2smFinalizer := "l2sm.operator.io/finalizer" + + // examine DeletionTimestamp to determine if object is under deletion + if overlay.ObjectMeta.DeletionTimestamp.IsZero() { + // The object is not being deleted, so if it does not have our finalizer, + // then lets add the finalizer and update the object. This is equivalent + // to registering our finalizer. + if !controllerutil.ContainsFinalizer(overlay, l2smFinalizer) { + controllerutil.AddFinalizer(overlay, l2smFinalizer) + if err := r.Update(ctx, overlay); err != nil { + return ctrl.Result{}, err + } + log.Info("Overlay created", "Overlay", overlay.Name) + + } + } else { + // The object is being deleted + if controllerutil.ContainsFinalizer(overlay, l2smFinalizer) { + // our finalizer is present, so lets handle any external dependency + if err := r.deleteExternalResources(ctx, overlay); err != nil { + // if fail to delete the external dependency here, return with error + // so that it can be retried. + return ctrl.Result{}, err + } + + // remove our finalizer from the list and update it. + controllerutil.RemoveFinalizer(overlay, l2smFinalizer) + if err := r.Update(ctx, overlay); err != nil { + return ctrl.Result{}, err + } + + } + log.Info("Overlay deleted", "Overlay", overlay.Name) + // Stop reconciliation as the item is being deleted + return ctrl.Result{}, nil + } + + var switchReplicaSets appsv1.ReplicaSetList + if err := r.List(ctx, &switchReplicaSets, client.InNamespace(req.Namespace), client.MatchingFields{replicaSetOwnerKeyOverlay: req.Name}); err != nil { + log.Error(err, "unable to list child ReplicaSets") + return ctrl.Result{}, err + } + + if len(switchReplicaSets.Items) == 0 { + if err := r.createExternalResources(ctx, overlay); err != nil { + log.Error(err, "unable to create ReplicaSet") + return ctrl.Result{}, err + } + log.Info("Overlay Launched") + return ctrl.Result{RequeueAfter: time.Second * 20}, nil + } else { + + //b, _ := json.Marshal(netEdgeDevice.Spec.Neighbors) + + } return ctrl.Result{}, nil } // SetupWithManager sets up the controller with the Manager. func (r *OverlayReconciler) SetupWithManager(mgr ctrl.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &appsv1.ReplicaSet{}, replicaSetOwnerKeyOverlay, func(rawObj client.Object) []string { + // grab the replica set object, extract the owner... + replicaSet := rawObj.(*appsv1.ReplicaSet) + owner := metav1.GetControllerOf(replicaSet) + if owner == nil { + return nil + } + // ...make sure it's a ReplicaSet... + if owner.APIVersion != apiGVStr || owner.Kind != "Overlay" { + return nil + } + + // ...and if so, return it + return []string{owner.Name} + }); err != nil { + return err + } return ctrl.NewControllerManagedBy(mgr). For(&l2smv1.Overlay{}). + Owns(&appsv1.ReplicaSet{}). Complete(r) } + +func (r *OverlayReconciler) deleteExternalResources(ctx context.Context, overlay *l2smv1.Overlay) error { + + return nil +} + +type TopologySwitchJson struct { + Nodes []NodeJson `json:"Nodes"` + Links []l2smv1.Link `json:"Links"` +} + +type NodeJson struct { + Name string `json:"name"` + NodeIP string `json:"nodeIP"` +} + +func (r *OverlayReconciler) createExternalResources(ctx context.Context, overlay *l2smv1.Overlay) error { + + // Create a ConfigMap to store the topology JSON + constructConfigMapForOverlay := func(overlay *l2smv1.Overlay) (*corev1.ConfigMap, error) { + + // Construct the TopologySwitchJson + topologySwitch := TopologySwitchJson{} + + overlayName := overlay.ObjectMeta.Name + + // Populate Nodes + for _, nodeName := range overlay.Spec.Topology.Nodes { + node := NodeJson{ + Name: nodeName, + NodeIP: fmt.Sprintf("l2sm-switch-%s-%s", overlayName, nodeName), + } + topologySwitch.Nodes = append(topologySwitch.Nodes, node) + } + + // Populate Links + for _, link := range overlay.Spec.Topology.Links { + topologySwitch.Links = append(topologySwitch.Links, link) + } + + // Convert TopologySwitchJson to JSON + topologyJSON, err := json.Marshal(topologySwitch) + if err != nil { + return nil, err + } + + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-topology", overlay.Name), + Namespace: overlay.Namespace, + }, + Data: map[string]string{ + "topology.json": string(topologyJSON), + }, + } + if err := controllerutil.SetControllerReference(overlay, configMap, r.Scheme); err != nil { + return nil, err + } + return configMap, nil + } + + configMap, _ := constructConfigMapForOverlay(overlay) + + // Create the ConfigMap in Kubernetes + if err := r.Client.Create(ctx, configMap); err != nil { + return err + } + + constructNodeResourcesForOverlay := func(overlay *l2smv1.Overlay) ([]*appsv1.ReplicaSet, []*corev1.Service, error) { + + // Define volume mounts to be added to each container + volumeMounts := []corev1.VolumeMount{ + { + Name: "topology", + MountPath: "/etc/l2sm/", + ReadOnly: true, + }, + } + + // Update containers to include the volume mount + containers := make([]corev1.Container, len(overlay.Spec.SwitchTemplate.Spec.Containers)) + for i, container := range overlay.Spec.SwitchTemplate.Spec.Containers { + container.VolumeMounts = append(container.VolumeMounts, volumeMounts...) + containers[i] = container + } + + // Define the volume using the created ConfigMap + volumes := []corev1.Volume{ + { + Name: "topology", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configMap.Name, + }, + Items: []corev1.KeyToPath{ + { + Key: "topology.json", + Path: "topology.json", + }, + }, + }, + }, + }, + } + + var replicaSets []*appsv1.ReplicaSet + var services []*corev1.Service + + for _, node := range overlay.Spec.Topology.Nodes { + + name := fmt.Sprintf("%s-%s-%s", "l2sm-switch", node, utils.GenerateHash(overlay)) + + replicaSet := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Labels: make(map[string]string), + Annotations: make(map[string]string), + Name: name, + Namespace: overlay.Namespace, + }, + Spec: appsv1.ReplicaSetSpec{ + Replicas: utils.Int32Ptr(1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": name, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": name, + }, + }, + Spec: corev1.PodSpec{ + InitContainers: overlay.Spec.SwitchTemplate.Spec.InitContainers, + Containers: containers, + Volumes: volumes, + HostNetwork: overlay.Spec.SwitchTemplate.Spec.HostNetwork, + NodeName: node, + }, + }, + }, + } + + for k, v := range overlay.Spec.SwitchTemplate.Annotations { + replicaSet.Annotations[k] = v + } + for k, v := range overlay.Spec.SwitchTemplate.Labels { + replicaSet.Labels[k] = v + } + if err := controllerutil.SetControllerReference(overlay, replicaSet, r.Scheme); err != nil { + return nil, nil, err + } + + replicaSets = append(replicaSets, replicaSet) + + // Create a headless service for the ReplicaSet + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("l2sm-switch-%s-%s", overlay.Name, node), + Namespace: overlay.Namespace, + Labels: map[string]string{"app": name}, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "None", + Selector: map[string]string{"app": name}, + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + }, + }, + }, + } + + if err := controllerutil.SetControllerReference(overlay, service, r.Scheme); err != nil { + return nil, nil, err + } + + services = append(services, service) + } + + return replicaSets, services, nil + } + + replicaSets, services, err := constructNodeResourcesForOverlay(overlay) + if err != nil { + return err + } + + for _, replicaSet := range replicaSets { + if err = r.Client.Create(ctx, replicaSet); err != nil { + return err + } + } + for _, service := range services { + if err = r.Client.Create(ctx, service); err != nil { + return err + } + } + + return nil +}