diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index e4657f0..9beffa8 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -18,13 +18,13 @@ Next install our CAPMS provider into the cluster. make push-to-capi-lab ``` -Before creating a cluster some manual steps are required beforehand: you need to allocate a node network. +Before creating a cluster the control plane IP needs to be created first: ```bash -make -C capi-lab node-network control-plane-ip +make -C capi-lab control-plane-ip ``` -A basic cluster configuration that relies on `config/clusterctl-templates/cluster-template.yaml` and uses the aforementioned node network can be generated and applied to the management cluster using a make target. +A basic cluster configuration that relies on `config/clusterctl-templates/cluster-template-calico.yaml` and uses the aforementioned IP can be generated and applied to the management cluster using a make target. ```bash make -C capi-lab apply-sample-cluster @@ -84,7 +84,8 @@ If you want to test the local changes you made to the provider, run: ```bash unset E2E_KUBECONFIG # ensure a new kind cluster is created -make docker-build-e2e test-e2e +# skip move tests as they won't have access to the docker image on your local machine +make docker-build-e2e test-e2e E2E_LABEL_FILTER="\!move" ``` This will automatically build and load your image. @@ -201,7 +202,6 @@ export METAL_API_URL= export METAL_PARTITION= export METAL_PROJECT_ID= -export METAL_NODE_NETWORK_ID= export CONTROL_PLANE_IP= export FIREWALL_MACHINE_IMAGE= @@ -225,11 +225,10 @@ export project_name= export tenant_name= ``` -Create project, node network and control plane ip if needed: +Create project and control plane ip if needed: ```bash metalctl project create --name $project_name --tenant $tenant_name --description "Cluster API test project" -metalctl network allocate --description "Node network for $CLUSTER_NAME" --name $CLUSTER_NAME --project $METAL_PROJECT_ID --partition $METAL_PARTITION metalctl network ip create --network internet --project $METAL_PROJECT_ID --name "$CLUSTER_NAME-vip" --type static -o template --template "{{ .ipaddress }}" ``` diff --git a/README.md b/README.md index 59acf6c..691df29 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,6 @@ Currently, we provide the following custom resources: We plan to cover more resources in the future: -- Node Networks - Complete Firewall Deployments using the [Firewall Controller Manager](https://github.com/metal-stack/firewall-controller-manager) - Improved configuration suggestion of CNIs @@ -62,20 +61,13 @@ clusterctl init --infrastructure metal-stack > **Manual steps needed:** > Due to the early development stage, manual actions are needed for the cluster to operate. Some metal-stack resources need to be created manually. -A node network needs to be created. +Allocate a VIP for the control plane. + ```bash export CLUSTER_NAME= export METAL_PARTITION= export METAL_PROJECT_ID= -metalctl network allocate --description "Node network for $CLUSTER_NAME" --name $CLUSTER_NAME --project $METAL_PROJECT_ID --partition $METAL_PARTITION -# export environment variable for use in the next steps -export METAL_NODE_NETWORK_ID=$(metalctl network list --name $CLUSTER_NAME -o template --template '{{ .id }}') -``` - -Allocate a VIP for the control plane. - -```bash export CONTROL_PLANE_IP=$(metalctl network ip create --network internet --project $METAL_PROJECT_ID --name "$CLUSTER_NAME-vip" --type static -o template --template "{{ .ipaddress }}") ``` @@ -83,7 +75,7 @@ For your first cluster, it is advised to start with our generated template. Ensu ```bash # display required environment variables -clusterctl generate cluster $CLUSTER_NAME --infrastructure metal-stack --list-variables +clusterctl generate cluster $CLUSTER_NAME --infrastructure metal-stack --list-variables --flavor calico # set additional environment variables export CONTROL_PLANE_MACHINE_IMAGE= @@ -94,7 +86,7 @@ export FIREWALL_MACHINE_IMAGE= export FIREWALL_MACHINE_SIZE= # generate manifest -clusterctl generate cluster $CLUSTER_NAME --kubernetes-version v1.32.9 --infrastructure metal-stack +clusterctl generate cluster $CLUSTER_NAME --kubernetes-version v1.32.9 --infrastructure metal-stack --flavor calico ``` Apply the generated manifest from the `clusterctl` output. @@ -103,51 +95,7 @@ Apply the generated manifest from the `clusterctl` output. kubectl apply -f ``` -Once your control plane and worker machines have been provisioned, you need to install your CNI of choice into your created cluster. This is required due to CAPI. An example is provided below: - -```bash -# get the kubeconfig -clusterctl get kubeconfig metal-test > capms-cluster.kubeconfig - -# install the calico operator -kubectl --kubeconfig=capms-cluster.kubeconfig create -f https://raw.githubusercontent.com/projectcalico/calico/v3.28.2/manifests/tigera-operator.yaml - -# install the calico CNI -cat < -export CLUSTER_NAME= -cat config/target-cluster/metal-ccm.yaml | envsubst | kubectl --kubeconfig capms-cluster.kubeconfig apply -f - -``` - -If you want to provide service's of type `LoadBalancer` through MetalLB by the `metal-ccm`, you need to deploy MetalLB: - -```bash -kubectl --kubeconfig capms-cluster.kubeconfig apply --kustomize capi-lab/metallb -``` +That's it! ## Frequently Asked Questions diff --git a/api/v1alpha1/metalstackcluster_types.go b/api/v1alpha1/metalstackcluster_types.go index 272fe66..9d7b78f 100644 --- a/api/v1alpha1/metalstackcluster_types.go +++ b/api/v1alpha1/metalstackcluster_types.go @@ -36,6 +36,7 @@ const ( ClusterControlPlaneEndpointDefaultPort = 443 ClusterControlPlaneIPEnsured = "ClusterControlPlaneIPEnsured" + ClusterNodeNetworkEnsured = "ClusterNodeNetworkEnsured" ) var ( @@ -55,7 +56,9 @@ type MetalStackClusterSpec struct { ProjectID string `json:"projectID"` // NodeNetworkID is the network ID in metal-stack in which the worker nodes and the firewall of the cluster are placed. - NodeNetworkID string `json:"nodeNetworkID"` + // If not provided this will automatically be acquired during reconcile. + // +optional + NodeNetworkID *string `json:"nodeNetworkID,omitempty"` // ControlPlaneIP is the ip address in metal-stack on which the control plane will be exposed. // If this ip and the control plane endpoint are not provided, an ephemeral ip will automatically be acquired during reconcile. @@ -165,6 +168,6 @@ func (c *MetalStackCluster) SetConditions(conditions []metav1.Condition) { c.Status.Conditions = conditions } -func (c *MetalStackCluster) GetClusterID() string { +func (c *MetalStackCluster) GetClusterName() string { return fmt.Sprintf("%s.%s", c.GetNamespace(), c.GetName()) } diff --git a/api/v1alpha1/metalstackcluster_types_test.go b/api/v1alpha1/metalstackcluster_types_test.go index d42150c..b0f4cb5 100644 --- a/api/v1alpha1/metalstackcluster_types_test.go +++ b/api/v1alpha1/metalstackcluster_types_test.go @@ -17,7 +17,7 @@ var _ = Describe("MetalStackCluster", func() { }, } - clusterID := cluster.GetClusterID() + clusterID := cluster.GetClusterName() Expect(utilvalidation.IsValidLabelValue(clusterID)).To(BeEmpty()) }) @@ -29,7 +29,7 @@ var _ = Describe("MetalStackCluster", func() { }, } - clusterID := cluster.GetClusterID() + clusterID := cluster.GetClusterName() Expect(clusterID).To(Equal("some-namespace.my-cluster")) }) }) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 7039d21..334043e 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -125,6 +125,11 @@ func (in *MetalStackClusterList) DeepCopyObject() runtime.Object { func (in *MetalStackClusterSpec) DeepCopyInto(out *MetalStackClusterSpec) { *out = *in out.ControlPlaneEndpoint = in.ControlPlaneEndpoint + if in.NodeNetworkID != nil { + in, out := &in.NodeNetworkID, &out.NodeNetworkID + *out = new(string) + **out = **in + } if in.ControlPlaneIP != nil { in, out := &in.ControlPlaneIP, &out.ControlPlaneIP *out = new(string) diff --git a/capi-lab/Makefile b/capi-lab/Makefile index b740749..04c3705 100644 --- a/capi-lab/Makefile +++ b/capi-lab/Makefile @@ -59,17 +59,12 @@ controller: kubectl --kubeconfig=$(KUBECONFIG) patch deployments.apps -n cap-metal-stack metal-stack-controller-manager --patch='{"spec":{"template":{"spec":{"containers":[{"name": "manager","imagePullPolicy":"IfNotPresent","image":"$(IMG)"}]}}}}' kubectl --kubeconfig=$(KUBECONFIG) delete pod -n cap-metal-stack -l control-plane=metal-stack-controller-manager -.PHONY: node-network -node-network: - metalctl network allocate --description "node network for $(CLUSTER_NAME) cluster" --name $(CLUSTER_NAME) --project 00000000-0000-0000-0000-000000000001 --partition mini-lab - .PHONY: control-plane-ip control-plane-ip: metalctl network ip create --network internet-mini-lab --project $(METAL_PROJECT_ID) --name "$(CLUSTER_NAME)-vip" --type static -o template --template "{{ .ipaddress }}" .PHONY: apply-sample-cluster apply-sample-cluster: - $(eval METAL_NODE_NETWORK_ID = $(shell metalctl network list --name $(CLUSTER_NAME) -o template --template '{{ .id }}')) $(eval CONTROL_PLANE_IP = $(shell metalctl network ip list --name "$(CLUSTER_NAME)-vip" -o template --template '{{ .ipaddress }}')) echo $(CLUSTER_NAME) clusterctl generate cluster $(CLUSTER_NAME) \ @@ -82,7 +77,6 @@ apply-sample-cluster: .PHONY: delete-sample-cluster delete-sample-cluster: - $(eval METAL_NODE_NETWORK_ID = $(shell metalctl network list --name $(CLUSTER_NAME) -o template --template '{{ .id }}')) $(eval CONTROL_PLANE_IP = $(shell metalctl network ip list --name "$(CLUSTER_NAME)-vip" -o template --template '{{ .ipaddress }}')) clusterctl generate cluster $(CLUSTER_NAME) \ --kubeconfig=$(KUBECONFIG) \ diff --git a/config/clusterctl-templates/cluster-template-calico-lab.yaml b/config/clusterctl-templates/cluster-template-calico-lab.yaml index dee5cd4..1e07261 100644 --- a/config/clusterctl-templates/cluster-template-calico-lab.yaml +++ b/config/clusterctl-templates/cluster-template-calico-lab.yaml @@ -31,7 +31,7 @@ metadata: spec: projectID: ${METAL_PROJECT_ID} partition: ${METAL_PARTITION} - nodeNetworkID: ${METAL_NODE_NETWORK_ID} + nodeNetworkID: ${METAL_NODE_NETWORK_ID:=null} controlPlaneIP: ${CONTROL_PLANE_IP} firewallDeploymentRef: name: ${CLUSTER_NAME} diff --git a/config/clusterctl-templates/cluster-template-calico.yaml b/config/clusterctl-templates/cluster-template-calico.yaml index 83e3ed0..9ed3ea4 100644 --- a/config/clusterctl-templates/cluster-template-calico.yaml +++ b/config/clusterctl-templates/cluster-template-calico.yaml @@ -29,7 +29,7 @@ metadata: spec: projectID: ${METAL_PROJECT_ID} partition: ${METAL_PARTITION} - nodeNetworkID: ${METAL_NODE_NETWORK_ID} + nodeNetworkID: ${METAL_NODE_NETWORK_ID:=null} controlPlaneIP: ${CONTROL_PLANE_IP} firewallDeploymentRef: name: ${CLUSTER_NAME} diff --git a/config/clusterctl-templates/cluster-template-pre-v1.33.yaml b/config/clusterctl-templates/cluster-template-pre-v1.33.yaml index a112f66..6fa2cd1 100644 --- a/config/clusterctl-templates/cluster-template-pre-v1.33.yaml +++ b/config/clusterctl-templates/cluster-template-pre-v1.33.yaml @@ -25,7 +25,7 @@ metadata: spec: projectID: ${METAL_PROJECT_ID} partition: ${METAL_PARTITION} - nodeNetworkID: ${METAL_NODE_NETWORK_ID} + nodeNetworkID: ${METAL_NODE_NETWORK_ID:=null} controlPlaneIP: ${CONTROL_PLANE_IP} firewallDeploymentRef: name: ${CLUSTER_NAME} diff --git a/config/clusterctl-templates/cluster-template.yaml b/config/clusterctl-templates/cluster-template.yaml index c1f7a0f..0620f0c 100644 --- a/config/clusterctl-templates/cluster-template.yaml +++ b/config/clusterctl-templates/cluster-template.yaml @@ -25,7 +25,7 @@ metadata: spec: projectID: ${METAL_PROJECT_ID} partition: ${METAL_PARTITION} - nodeNetworkID: ${METAL_NODE_NETWORK_ID} + nodeNetworkID: ${METAL_NODE_NETWORK_ID:=null} controlPlaneIP: ${CONTROL_PLANE_IP} firewallDeploymentRef: name: ${CLUSTER_NAME} diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_metalstackclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_metalstackclusters.yaml index 2808ee4..9ada28d 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_metalstackclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_metalstackclusters.yaml @@ -104,8 +104,9 @@ spec: - name type: object nodeNetworkID: - description: NodeNetworkID is the network ID in metal-stack in which - the worker nodes and the firewall of the cluster are placed. + description: |- + NodeNetworkID is the network ID in metal-stack in which the worker nodes and the firewall of the cluster are placed. + If not provided this will automatically be acquired during reconcile. type: string partition: description: Partition is the data center partition in which the resources @@ -116,7 +117,6 @@ spec: in which the associated metal-stack resources are created. type: string required: - - nodeNetworkID - partition - projectID type: object diff --git a/internal/controller/metalstackcluster_controller.go b/internal/controller/metalstackcluster_controller.go index ca65812..95ebcdd 100644 --- a/internal/controller/metalstackcluster_controller.go +++ b/internal/controller/metalstackcluster_controller.go @@ -309,6 +309,26 @@ func (r *MetalStackClusterReconciler) metalStackFirewallToMetalStackCluster(log } func (r *clusterReconciler) reconcile() error { + nodeNetworkID, err := r.ensureNodeNetwork() + if err != nil { + conditions.Set(r.infraCluster, metav1.Condition{ + Type: v1alpha1.ClusterNodeNetworkEnsured, + Status: metav1.ConditionFalse, + Reason: "InternalError", + Message: err.Error(), + }) + return fmt.Errorf("unable to ensure node network: %w", err) + } + conditions.Set(r.infraCluster, metav1.Condition{ + Type: v1alpha1.ClusterNodeNetworkEnsured, + Status: metav1.ConditionTrue, + Reason: "ClusterNodeNetworkEnsured", + Message: "Node network ensured", + }) + r.infraCluster.Spec.NodeNetworkID = &nodeNetworkID + + r.log.Info("reconciled node network", "network-id", nodeNetworkID) + if r.infraCluster.Spec.FirewallDeploymentRef != nil { err := r.ensureFirewallDeployment() if err != nil { @@ -390,12 +410,53 @@ func (r *clusterReconciler) delete() error { } r.infraCluster.Spec.ControlPlaneIP = nil + err = r.deleteNodeNetwork() + if err != nil { + return fmt.Errorf("unable to delete node network: %w", err) + } + r.infraCluster.Spec.NodeNetworkID = nil + r.log.Info("deletion finished, removing finalizer") controllerutil.RemoveFinalizer(r.infraCluster, v1alpha1.ClusterFinalizer) return err } +func (r *clusterReconciler) ensureNodeNetwork() (string, error) { + if r.infraCluster.Spec.NodeNetworkID != nil { + return *r.infraCluster.Spec.NodeNetworkID, nil + } + + resp, err := r.metalClient.Network().AllocateNetwork(network.NewAllocateNetworkParams().WithBody(&models.V1NetworkAllocateRequest{ + Projectid: r.infraCluster.Spec.ProjectID, + Partitionid: r.infraCluster.Spec.Partition, + Name: r.infraCluster.GetName(), + Description: fmt.Sprintf("%s/%s", r.infraCluster.GetNamespace(), r.infraCluster.GetName()), + Labels: map[string]string{ + tag.ClusterID: r.infraCluster.GetClusterName(), + }, + }).WithContext(r.ctx), nil) + if err != nil { + return "", fmt.Errorf("error creating node network: %w", err) + } + + return *resp.Payload.ID, nil +} + +func (r *clusterReconciler) deleteNodeNetwork() error { + if r.infraCluster.Spec.NodeNetworkID == nil { + return nil + } + + _, err := r.metalClient.Network().FreeNetwork(network.NewFreeNetworkParams().WithID(*r.infraCluster.Spec.NodeNetworkID).WithContext(r.ctx), nil) + if err != nil { + return err + } + r.log.Info("deleted node network") + + return nil +} + func (r *clusterReconciler) ensureFirewallDeployment() error { fwdeploy := &v1alpha1.MetalStackFirewallDeployment{} err := r.client.Get(r.ctx, types.NamespacedName{ @@ -445,12 +506,12 @@ func (r *clusterReconciler) ensureControlPlaneIP() (string, error) { defaultNetwork := nwResp.Payload[0] resp, err := r.metalClient.IP().AllocateIP(ipmodels.NewAllocateIPParams().WithBody(&models.V1IPAllocateRequest{ - Description: fmt.Sprintf("%s control plane ip", r.infraCluster.GetClusterID()), + Description: fmt.Sprintf("%s control plane ip", r.infraCluster.GetClusterName()), Name: r.infraCluster.GetName() + "-control-plane", Networkid: defaultNetwork.ID, Projectid: &r.infraCluster.Spec.ProjectID, Tags: []string{ - tag.New(tag.ClusterID, r.infraCluster.GetClusterID()), + tag.New(tag.ClusterID, r.infraCluster.GetClusterName()), v1alpha1.TagControlPlanePurpose, }, Type: ptr.To(models.V1IPBaseTypeEphemeral), diff --git a/internal/controller/metalstackcluster_controller_test.go b/internal/controller/metalstackcluster_controller_test.go index 2667d64..4962785 100644 --- a/internal/controller/metalstackcluster_controller_test.go +++ b/internal/controller/metalstackcluster_controller_test.go @@ -106,7 +106,7 @@ var _ = Describe("MetalStackCluster Controller", func() { resource.Spec = v1alpha1.MetalStackClusterSpec{ ControlPlaneEndpoint: v1alpha1.APIEndpoint{}, ProjectID: "test-project", - NodeNetworkID: "node-network-id", + NodeNetworkID: nil, ControlPlaneIP: nil, Partition: "test-partition", } @@ -223,7 +223,7 @@ var _ = Describe("MetalStackCluster Controller", func() { resource.Spec = v1alpha1.MetalStackClusterSpec{ ControlPlaneEndpoint: v1alpha1.APIEndpoint{}, ProjectID: "test-project", - NodeNetworkID: "node-network-id", + NodeNetworkID: nil, ControlPlaneIP: nil, Partition: "test-partition", } @@ -260,11 +260,11 @@ var _ = Describe("MetalStackCluster Controller", func() { IP: func(m *mock.Mock) { m.On("AllocateIP", testcommon.MatchIgnoreContext(testingT, metalip.NewAllocateIPParams().WithBody(&models.V1IPAllocateRequest{ Tags: []string{ - "cluster.metal-stack.io/id=" + resource.GetClusterID(), + "cluster.metal-stack.io/id=" + resource.GetClusterName(), "metal-stack.infrastructure.cluster.x-k8s.io/purpose=control-plane", }, Name: resource.Name + "-control-plane", - Description: resource.GetClusterID() + " control plane ip", + Description: resource.GetClusterName() + " control plane ip", Networkid: ptr.To("internet"), Projectid: ptr.To("test-project"), Type: ptr.To("ephemeral"), @@ -275,6 +275,26 @@ var _ = Describe("MetalStackCluster Controller", func() { }, nil) }, Network: func(m *mock.Mock) { + m.On("AllocateNetwork", testcommon.MatchIgnoreContext(testingT, metalnetwork.NewAllocateNetworkParams().WithBody(&models.V1NetworkAllocateRequest{ + Name: resource.Name, + Description: resource.Namespace + "/" + resource.Name, + Labels: map[string]string{ + "cluster.metal-stack.io/id": resource.GetClusterName(), + }, + Partitionid: "test-partition", + Projectid: "test-project", + })), nil).Return(&metalnetwork.AllocateNetworkCreated{ + Payload: &models.V1NetworkResponse{ + Labels: map[string]string{ + "cluster.metal-stack.io/id": resource.GetClusterName(), + }, + Partitionid: "test-partition", + Projectid: "test-project", + ID: ptr.To("test-network"), + Prefixes: []string{"192.168.42.0/24"}, + }, + }, nil) + m.On("FindNetworks", testcommon.MatchIgnoreContext(testingT, metalnetwork.NewFindNetworksParams().WithBody(&models.V1NetworkFindRequest{ Labels: map[string]string{ "network.metal-stack.io/default": "", @@ -314,6 +334,10 @@ var _ = Describe("MetalStackCluster Controller", func() { Expect(k8sClient.Get(ctx, typeNamespacedName, resource)).To(Succeed()) + Expect(resource.Status.Conditions).To(ContainElement(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(v1alpha1.ClusterNodeNetworkEnsured), + "Status": Equal(metav1.ConditionTrue), + }))) Expect(resource.Status.Conditions).To(ContainElement(MatchFields(IgnoreExtras, Fields{ "Type": Equal(v1alpha1.ClusterControlPlaneIPEnsured), "Status": Equal(metav1.ConditionTrue), @@ -340,7 +364,7 @@ var _ = Describe("MetalStackCluster Controller", func() { resource.Spec = v1alpha1.MetalStackClusterSpec{ ControlPlaneEndpoint: v1alpha1.APIEndpoint{}, ProjectID: "test-project", - NodeNetworkID: nodeNetworkID, + NodeNetworkID: &nodeNetworkID, ControlPlaneIP: &controlPlaneIP, Partition: "test-partition", } @@ -387,6 +411,10 @@ var _ = Describe("MetalStackCluster Controller", func() { return resource.Status.Conditions }, "20s").Should(ContainElements( + MatchFields(IgnoreExtras, Fields{ + "Type": Equal(v1alpha1.ClusterNodeNetworkEnsured), + "Status": Equal(metav1.ConditionTrue), + }), MatchFields(IgnoreExtras, Fields{ "Type": Equal(v1alpha1.ClusterControlPlaneIPEnsured), "Status": Equal(metav1.ConditionTrue), @@ -428,6 +456,10 @@ var _ = Describe("MetalStackCluster Controller", func() { return resource.Status.Conditions }, "20s").Should(ContainElements( + MatchFields(IgnoreExtras, Fields{ + "Type": Equal(v1alpha1.ClusterNodeNetworkEnsured), + "Status": Equal(metav1.ConditionTrue), + }), MatchFields(IgnoreExtras, Fields{ "Type": Equal(v1alpha1.ClusterControlPlaneIPEnsured), "Status": Equal(metav1.ConditionTrue), diff --git a/internal/controller/metalstackfirewalldeployment_controller.go b/internal/controller/metalstackfirewalldeployment_controller.go index 41f2844..d7c09ba 100644 --- a/internal/controller/metalstackfirewalldeployment_controller.go +++ b/internal/controller/metalstackfirewalldeployment_controller.go @@ -258,7 +258,7 @@ func (r *MetalStackFirewallDeploymentReconciler) metalStackFirewallTemplateToMet } func (r *firewallDeploymentReconciler) reconcile() error { - if r.infraCluster.Spec.NodeNetworkID == "" { + if r.infraCluster.Spec.NodeNetworkID == nil { conditions.Set(r.infraCluster, metav1.Condition{ Type: v1alpha1.ClusterFirewallDeploymentEnsured, Status: metav1.ConditionFalse, @@ -373,7 +373,7 @@ func (r *firewallDeploymentReconciler) ensureFirewallDeployment() error { var ( name = fmt.Sprintf("%s-firewall", r.infraCluster.GetName()) tags = []string{ - tag.New(tag.ClusterID, r.infraCluster.Spec.NodeNetworkID), + tag.New(tag.ClusterID, *r.infraCluster.Spec.NodeNetworkID), tag.New(v1alpha1.TagInfraClusterResource, fmt.Sprintf("%s.%s", r.infraCluster.Namespace, r.infraCluster.Name)), tag.New(fcmv2.FirewallControllerManagedByAnnotation, "cluster-api-provider-metal-stack"), tag.New((v1alpha1.TagFirewallDeploymentResource), fmt.Sprintf("%s.%s", r.firewallDeployment.Namespace, r.firewallDeployment.Name)), @@ -385,8 +385,8 @@ func (r *firewallDeploymentReconciler) ensureFirewallDeployment() error { } networkIDs := r.firewallTemplate.Spec.Networks - if !slices.Contains(networkIDs, r.infraCluster.Spec.NodeNetworkID) { - networkIDs = append(networkIDs, r.infraCluster.Spec.NodeNetworkID) + if !slices.Contains(networkIDs, *r.infraCluster.Spec.NodeNetworkID) { + networkIDs = append(networkIDs, *r.infraCluster.Spec.NodeNetworkID) } networks := make([]*models.V1MachineAllocationNetwork, 0, len(networkIDs)) @@ -457,7 +457,7 @@ func (r *firewallDeploymentReconciler) deleteFirewallDeployment() error { var ( tags = []string{ - tag.New(tag.ClusterID, r.infraCluster.Spec.NodeNetworkID), + tag.New(tag.ClusterID, *r.infraCluster.Spec.NodeNetworkID), tag.New(v1alpha1.TagInfraClusterResource, fmt.Sprintf("%s.%s", r.infraCluster.Namespace, r.infraCluster.Name)), tag.New(fcmv2.FirewallControllerManagedByAnnotation, "cluster-api-provider-metal-stack"), tag.New((v1alpha1.TagFirewallDeploymentResource), fmt.Sprintf("%s.%s", r.firewallDeployment.Namespace, r.firewallDeployment.Name)), diff --git a/internal/controller/metalstackfirewalldeployment_controller_test.go b/internal/controller/metalstackfirewalldeployment_controller_test.go index 543b218..24f88c5 100644 --- a/internal/controller/metalstackfirewalldeployment_controller_test.go +++ b/internal/controller/metalstackfirewalldeployment_controller_test.go @@ -129,7 +129,7 @@ var _ = Describe("MetalStackFirewall Controller", func() { FirewallDeploymentRef: &v1alpha1.MetalStackFirewallDeploymentRef{ Name: resourceName, }, - NodeNetworkID: "some-network-id", + NodeNetworkID: ptr.To("some-network-id"), Partition: "some-partition", }, } diff --git a/internal/controller/metalstackmachine_controller.go b/internal/controller/metalstackmachine_controller.go index 81690e7..aaf95a5 100644 --- a/internal/controller/metalstackmachine_controller.go +++ b/internal/controller/metalstackmachine_controller.go @@ -328,6 +328,11 @@ func (r *MetalStackMachineReconciler) machineToMetalStackMachine(log logr.Logger } func (r *machineReconciler) reconcile() (ctrl.Result, error) { + if r.infraCluster.Spec.NodeNetworkID == nil { + // this should not happen because before setting this id the cluster status should not become ready, but we check it anyway + return ctrl.Result{}, errors.New("waiting until node network id was set to infrastructure cluster status") + } + if r.infraCluster.Spec.ControlPlaneEndpoint.Host == "" { return ctrl.Result{}, errors.New("waiting until control plane ip was set to infrastructure cluster spec") } @@ -492,7 +497,7 @@ func (r *machineReconciler) create() (*models.V1MachineResponse, error) { nws = []*models.V1MachineAllocationNetwork{ { Autoacquire: ptr.To(true), - Networkid: &r.infraCluster.Spec.NodeNetworkID, + Networkid: r.infraCluster.Spec.NodeNetworkID, }, } ) @@ -500,7 +505,7 @@ func (r *machineReconciler) create() (*models.V1MachineResponse, error) { resp, err := r.metalClient.Machine().AllocateMachine(metalmachine.NewAllocateMachineParamsWithContext(r.ctx).WithBody(&models.V1MachineAllocateRequest{ Partitionid: &r.infraCluster.Spec.Partition, Projectid: &r.infraCluster.Spec.ProjectID, - PlacementTags: []string{tag.New(tag.ClusterID, r.infraCluster.GetClusterID())}, + PlacementTags: []string{tag.New(tag.ClusterID, r.infraCluster.GetClusterName())}, Tags: append(r.machineTags(), r.additionalMachineTags()...), Name: r.infraMachine.Name, Hostname: r.infraMachine.Name, @@ -629,7 +634,7 @@ func (r *machineReconciler) patchMachineLabels(m *models.V1MachineResponse) { func (r *machineReconciler) machineTags() []string { tags := []string{ - tag.New(tag.ClusterID, r.infraCluster.Spec.NodeNetworkID), + tag.New(tag.ClusterID, *r.infraCluster.Spec.NodeNetworkID), tag.New(v1alpha1.TagInfraClusterResource, fmt.Sprintf("%s.%s", r.infraCluster.Namespace, r.infraCluster.Name)), tag.New(v1alpha1.TagInfraMachineResource, fmt.Sprintf("%s.%s", r.infraMachine.Namespace, r.infraMachine.Name)), } diff --git a/test/e2e/frmwrk/cluster_upgrade_kubernetes_test.go b/test/e2e/frmwrk/cluster_upgrade_kubernetes_test.go index dcf59e0..e3c9767 100644 --- a/test/e2e/frmwrk/cluster_upgrade_kubernetes_test.go +++ b/test/e2e/frmwrk/cluster_upgrade_kubernetes_test.go @@ -43,7 +43,7 @@ var _ = Describe("Upgrade Kubernetes Cluster Version", Ordered, Label("upgrade") WorkerMachineImage: e2eCtx.envOrVar("E2E_WORKER_MACHINE_IMAGE_PREFIX") + fromKubernetesVersion, WorkerMachineCount: 0, }) - ec.SetupMetalStackPreconditions(ctx) + ec.SetupControlPlaneIP(ctx) ec.SetupNamespace(ctx) cfg = e2eCtx.E2EConfig.DeepCopy() diff --git a/test/e2e/frmwrk/shared_cases.go b/test/e2e/frmwrk/shared_cases.go index e9f7984..ef62f96 100644 --- a/test/e2e/frmwrk/shared_cases.go +++ b/test/e2e/frmwrk/shared_cases.go @@ -12,7 +12,7 @@ import ( func createE2ECluster(ctx context.Context, e2eCtx *E2EContext, cfg ClusterConfig) *E2ECluster { ec := e2eCtx.NewE2ECluster(cfg) - ec.SetupMetalStackPreconditions(ctx) + ec.SetupControlPlaneIP(ctx) ec.SetupNamespace(ctx) ec.GenerateAndApplyClusterTemplate(ctx) diff --git a/test/e2e/frmwrk/shared_cluster.go b/test/e2e/frmwrk/shared_cluster.go index e819349..bdc208d 100644 --- a/test/e2e/frmwrk/shared_cluster.go +++ b/test/e2e/frmwrk/shared_cluster.go @@ -13,7 +13,6 @@ import ( . "github.com/onsi/gomega" //nolint:staticcheck metalip "github.com/metal-stack/metal-go/api/client/ip" - metalnetwork "github.com/metal-stack/metal-go/api/client/network" metalmodels "github.com/metal-stack/metal-go/api/models" corev1 "k8s.io/api/core/v1" @@ -81,7 +80,6 @@ func (e2e *E2ECluster) Variables() map[string]string { vars["NAMESPACE"] = e2e.NamespaceName vars["METAL_PROJECT_ID"] = e2e.E2EContext.Environment.projectID vars["METAL_PARTITION"] = e2e.E2EContext.Environment.partition - vars["METAL_NODE_NETWORK_ID"] = *e2e.Refs.NodeNetwork.ID vars["FIREWALL_MACHINE_SIZE"] = e2e.FirewallSize vars["FIREWALL_MACHINE_IMAGE"] = e2e.FirewallImage vars["FIREWALL_MACHINE_NETWORKS"] = "[" + strings.Join(e2e.FirewallNetworks, ",") + "]" @@ -91,6 +89,10 @@ func (e2e *E2ECluster) Variables() map[string]string { vars["WORKER_MACHINE_SIZE"] = e2e.WorkerMachineSize vars["WORKER_MACHINE_IMAGE"] = e2e.WorkerMachineImage + if e2e.Refs.NodeNetwork != nil { + vars["METAL_NODE_NETWORK_ID"] = *e2e.Refs.NodeNetwork.ID + } + return vars } @@ -124,51 +126,14 @@ func (e2e *E2ECluster) teardownNamespace(ctx context.Context) { e2e.Refs.Namespace = nil } -func (e2e *E2ECluster) SetupMetalStackPreconditions(ctx context.Context) { - By("Setup Preconditions") - e2e.setupNodeNetwork(ctx) - e2e.setupControlPlaneIP(ctx) -} - func (e2e *E2ECluster) Teardown(ctx context.Context) { e2e.teardownAddons(ctx) e2e.teardownCluster(ctx) e2e.teardownControlPlaneIP(ctx) - e2e.teardownNodeNetwork(ctx) e2e.teardownNamespace(ctx) } -func (e2e *E2ECluster) setupNodeNetwork(ctx context.Context) { - By("Setup Node Network") - - nar := &metalmodels.V1NetworkAllocateRequest{ - Partitionid: e2e.E2EContext.Environment.partition, - Projectid: e2e.E2EContext.Environment.projectID, - Name: e2e.ClusterName + "-node", - Description: fmt.Sprintf("Node network for %s", e2e.ClusterName), - Labels: map[string]string{ - "e2e-test": e2e.SpecName, - capmsv1alpha1.TagInfraClusterResource: e2e.NamespaceName + "." + e2e.ClusterName, - }, - } - net, err := e2e.E2EContext.Environment.Metal.Network().AllocateNetwork(metalnetwork.NewAllocateNetworkParamsWithContext(ctx).WithBody(nar), nil) - Expect(err).ToNot(HaveOccurred(), "failed to allocate node network") - - e2e.Refs.NodeNetwork = net.Payload -} - -func (e2e *E2ECluster) teardownNodeNetwork(ctx context.Context) { - if e2e.Refs.NodeNetwork == nil || e2e.Refs.NodeNetwork.ID == nil { - return - } - - _, err := e2e.E2EContext.Environment.Metal.Network().FreeNetwork(metalnetwork.NewFreeNetworkParamsWithContext(ctx).WithID(*e2e.Refs.NodeNetwork.ID), nil) - Expect(err).ToNot(HaveOccurred(), "failed to delete node network") - - e2e.Refs.NodeNetwork = nil -} - -func (e2e *E2ECluster) setupControlPlaneIP(ctx context.Context) { +func (e2e *E2ECluster) SetupControlPlaneIP(ctx context.Context) { if e2e.ControlPlaneIP != "" { return } @@ -212,7 +177,6 @@ func (e2e *E2ECluster) GenerateAndApplyClusterTemplate(ctx context.Context) { By("Generate cluster template") Expect(e2e.Refs.Namespace).NotTo(BeNil(), "namespace not created yet") - Expect(e2e.Refs.NodeNetwork).NotTo(BeNil(), "node network not created yet") workloadTempl := clusterctl.ConfigCluster(ctx, clusterctl.ConfigClusterInput{ Namespace: e2e.NamespaceName,